summaryrefslogtreecommitdiffstats
path: root/swift
diff options
context:
space:
mode:
authorPeter Portante <peter.portante@redhat.com>2012-10-16 23:27:21 -0400
committerAnand Avati <avati@redhat.com>2012-10-17 11:04:05 -0700
commitdbe793bcde5d0ba936eceb19bcb8a6f376a0dbc4 (patch)
treeff281eec3636f11366bc89ee2b82f865d54814b5 /swift
parent59dfcf15578e08731f80c1f0c88cd4b7cd79d23d (diff)
object-storage: Refactor code to reduce Swift diffs carried
The upstream Swift code base contains the following commit which adds the ability to modify constraint values using the swift.conf file: https://github.com/openstack/swift/commit/a2ac5efaa64f57fbbe059066c6c4636dfd0715c2 These changes rely on the above commit being back-ported to Swift 1.4.8 (see https://github.com/portante/swift/commit/fc2421b04022ac6bbe9d5014362ec5f99f94c5e0). As a result, a good number of differences we carry can be removed, since the provided swift.conf file now contains the values we need. Along with these changes, we add middleware to get constraints loaded properly (via monkey patching) until subclassing is implemented in a future commit. We further simplify the diffs to a minimal set by storing the timestamp in file system metadata and moving new files to the plugins/middleware directory. Note that the original "gluster" middleware was in the swift.diff file, and has been renamed to "glusterfs" and moved to the new plugins/middleware directory. The "gluster" middleware is now a temporary way to get the constraints properly loaded (both of these middleware pieces should disappear in future commits when we refactor further to subclass the Swift objects instead of patching them). Change-Id: I9dc00d6b6cdd64e277896d75c2fb06431c3e69cb BUG: 862052 Signed-off-by: Peter Portante <peter.portante@redhat.com> Reviewed-on: http://review.gluster.org/4093 Tested-by: Peter Portante <pportant@redhat.com> Reviewed-by: Anand Avati <avati@redhat.com>
Diffstat (limited to 'swift')
-rw-r--r--swift/1.4.8/gluster-swift-plugin.spec9
-rw-r--r--swift/1.4.8/plugins/DiskDir.py111
-rw-r--r--swift/1.4.8/plugins/__init__.py16
-rw-r--r--swift/1.4.8/plugins/conf/account-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/container-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/object-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/proxy-server.conf5
-rw-r--r--swift/1.4.8/plugins/conf/swift.conf85
-rw-r--r--swift/1.4.8/plugins/constraints.py98
-rw-r--r--swift/1.4.8/plugins/middleware/__init__.py0
-rw-r--r--swift/1.4.8/plugins/middleware/gluster.py40
-rw-r--r--swift/1.4.8/plugins/middleware/glusterfs.py55
-rw-r--r--swift/1.4.8/plugins/utils.py36
-rw-r--r--swift/1.4.8/swift.diff452
14 files changed, 379 insertions, 546 deletions
diff --git a/swift/1.4.8/gluster-swift-plugin.spec b/swift/1.4.8/gluster-swift-plugin.spec
index 746f75c5f5d..181666a3608 100644
--- a/swift/1.4.8/gluster-swift-plugin.spec
+++ b/swift/1.4.8/gluster-swift-plugin.spec
@@ -13,7 +13,7 @@
%define _confdir /etc/swift
%define _swiftdir /usr/lib/python2.6/site-packages/swift
%define _ufo_version 1.0
-%define _ufo_release 3
+%define _ufo_release 5
Summary : GlusterFS Unified File and Object Storage.
Name : gluster-swift-plugin
@@ -42,7 +42,7 @@ storage costs.
%install
rm -rf %{buildroot}
-mkdir -p %{buildroot}/%{_swiftdir}/plugins
+mkdir -p %{buildroot}/%{_swiftdir}/plugins/middleware
mkdir -p %{buildroot}/%{_confdir}/
cp constraints.py %{buildroot}/%{_swiftdir}/plugins
@@ -52,9 +52,14 @@ cp Glusterfs.py %{buildroot}/%{_swiftdir}/plugins
cp __init__.py %{buildroot}/%{_swiftdir}/plugins
cp utils.py %{buildroot}/%{_swiftdir}/plugins
+cp middleware/__init__.py %{buildroot}/%{_swiftdir}/plugins/middleware
+cp middleware/gluster.py %{buildroot}/%{_swiftdir}/plugins/middleware
+cp middleware/glusterfs.py %{buildroot}/%{_swiftdir}/plugins/middleware
+
cp -r conf/* %{buildroot}/%{_confdir}/
%files
%defattr(-,root,root)
+%{_swiftdir}/plugins/middleware
%{_swiftdir}/plugins
%{_confdir}/
diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py
index 28671be3cd9..e8024768d8b 100644
--- a/swift/1.4.8/plugins/DiskDir.py
+++ b/swift/1.4.8/plugins/DiskDir.py
@@ -18,7 +18,7 @@ import os
from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \
validate_account, validate_container, check_valid_account, is_marker, \
get_container_details, get_account_details, create_container_metadata, \
- create_account_metadata, DEFAULT_GID, DEFAULT_UID, get_account_details, \
+ create_account_metadata, DEFAULT_GID, DEFAULT_UID, \
validate_object, create_object_metadata, read_metadata, write_metadata
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
@@ -38,6 +38,26 @@ def strip_obj_storage_path(path, string='/mnt/gluster-object'):
DATADIR = 'containers'
+def _read_metadata(dd):
+ """ Filter read metadata so that it always returns a tuple that includes
+ some kind of timestamp. With 1.4.8 of the Swift integration the
+ timestamps were not stored. Here we fabricate timestamps for volumes
+ where the existing data has no timestamp (that is, stored data is not
+ a tuple), allowing us a measure of backward compatibility.
+
+ FIXME: At this time it does not appear that the timestamps on each
+ metadata are used for much, so this should not hurt anything.
+ """
+ metadata_i = read_metadata(dd)
+ metadata = {}
+ timestamp = 0
+ for key, value in metadata_i.iteritems():
+ if not isinstance(value, tuple):
+ value = (value, timestamp)
+ metadata[key] = value
+ return metadata
+
+
class DiskCommon(object):
def is_deleted(self):
return not os.path.exists(self.datadir)
@@ -112,6 +132,7 @@ class DiskCommon(object):
write_metadata(acc_path, metadata)
self.metadata = metadata
+
class DiskDir(DiskCommon):
"""
Manage object files on disk.
@@ -153,7 +174,7 @@ class DiskDir(DiskCommon):
self.dir_exists = os.path.exists(self.datadir)
if self.dir_exists:
try:
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
@@ -161,19 +182,19 @@ class DiskDir(DiskCommon):
if container:
if not self.metadata:
create_container_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
def empty(self):
return dir_empty(self.datadir)
@@ -187,7 +208,6 @@ class DiskDir(DiskCommon):
rmdirs(self.datadir)
self.dir_exists = False
-
def put_metadata(self, metadata):
"""
Write metadata to directory/container.
@@ -209,34 +229,40 @@ class DiskDir(DiskCommon):
self.dir_exists = True
def put_obj(self, content_length, timestamp):
- self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) + 1
+ ocnt = self.metadata[X_OBJECTS_COUNT][0]
+ self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp)
self.metadata[X_PUT_TIMESTAMP] = timestamp
- self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) + int(content_length)
+ bused = self.metadata[X_BYTES_USED][0]
+ self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp)
#TODO: define update_metadata instad of writing whole metadata again.
self.put_metadata(self.metadata)
def delete_obj(self, content_length):
- self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) - 1
- self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) - int(content_length)
+ ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0]
+ self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp)
+ bused, timestamp = self.metadata[X_BYTES_USED]
+ self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp)
self.put_metadata(self.metadata)
def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
"""
For account server.
"""
- self.metadata[X_OBJECTS_COUNT] = 0
- self.metadata[X_BYTES_USED] = 0
- self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) + 1
- self.metadata[X_PUT_TIMESTAMP] = 1
+ self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp)
+ self.metadata[X_BYTES_USED] = (0, put_timestamp)
+ ccnt = self.metadata[X_CONTAINER_COUNT][0]
+ self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp)
+ self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp)
self.put_metadata(self.metadata)
def delete_container(self, object_count, bytes_used):
"""
For account server.
"""
- self.metadata[X_OBJECTS_COUNT] = 0
- self.metadata[X_BYTES_USED] = 0
- self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) - 1
+ self.metadata[X_OBJECTS_COUNT] = (0, 0)
+ self.metadata[X_BYTES_USED] = (0, 0)
+ ccnt, timestamp = self.metadata[X_CONTAINER_COUNT]
+ self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp)
self.put_metadata(self.metadata)
def unlink(self):
@@ -264,10 +290,10 @@ class DiskDir(DiskCommon):
objects, object_count, bytes_used = get_container_details(self.datadir)
- if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
- int(self.metadata[X_BYTES_USED]) != bytes_used:
- self.metadata[X_OBJECTS_COUNT] = object_count
- self.metadata[X_BYTES_USED] = bytes_used
+ if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
+ int(self.metadata[X_BYTES_USED][0]) != bytes_used:
+ self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
+ self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
if objects:
@@ -316,11 +342,10 @@ class DiskDir(DiskCommon):
bytes_used = 0
objects, object_count, bytes_used = get_container_details(self.datadir)
-
- if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
- int(self.metadata[X_BYTES_USED]) != bytes_used:
- self.metadata[X_OBJECTS_COUNT] = object_count
- self.metadata[X_BYTES_USED] = bytes_used
+ if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
+ int(self.metadata[X_BYTES_USED][0]) != bytes_used:
+ self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
+ self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
def update_container_count(self):
@@ -329,8 +354,8 @@ class DiskDir(DiskCommon):
containers, container_count = get_account_details(self.datadir)
- if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
- self.metadata[X_CONTAINER_COUNT] = container_count
+ if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
+ self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
self.update_account(self.metadata)
def get_info(self, include_metadata=False):
@@ -350,13 +375,13 @@ class DiskDir(DiskCommon):
metadata = {}
if os.path.exists(self.datadir):
- metadata = read_metadata(self.datadir)
+ metadata = _read_metadata(self.datadir)
data = {'account' : self.account, 'container' : self.name,
- 'object_count' : metadata.get(X_OBJECTS_COUNT, '0'),
- 'bytes_used' : metadata.get(X_BYTES_USED, '0'),
+ 'object_count' : metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
+ 'bytes_used' : metadata.get(X_BYTES_USED, ('0',0))[0],
'hash': '', 'id' : '', 'created_at' : '1',
- 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'),
+ 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
'delete_timestamp' : '1',
'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
'reported_object_count' : '1', 'reported_bytes_used' : '1'}
@@ -401,7 +426,7 @@ class DiskAccount(DiskDir):
self.datadir = os.path.join(self.root, self.account)
if not check_mount(root, account):
check_valid_account(account, fs_object)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
if not self.metadata or not validate_account(self.metadata):
self.metadata = create_account_metadata(self.datadir)
@@ -419,8 +444,8 @@ class DiskAccount(DiskDir):
containers, container_count = get_account_details(self.datadir)
- if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
- self.metadata[X_CONTAINER_COUNT] = container_count
+ if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
+ self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
self.update_account(self.metadata)
if containers:
@@ -447,13 +472,13 @@ class DiskAccount(DiskDir):
list_item = []
metadata = None
list_item.append(cont)
- metadata = read_metadata(self.datadir + '/' + cont)
+ metadata = _read_metadata(self.datadir + '/' + cont)
if not metadata or not validate_container(metadata):
metadata = create_container_metadata(self.datadir + '/' + cont)
if metadata:
- list_item.append(metadata[X_OBJECTS_COUNT])
- list_item.append(metadata[X_BYTES_USED])
+ list_item.append(metadata[X_OBJECTS_COUNT][0])
+ list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
@@ -468,15 +493,15 @@ class DiskAccount(DiskDir):
"""
metadata = {}
if (os.path.exists(self.datadir)):
- metadata = read_metadata(self.datadir)
+ metadata = _read_metadata(self.datadir)
if not metadata:
metadata = create_account_metadata(self.datadir)
data = {'account' : self.account, 'created_at' : '1',
'put_timestamp' : '1', 'delete_timestamp' : '1',
- 'container_count' : metadata.get(X_CONTAINER_COUNT, 0),
- 'object_count' : metadata.get(X_OBJECTS_COUNT, 0),
- 'bytes_used' : metadata.get(X_BYTES_USED, 0),
+ 'container_count' : metadata.get(X_CONTAINER_COUNT, (0,0))[0],
+ 'object_count' : metadata.get(X_OBJECTS_COUNT, (0,0))[0],
+ 'bytes_used' : metadata.get(X_BYTES_USED, (0,0))[0],
'hash' : '', 'id' : ''}
if include_metadata:
diff --git a/swift/1.4.8/plugins/__init__.py b/swift/1.4.8/plugins/__init__.py
index 3d98c960c14..e69de29bb2d 100644
--- a/swift/1.4.8/plugins/__init__.py
+++ b/swift/1.4.8/plugins/__init__.py
@@ -1,16 +0,0 @@
-# Copyright (c) 2011 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from Glusterfs import Glusterfs
diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf b/swift/1.4.8/plugins/conf/account-server/1.conf
index 54cbf654003..a82bc35eaa4 100644
--- a/swift/1.4.8/plugins/conf/account-server/1.conf
+++ b/swift/1.4.8/plugins/conf/account-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster account-server
+pipeline = glusterfs account-server
[app:account-server]
use = egg:swift#account
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[account-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf b/swift/1.4.8/plugins/conf/container-server/1.conf
index 9e776838f58..e6d313c3872 100644
--- a/swift/1.4.8/plugins/conf/container-server/1.conf
+++ b/swift/1.4.8/plugins/conf/container-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster container-server
+pipeline = glusterfs container-server
[app:container-server]
use = egg:swift#container
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[container-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf b/swift/1.4.8/plugins/conf/object-server/1.conf
index f191cefcf01..665aa94be8b 100644
--- a/swift/1.4.8/plugins/conf/object-server/1.conf
+++ b/swift/1.4.8/plugins/conf/object-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster object-server
+pipeline = glusterfs object-server
[app:object-server]
use = egg:swift#object
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[object-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf b/swift/1.4.8/plugins/conf/proxy-server.conf
index 1fcde8e0d3a..9bec6425355 100644
--- a/swift/1.4.8/plugins/conf/proxy-server.conf
+++ b/swift/1.4.8/plugins/conf/proxy-server.conf
@@ -4,7 +4,7 @@ user = root
log_facility = LOG_LOCAL1
[pipeline:main]
-pipeline = healthcheck cache tempauth proxy-server
+pipeline = healthcheck cache tempauth gluster proxy-server
[app:proxy-server]
use = egg:swift#proxy
@@ -19,3 +19,6 @@ use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
+
+[filter:gluster]
+use = egg:swift#gluster \ No newline at end of file
diff --git a/swift/1.4.8/plugins/conf/swift.conf b/swift/1.4.8/plugins/conf/swift.conf
index f9864e35231..e506b6f54a8 100644
--- a/swift/1.4.8/plugins/conf/swift.conf
+++ b/swift/1.4.8/plugins/conf/swift.conf
@@ -1,7 +1,92 @@
[DEFAULT]
Enable_plugin = yes
+
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
swift_hash_path_suffix = gluster
+
+# The swift-constraints section sets the basic constraints on data
+# saved in the swift cluster.
+
+[swift-constraints]
+
+# max_file_size is the largest "normal" object that can be saved in
+# the cluster. This is also the limit on the size of each segment of
+# a "large" object when using the large object manifest support.
+# This value is set in bytes. Setting it to lower than 1MiB will cause
+# some tests to fail. It is STRONGLY recommended to leave this value at
+# the default (5 * 2**30 + 2).
+
+# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting
+# web service handle such a size? I think with UFO, we need to keep with the
+# default size from Swift and encourage users to research what size their web
+# services infrastructure can handle.
+
+max_file_size = 18446744073709551616
+
+
+# max_meta_name_length is the max number of bytes in the utf8 encoding
+# of the name portion of a metadata header.
+
+#max_meta_name_length = 128
+
+
+# max_meta_value_length is the max number of bytes in the utf8 encoding
+# of a metadata value
+
+#max_meta_value_length = 256
+
+
+# max_meta_count is the max number of metadata keys that can be stored
+# on a single account, container, or object
+
+#max_meta_count = 90
+
+
+# max_meta_overall_size is the max number of bytes in the utf8 encoding
+# of the metadata (keys + values)
+
+#max_meta_overall_size = 4096
+
+
+# max_object_name_length is the max number of bytes in the utf8 encoding of an
+# object name: Gluster FS can handle much longer file names, but the length
+# between the slashes of the URL is handled below. Remember that most web
+# clients can't handle anything greater than 2048, and those that do are
+# rather clumsy.
+
+max_object_name_length = 2048
+
+# max_object_name_component_length (GlusterFS) is the max number of bytes in
+# the utf8 encoding of an object name component (the part between the
+# slashes); this is a limit imposed by the underlying file system (for XFS it
+# is 255 bytes).
+
+max_object_name_component_length = 255
+
+# container_listing_limit is the default (and max) number of items
+# returned for a container listing request
+
+#container_listing_limit = 10000
+
+
+# account_listing_limit is the default (and max) number of items returned
+# for an account listing request
+
+#account_listing_limit = 10000
+
+
+# max_account_name_length is the max number of bytes in the utf8 encoding of
+# an account name: Gluster FS Filename limit (XFS limit?), must be the same
+# size as max_object_name_component_length above.
+
+max_account_name_length = 255
+
+
+# max_container_name_length is the max number of bytes in the utf8 encoding
+# of a container name: Gluster FS Filename limit (XFS limit?), must be the same
+# size as max_object_name_component_length above.
+
+max_container_name_length = 255
diff --git a/swift/1.4.8/plugins/constraints.py b/swift/1.4.8/plugins/constraints.py
index 26640717684..d522090829f 100644
--- a/swift/1.4.8/plugins/constraints.py
+++ b/swift/1.4.8/plugins/constraints.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2011 Red Hat, Inc.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,46 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
-import logging
-from swift.common.constraints import check_utf8, check_metadata
+import swift.common.constraints
-from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
- HTTPRequestEntityTooLarge
+MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int(
+ 'max_object_name_component_length', 255)
-
-#: Max file size allowed for objects
-MAX_FILE_SIZE = 0xffffffffffffffff
-#: Max length of the name of a key for metadata
-MAX_META_NAME_LENGTH = 128
-#: Max length of the value of a key for metadata
-MAX_META_VALUE_LENGTH = 256
-#: Max number of metadata items
-MAX_META_COUNT = 90
-#: Max overall size of metadata
-MAX_META_OVERALL_SIZE = 4096
-#: Max object name length
-MAX_OBJECT_NAME_LENGTH = 255
-#: Max object list length of a get request for a container
-CONTAINER_LISTING_LIMIT = 10000
-#: Max container list length of a get request for an account
-ACCOUNT_LISTING_LIMIT = 10000
-MAX_ACCOUNT_NAME_LENGTH = 255
-MAX_CONTAINER_NAME_LENGTH = 255
-
-def validate_obj_name(obj):
- if len(obj) > MAX_OBJECT_NAME_LENGTH:
- logging.error('Object name too long %s' % obj)
- return False
+def validate_obj_name_component(obj):
+ if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH:
+ return 'too long (%d)' % len(obj)
if obj == '.' or obj == '..':
- logging.error('Object name cannot be . or .. %s' % obj)
- return False
+ return 'cannot be . or ..'
+ return ''
- return True
+# Save the original
+__check_object_creation = swift.common.constraints.check_object_creation
-def check_object_creation(req, object_name):
+# Define our new one which invokes the original
+def gluster_check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
+ Monkey patches swift.common.constraints.check_object_creation, invoking
+ the original, and then adding an additional check for individual object
+ name components.
:param req: HTTP request object
:param object_name: name of object to be created
@@ -62,38 +44,18 @@ def check_object_creation(req, object_name):
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
- if req.content_length and req.content_length > MAX_FILE_SIZE:
- return HTTPRequestEntityTooLarge(body='Your request is too large.',
- request=req,
- content_type='text/plain')
- if req.content_length is None and \
- req.headers.get('transfer-encoding') != 'chunked':
- return HTTPLengthRequired(request=req)
- if 'X-Copy-From' in req.headers and req.content_length:
- return HTTPBadRequest(body='Copy requests require a zero byte body',
- request=req, content_type='text/plain')
- for obj in object_name.split('/'):
- if not validate_obj_name(obj):
- return HTTPBadRequest(body='Invalid object name %s' %
- (obj), request=req,
- content_type='text/plain')
- if 'Content-Type' not in req.headers:
- return HTTPBadRequest(request=req, content_type='text/plain',
- body='No content type')
- if not check_utf8(req.headers['Content-Type']):
- return HTTPBadRequest(request=req, body='Invalid Content-Type',
- content_type='text/plain')
- if 'x-object-manifest' in req.headers:
- value = req.headers['x-object-manifest']
- container = prefix = None
- try:
- container, prefix = value.split('/', 1)
- except ValueError:
- pass
- if not container or not prefix or '?' in value or '&' in value or \
- prefix[0] == '/':
- return HTTPBadRequest(
- request=req,
- body='X-Object-Manifest must in the format container/prefix')
- return check_metadata(req, 'object')
+ ret = __check_object_creation(req, object_name)
+
+ if ret is None:
+ for obj in object_name.split('/'):
+ reason = validate_obj_name(obj)
+ if reason:
+ bdy = 'Invalid object name "%s", component "%s" %s' \
+ % (object_name, obj, reason)
+ ret = HTTPBadRequest(body=bdy,
+ request=req,
+ content_type='text/plain')
+
+ return ret
+swift.common.constraints.check_object_creation = gluster_check_object_creation
diff --git a/swift/1.4.8/plugins/middleware/__init__.py b/swift/1.4.8/plugins/middleware/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/__init__.py
diff --git a/swift/1.4.8/plugins/middleware/gluster.py b/swift/1.4.8/plugins/middleware/gluster.py
new file mode 100644
index 00000000000..e0e65e35312
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/gluster.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Noop Middleware that simply allows us to monkey patch the constraints
+import swift.plugins.constraints
+
+class Gluster(object):
+ """
+ Noop middleware for use with the proxy server to get paste.deploy to load
+ this middleware such that the plugin constraints monkey patch the common
+ constraints ahead of their use.
+ """
+ def __init__(self, app, conf):
+ self.app = app
+ self.conf = conf
+
+ def __call__(self, env, start_response):
+ return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+ """Returns a WSGI filter app for use with paste.deploy."""
+ conf = global_conf.copy()
+ conf.update(local_conf)
+
+ def gluster_filter(app):
+ return Gluster(app, conf)
+ return gluster_filter
diff --git a/swift/1.4.8/plugins/middleware/glusterfs.py b/swift/1.4.8/plugins/middleware/glusterfs.py
new file mode 100644
index 00000000000..7472146406c
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/glusterfs.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Monkey patch constraints
+import swift.plugins.constraints
+from swift.plugins.Glusterfs import Glusterfs
+from ConfigParser import ConfigParser
+
+fs_conf = ConfigParser()
+if fs_conf.read('/etc/swift/fs.conf'):
+ try:
+ mount_path = fs_conf.get ('DEFAULT', 'mount_path')
+ except NoSectionError, NoOptionError:
+ # FIXME - How to log during module initialization
+ logger.exception(_('ERROR mount_path not present'))
+ mount_path = ''
+else:
+ mount_path = ''
+
+
+class Gluster(object):
+ """
+ Update the environment with keys that reflect GlusterFS middleware enabled
+ """
+ def __init__(self, app, conf):
+ self.app = app
+ self.conf = conf
+
+ def __call__(self, env, start_response):
+ env['Gluster_enabled'] = True
+ env['fs_object'] = Glusterfs()
+ env['root'] = mount_path
+ return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+ """Returns a WSGI filter app for use with paste.deploy."""
+ conf = global_conf.copy()
+ conf.update(local_conf)
+
+ def gluster_filter(app):
+ return Gluster(app, conf)
+ return gluster_filter
diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py
index 1c7eeb92f54..59e704702cd 100644
--- a/swift/1.4.8/plugins/utils.py
+++ b/swift/1.4.8/plugins/utils.py
@@ -18,8 +18,9 @@ import os
import errno
import xattr
from hashlib import md5
-from swift.common.utils import normalize_timestamp
+from swift.common.utils import normalize_timestamp, TRUE_VALUES
import cPickle as pickle
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
X_CONTENT_TYPE = 'Content-Type'
X_CONTENT_LENGTH = 'Content-Length'
@@ -174,6 +175,18 @@ def do_rename(old_path, new_path):
raise
return True
+def _add_timestamp(metadata_i):
+ # At this point we have a simple key/value dictionary, turn it into
+ # key/(value,timestamp) pairs.
+ timestamp = 0
+ metadata = {}
+ for key, value_i in metadata_i.iteritems():
+ if not isinstance(value_i, tuple):
+ metadata[key] = (value_i, timestamp)
+ else:
+ metadata[key] = value_i
+ return metadata
+
def read_metadata(path):
"""
Helper function to read the pickled metadata from a File/Directory .
@@ -301,7 +314,8 @@ def validate_container(metadata):
#logging.warn('validate_container: Metadata missing entries: %s' % metadata)
return False
- if metadata[X_TYPE] == CONTAINER:
+ (value, timestamp) = metadata[X_TYPE]
+ if value == CONTAINER:
return True
logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,))
@@ -321,7 +335,8 @@ def validate_account(metadata):
#logging.warn('validate_account: Metadata missing entries: %s' % metadata)
return False
- if metadata[X_TYPE] == ACCOUNT:
+ (value, timestamp) = metadata[X_TYPE]
+ if value == ACCOUNT:
return True
logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,))
@@ -552,7 +567,7 @@ def get_container_metadata(cont_path, memcache=None):
X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
- return metadata
+ return _add_timestamp(metadata)
def get_account_metadata(acc_path, memcache=None):
containers = []
@@ -564,7 +579,7 @@ def get_account_metadata(acc_path, memcache=None):
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
- return metadata
+ return _add_timestamp(metadata)
def restore_object(obj_path, metadata):
meta = read_metadata(obj_path)
@@ -623,3 +638,14 @@ def get_account_list(fs_object):
def get_account_id(account):
return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest()
+
+__swift_conf = ConfigParser()
+__swift_conf.read(os.path.join('/etc/swift', 'swift.conf'))
+try:
+ _plugin_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES
+except NoOptionError, NoSectionError:
+ _plugin_enabled = False
+del __swift_conf
+
+def plugin_enabled():
+ return _plugin_enabled
diff --git a/swift/1.4.8/swift.diff b/swift/1.4.8/swift.diff
index 943ec9f037e..0024d7b8a3a 100644
--- a/swift/1.4.8/swift.diff
+++ b/swift/1.4.8/swift.diff
@@ -1,51 +1,44 @@
diff --git a/setup.py b/setup.py
-index d195d34..b5b5ca2 100644
+index d195d34..ab236ee 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
#!/usr/bin/python
# Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
++# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
-@@ -94,6 +95,7 @@ setup(
+@@ -94,6 +95,8 @@ setup(
'tempurl=swift.common.middleware.tempurl:filter_factory',
'formpost=swift.common.middleware.formpost:filter_factory',
'name_check=swift.common.middleware.name_check:filter_factory',
-+ 'gluster=swift.common.middleware.gluster:filter_factory',
++ 'gluster=swift.plugins.middleware.gluster:filter_factory',
++ 'glusterfs=swift.plugins.middleware.glusterfs:filter_factory',
],
},
)
diff --git a/swift/account/server.py b/swift/account/server.py
-index 800b3c0..cb17970 100644
+index 800b3c0..77f9879 100644
--- a/swift/account/server.py
+++ b/swift/account/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
++# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
-@@ -31,7 +32,7 @@ import simplejson
-
- from swift.common.db import AccountBroker
- from swift.common.utils import get_logger, get_param, hash_path, \
-- normalize_timestamp, split_path, storage_directory
-+ normalize_timestamp, split_path, storage_directory, plugin_enabled
+@@ -35,6 +36,9 @@ from swift.common.utils import get_logger, get_param, hash_path, \
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8
from swift.common.db_replicator import ReplicatorRpc
-@@ -39,6 +40,8 @@ from swift.common.db_replicator import ReplicatorRpc
-
- DATADIR = 'accounts'
-
++from swift.plugins.utils import plugin_enabled
+if plugin_enabled():
+ from swift.plugins.DiskDir import DiskAccount
- class AccountController(object):
- """WSGI controller for the account server."""
-@@ -52,8 +55,12 @@ class AccountController(object):
+
+ DATADIR = 'accounts'
+@@ -52,8 +56,12 @@ class AccountController(object):
self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
@@ -58,26 +51,7 @@ index 800b3c0..cb17970 100644
hsh = hash_path(account)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
-@@ -121,9 +128,15 @@ class AccountController(object):
- if broker.is_deleted():
- return HTTPConflict(request=req)
- metadata = {}
-- metadata.update((key, (value, timestamp))
-- for key, value in req.headers.iteritems()
-- if key.lower().startswith('x-account-meta-'))
-+ if not self.fs_object:
-+ metadata.update((key, (value, timestamp))
-+ for key, value in req.headers.iteritems()
-+ if key.lower().startswith('x-account-meta-'))
-+ else:
-+ metadata.update((key, value)
-+ for key, value in req.headers.iteritems()
-+ if key.lower().startswith('x-account-meta-'))
-+
- if metadata:
- broker.update_metadata(metadata)
- if created:
-@@ -153,6 +166,9 @@ class AccountController(object):
+@@ -153,6 +161,9 @@ class AccountController(object):
broker.stale_reads_ok = True
if broker.is_deleted():
return HTTPNotFound(request=req)
@@ -87,69 +61,7 @@ index 800b3c0..cb17970 100644
info = broker.get_info()
headers = {
'X-Account-Container-Count': info['container_count'],
-@@ -164,9 +180,16 @@ class AccountController(object):
- container_ts = broker.get_container_timestamp(container)
- if container_ts is not None:
- headers['X-Container-Timestamp'] = container_ts
-- headers.update((key, value)
-- for key, (value, timestamp) in broker.metadata.iteritems()
-- if value != '')
-+ if not self.fs_object:
-+ headers.update((key, value)
-+ for key, (value, timestamp) in broker.metadata.iteritems()
-+ if value != '')
-+ else:
-+ headers.update((key, value)
-+ for key, value in broker.metadata.iteritems()
-+ if value != '')
-+
-+
- return HTTPNoContent(request=req, headers=headers)
-
- def GET(self, req):
-@@ -190,9 +213,15 @@ class AccountController(object):
- 'X-Account-Bytes-Used': info['bytes_used'],
- 'X-Timestamp': info['created_at'],
- 'X-PUT-Timestamp': info['put_timestamp']}
-- resp_headers.update((key, value)
-- for key, (value, timestamp) in broker.metadata.iteritems()
-- if value != '')
-+ if not self.fs_object:
-+ resp_headers.update((key, value)
-+ for key, (value, timestamp) in broker.metadata.iteritems()
-+ if value != '')
-+ else:
-+ resp_headers.update((key, value)
-+ for key, value in broker.metadata.iteritems()
-+ if value != '')
-+
- try:
- prefix = get_param(req, 'prefix')
- delimiter = get_param(req, 'delimiter')
-@@ -224,6 +253,7 @@ class AccountController(object):
- content_type='text/plain', request=req)
- account_list = broker.list_containers_iter(limit, marker, end_marker,
- prefix, delimiter)
-+
- if out_content_type == 'application/json':
- json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
- json_pattern = '{' + ','.join(json_pattern) + '}'
-@@ -298,15 +328,29 @@ class AccountController(object):
- return HTTPNotFound(request=req)
- timestamp = normalize_timestamp(req.headers['x-timestamp'])
- metadata = {}
-- metadata.update((key, (value, timestamp))
-- for key, value in req.headers.iteritems()
-- if key.lower().startswith('x-account-meta-'))
-+ if not self.fs_object:
-+ metadata.update((key, (value, timestamp))
-+ for key, value in req.headers.iteritems()
-+ if key.lower().startswith('x-account-meta-'))
-+ else:
-+ metadata.update((key, value)
-+ for key, value in req.headers.iteritems()
-+ if key.lower().startswith('x-account-meta-'))
- if metadata:
+@@ -305,8 +316,17 @@ class AccountController(object):
broker.update_metadata(metadata)
return HTTPNoContent(request=req)
@@ -167,120 +79,27 @@ index 800b3c0..cb17970 100644
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
-diff --git a/swift/common/middleware/gluster.py b/swift/common/middleware/gluster.py
-new file mode 100644
-index 0000000..341285d
---- /dev/null
-+++ b/swift/common/middleware/gluster.py
-@@ -0,0 +1,55 @@
-+# Copyright (c) 2011 Red Hat, Inc.
-+#
-+# Licensed under the Apache License, Version 2.0 (the "License");
-+# you may not use this file except in compliance with the License.
-+# You may obtain a copy of the License at
-+#
-+# http://www.apache.org/licenses/LICENSE-2.0
-+#
-+# Unless required by applicable law or agreed to in writing, software
-+# distributed under the License is distributed on an "AS IS" BASIS,
-+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-+# implied.
-+# See the License for the specific language governing permissions and
-+# limitations under the License.
-+
-+from swift.common.utils import get_logger, plugin_enabled
-+from swift import plugins
-+from ConfigParser import ConfigParser
-+
-+class Gluster_plugin(object):
-+ """
-+ Update the environment with keys that reflect Gluster_plugin enabled
-+ """
-+
-+ def __init__(self, app, conf):
-+ self.app = app
-+ self.conf = conf
-+ self.fs_name = 'Glusterfs'
-+ self.logger = get_logger(conf, log_route='gluster')
-+
-+ def __call__(self, env, start_response):
-+ if not plugin_enabled():
-+ return self.app(env, start_response)
-+ env['Gluster_enabled'] =True
-+ fs_object = getattr(plugins, self.fs_name, False)
-+ if not fs_object:
-+ raise Exception('%s plugin not found', self.fs_name)
-+
-+ env['fs_object'] = fs_object()
-+ fs_conf = ConfigParser()
-+ if fs_conf.read('/etc/swift/fs.conf'):
-+ try:
-+ env['root'] = fs_conf.get ('DEFAULT', 'mount_path')
-+ except NoSectionError, NoOptionError:
-+ self.logger.exception(_('ERROR mount_path not present'))
-+ return self.app(env, start_response)
-+
-+def filter_factory(global_conf, **local_conf):
-+ """Returns a WSGI filter app for use with paste.deploy."""
-+ conf = global_conf.copy()
-+ conf.update(local_conf)
-+
-+ def gluster_filter(app):
-+ return Gluster_plugin(app, conf)
-+ return gluster_filter
-diff --git a/swift/common/utils.py b/swift/common/utils.py
-index 47edce8..03701ce 100644
---- a/swift/common/utils.py
-+++ b/swift/common/utils.py
-@@ -1,4 +1,5 @@
- # Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
-@@ -1138,3 +1139,11 @@ def streq_const_time(s1, s2):
- for (a, b) in zip(s1, s2):
- result |= ord(a) ^ ord(b)
- return result == 0
-+
-+def plugin_enabled():
-+ swift_conf = ConfigParser()
-+ swift_conf.read(os.path.join('/etc/swift', 'swift.conf'))
-+ try:
-+ return swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES
-+ except NoOptionError, NoSectionError:
-+ return False
diff --git a/swift/container/server.py b/swift/container/server.py
-index 8a18cfd..93943a3 100644
+index 8a18cfd..952b8cd 100644
--- a/swift/container/server.py
+++ b/swift/container/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
++# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
-@@ -31,7 +32,8 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
-
- from swift.common.db import ContainerBroker
- from swift.common.utils import get_logger, get_param, hash_path, \
-- normalize_timestamp, storage_directory, split_path, validate_sync_to
-+ normalize_timestamp, storage_directory, split_path, validate_sync_to, \
-+ plugin_enabled
- from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
- check_mount, check_float, check_utf8
+@@ -37,6 +38,9 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
from swift.common.bufferedhttp import http_connect
-@@ -40,6 +42,9 @@ from swift.common.db_replicator import ReplicatorRpc
-
- DATADIR = 'containers'
-
+ from swift.common.exceptions import ConnectionTimeout
+ from swift.common.db_replicator import ReplicatorRpc
++from swift.plugins.utils import plugin_enabled
+if plugin_enabled():
+ from swift.plugins.DiskDir import DiskDir
-+
- class ContainerController(object):
- """WSGI Controller for the container server."""
-@@ -62,6 +67,7 @@ class ContainerController(object):
+ DATADIR = 'containers'
+
+@@ -62,6 +66,7 @@ class ContainerController(object):
ContainerBroker, self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
@@ -288,7 +107,7 @@ index 8a18cfd..93943a3 100644
def _get_container_broker(self, drive, part, account, container):
"""
-@@ -73,6 +79,11 @@ class ContainerController(object):
+@@ -73,6 +78,10 @@ class ContainerController(object):
:param container: container name
:returns: ContainerBroker object
"""
@@ -296,128 +115,20 @@ index 8a18cfd..93943a3 100644
+ return DiskDir(self.root, drive, part, account,
+ container, self.logger,
+ fs_object = self.fs_object)
-+
hsh = hash_path(account, container)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
-@@ -211,10 +222,18 @@ class ContainerController(object):
- if broker.is_deleted():
- return HTTPConflict(request=req)
- metadata = {}
-- metadata.update((key, (value, timestamp))
-- for key, value in req.headers.iteritems()
-- if key.lower() in self.save_headers or
-- key.lower().startswith('x-container-meta-'))
-+ #Note: check the structure of req.headers
-+ if not self.fs_object:
-+ metadata.update((key, (value, timestamp))
-+ for key, value in req.headers.iteritems()
-+ if key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-'))
-+ else:
-+ metadata.update((key, value)
-+ for key, value in req.headers.iteritems()
-+ if key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-'))
-+
- if metadata:
- if 'X-Container-Sync-To' in metadata:
- if 'X-Container-Sync-To' not in broker.metadata or \
-@@ -222,6 +241,7 @@ class ContainerController(object):
- broker.metadata['X-Container-Sync-To'][0]:
- broker.set_x_container_sync_points(-1, -1)
- broker.update_metadata(metadata)
-+
- resp = self.account_update(req, account, container, broker)
- if resp:
- return resp
-@@ -245,6 +265,11 @@ class ContainerController(object):
+@@ -245,6 +254,9 @@ class ContainerController(object):
broker.stale_reads_ok = True
if broker.is_deleted():
return HTTPNotFound(request=req)
-+
+ if self.fs_object and not self.fs_object.object_only:
+ broker.list_objects_iter(None, None, None, None,
+ None, None)
-+
info = broker.get_info()
headers = {
'X-Container-Object-Count': info['object_count'],
-@@ -252,10 +277,17 @@ class ContainerController(object):
- 'X-Timestamp': info['created_at'],
- 'X-PUT-Timestamp': info['put_timestamp'],
- }
-- headers.update((key, value)
-- for key, (value, timestamp) in broker.metadata.iteritems()
-- if value != '' and (key.lower() in self.save_headers or
-- key.lower().startswith('x-container-meta-')))
-+ if not self.fs_object:
-+ headers.update((key, value)
-+ for key, (value, timestamp) in broker.metadata.iteritems()
-+ if value != '' and (key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-')))
-+ else:
-+ headers.update((key, value)
-+ for key, value in broker.metadata.iteritems()
-+ if value != '' and (key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-')))
-+
- return HTTPNoContent(request=req, headers=headers)
-
- def GET(self, req):
-@@ -268,6 +300,7 @@ class ContainerController(object):
- request=req)
- if self.mount_check and not check_mount(self.root, drive):
- return Response(status='507 %s is not mounted' % drive)
-+
- broker = self._get_container_broker(drive, part, account, container)
- broker.pending_timeout = 0.1
- broker.stale_reads_ok = True
-@@ -280,10 +313,17 @@ class ContainerController(object):
- 'X-Timestamp': info['created_at'],
- 'X-PUT-Timestamp': info['put_timestamp'],
- }
-- resp_headers.update((key, value)
-- for key, (value, timestamp) in broker.metadata.iteritems()
-- if value != '' and (key.lower() in self.save_headers or
-- key.lower().startswith('x-container-meta-')))
-+ if not self.fs_object:
-+ resp_headers.update((key, value)
-+ for key, (value, timestamp) in broker.metadata.iteritems()
-+ if value != '' and (key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-')))
-+ else:
-+ resp_headers.update((key, value)
-+ for key, value in broker.metadata.iteritems()
-+ if value != '' and (key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-')))
-+
- try:
- path = get_param(req, 'path')
- prefix = get_param(req, 'prefix')
-@@ -414,10 +454,17 @@ class ContainerController(object):
- return HTTPNotFound(request=req)
- timestamp = normalize_timestamp(req.headers['x-timestamp'])
- metadata = {}
-- metadata.update((key, (value, timestamp))
-- for key, value in req.headers.iteritems()
-- if key.lower() in self.save_headers or
-- key.lower().startswith('x-container-meta-'))
-+ if not self.fs_object:
-+ metadata.update((key, (value, timestamp))
-+ for key, value in req.headers.iteritems()
-+ if key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-'))
-+ else:
-+ metadata.update((key, value)
-+ for key, value in req.headers.iteritems()
-+ if key.lower() in self.save_headers or
-+ key.lower().startswith('x-container-meta-'))
-+
- if metadata:
- if 'X-Container-Sync-To' in metadata:
- if 'X-Container-Sync-To' not in broker.metadata or \
-@@ -427,8 +474,19 @@ class ContainerController(object):
+@@ -427,8 +439,19 @@ class ContainerController(object):
broker.update_metadata(metadata)
return HTTPNoContent(request=req)
@@ -438,62 +149,37 @@ index 8a18cfd..93943a3 100644
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
diff --git a/swift/obj/server.py b/swift/obj/server.py
-index 9cca16b..a45daff 100644
+index 9cca16b..82eaa40 100644
--- a/swift/obj/server.py
+++ b/swift/obj/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
++# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
-@@ -26,6 +27,7 @@ from hashlib import md5
- from tempfile import mkstemp
- from urllib import unquote
- from contextlib import contextmanager
-+from ConfigParser import ConfigParser
-
- from webob import Request, Response, UTC
- from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
-@@ -37,16 +39,23 @@ from eventlet import sleep, Timeout, tpool
-
- from swift.common.utils import mkdirs, normalize_timestamp, \
- storage_directory, hash_path, renamer, fallocate, \
-- split_path, drop_buffer_cache, get_logger, write_pickle
-+ split_path, drop_buffer_cache, get_logger, write_pickle, \
-+ plugin_enabled
- from swift.common.bufferedhttp import http_connect
--from swift.common.constraints import check_object_creation, check_mount, \
-- check_float, check_utf8
-+if plugin_enabled():
-+ from swift.plugins.constraints import check_object_creation
-+ from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \
-+ OBJECT, DIR_TYPE, FILE_TYPE
-+else:
-+ from swift.common.constraints import check_object_creation
-+
-+from swift.common.constraints import check_mount, check_float, check_utf8
-+
- from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
+@@ -45,6 +46,10 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
DiskFileNotExist
from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \
quarantine_renamer
++from swift.plugins.utils import plugin_enabled
++if plugin_enabled():
++ from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \
++ OBJECT, DIR_TYPE, FILE_TYPE
+
--
DATADIR = 'objects'
- ASYNCDIR = 'async_pending'
- PICKLE_PROTOCOL = 2
-@@ -339,6 +348,9 @@ class DiskFile(object):
- raise
+@@ -340,6 +345,9 @@ class DiskFile(object):
raise DiskFileNotExist('Data File does not exist.')
+
+if plugin_enabled():
+ from swift.plugins.DiskFile import Gluster_DiskFile
+
-
class ObjectController(object):
"""Implements the WSGI application for the Swift Object Server."""
-@@ -377,6 +389,17 @@ class ObjectController(object):
+
+@@ -377,6 +385,17 @@ class ObjectController(object):
'expiring_objects'
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
@@ -511,7 +197,7 @@ index 9cca16b..a45daff 100644
def async_update(self, op, account, container, obj, host, partition,
contdevice, headers_out, objdevice):
-@@ -493,7 +516,7 @@ class ObjectController(object):
+@@ -493,7 +512,7 @@ class ObjectController(object):
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
@@ -520,7 +206,7 @@ index 9cca16b..a45daff 100644
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if 'X-Delete-At' in file.metadata and \
-@@ -548,7 +571,7 @@ class ObjectController(object):
+@@ -548,7 +567,7 @@ class ObjectController(object):
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
@@ -529,7 +215,7 @@ index 9cca16b..a45daff 100644
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
-@@ -580,12 +603,29 @@ class ObjectController(object):
+@@ -580,12 +599,26 @@ class ObjectController(object):
if 'etag' in request.headers and \
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
@@ -545,27 +231,24 @@ index 9cca16b..a45daff 100644
+ if not self.fs_object:
+ metadata = {
+ 'X-Timestamp': request.headers['x-timestamp'],
-+ 'Content-Type': request.headers['content-type'],
++ 'Content-Type': content_type,
+ 'ETag': etag,
+ 'Content-Length': str(os.fstat(fd).st_size),
+ }
+ else:
++ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE
+ metadata = {
+ 'X-Timestamp': request.headers['x-timestamp'],
-+ 'Content-Type': request.headers['content-type'],
++ 'Content-Type': content_type,
+ 'ETag': etag,
+ 'Content-Length': str(os.fstat(fd).st_size),
+ X_TYPE: OBJECT,
-+ X_OBJECT_TYPE: FILE,
++ X_OBJECT_TYPE: x_object_type,
+ }
-+
-+ if self.fs_object and \
-+ request.headers['content-type'].lower() == DIR_TYPE:
-+ metadata.update({X_OBJECT_TYPE: MARKER_DIR})
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-') and
len(val[0]) > 14)
-@@ -612,7 +652,7 @@ class ObjectController(object):
+@@ -612,7 +645,7 @@ class ObjectController(object):
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
@@ -574,7 +257,7 @@ index 9cca16b..a45daff 100644
resp = HTTPCreated(request=request, etag=etag)
return resp
-@@ -626,9 +666,9 @@ class ObjectController(object):
+@@ -626,9 +659,9 @@ class ObjectController(object):
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
@@ -587,7 +270,7 @@ index 9cca16b..a45daff 100644
if file.is_deleted() or ('X-Delete-At' in file.metadata and
int(file.metadata['X-Delete-At']) <= time.time()):
if request.headers.get('if-match') == '*':
-@@ -702,7 +742,7 @@ class ObjectController(object):
+@@ -702,7 +735,7 @@ class ObjectController(object):
return resp
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
@@ -596,7 +279,7 @@ index 9cca16b..a45daff 100644
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if file.is_deleted() or ('X-Delete-At' in file.metadata and
int(file.metadata['X-Delete-At']) <= time.time()):
-@@ -744,7 +784,7 @@ class ObjectController(object):
+@@ -744,7 +777,7 @@ class ObjectController(object):
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
response_class = HTTPNoContent
@@ -605,7 +288,7 @@ index 9cca16b..a45daff 100644
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if 'x-if-delete-at' in request.headers and \
int(request.headers['x-if-delete-at']) != \
-@@ -797,9 +837,18 @@ class ObjectController(object):
+@@ -797,9 +830,18 @@ class ObjectController(object):
raise hashes
return Response(body=pickle.dumps(hashes))
@@ -624,38 +307,3 @@ index 9cca16b..a45daff 100644
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
-diff --git a/swift/proxy/server.py b/swift/proxy/server.py
-index 17613b8..d277d28 100644
---- a/swift/proxy/server.py
-+++ b/swift/proxy/server.py
-@@ -1,4 +1,5 @@
- # Copyright (c) 2010-2012 OpenStack, LLC.
-+# Copyright (c) 2011 Red Hat, Inc.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
-@@ -53,11 +54,20 @@ from webob import Request, Response
-
- from swift.common.ring import Ring
- from swift.common.utils import cache_from_env, ContextPool, get_logger, \
-- get_remote_client, normalize_timestamp, split_path, TRUE_VALUES
-+ get_remote_client, normalize_timestamp, split_path, TRUE_VALUES, \
-+ plugin_enabled
- from swift.common.bufferedhttp import http_connect
--from swift.common.constraints import check_metadata, check_object_creation, \
-- check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \
-- MAX_CONTAINER_NAME_LENGTH, MAX_FILE_SIZE
-+
-+if plugin_enabled():
-+ from swift.plugins.constraints import check_object_creation, \
-+ MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_FILE_SIZE
-+else:
-+ from swift.common.constraints import check_object_creation, \
-+ MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_FILE_SIZE
-+
-+from swift.common.constraints import check_metadata, check_utf8, \
-+ CONTAINER_LISTING_LIMIT
-+
- from swift.common.exceptions import ChunkReadTimeout, \
- ChunkWriteTimeout, ConnectionTimeout
-