From dbe793bcde5d0ba936eceb19bcb8a6f376a0dbc4 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 16 Oct 2012 23:27:21 -0400 Subject: object-storage: Refactor code to reduce Swift diffs carried The upstream Swift code base contains the following commit which adds the ability to modify constraint values using the swift.conf file: https://github.com/openstack/swift/commit/a2ac5efaa64f57fbbe059066c6c4636dfd0715c2 These changes rely on the above commit being back-ported to Swift 1.4.8 (see https://github.com/portante/swift/commit/fc2421b04022ac6bbe9d5014362ec5f99f94c5e0). As a result, a good number of differences we carry can be removed, since the provided swift.conf file now contains the values we need. Along with these changes, we add middleware to get constraints loaded properly (via monkey patching) until subclassing is implemented in a future commit. We further simplify the diffs to a minimal set by storing the timestamp in file system metadata and moving new files to the plugins/middleware directory. Note that the original "gluster" middleware was in the swift.diff file, and has been renamed to "glusterfs" and moved to the new plugins/middleware directory. The "gluster" middleware is now a temporary way to get the constraints properly loaded (both of these middleware pieces should disappear in future commits when we refactor further to subclass the Swift objects instead of patching them). Change-Id: I9dc00d6b6cdd64e277896d75c2fb06431c3e69cb BUG: 862052 Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/4093 Tested-by: Peter Portante Reviewed-by: Anand Avati --- swift/1.4.8/plugins/DiskDir.py | 111 +++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 43 deletions(-) (limited to 'swift/1.4.8/plugins/DiskDir.py') diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py index 28671be3cd9..e8024768d8b 100644 --- a/swift/1.4.8/plugins/DiskDir.py +++ b/swift/1.4.8/plugins/DiskDir.py @@ -18,7 +18,7 @@ import os from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \ validate_account, validate_container, check_valid_account, is_marker, \ get_container_details, get_account_details, create_container_metadata, \ - create_account_metadata, DEFAULT_GID, DEFAULT_UID, get_account_details, \ + create_account_metadata, DEFAULT_GID, DEFAULT_UID, \ validate_object, create_object_metadata, read_metadata, write_metadata from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ @@ -38,6 +38,26 @@ def strip_obj_storage_path(path, string='/mnt/gluster-object'): DATADIR = 'containers' +def _read_metadata(dd): + """ Filter read metadata so that it always returns a tuple that includes + some kind of timestamp. With 1.4.8 of the Swift integration the + timestamps were not stored. Here we fabricate timestamps for volumes + where the existing data has no timestamp (that is, stored data is not + a tuple), allowing us a measure of backward compatibility. + + FIXME: At this time it does not appear that the timestamps on each + metadata are used for much, so this should not hurt anything. + """ + metadata_i = read_metadata(dd) + metadata = {} + timestamp = 0 + for key, value in metadata_i.iteritems(): + if not isinstance(value, tuple): + value = (value, timestamp) + metadata[key] = value + return metadata + + class DiskCommon(object): def is_deleted(self): return not os.path.exists(self.datadir) @@ -112,6 +132,7 @@ class DiskCommon(object): write_metadata(acc_path, metadata) self.metadata = metadata + class DiskDir(DiskCommon): """ Manage object files on disk. @@ -153,7 +174,7 @@ class DiskDir(DiskCommon): self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) except EOFError: create_container_metadata(self.datadir) else: @@ -161,19 +182,19 @@ class DiskDir(DiskCommon): if container: if not self.metadata: create_container_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not self.metadata: create_account_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not validate_account(self.metadata): create_account_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) def empty(self): return dir_empty(self.datadir) @@ -187,7 +208,6 @@ class DiskDir(DiskCommon): rmdirs(self.datadir) self.dir_exists = False - def put_metadata(self, metadata): """ Write metadata to directory/container. @@ -209,34 +229,40 @@ class DiskDir(DiskCommon): self.dir_exists = True def put_obj(self, content_length, timestamp): - self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) + 1 + ocnt = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) self.metadata[X_PUT_TIMESTAMP] = timestamp - self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) + int(content_length) + bused = self.metadata[X_BYTES_USED][0] + self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) #TODO: define update_metadata instad of writing whole metadata again. self.put_metadata(self.metadata) def delete_obj(self, content_length): - self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) - 1 - self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) - int(content_length) + ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) + bused, timestamp = self.metadata[X_BYTES_USED] + self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) self.put_metadata(self.metadata) def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): """ For account server. """ - self.metadata[X_OBJECTS_COUNT] = 0 - self.metadata[X_BYTES_USED] = 0 - self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) + 1 - self.metadata[X_PUT_TIMESTAMP] = 1 + self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) + self.metadata[X_BYTES_USED] = (0, put_timestamp) + ccnt = self.metadata[X_CONTAINER_COUNT][0] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) + self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) self.put_metadata(self.metadata) def delete_container(self, object_count, bytes_used): """ For account server. """ - self.metadata[X_OBJECTS_COUNT] = 0 - self.metadata[X_BYTES_USED] = 0 - self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) - 1 + self.metadata[X_OBJECTS_COUNT] = (0, 0) + self.metadata[X_BYTES_USED] = (0, 0) + ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) self.put_metadata(self.metadata) def unlink(self): @@ -264,10 +290,10 @@ class DiskDir(DiskCommon): objects, object_count, bytes_used = get_container_details(self.datadir) - if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ - int(self.metadata[X_BYTES_USED]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = object_count - self.metadata[X_BYTES_USED] = bytes_used + if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ + int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) if objects: @@ -316,11 +342,10 @@ class DiskDir(DiskCommon): bytes_used = 0 objects, object_count, bytes_used = get_container_details(self.datadir) - - if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ - int(self.metadata[X_BYTES_USED]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = object_count - self.metadata[X_BYTES_USED] = bytes_used + if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ + int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) def update_container_count(self): @@ -329,8 +354,8 @@ class DiskDir(DiskCommon): containers, container_count = get_account_details(self.datadir) - if int(self.metadata[X_CONTAINER_COUNT]) != container_count: - self.metadata[X_CONTAINER_COUNT] = container_count + if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) self.update_account(self.metadata) def get_info(self, include_metadata=False): @@ -350,13 +375,13 @@ class DiskDir(DiskCommon): metadata = {} if os.path.exists(self.datadir): - metadata = read_metadata(self.datadir) + metadata = _read_metadata(self.datadir) data = {'account' : self.account, 'container' : self.name, - 'object_count' : metadata.get(X_OBJECTS_COUNT, '0'), - 'bytes_used' : metadata.get(X_BYTES_USED, '0'), + 'object_count' : metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], + 'bytes_used' : metadata.get(X_BYTES_USED, ('0',0))[0], 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'), + 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], 'delete_timestamp' : '1', 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', 'reported_object_count' : '1', 'reported_bytes_used' : '1'} @@ -401,7 +426,7 @@ class DiskAccount(DiskDir): self.datadir = os.path.join(self.root, self.account) if not check_mount(root, account): check_valid_account(account, fs_object) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) if not self.metadata or not validate_account(self.metadata): self.metadata = create_account_metadata(self.datadir) @@ -419,8 +444,8 @@ class DiskAccount(DiskDir): containers, container_count = get_account_details(self.datadir) - if int(self.metadata[X_CONTAINER_COUNT]) != container_count: - self.metadata[X_CONTAINER_COUNT] = container_count + if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) self.update_account(self.metadata) if containers: @@ -447,13 +472,13 @@ class DiskAccount(DiskDir): list_item = [] metadata = None list_item.append(cont) - metadata = read_metadata(self.datadir + '/' + cont) + metadata = _read_metadata(self.datadir + '/' + cont) if not metadata or not validate_container(metadata): metadata = create_container_metadata(self.datadir + '/' + cont) if metadata: - list_item.append(metadata[X_OBJECTS_COUNT]) - list_item.append(metadata[X_BYTES_USED]) + list_item.append(metadata[X_OBJECTS_COUNT][0]) + list_item.append(metadata[X_BYTES_USED][0]) list_item.append(0) account_list.append(list_item) @@ -468,15 +493,15 @@ class DiskAccount(DiskDir): """ metadata = {} if (os.path.exists(self.datadir)): - metadata = read_metadata(self.datadir) + metadata = _read_metadata(self.datadir) if not metadata: metadata = create_account_metadata(self.datadir) data = {'account' : self.account, 'created_at' : '1', 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : metadata.get(X_CONTAINER_COUNT, 0), - 'object_count' : metadata.get(X_OBJECTS_COUNT, 0), - 'bytes_used' : metadata.get(X_BYTES_USED, 0), + 'container_count' : metadata.get(X_CONTAINER_COUNT, (0,0))[0], + 'object_count' : metadata.get(X_OBJECTS_COUNT, (0,0))[0], + 'bytes_used' : metadata.get(X_BYTES_USED, (0,0))[0], 'hash' : '', 'id' : ''} if include_metadata: -- cgit