object-storage: swift-diff removal: hide obj only

Towards swift-diff removal: hide object_only behavior inside DiskDir class
objects. See BZ 870589 (https://bugzilla.redhat.com/show_bug.cgi?id=870589)

One of the last few reasons for having the swift diff is the need to update
the account and container information from the file system when the
configuration is not for object-only use (meaning, other updates to the file
system can come from clients other than UFO, which means UFO currently has to
generate the necessary metadata for each request instead of caching it).

This change moves that logic into the DiskDir class methods so that the
container and account server code does not have to track that.

Change-Id: If0970c77c72245facd09c2035083e681d6d80303
BUG: 870589
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4176
Reviewed-by: Pete Zaitcev <zaitcev@redhat.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Mohammed Junaid <junaid@redhat.com>

4 files changed, 47 insertions, 72 deletions

diff --git a/swift/1.4.8/gluster-swift-plugin.spec b/swift/1.4.8/gluster-swift-plugin.spec
index 50731bc844f..0dbf187e092 100644
--- a/swift/1.4.8/gluster-swift-plugin.spec
+++ b/swift/1.4.8/gluster-swift-plugin.spec
@@ -13,7 +13,7 @@
 %define _confdir     /etc/swift
 %define _swiftdir    /usr/lib/python2.6/site-packages/swift
 %define _ufo_version 1.0
-%define _ufo_release 6
+%define _ufo_release 7
 
 Summary  : GlusterFS Unified File and Object Storage.
 Name     : gluster-swift-plugin
diff --git a/swift/1.4.8/gluster-swift.spec b/swift/1.4.8/gluster-swift.spec
index 47fafcb3203..02dfe814934 100644
--- a/swift/1.4.8/gluster-swift.spec
+++ b/swift/1.4.8/gluster-swift.spec
@@ -4,7 +4,7 @@
 
 Name:             gluster-swift
 Version:          1.4.8
-Release:          6%{?dist}
+Release:          7%{?dist}
 Summary:          OpenStack Object Storage (swift)
 
 Group:            Development/Languages
@@ -152,7 +152,7 @@ mkdir -p doc/build
 #SPHINX_DEBUG=1 sphinx-1.0-build -b html doc/source doc/build/html
 #SPHINX_DEBUG=1 sphinx-1.0-build -b man doc/source doc/build/man
 #%endif
-# Fix hidden-file-or-dir warning 
+# Fix hidden-file-or-dir warning
 #rm doc/build/html/.buildinfo
 
 %install
@@ -324,7 +324,7 @@ fi
 
 %files doc
 %defattr(-,root,root,-)
-%doc LICENSE 
+%doc LICENSE
 #%doc doc/build/html
 
 %changelog
@@ -369,7 +369,7 @@ fi
 - Update to 1.4.0
 
 * Fri May 20 2011 David Nalley <david@gnsa.us> - 1.3.0-1
-- Update to 1.3.0 
+- Update to 1.3.0
 
 * Tue Feb 08 2011 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.1.0-2
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild
diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py
index af924f1fef9..6887bec6c13 100644
--- a/swift/1.4.8/plugins/DiskDir.py
+++ b/swift/1.4.8/plugins/DiskDir.py
@@ -17,11 +17,13 @@ import os, errno
 
 from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \
      validate_account, validate_container, check_valid_account, is_marker, \
-     get_container_details, get_account_details, create_container_metadata, \
+     get_container_details, get_account_details, get_container_metadata, \
+     create_container_metadata, \
      create_account_metadata, DEFAULT_GID, DEFAULT_UID, validate_object, \
      create_object_metadata, read_metadata, write_metadata, X_CONTENT_TYPE, \
      X_CONTENT_LENGTH, X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, \
      X_OBJECTS_COUNT, X_BYTES_USED, X_CONTAINER_COUNT, CONTAINER
+from swift.plugins import Glusterfs
 
 from swift.common.constraints import CONTAINER_LISTING_LIMIT, check_mount
 from swift.common.utils import normalize_timestamp, TRUE_VALUES
@@ -166,6 +168,8 @@ class DiskDir(DiskCommon):
         assert logger is not None
         self.logger = logger
         self.metadata = {}
+        self.container_info = None
+        self.object_info = None
         self.uid = int(uid)
         self.gid = int(gid)
         self.db_file = _db_file
@@ -281,13 +285,9 @@ class DiskDir(DiskCommon):
         if delimiter and not prefix:
             prefix = ''
 
-        objects, object_count, bytes_used = get_container_details(self.datadir)
+        self.update_object_count()
 
-        if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
-           int(self.metadata[X_BYTES_USED][0]) != bytes_used:
-            self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
-            self.metadata[X_BYTES_USED] = (bytes_used, 0)
-            self.update_container(self.metadata)
+        objects, object_count, bytes_used = self.object_info
 
         if objects:
             objects.sort()
@@ -326,32 +326,30 @@ class DiskDir(DiskCommon):
 
         return container_list
 
-    def update_container(self, metadata):
-        cont_path = self.datadir
-        write_metadata(cont_path, metadata)
-        self.metadata = metadata
-
     def update_object_count(self):
-        objects = []
-        object_count = 0
-        bytes_used = 0
-        objects, object_count, bytes_used = get_container_details(self.datadir)
+        if not self.object_info:
+            self.object_info = get_container_details(self.datadir)
 
-        if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
-           int(self.metadata[X_BYTES_USED][0]) != bytes_used:
+        objects, object_count, bytes_used = self.object_info
+
+        if X_OBJECTS_COUNT not in self.metadata \
+                or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
+                or X_BYTES_USED not in self.metadata \
+                or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
             self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
             self.metadata[X_BYTES_USED] = (bytes_used, 0)
-            self.update_container(self.metadata)
+            write_metadata(self.datadir, self.metadata)
 
     def update_container_count(self):
-        containers = []
-        container_count = 0
+        if not self.container_info:
+            self.container_info = get_account_details(self.datadir)
 
-        containers, container_count = get_account_details(self.datadir)
+        containers, container_count = self.container_info
 
-        if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
+        if X_CONTAINER_COUNT not in self.metadata \
+                or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
             self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
-            self.update_container(self.metadata)
+            write_metadata(self.datadir, self.metadata)
 
     def get_info(self, include_metadata=False):
         """
@@ -366,21 +364,21 @@ class DiskDir(DiskCommon):
         # TODO: delete_timestamp, reported_put_timestamp
         #       reported_delete_timestamp, reported_object_count,
         #       reported_bytes_used, created_at
-
-        metadata = {}
-        if os.path.exists(self.datadir):
-            metadata = _read_metadata(self.datadir)
+        if not Glusterfs.OBJECT_ONLY:
+            # If we are not configured for object only environments, we should
+            # update the object counts in case they changed behind our back.
+            self.update_object_count()
 
         data = {'account' : self.account, 'container' : self.container,
-                'object_count' : metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
-                'bytes_used' : metadata.get(X_BYTES_USED, ('0',0))[0],
+                'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
+                'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0],
                 'hash': '', 'id' : '', 'created_at' : '1',
-                'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
+                'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
                 'delete_timestamp' : '1',
                 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
                 'reported_object_count' : '1', 'reported_bytes_used' : '1'}
         if include_metadata:
-            data['metadata'] = metadata
+            data['metadata'] = self.metadata
         return data
 
     def put_object(self, name, timestamp, size, content_type,
@@ -432,11 +430,9 @@ class DiskAccount(DiskDir):
         if delimiter and not prefix:
             prefix = ''
 
-        containers, container_count = get_account_details(self.datadir)
+        self.update_container_count()
 
-        if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
-            self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
-            self.update_container(self.metadata)
+        containers, container_count = self.container_info
 
         if containers:
             containers.sort()
@@ -483,19 +479,18 @@ class DiskAccount(DiskDir):
                   delete_timestamp, container_count, object_count,
                   bytes_used, hash, id
         """
-        metadata = {}
-        if (os.path.exists(self.datadir)):
-            metadata = _read_metadata(self.datadir)
-            if not metadata:
-                metadata = create_account_metadata(self.datadir)
+        if not Glusterfs.OBJECT_ONLY:
+            # If we are not configured for object only environments, we should
+            # update the container counts in case they changed behind our back.
+            self.update_container_count()
 
         data = {'account' : self.account, 'created_at' : '1',
                 'put_timestamp' : '1', 'delete_timestamp' : '1',
-                'container_count' : metadata.get(X_CONTAINER_COUNT, (0,0))[0],
-                'object_count' : metadata.get(X_OBJECTS_COUNT, (0,0))[0],
-                'bytes_used' : metadata.get(X_BYTES_USED, (0,0))[0],
+                'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0],
+                'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0],
+                'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0],
                 'hash' : '', 'id' : ''}
 
         if include_metadata:
-            data['metadata'] = metadata
+            data['metadata'] = self.metadata
         return data
diff --git a/swift/1.4.8/swift.diff b/swift/1.4.8/swift.diff
index 5ce314f5525..c29199461e2 100644
--- a/swift/1.4.8/swift.diff
+++ b/swift/1.4.8/swift.diff
@@ -18,7 +18,7 @@ index d195d34..ef625ff 100644
          },
      )
 diff --git a/swift/account/server.py b/swift/account/server.py
-index 800b3c0..78fbf07 100644
+index 800b3c0..c34d967 100644
 --- a/swift/account/server.py
 +++ b/swift/account/server.py
 @@ -1,4 +1,5 @@
@@ -63,18 +63,8 @@ index 800b3c0..78fbf07 100644
          hsh = hash_path(account)
          db_dir = storage_directory(DATADIR, part, hsh)
          db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
-@@ -153,6 +164,9 @@ class AccountController(object):
-             broker.stale_reads_ok = True
-         if broker.is_deleted():
-             return HTTPNotFound(request=req)
-+        if Gluster_enabled() and not Glusterfs.OBJECT_ONLY:
-+            broker.list_containers_iter(None, None,None,
-+                                        None, None)
-         info = broker.get_info()
-         headers = {
-             'X-Account-Container-Count': info['container_count'],
 diff --git a/swift/container/server.py b/swift/container/server.py
-index 8a18cfd..e39f5ef 100644
+index 8a18cfd..f5346ce 100644
 --- a/swift/container/server.py
 +++ b/swift/container/server.py
 @@ -1,4 +1,5 @@
@@ -121,16 +111,6 @@ index 8a18cfd..e39f5ef 100644
          hsh = hash_path(account, container)
          db_dir = storage_directory(DATADIR, part, hsh)
          db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
-@@ -245,6 +257,9 @@ class ContainerController(object):
-         broker.stale_reads_ok = True
-         if broker.is_deleted():
-             return HTTPNotFound(request=req)
-+        if Gluster_enabled() and not Glusterfs.OBJECT_ONLY:
-+            broker.list_objects_iter(None, None, None, None,
-+                                     None, None)
-         info = broker.get_info()
-         headers = {
-             'X-Container-Object-Count': info['object_count'],
 diff --git a/swift/obj/server.py b/swift/obj/server.py
 index 9cca16b..7a671c2 100644
 --- a/swift/obj/server.py