From 9e6fbbcee91105ea8e5fa67a86b4b5d3054e32aa Mon Sep 17 00:00:00 2001 From: Luis Pabon Date: Fri, 26 Jul 2013 15:56:26 -0400 Subject: perf: Container and account performance inc * Container and accounts performance increase by removing the need to update either the object count or the container count, respectively. New hidden configuratoins added to re-enable the functionality. * object_only configuratoin removed from fs.conf and replaced with a hidden configuration. The new hidden configuration reports gratuituosly crated directories as objects, to support a compatibility behavior with previous version 1.4.8 (even though it was incorrect). BUG 988969: https://bugzilla.redhat.com/show_bug.cgi?id=988969 Change-Id: Idca20b8629ec38606ff5692fe62bc2cadabffc86 Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/5403 Reviewed-by: Peter Portante Tested-by: Peter Portante Reviewed-on: http://review.gluster.org/5504 --- gluster/swift/common/DiskDir.py | 21 ++++--------------- gluster/swift/common/Glusterfs.py | 44 ++++++++++++++++++++++++++++++++------- gluster/swift/common/utils.py | 5 ++++- 3 files changed, 45 insertions(+), 25 deletions(-) (limited to 'gluster/swift') diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py index 4c835d3..556907f 100644 --- a/gluster/swift/common/DiskDir.py +++ b/gluster/swift/common/DiskDir.py @@ -373,7 +373,7 @@ class DiskDir(DiskCommon): # test cases working for now. if e.errno != errno.ENOENT: raise - if Glusterfs.OBJECT_ONLY and metadata \ + if not Glusterfs._implicit_dir_objects and metadata \ and metadata[X_CONTENT_TYPE] == DIR_TYPE \ and not dir_is_object(metadata): continue @@ -412,16 +412,8 @@ class DiskDir(DiskCommon): reported_put_timestamp, reported_delete_timestamp, reported_object_count, and reported_bytes_used. """ - if self._dir_exists: - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, - # we should update the object counts in case they changed - # behind our back. - self._update_object_count() - else: - # FIXME: to facilitate testing, we need to update all - # the time - self._update_object_count() + if self._dir_exists and Glusterfs._container_update_object_count: + self._update_object_count() data = {'account': self.account, 'container': self.container, 'object_count': self.metadata.get( @@ -697,12 +689,7 @@ class DiskAccount(DiskCommon): delete_timestamp, container_count, object_count, bytes_used, hash, id """ - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the container counts in case they changed behind our back. - self._update_container_count() - else: - # FIXME: to facilitate testing, we need to update all the time + if Glusterfs._account_update_container_count: self._update_container_count() data = {'account': self.account, 'created_at': '1', diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py index 01cfcc0..9ff54ba 100644 --- a/gluster/swift/common/Glusterfs.py +++ b/gluster/swift/common/Glusterfs.py @@ -31,23 +31,19 @@ from gluster.swift.common.exceptions import GlusterfsException, \ # _fs_conf = ConfigParser() MOUNT_IP = 'localhost' -OBJECT_ONLY = True RUN_DIR = '/var/run/swift' SWIFT_DIR = '/etc/swift' _do_getsize = False _allow_mount_per_server = False +_implicit_dir_objects = False +_container_update_object_count = False +_account_update_container_count = False if _fs_conf.read(os.path.join(SWIFT_DIR, 'fs.conf')): try: MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', MOUNT_IP) except (NoSectionError, NoOptionError): pass - try: - OBJECT_ONLY = _fs_conf.get('DEFAULT', - 'object_only', - "yes") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass try: RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', RUN_DIR) except (NoSectionError, NoOptionError): @@ -68,6 +64,40 @@ if _fs_conf.read(os.path.join(SWIFT_DIR, 'fs.conf')): except (NoSectionError, NoOptionError): pass + # -- Hidden configuration option -- + # Report gratuitously created directories as objects + # Directories can be gratuitously created on the path to a given + # object. This option turn on or off the reporting of those directories. + # It defaults to False so that only those directories explicitly + # created by the object server PUT REST API are reported + try: + _implicit_dir_objects = \ + _fs_conf.get('DEFAULT', + 'implicit_dir_objects', + "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + + # -- Hidden configuration option -- + # Due to the impact on performance, this option is disabled by default + try: + _container_update_object_count = \ + _fs_conf.get('DEFAULT', + 'container_update_object_count', + "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + + # -- Hidden configuration option -- + # Due to the impact on performance, this option is disabled by default + try: + _account_update_container_count = \ + _fs_conf.get('DEFAULT', + 'account_update_container_count', + "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + NAME = 'glusterfs' diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index 5152861..522d307 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -243,7 +243,10 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, obj_path = path.replace(cont_path, '').strip(os.path.sep) for obj_name in src_list: - if not reg_file and Glusterfs.OBJECT_ONLY: + # If it is not a reg_file then it is a directory. + if not reg_file and not Glusterfs._implicit_dir_objects: + # Now check if this is a dir object or a gratuiously crated + # directory metadata = \ read_metadata(os.path.join(cont_path, obj_path, obj_name)) if not dir_is_object(metadata): -- cgit