From 8534cd56d8633c9468f337372d78c436263f36f6 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Fri, 19 Oct 2012 01:35:46 -0400 Subject: Reduce the number of stat/fstat system calls made This address BZ 868086, https://bugzilla.redhat.com/show_bug.cgi?id=868086 This refactoring consolidates the code a bit, and uses os.stat calls to get all the information in one call when gathering the metadata for an object. The five stat system calls (invoked via all the os.path.*() calls) have been reduced to one. We also added a unit test for the one new behavior where get_object_metadata() will now throw an OSError exception if it has a problem stat'ing a file that exists. Change-Id: Iad5410c77938af68a47be757a3170abd201adeb0 BUG: 868086 Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/4112 Reviewed-by: Kaleb KEITHLEY Reviewed-by: Mohammed Junaid Tested-by: Anand Avati --- swift/1.4.8/plugins/utils.py | 64 +++++++++++------------------ swift/1.4.8/test/unit/plugins/test_utils.py | 9 ++++ 2 files changed, 34 insertions(+), 39 deletions(-) (limited to 'swift') diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py index 289420cb9..a25cbc3b8 100644 --- a/swift/1.4.8/plugins/utils.py +++ b/swift/1.4.8/plugins/utils.py @@ -525,50 +525,36 @@ def get_account_details(acc_path, memcache=None): return get_account_details_from_fs(acc_path, memcache) def get_etag(path): - etag = None - if os.path.exists(path): - etag = md5() - if not os.path.isdir(path): - fp = open(path, 'rb') - if fp: - while True: - chunk = fp.read(CHUNK_SIZE) - if chunk: - etag.update(chunk) - else: - break - fp.close() - - etag = etag.hexdigest() - - return etag - + etag = md5() + with open(path, 'rb') as fp: + while True: + chunk = fp.read(CHUNK_SIZE) + if chunk: + etag.update(chunk) + else: + break + return etag.hexdigest() def get_object_metadata(obj_path): """ Return metadata of object. """ - metadata = {} - if os.path.exists(obj_path): - if not os.path.isdir(obj_path): - metadata = { - X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)), - X_CONTENT_TYPE: FILE_TYPE, - X_ETAG: get_etag(obj_path), - X_CONTENT_LENGTH: os.path.getsize(obj_path), - X_TYPE: OBJECT, - X_OBJECT_TYPE: FILE, - } - else: - metadata = { - X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)), - X_CONTENT_TYPE: DIR_TYPE, - X_ETAG: get_etag(obj_path), - X_CONTENT_LENGTH: 0, - X_TYPE: OBJECT, - X_OBJECT_TYPE: DIR, - } - + try: + stats = os.stat(obj_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + metadata = {} + else: + is_dir = (stats.st_mode & 0040000) != 0 + metadata = { + X_TYPE: OBJECT, + X_TIMESTAMP: normalize_timestamp(stats.st_ctime), + X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, + X_OBJECT_TYPE: DIR if is_dir else FILE, + X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, + X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path), + } return metadata def _add_timestamp(metadata_i): diff --git a/swift/1.4.8/test/unit/plugins/test_utils.py b/swift/1.4.8/test/unit/plugins/test_utils.py index 944d11892..70081a7d2 100644 --- a/swift/1.4.8/test/unit/plugins/test_utils.py +++ b/swift/1.4.8/test/unit/plugins/test_utils.py @@ -312,6 +312,15 @@ class TestUtils(unittest.TestCase): md = utils.get_object_metadata("/tmp/doesNotEx1st") assert md == {} + def test_get_object_metadata_err(self): + tf = tempfile.NamedTemporaryFile() + try: + md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) + except OSError as e: + assert e.errno != errno.ENOENT + else: + self.fail("Expected exception") + obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) -- cgit