From 189eaecf61091145389142b961a1c9f190299f60 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Sat, 12 Jan 2013 22:39:24 -0500 Subject: object-storage: yield during directory tree walks Have the co-routine (greenlet) yield during long running directory tree walks to avoid starving out other greenlets that might be available to run. A directory walk involves statting every file in the tree to get it size. For large numbers of files, this can get expensive. So we yield after every stat call, and after processing every directory. Change-Id: I07f1dfeef6a09b5817e0c237ecc748c491d52a31 BUG: 894674 Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/4380 Reviewed-by: Kaleb KEITHLEY Reviewed-by: Jeff Darcy Reviewed-by: Mohammed Junaid Tested-by: Mohammed Junaid --- ufo/gluster/swift/common/utils.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ufo/gluster/swift/common/utils.py b/ufo/gluster/swift/common/utils.py index d35abe58..795ddfa9 100644 --- a/ufo/gluster/swift/common/utils.py +++ b/ufo/gluster/swift/common/utils.py @@ -19,6 +19,7 @@ import errno import xattr import random from hashlib import md5 +from eventlet import sleep import cPickle as pickle from ConfigParser import ConfigParser, NoSectionError, NoOptionError from swift.common.utils import normalize_timestamp, TRUE_VALUES @@ -233,27 +234,30 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, # strip the prefix off, also stripping the leading and trailing slashes obj_path = path.replace(cont_path, '').strip(os.path.sep) - for i in src_list: + for obj_name in src_list: if obj_path: - obj_list.append(os.path.join(obj_path, i)) + obj_list.append(os.path.join(obj_path, obj_name)) else: - obj_list.append(i) + obj_list.append(obj_name) object_count += 1 if reg_file: - bytes_used += os.path.getsize(path + '/' + i) + bytes_used += os.path.getsize(os.path.join(path, obj_name)) + sleep() return object_count, bytes_used def update_list(path, cont_path, dirs=[], files=[], object_count=0, bytes_used=0, obj_list=[]): - object_count, bytes_used = _update_list(path, cont_path, files, True, - object_count, bytes_used, - obj_list) - object_count, bytes_used = _update_list(path, cont_path, dirs, False, - object_count, bytes_used, - obj_list) + if files: + object_count, bytes_used = _update_list(path, cont_path, files, True, + object_count, bytes_used, + obj_list) + if dirs: + object_count, bytes_used = _update_list(path, cont_path, dirs, False, + object_count, bytes_used, + obj_list) return object_count, bytes_used @@ -281,6 +285,7 @@ def _get_container_details_from_fs(cont_path): obj_list) dir_list.append((path, do_stat(path).st_mtime)) + sleep() return ContainerDetails(bytes_used, object_count, obj_list, dir_list) -- cgit