summaryrefslogtreecommitdiffstats
path: root/gluster
diff options
context:
space:
mode:
authorPeter Portante <peter.portante@redhat.com>2013-05-23 16:34:24 -0400
committerLuis Pabon <lpabon@redhat.com>2013-05-24 12:08:18 -0700
commitb29164198523591a69b234e869b7a1b94bd4f08e (patch)
tree04a3787ea446cdc42d282d5483ad21841c880f70 /gluster
parent3ff44850b4b0151b55028d45c8c77a521c478b2c (diff)
Add DiskDir unit test skeleton and pep8 filter
The new DiskDir unit test skeleton is quite incomplete, but gets the DiskDir module on the board for modules covered, explicitly exposing the fact that we need to get test coverage. This is a first step. At the same time, we also update all the modules we have applying the fix for pep8 errors now run under tox. We can then add a Jenkins pre-commit job to fail on pep8 errors. This brings our code to parity with what they are doing in OpenStack Swift. Change-Id: Ia0565606512efda6e73f67bd00269177b89db858 Signed-off-by: Peter Portante <peter.portante@redhat.com> Reviewed-on: http://review.gluster.org/5080 Reviewed-by: Luis Pabon <lpabon@redhat.com> Tested-by: Luis Pabon <lpabon@redhat.com>
Diffstat (limited to 'gluster')
-rw-r--r--gluster/swift/__init__.py1
-rw-r--r--gluster/swift/account/server.py2
-rw-r--r--gluster/swift/common/DiskDir.py95
-rw-r--r--gluster/swift/common/DiskFile.py43
-rw-r--r--gluster/swift/common/Glusterfs.py38
-rw-r--r--gluster/swift/common/constraints.py10
-rw-r--r--gluster/swift/common/exceptions.py5
-rw-r--r--gluster/swift/common/fs_utils.py30
-rw-r--r--gluster/swift/common/ring.py12
-rw-r--r--gluster/swift/common/utils.py93
-rw-r--r--gluster/swift/container/server.py2
-rw-r--r--gluster/swift/obj/server.py4
-rw-r--r--gluster/swift/proxy/server.py3
13 files changed, 203 insertions, 135 deletions
diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py
index 14bb4e8..fa360b6 100644
--- a/gluster/swift/__init__.py
+++ b/gluster/swift/__init__.py
@@ -1,5 +1,6 @@
""" Gluster for Swift """
+
class PkgInfo(object):
def __init__(self, canonical_version, name, final):
self.canonical_version = canonical_version
diff --git a/gluster/swift/account/server.py b/gluster/swift/account/server.py
index aeaabc9..3c6d616 100644
--- a/gluster/swift/account/server.py
+++ b/gluster/swift/account/server.py
@@ -17,7 +17,7 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.account import server
from gluster.swift.common.DiskDir import DiskAccount
diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py
index 18d08cc..5d67303 100644
--- a/gluster/swift/common/DiskDir.py
+++ b/gluster/swift/common/DiskDir.py
@@ -13,36 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os, errno
-
-from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \
- mkdirs, validate_account, validate_container, is_marker, \
- get_container_details, get_account_details, get_container_metadata, \
- create_container_metadata, create_account_metadata, DEFAULT_GID, \
- DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
- write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
- X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
- X_CONTAINER_COUNT, CONTAINER, os_path
+import os
+
+from gluster.swift.common.fs_utils import dir_empty, rmdirs, mkdirs, os_path
+from gluster.swift.common.utils import clean_metadata, validate_account, \
+ validate_container, get_container_details, get_account_details, \
+ create_container_metadata, create_account_metadata, DEFAULT_GID, \
+ DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
+ write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
+ X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
+ X_CONTAINER_COUNT
from gluster.swift.common import Glusterfs
-from swift.common.constraints import CONTAINER_LISTING_LIMIT
-from swift.common.utils import normalize_timestamp, TRUE_VALUES
-
DATADIR = 'containers'
-# Create a dummy db_file in /etc/swift
-_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no')
-if _unittests_enabled in TRUE_VALUES:
- _tmp_dir = '/tmp/gluster_unit_tests'
- try:
- os.mkdir(_tmp_dir)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- _db_file = os.path.join(_tmp_dir, 'db_file.db')
-else:
- _db_file = '/etc/swift/db_file.db'
+# Create a dummy db_file in Glusterfs.RUN_DIR
+_db_file = os.path.join(Glusterfs.RUN_DIR, 'db_file.db')
if not os.path.exists(_db_file):
file(_db_file, 'w+')
@@ -91,7 +78,7 @@ class DiskCommon(object):
Accept sorted list.
Objects should start with prefix.
"""
- filtered_objs=[]
+ filtered_objs = []
for object_name in objects:
tmp_obj = object_name.replace(prefix, '', 1)
sufix = tmp_obj.split(delimiter, 1)
@@ -106,8 +93,7 @@ class DiskCommon(object):
TODO: We can traverse in reverse order to optimize.
Accept sorted list.
"""
- filtered_objs=[]
- found = 0
+ filtered_objs = []
if objects[-1] < marker:
return filtered_objs
for object_name in objects:
@@ -120,7 +106,7 @@ class DiskCommon(object):
"""
Accept sorted list.
"""
- filtered_objs=[]
+ filtered_objs = []
for object_name in objects:
if object_name < end_marker:
filtered_objs.append(object_name)
@@ -130,7 +116,7 @@ class DiskCommon(object):
return filtered_objs
def filter_limit(self, objects, limit):
- filtered_objs=[]
+ filtered_objs = []
for i in range(0, limit):
filtered_objs.append(objects[i])
@@ -232,7 +218,8 @@ class DiskDir(DiskCommon):
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp)
self.metadata[X_PUT_TIMESTAMP] = timestamp
bused = self.metadata[X_BYTES_USED][0]
- self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp)
+ self.metadata[X_BYTES_USED] = (int(bused) + int(content_length),
+ timestamp)
#TODO: define update_metadata instad of writing whole metadata again.
self.put_metadata(self.metadata)
@@ -240,10 +227,12 @@ class DiskDir(DiskCommon):
ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0]
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp)
bused, timestamp = self.metadata[X_BYTES_USED]
- self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp)
+ self.metadata[X_BYTES_USED] = (int(bused) - int(content_length),
+ timestamp)
self.put_metadata(self.metadata)
- def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
+ def put_container(self, container, put_timestamp, del_timestamp,
+ object_count, bytes_used):
"""
For account server.
"""
@@ -363,20 +352,22 @@ class DiskDir(DiskCommon):
# update the object counts in case they changed behind our back.
self.update_object_count()
- data = {'account' : self.account, 'container' : self.container,
- 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
- 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0],
- 'hash': '', 'id' : '', 'created_at' : '1',
- 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
- 'delete_timestamp' : '1',
- 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
- 'reported_object_count' : '1', 'reported_bytes_used' : '1'}
+ data = {'account': self.account, 'container': self.container,
+ 'object_count': self.metadata.get(
+ X_OBJECTS_COUNT, ('0', 0))[0],
+ 'bytes_used': self.metadata.get(X_BYTES_USED, ('0', 0))[0],
+ 'hash': '', 'id': '', 'created_at': '1',
+ 'put_timestamp': self.metadata.get(
+ X_PUT_TIMESTAMP, ('0', 0))[0],
+ 'delete_timestamp': '1',
+ 'reported_put_timestamp': '1',
+ 'reported_delete_timestamp': '1',
+ 'reported_object_count': '1', 'reported_bytes_used': '1'}
if include_metadata:
data['metadata'] = self.metadata
return data
- def put_object(self, name, timestamp, size, content_type,
- etag, deleted=0):
+ def put_object(self, name, timestamp, size, content_type, etag, deleted=0):
# TODO: Implement the specifics of this func.
pass
@@ -401,7 +392,8 @@ class DiskDir(DiskCommon):
self.unlink()
def update_metadata(self, metadata):
- assert self.metadata, "Valid container/account metadata should have been created by now"
+ assert self.metadata, "Valid container/account metadata should have" \
+ " been created by now"
if metadata:
new_metadata = self.metadata.copy()
new_metadata.update(metadata)
@@ -478,12 +470,13 @@ class DiskAccount(DiskDir):
# update the container counts in case they changed behind our back.
self.update_container_count()
- data = {'account' : self.account, 'created_at' : '1',
- 'put_timestamp' : '1', 'delete_timestamp' : '1',
- 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0],
- 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0],
- 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0],
- 'hash' : '', 'id' : ''}
+ data = {'account': self.account, 'created_at': '1',
+ 'put_timestamp': '1', 'delete_timestamp': '1',
+ 'container_count': self.metadata.get(
+ X_CONTAINER_COUNT, (0, 0))[0],
+ 'object_count': self.metadata.get(X_OBJECTS_COUNT, (0, 0))[0],
+ 'bytes_used': self.metadata.get(X_BYTES_USED, (0, 0))[0],
+ 'hash': '', 'id': ''}
if include_metadata:
data['metadata'] = self.metadata
@@ -493,4 +486,4 @@ class DiskAccount(DiskDir):
cont_path = os.path.join(self.datadir, container)
metadata = read_metadata(cont_path)
- return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None
+ return int(metadata.get(X_PUT_TIMESTAMP, ('0', 0))[0]) or None
diff --git a/gluster/swift/common/DiskFile.py b/gluster/swift/common/DiskFile.py
index 900bd49..577147a 100644
--- a/gluster/swift/common/DiskFile.py
+++ b/gluster/swift/common/DiskFile.py
@@ -18,16 +18,16 @@ import errno
import random
from hashlib import md5
from contextlib import contextmanager
-from swift.common.utils import normalize_timestamp, renamer
+from swift.common.utils import renamer
from swift.common.exceptions import DiskFileNotExist
from gluster.swift.common.exceptions import AlreadyExistsAsDir
-from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \
- create_object_metadata, do_open, do_close, do_unlink, do_chown, \
- do_listdir, read_metadata, write_metadata, os_path, do_fsync
-from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \
- X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \
- X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \
- FILE_TYPE, DEFAULT_UID, DEFAULT_GID
+from gluster.swift.common.fs_utils import mkdirs, rmdirs, do_open, do_close, \
+ do_unlink, do_chown, os_path, do_fsync
+from gluster.swift.common.utils import read_metadata, write_metadata, \
+ validate_object, create_object_metadata
+from gluster.swift.common.utils import X_CONTENT_LENGTH, X_CONTENT_TYPE, \
+ X_TIMESTAMP, X_TYPE, X_OBJECT_TYPE, FILE, MARKER_DIR, OBJECT, DIR_TYPE, \
+ FILE_TYPE, DEFAULT_UID, DEFAULT_GID
import logging
from swift.obj.server import DiskFile
@@ -42,17 +42,18 @@ def _adjust_metadata(metadata):
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
# metadata values.
- content_type = metadata['Content-Type']
+ content_type = metadata[X_CONTENT_TYPE]
if not content_type:
# FIXME: How can this be that our caller supplied us with metadata
# that has a content type that evaluates to False?
#
# FIXME: If the file exists, we would already know it is a
# directory. So why are we assuming it is a file object?
- metadata['Content-Type'] = FILE_TYPE
+ metadata[X_CONTENT_TYPE] = FILE_TYPE
x_object_type = FILE
else:
- x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE
+ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE \
+ else FILE
metadata[X_TYPE] = OBJECT
metadata[X_OBJECT_TYPE] = x_object_type
return metadata
@@ -184,7 +185,8 @@ class Gluster_DiskFile(DiskFile):
if tombstone:
# We don't write tombstone files. So do nothing.
return
- assert self.data_file is not None, "put_metadata: no file to put metadata into"
+ assert self.data_file is not None, \
+ "put_metadata: no file to put metadata into"
metadata = _adjust_metadata(metadata)
write_metadata(self.data_file, metadata)
self.metadata = metadata
@@ -192,8 +194,8 @@ class Gluster_DiskFile(DiskFile):
def put(self, fd, metadata, extension='.data'):
"""
- Finalize writing the file on disk, and renames it from the temp file to
- the real location. This should be called after the data has been
+ Finalize writing the file on disk, and renames it from the temp file
+ to the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
@@ -202,7 +204,6 @@ class Gluster_DiskFile(DiskFile):
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
- extension = ''
metadata = _adjust_metadata(metadata)
@@ -220,7 +221,6 @@ class Gluster_DiskFile(DiskFile):
msg = 'File object exists as a directory: %s' % self.data_file
raise AlreadyExistsAsDir(msg)
- timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(self.tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
@@ -248,7 +248,7 @@ class Gluster_DiskFile(DiskFile):
:param timestamp: timestamp to compare with each file
"""
- if not self.metadata or self.metadata['X-Timestamp'] >= timestamp:
+ if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
return
assert self.data_file, \
@@ -257,7 +257,8 @@ class Gluster_DiskFile(DiskFile):
if self._is_dir:
# Marker directory object
if not rmdirs(self.data_file):
- logging.error('Unable to delete dir object: %s', self.data_file)
+ logging.error('Unable to delete dir object: %s',
+ self.data_file)
return
else:
# File object
@@ -283,7 +284,7 @@ class Gluster_DiskFile(DiskFile):
file_size = 0
if self.data_file:
file_size = os_path.getsize(self.data_file)
- if X_CONTENT_LENGTH in self.metadata:
+ if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
@@ -314,11 +315,11 @@ class Gluster_DiskFile(DiskFile):
path = self._container_path
subdir_list = self._obj_path.split(os.path.sep)
for i in range(len(subdir_list)):
- path = os.path.join(path, subdir_list[i]);
+ path = os.path.join(path, subdir_list[i])
if not os_path.exists(path):
self._create_dir_object(path)
- tmpfile = '.' + self._obj + '.' + md5(self._obj + \
+ tmpfile = '.' + self._obj + '.' + md5(self._obj +
str(random.random())).hexdigest()
self.tmppath = os.path.join(self.datadir, tmpfile)
diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py
index ba6bcf1..819a747 100644
--- a/gluster/swift/common/Glusterfs.py
+++ b/gluster/swift/common/Glusterfs.py
@@ -13,8 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+import fcntl
+import time
+import errno
import logging
-import os, sys, fcntl, time, errno
+
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.common.utils import TRUE_VALUES, search_tree
from gluster.swift.common.fs_utils import mkdirs
@@ -25,26 +29,29 @@ from gluster.swift.common.fs_utils import mkdirs
_fs_conf = ConfigParser()
MOUNT_IP = 'localhost'
OBJECT_ONLY = False
-RUN_DIR='/var/run/swift'
+RUN_DIR = '/var/run/swift'
SWIFT_DIR = '/etc/swift'
_do_getsize = False
-if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')):
+if _fs_conf.read(os.path.join(SWIFT_DIR, 'fs.conf')):
try:
- MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost')
+ MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', MOUNT_IP)
except (NoSectionError, NoOptionError):
pass
try:
- OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
+ OBJECT_ONLY = _fs_conf.get('DEFAULT',
+ 'object_only',
+ "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
try:
- RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift')
+ RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', RUN_DIR)
except (NoSectionError, NoOptionError):
pass
try:
- _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \
- "no") in TRUE_VALUES
+ _do_getsize = _fs_conf.get('DEFAULT',
+ 'accurate_size_in_listing',
+ "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
@@ -58,9 +65,11 @@ def _busy_wait(full_mount_path):
if os.path.ismount(full_mount_path):
return True
time.sleep(2)
- logging.error('Busy wait for mount timed out for mount %s', full_mount_path)
+ logging.error('Busy wait for mount timed out for mount %s',
+ full_mount_path)
return False
+
def mount(root, drive):
# FIXME: Possible thundering herd problem here
@@ -77,15 +86,15 @@ def mount(root, drive):
if not os.path.isdir(full_mount_path):
mkdirs(full_mount_path)
- lck_file = os.path.join(RUN_DIR, '%s.lock' %drive);
+ lck_file = os.path.join(RUN_DIR, '%s.lock' % drive)
if not os.path.exists(RUN_DIR):
mkdirs(RUN_DIR)
- fd = os.open(lck_file, os.O_CREAT|os.O_RDWR)
+ fd = os.open(lck_file, os.O_CREAT | os.O_RDWR)
with os.fdopen(fd, 'r+b') as f:
try:
- fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
+ fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as ex:
if ex.errno in (errno.EACCES, errno.EAGAIN):
# This means that some other process is mounting the
@@ -93,13 +102,14 @@ def mount(root, drive):
return _busy_wait(full_mount_path)
else:
raise ex
- mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \
+ mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export,
full_mount_path)
if os.system(mnt_cmd) or not _busy_wait(full_mount_path):
logging.error('Mount failed %s: %s', NAME, mnt_cmd)
return False
return True
+
def unmount(full_mount_path):
# FIXME: Possible thundering herd problem here
@@ -107,6 +117,7 @@ def unmount(full_mount_path):
if os.system(umnt_cmd):
logging.error('Unable to unmount %s %s' % (full_mount_path, NAME))
+
def _get_export_list():
cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP
@@ -126,6 +137,7 @@ def _get_export_list():
return export_list
+
def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"):
"""Read the object-server's configuration file and return
the device value"""
diff --git a/gluster/swift/common/constraints.py b/gluster/swift/common/constraints.py
index 11f626b..d1c990d 100644
--- a/gluster/swift/common/constraints.py
+++ b/gluster/swift/common/constraints.py
@@ -23,11 +23,12 @@ from gluster.swift.common import Glusterfs, ring
if hasattr(swift.common.constraints, 'constraints_conf_int'):
MAX_OBJECT_NAME_COMPONENT_LENGTH = \
- swift.common.constraints.constraints_conf_int(
- 'max_object_name_component_length', 255)
+ swift.common.constraints.constraints_conf_int(
+ 'max_object_name_component_length', 255)
else:
MAX_OBJECT_NAME_COMPONENT_LENGTH = 255
+
def validate_obj_name_component(obj):
if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH:
return 'too long (%d)' % len(obj)
@@ -38,6 +39,7 @@ def validate_obj_name_component(obj):
# Save the original check object creation
__check_object_creation = swift.common.constraints.check_object_creation
+
# Define our new one which invokes the original
def gluster_check_object_creation(req, object_name):
"""
@@ -61,7 +63,7 @@ def gluster_check_object_creation(req, object_name):
reason = validate_obj_name_component(obj)
if reason:
bdy = 'Invalid object name "%s", component "%s" %s' \
- % (object_name, obj, reason)
+ % (object_name, obj, reason)
ret = HTTPBadRequest(body=bdy,
request=req,
content_type='text/plain')
@@ -74,6 +76,7 @@ swift.common.constraints.check_object_creation = gluster_check_object_creation
# Save the original check mount
__check_mount = swift.common.constraints.check_mount
+
# Define our new one which invokes the original
def gluster_check_mount(root, drive):
# FIXME: Potential performance optimization here to not call the original
@@ -84,6 +87,7 @@ def gluster_check_mount(root, drive):
return Glusterfs.mount(root, drive)
+
# Replace the original check mount with ours
swift.common.constraints.check_mount = gluster_check_mount
diff --git a/gluster/swift/common/exceptions.py b/gluster/swift/common/exceptions.py
index d9357db..75a95ec 100644
--- a/gluster/swift/common/exceptions.py
+++ b/gluster/swift/common/exceptions.py
@@ -13,15 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
class GlusterfsException(Exception):
pass
+
class FileOrDirNotFoundError(GlusterfsException):
pass
+
class NotDirectoryError(GlusterfsException):
pass
+
class AlreadyExistsAsDir(GlusterfsException):
pass
-
diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py
index 0613a26..de450a5 100644
--- a/gluster/swift/common/fs_utils.py
+++ b/gluster/swift/common/fs_utils.py
@@ -16,14 +16,16 @@
import logging
import os
import errno
-import os.path as os_path
+import os.path as os_path # noqa
from eventlet import tpool
from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
NotDirectoryError
+
def do_walk(*args, **kwargs):
return os.walk(*args, **kwargs)
+
def do_write(fd, msg):
try:
cnt = os.write(fd, msg)
@@ -32,6 +34,7 @@ def do_write(fd, msg):
raise
return cnt
+
def do_mkdir(path):
try:
os.mkdir(path)
@@ -41,15 +44,18 @@ def do_mkdir(path):
raise
return True
+
def do_makedirs(path):
try:
os.makedirs(path)
except OSError as err:
if err.errno != errno.EEXIST:
- logging.exception("Makedirs failed on %s err: %s", path, err.strerror)
+ logging.exception("Makedirs failed on %s err: %s",
+ path, err.strerror)
raise
return True
+
def do_listdir(path):
try:
buf = os.listdir(path)
@@ -58,6 +64,7 @@ def do_listdir(path):
raise
return buf
+
def do_chown(path, uid, gid):
try:
os.chown(path, uid, gid)
@@ -66,6 +73,7 @@ def do_chown(path, uid, gid):
raise
return True
+
def do_stat(path):
try:
#Check for fd.
@@ -78,6 +86,7 @@ def do_stat(path):
raise
return buf
+
def do_open(path, mode):
if isinstance(mode, int):
try:
@@ -93,6 +102,7 @@ def do_open(path, mode):
raise
return fd
+
def do_close(fd):
#fd could be file or int type.
try:
@@ -105,16 +115,19 @@ def do_close(fd):
raise
return True
-def do_unlink(path, log = True):
+
+def do_unlink(path, log=True):
try:
os.unlink(path)
except OSError as err:
if err.errno != errno.ENOENT:
if log:
- logging.exception("Unlink failed on %s err: %s", path, err.strerror)
+ logging.exception("Unlink failed on %s err: %s",
+ path, err.strerror)
raise
return True
+
def do_rmdir(path):
try:
os.rmdir(path)
@@ -127,15 +140,17 @@ def do_rmdir(path):
res = True
return res
+
def do_rename(old_path, new_path):
try:
os.rename(old_path, new_path)
except OSError as err:
- logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \
- err.strerror)
+ logging.exception("Rename failed on %s to %s err: %s",
+ old_path, new_path, err.strerror)
raise
return True
+
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path
@@ -146,6 +161,7 @@ def mkdirs(path):
if not os.path.isdir(path):
do_makedirs(path)
+
def dir_empty(path):
"""
Return true if directory/container is empty.
@@ -159,6 +175,7 @@ def dir_empty(path):
raise FileOrDirNotFoundError()
raise NotDirectoryError()
+
def rmdirs(path):
if not os.path.isdir(path):
return False
@@ -170,6 +187,7 @@ def rmdirs(path):
return False
return True
+
def do_fsync(fd):
try:
tpool.execute(os.fsync, fd)
diff --git a/gluster/swift/common/ring.py b/gluster/swift/common/ring.py
index 06aab8d..f88af4e 100644
--- a/gluster/swift/common/ring.py
+++ b/gluster/swift/common/ring.py
@@ -36,13 +36,12 @@ if conf_files and _conf.read(conf_file):
if not reseller_prefix.endswith('_'):
reseller_prefix = reseller_prefix + '_'
+
class Ring(ring.Ring):
def _get_part_nodes(self, part):
seen_ids = set()
- nodes = [dev for dev in self._devs \
- if dev['device'] == self.acc_name \
- and not (dev['id'] in seen_ids \
- or seen_ids.add(dev['id']))]
+ nodes = [dev for dev in self._devs if dev['device'] == self.acc_name
+ and not (dev['id'] in seen_ids or seen_ids.add(dev['id']))]
if not nodes:
nodes = [self.false_node]
return nodes
@@ -86,8 +85,8 @@ class Ring(ring.Ring):
hardware description
====== ===============================================================
"""
- self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \
- 'meta': '', 'device': 'volume_not_in_ring', \
+ self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1',
+ 'id': 0, 'meta': '', 'device': 'volume_not_in_ring',
'port': 6012}
if account.startswith(reseller_prefix):
self.acc_name = account.replace(reseller_prefix, '', 1)
@@ -97,7 +96,6 @@ class Ring(ring.Ring):
part = 0
return part, self._get_part_nodes(part)
-
def get_more_nodes(self, part):
"""
Generator to get extra nodes for a partition for hinted handoff.
diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index f2cd8de..2cbb7f3 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -21,9 +21,9 @@ import random
from hashlib import md5
from eventlet import sleep
import cPickle as pickle
-from ConfigParser import ConfigParser, NoSectionError, NoOptionError
-from swift.common.utils import normalize_timestamp, TRUE_VALUES
-from gluster.swift.common.fs_utils import *
+from swift.common.utils import normalize_timestamp
+from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \
+ do_stat, do_listdir, do_walk
from gluster.swift.common import Glusterfs
X_CONTENT_TYPE = 'Content-Type'
@@ -54,8 +54,11 @@ DEFAULT_GID = -1
PICKLE_PROTOCOL = 2
CHUNK_SIZE = 65536
MEMCACHE_KEY_PREFIX = 'gluster.swift.'
-MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.'
-MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.'
+MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \
+ 'account.details.'
+MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \
+ 'container.details.'
+
def read_metadata(path):
"""
@@ -70,7 +73,8 @@ def read_metadata(path):
key = 0
while metadata is None:
try:
- metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or '')))
+ metadata_s += xattr.getxattr(path,
+ '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
if key > 0:
@@ -108,6 +112,7 @@ def read_metadata(path):
key += 1
return metadata
+
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
@@ -120,13 +125,17 @@ def write_metadata(path, metadata):
key = 0
while metastr:
try:
- xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE])
+ xattr.setxattr(path,
+ '%s%s' % (METADATA_KEY, key or ''),
+ metastr[:MAX_XATTR_SIZE])
except IOError as err:
- logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err))
+ logging.exception("setxattr failed on %s key %s err: %s",
+ path, key, str(err))
raise
metastr = metastr[MAX_XATTR_SIZE:]
key += 1
+
def clean_metadata(path):
key = 0
while True:
@@ -138,21 +147,25 @@ def clean_metadata(path):
raise
key += 1
+
def check_user_xattr(path):
if not os_path.exists(path):
return False
try:
xattr.setxattr(path, 'user.test.key1', 'value1')
except IOError as err:
- logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
+ logging.exception("check_user_xattr: set failed on %s err: %s",
+ path, str(err))
raise
try:
xattr.removexattr(path, 'user.test.key1')
except IOError as err:
- logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err))
+ logging.exception("check_user_xattr: remove failed on %s err: %s",
+ path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
+
def validate_container(metadata):
if not metadata:
logging.warn('validate_container: No metadata')
@@ -163,16 +176,17 @@ def validate_container(metadata):
X_PUT_TIMESTAMP not in metadata.keys() or \
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys():
- #logging.warn('validate_container: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == CONTAINER:
return True
- logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,))
+ logging.warn('validate_container: metadata type is not CONTAINER (%r)',
+ value)
return False
+
def validate_account(metadata):
if not metadata:
logging.warn('validate_account: No metadata')
@@ -184,16 +198,17 @@ def validate_account(metadata):
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys() or \
X_CONTAINER_COUNT not in metadata.keys():
- #logging.warn('validate_account: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == ACCOUNT:
return True
- logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,))
+ logging.warn('validate_account: metadata type is not ACCOUNT (%r)',
+ value)
return False
+
def validate_object(metadata):
if not metadata:
logging.warn('validate_object: No metadata')
@@ -205,22 +220,24 @@ def validate_object(metadata):
X_CONTENT_LENGTH not in metadata.keys() or \
X_TYPE not in metadata.keys() or \
X_OBJECT_TYPE not in metadata.keys():
- #logging.warn('validate_object: Metadata missing entries: %s' % metadata)
return False
if metadata[X_TYPE] == OBJECT:
return True
- logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],))
+ logging.warn('validate_object: metadata type is not OBJECT (%r)',
+ metadata[X_TYPE])
return False
+
def is_marker(metadata):
if not metadata:
logging.warn('is_marker: No metadata')
return False
if X_OBJECT_TYPE not in metadata.keys():
- logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata)
+ logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s',
+ metadata)
return False
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
@@ -228,6 +245,7 @@ def is_marker(metadata):
else:
return False
+
def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
bytes_used=0, obj_list=[]):
# strip the prefix off, also stripping the leading and trailing slashes
@@ -247,6 +265,7 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
return object_count, bytes_used
+
def update_list(path, cont_path, dirs=[], files=[], object_count=0,
bytes_used=0, obj_list=[]):
if files:
@@ -279,15 +298,16 @@ def _get_container_details_from_fs(cont_path):
if os_path.isdir(cont_path):
for (path, dirs, files) in do_walk(cont_path):
- object_count, bytes_used = update_list(path, cont_path, dirs, files,
- object_count, bytes_used,
- obj_list)
+ object_count, bytes_used = update_list(path, cont_path, dirs,
+ files, object_count,
+ bytes_used, obj_list)
dir_list.append((path, do_stat(path).st_mtime))
sleep()
return ContainerDetails(bytes_used, object_count, obj_list, dir_list)
+
def get_container_details(cont_path, memcache=None):
"""
Return object_list, object_count and bytes_used.
@@ -344,6 +364,7 @@ def _get_account_details_from_fs(acc_path, acc_stats):
return AccountDetails(acc_stats.st_mtime, container_count, container_list)
+
def get_account_details(acc_path, memcache=None):
"""
Return container_list and container_count.
@@ -369,6 +390,7 @@ def get_account_details(acc_path, memcache=None):
memcache.set(mkey, ad)
return ad.container_list, ad.container_count
+
def _get_etag(path):
etag = md5()
with open(path, 'rb') as fp:
@@ -380,6 +402,7 @@ def _get_etag(path):
break
return etag.hexdigest()
+
def get_object_metadata(obj_path):
"""
Return metadata of object.
@@ -398,10 +421,10 @@ def get_object_metadata(obj_path):
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
- X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path),
- }
+ X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path)}
return metadata
+
def _add_timestamp(metadata_i):
# At this point we have a simple key/value dictionary, turn it into
# key/(value,timestamp) pairs.
@@ -414,30 +437,38 @@ def _add_timestamp(metadata_i):
metadata[key] = value_i
return metadata
+
def get_container_metadata(cont_path, memcache=None):
objects = []
object_count = 0
bytes_used = 0
- objects, object_count, bytes_used = get_container_details(cont_path, memcache)
+ objects, object_count, bytes_used = get_container_details(cont_path,
+ memcache)
metadata = {X_TYPE: CONTAINER,
- X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)),
- X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)),
+ X_TIMESTAMP: normalize_timestamp(
+ os_path.getctime(cont_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(
+ os_path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
return _add_timestamp(metadata)
+
def get_account_metadata(acc_path, memcache=None):
containers = []
container_count = 0
containers, container_count = get_account_details(acc_path, memcache)
metadata = {X_TYPE: ACCOUNT,
- X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)),
- X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)),
+ X_TIMESTAMP: normalize_timestamp(
+ os_path.getctime(acc_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(
+ os_path.getmtime(acc_path)),
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
return _add_timestamp(metadata)
+
def restore_metadata(path, metadata):
meta_orig = read_metadata(path)
if meta_orig:
@@ -449,18 +480,22 @@ def restore_metadata(path, metadata):
write_metadata(path, meta_new)
return meta_new
+
def create_object_metadata(obj_path):
metadata = get_object_metadata(obj_path)
return restore_metadata(obj_path, metadata)
+
def create_container_metadata(cont_path, memcache=None):
metadata = get_container_metadata(cont_path, memcache)
return restore_metadata(cont_path, metadata)
+
def create_account_metadata(acc_path, memcache=None):
metadata = get_account_metadata(acc_path, memcache)
return restore_metadata(acc_path, metadata)
+
def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
"""
Ensure that a pickle file gets written to disk. The file is first written
@@ -479,7 +514,8 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
"""
dirname = os.path.dirname(dest)
basename = os.path.basename(dest)
- tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest()
+ tmpname = '.' + basename + '.' + \
+ md5(basename + str(random.random())).hexdigest()
tmppath = os.path.join(dirname, tmpname)
with open(tmppath, 'wb') as fo:
pickle.dump(obj, fo, pickle_protocol)
@@ -491,6 +527,7 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
do_fsync(fo)
do_rename(tmppath, dest)
+
# Over-ride Swift's utils.write_pickle with ours
import swift.common.utils
swift.common.utils.write_pickle = write_pickle
diff --git a/gluster/swift/container/server.py b/gluster/swift/container/server.py
index c5792aa..ee284c9 100644
--- a/gluster/swift/container/server.py
+++ b/gluster/swift/container/server.py
@@ -17,7 +17,7 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.container import server
from gluster.swift.common.DiskDir import DiskDir
diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py
index 1c2b6cb..1223036 100644
--- a/gluster/swift/obj/server.py
+++ b/gluster/swift/obj/server.py
@@ -17,8 +17,8 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
-import gluster.swift.common.utils
+import gluster.swift.common.constraints # noqa
+import gluster.swift.common.utils # noqa
from swift.obj import server
from gluster.swift.common.DiskFile import Gluster_DiskFile
diff --git a/gluster/swift/proxy/server.py b/gluster/swift/proxy/server.py
index 792a97d..df9b245 100644
--- a/gluster/swift/proxy/server.py
+++ b/gluster/swift/proxy/server.py
@@ -16,10 +16,11 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.proxy import server
+
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()