summaryrefslogtreecommitdiffstats
path: root/gluster
diff options
context:
space:
mode:
Diffstat (limited to 'gluster')
-rw-r--r--gluster/swift/__init__.py1
-rw-r--r--gluster/swift/account/server.py2
-rw-r--r--gluster/swift/common/DiskDir.py95
-rw-r--r--gluster/swift/common/DiskFile.py43
-rw-r--r--gluster/swift/common/Glusterfs.py38
-rw-r--r--gluster/swift/common/constraints.py10
-rw-r--r--gluster/swift/common/exceptions.py5
-rw-r--r--gluster/swift/common/fs_utils.py30
-rw-r--r--gluster/swift/common/ring.py12
-rw-r--r--gluster/swift/common/utils.py93
-rw-r--r--gluster/swift/container/server.py2
-rw-r--r--gluster/swift/obj/server.py4
-rw-r--r--gluster/swift/proxy/server.py3
13 files changed, 203 insertions, 135 deletions
diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py
index 14bb4e8..fa360b6 100644
--- a/gluster/swift/__init__.py
+++ b/gluster/swift/__init__.py
@@ -1,5 +1,6 @@
""" Gluster for Swift """
+
class PkgInfo(object):
def __init__(self, canonical_version, name, final):
self.canonical_version = canonical_version
diff --git a/gluster/swift/account/server.py b/gluster/swift/account/server.py
index aeaabc9..3c6d616 100644
--- a/gluster/swift/account/server.py
+++ b/gluster/swift/account/server.py
@@ -17,7 +17,7 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.account import server
from gluster.swift.common.DiskDir import DiskAccount
diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py
index 18d08cc..5d67303 100644
--- a/gluster/swift/common/DiskDir.py
+++ b/gluster/swift/common/DiskDir.py
@@ -13,36 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os, errno
-
-from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \
- mkdirs, validate_account, validate_container, is_marker, \
- get_container_details, get_account_details, get_container_metadata, \
- create_container_metadata, create_account_metadata, DEFAULT_GID, \
- DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
- write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
- X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
- X_CONTAINER_COUNT, CONTAINER, os_path
+import os
+
+from gluster.swift.common.fs_utils import dir_empty, rmdirs, mkdirs, os_path
+from gluster.swift.common.utils import clean_metadata, validate_account, \
+ validate_container, get_container_details, get_account_details, \
+ create_container_metadata, create_account_metadata, DEFAULT_GID, \
+ DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
+ write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
+ X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
+ X_CONTAINER_COUNT
from gluster.swift.common import Glusterfs
-from swift.common.constraints import CONTAINER_LISTING_LIMIT
-from swift.common.utils import normalize_timestamp, TRUE_VALUES
-
DATADIR = 'containers'
-# Create a dummy db_file in /etc/swift
-_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no')
-if _unittests_enabled in TRUE_VALUES:
- _tmp_dir = '/tmp/gluster_unit_tests'
- try:
- os.mkdir(_tmp_dir)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- _db_file = os.path.join(_tmp_dir, 'db_file.db')
-else:
- _db_file = '/etc/swift/db_file.db'
+# Create a dummy db_file in Glusterfs.RUN_DIR
+_db_file = os.path.join(Glusterfs.RUN_DIR, 'db_file.db')
if not os.path.exists(_db_file):
file(_db_file, 'w+')
@@ -91,7 +78,7 @@ class DiskCommon(object):
Accept sorted list.
Objects should start with prefix.
"""
- filtered_objs=[]
+ filtered_objs = []
for object_name in objects:
tmp_obj = object_name.replace(prefix, '', 1)
sufix = tmp_obj.split(delimiter, 1)
@@ -106,8 +93,7 @@ class DiskCommon(object):
TODO: We can traverse in reverse order to optimize.
Accept sorted list.
"""
- filtered_objs=[]
- found = 0
+ filtered_objs = []
if objects[-1] < marker:
return filtered_objs
for object_name in objects:
@@ -120,7 +106,7 @@ class DiskCommon(object):
"""
Accept sorted list.
"""
- filtered_objs=[]
+ filtered_objs = []
for object_name in objects:
if object_name < end_marker:
filtered_objs.append(object_name)
@@ -130,7 +116,7 @@ class DiskCommon(object):
return filtered_objs
def filter_limit(self, objects, limit):
- filtered_objs=[]
+ filtered_objs = []
for i in range(0, limit):
filtered_objs.append(objects[i])
@@ -232,7 +218,8 @@ class DiskDir(DiskCommon):
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp)
self.metadata[X_PUT_TIMESTAMP] = timestamp
bused = self.metadata[X_BYTES_USED][0]
- self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp)
+ self.metadata[X_BYTES_USED] = (int(bused) + int(content_length),
+ timestamp)
#TODO: define update_metadata instad of writing whole metadata again.
self.put_metadata(self.metadata)
@@ -240,10 +227,12 @@ class DiskDir(DiskCommon):
ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0]
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp)
bused, timestamp = self.metadata[X_BYTES_USED]
- self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp)
+ self.metadata[X_BYTES_USED] = (int(bused) - int(content_length),
+ timestamp)
self.put_metadata(self.metadata)
- def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
+ def put_container(self, container, put_timestamp, del_timestamp,
+ object_count, bytes_used):
"""
For account server.
"""
@@ -363,20 +352,22 @@ class DiskDir(DiskCommon):
# update the object counts in case they changed behind our back.
self.update_object_count()
- data = {'account' : self.account, 'container' : self.container,
- 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
- 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0],
- 'hash': '', 'id' : '', 'created_at' : '1',
- 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
- 'delete_timestamp' : '1',
- 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
- 'reported_object_count' : '1', 'reported_bytes_used' : '1'}
+ data = {'account': self.account, 'container': self.container,
+ 'object_count': self.metadata.get(
+ X_OBJECTS_COUNT, ('0', 0))[0],
+ 'bytes_used': self.metadata.get(X_BYTES_USED, ('0', 0))[0],
+ 'hash': '', 'id': '', 'created_at': '1',
+ 'put_timestamp': self.metadata.get(
+ X_PUT_TIMESTAMP, ('0', 0))[0],
+ 'delete_timestamp': '1',
+ 'reported_put_timestamp': '1',
+ 'reported_delete_timestamp': '1',
+ 'reported_object_count': '1', 'reported_bytes_used': '1'}
if include_metadata:
data['metadata'] = self.metadata
return data
- def put_object(self, name, timestamp, size, content_type,
- etag, deleted=0):
+ def put_object(self, name, timestamp, size, content_type, etag, deleted=0):
# TODO: Implement the specifics of this func.
pass
@@ -401,7 +392,8 @@ class DiskDir(DiskCommon):
self.unlink()
def update_metadata(self, metadata):
- assert self.metadata, "Valid container/account metadata should have been created by now"
+ assert self.metadata, "Valid container/account metadata should have" \
+ " been created by now"
if metadata:
new_metadata = self.metadata.copy()
new_metadata.update(metadata)
@@ -478,12 +470,13 @@ class DiskAccount(DiskDir):
# update the container counts in case they changed behind our back.
self.update_container_count()
- data = {'account' : self.account, 'created_at' : '1',
- 'put_timestamp' : '1', 'delete_timestamp' : '1',
- 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0],
- 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0],
- 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0],
- 'hash' : '', 'id' : ''}
+ data = {'account': self.account, 'created_at': '1',
+ 'put_timestamp': '1', 'delete_timestamp': '1',
+ 'container_count': self.metadata.get(
+ X_CONTAINER_COUNT, (0, 0))[0],
+ 'object_count': self.metadata.get(X_OBJECTS_COUNT, (0, 0))[0],
+ 'bytes_used': self.metadata.get(X_BYTES_USED, (0, 0))[0],
+ 'hash': '', 'id': ''}
if include_metadata:
data['metadata'] = self.metadata
@@ -493,4 +486,4 @@ class DiskAccount(DiskDir):
cont_path = os.path.join(self.datadir, container)
metadata = read_metadata(cont_path)
- return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None
+ return int(metadata.get(X_PUT_TIMESTAMP, ('0', 0))[0]) or None
diff --git a/gluster/swift/common/DiskFile.py b/gluster/swift/common/DiskFile.py
index 900bd49..577147a 100644
--- a/gluster/swift/common/DiskFile.py
+++ b/gluster/swift/common/DiskFile.py
@@ -18,16 +18,16 @@ import errno
import random
from hashlib import md5
from contextlib import contextmanager
-from swift.common.utils import normalize_timestamp, renamer
+from swift.common.utils import renamer
from swift.common.exceptions import DiskFileNotExist
from gluster.swift.common.exceptions import AlreadyExistsAsDir
-from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \
- create_object_metadata, do_open, do_close, do_unlink, do_chown, \
- do_listdir, read_metadata, write_metadata, os_path, do_fsync
-from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \
- X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \
- X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \
- FILE_TYPE, DEFAULT_UID, DEFAULT_GID
+from gluster.swift.common.fs_utils import mkdirs, rmdirs, do_open, do_close, \
+ do_unlink, do_chown, os_path, do_fsync
+from gluster.swift.common.utils import read_metadata, write_metadata, \
+ validate_object, create_object_metadata
+from gluster.swift.common.utils import X_CONTENT_LENGTH, X_CONTENT_TYPE, \
+ X_TIMESTAMP, X_TYPE, X_OBJECT_TYPE, FILE, MARKER_DIR, OBJECT, DIR_TYPE, \
+ FILE_TYPE, DEFAULT_UID, DEFAULT_GID
import logging
from swift.obj.server import DiskFile
@@ -42,17 +42,18 @@ def _adjust_metadata(metadata):
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
# metadata values.
- content_type = metadata['Content-Type']
+ content_type = metadata[X_CONTENT_TYPE]
if not content_type:
# FIXME: How can this be that our caller supplied us with metadata
# that has a content type that evaluates to False?
#
# FIXME: If the file exists, we would already know it is a
# directory. So why are we assuming it is a file object?
- metadata['Content-Type'] = FILE_TYPE
+ metadata[X_CONTENT_TYPE] = FILE_TYPE
x_object_type = FILE
else:
- x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE
+ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE \
+ else FILE
metadata[X_TYPE] = OBJECT
metadata[X_OBJECT_TYPE] = x_object_type
return metadata
@@ -184,7 +185,8 @@ class Gluster_DiskFile(DiskFile):
if tombstone:
# We don't write tombstone files. So do nothing.
return
- assert self.data_file is not None, "put_metadata: no file to put metadata into"
+ assert self.data_file is not None, \
+ "put_metadata: no file to put metadata into"
metadata = _adjust_metadata(metadata)
write_metadata(self.data_file, metadata)
self.metadata = metadata
@@ -192,8 +194,8 @@ class Gluster_DiskFile(DiskFile):
def put(self, fd, metadata, extension='.data'):
"""
- Finalize writing the file on disk, and renames it from the temp file to
- the real location. This should be called after the data has been
+ Finalize writing the file on disk, and renames it from the temp file
+ to the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
@@ -202,7 +204,6 @@ class Gluster_DiskFile(DiskFile):
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
- extension = ''
metadata = _adjust_metadata(metadata)
@@ -220,7 +221,6 @@ class Gluster_DiskFile(DiskFile):
msg = 'File object exists as a directory: %s' % self.data_file
raise AlreadyExistsAsDir(msg)
- timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(self.tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
@@ -248,7 +248,7 @@ class Gluster_DiskFile(DiskFile):
:param timestamp: timestamp to compare with each file
"""
- if not self.metadata or self.metadata['X-Timestamp'] >= timestamp:
+ if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
return
assert self.data_file, \
@@ -257,7 +257,8 @@ class Gluster_DiskFile(DiskFile):
if self._is_dir:
# Marker directory object
if not rmdirs(self.data_file):
- logging.error('Unable to delete dir object: %s', self.data_file)
+ logging.error('Unable to delete dir object: %s',
+ self.data_file)
return
else:
# File object
@@ -283,7 +284,7 @@ class Gluster_DiskFile(DiskFile):
file_size = 0
if self.data_file:
file_size = os_path.getsize(self.data_file)
- if X_CONTENT_LENGTH in self.metadata:
+ if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
@@ -314,11 +315,11 @@ class Gluster_DiskFile(DiskFile):
path = self._container_path
subdir_list = self._obj_path.split(os.path.sep)
for i in range(len(subdir_list)):
- path = os.path.join(path, subdir_list[i]);
+ path = os.path.join(path, subdir_list[i])
if not os_path.exists(path):
self._create_dir_object(path)
- tmpfile = '.' + self._obj + '.' + md5(self._obj + \
+ tmpfile = '.' + self._obj + '.' + md5(self._obj +
str(random.random())).hexdigest()
self.tmppath = os.path.join(self.datadir, tmpfile)
diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py
index ba6bcf1..819a747 100644
--- a/gluster/swift/common/Glusterfs.py
+++ b/gluster/swift/common/Glusterfs.py
@@ -13,8 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+import fcntl
+import time
+import errno
import logging
-import os, sys, fcntl, time, errno
+
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.common.utils import TRUE_VALUES, search_tree
from gluster.swift.common.fs_utils import mkdirs
@@ -25,26 +29,29 @@ from gluster.swift.common.fs_utils import mkdirs
_fs_conf = ConfigParser()
MOUNT_IP = 'localhost'
OBJECT_ONLY = False
-RUN_DIR='/var/run/swift'
+RUN_DIR = '/var/run/swift'
SWIFT_DIR = '/etc/swift'
_do_getsize = False
-if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')):
+if _fs_conf.read(os.path.join(SWIFT_DIR, 'fs.conf')):
try:
- MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost')
+ MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', MOUNT_IP)
except (NoSectionError, NoOptionError):
pass
try:
- OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
+ OBJECT_ONLY = _fs_conf.get('DEFAULT',
+ 'object_only',
+ "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
try:
- RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift')
+ RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', RUN_DIR)
except (NoSectionError, NoOptionError):
pass
try:
- _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \
- "no") in TRUE_VALUES
+ _do_getsize = _fs_conf.get('DEFAULT',
+ 'accurate_size_in_listing',
+ "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
@@ -58,9 +65,11 @@ def _busy_wait(full_mount_path):
if os.path.ismount(full_mount_path):
return True
time.sleep(2)
- logging.error('Busy wait for mount timed out for mount %s', full_mount_path)
+ logging.error('Busy wait for mount timed out for mount %s',
+ full_mount_path)
return False
+
def mount(root, drive):
# FIXME: Possible thundering herd problem here
@@ -77,15 +86,15 @@ def mount(root, drive):
if not os.path.isdir(full_mount_path):
mkdirs(full_mount_path)
- lck_file = os.path.join(RUN_DIR, '%s.lock' %drive);
+ lck_file = os.path.join(RUN_DIR, '%s.lock' % drive)
if not os.path.exists(RUN_DIR):
mkdirs(RUN_DIR)
- fd = os.open(lck_file, os.O_CREAT|os.O_RDWR)
+ fd = os.open(lck_file, os.O_CREAT | os.O_RDWR)
with os.fdopen(fd, 'r+b') as f:
try:
- fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
+ fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as ex:
if ex.errno in (errno.EACCES, errno.EAGAIN):
# This means that some other process is mounting the
@@ -93,13 +102,14 @@ def mount(root, drive):
return _busy_wait(full_mount_path)
else:
raise ex
- mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \
+ mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export,
full_mount_path)
if os.system(mnt_cmd) or not _busy_wait(full_mount_path):
logging.error('Mount failed %s: %s', NAME, mnt_cmd)
return False
return True
+
def unmount(full_mount_path):
# FIXME: Possible thundering herd problem here
@@ -107,6 +117,7 @@ def unmount(full_mount_path):
if os.system(umnt_cmd):
logging.error('Unable to unmount %s %s' % (full_mount_path, NAME))
+
def _get_export_list():
cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP
@@ -126,6 +137,7 @@ def _get_export_list():
return export_list
+
def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"):
"""Read the object-server's configuration file and return
the device value"""
diff --git a/gluster/swift/common/constraints.py b/gluster/swift/common/constraints.py
index 11f626b..d1c990d 100644
--- a/gluster/swift/common/constraints.py
+++ b/gluster/swift/common/constraints.py
@@ -23,11 +23,12 @@ from gluster.swift.common import Glusterfs, ring
if hasattr(swift.common.constraints, 'constraints_conf_int'):
MAX_OBJECT_NAME_COMPONENT_LENGTH = \
- swift.common.constraints.constraints_conf_int(
- 'max_object_name_component_length', 255)
+ swift.common.constraints.constraints_conf_int(
+ 'max_object_name_component_length', 255)
else:
MAX_OBJECT_NAME_COMPONENT_LENGTH = 255
+
def validate_obj_name_component(obj):
if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH:
return 'too long (%d)' % len(obj)
@@ -38,6 +39,7 @@ def validate_obj_name_component(obj):
# Save the original check object creation
__check_object_creation = swift.common.constraints.check_object_creation
+
# Define our new one which invokes the original
def gluster_check_object_creation(req, object_name):
"""
@@ -61,7 +63,7 @@ def gluster_check_object_creation(req, object_name):
reason = validate_obj_name_component(obj)
if reason:
bdy = 'Invalid object name "%s", component "%s" %s' \
- % (object_name, obj, reason)
+ % (object_name, obj, reason)
ret = HTTPBadRequest(body=bdy,
request=req,
content_type='text/plain')
@@ -74,6 +76,7 @@ swift.common.constraints.check_object_creation = gluster_check_object_creation
# Save the original check mount
__check_mount = swift.common.constraints.check_mount
+
# Define our new one which invokes the original
def gluster_check_mount(root, drive):
# FIXME: Potential performance optimization here to not call the original
@@ -84,6 +87,7 @@ def gluster_check_mount(root, drive):
return Glusterfs.mount(root, drive)
+
# Replace the original check mount with ours
swift.common.constraints.check_mount = gluster_check_mount
diff --git a/gluster/swift/common/exceptions.py b/gluster/swift/common/exceptions.py
index d9357db..75a95ec 100644
--- a/gluster/swift/common/exceptions.py
+++ b/gluster/swift/common/exceptions.py
@@ -13,15 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
class GlusterfsException(Exception):
pass
+
class FileOrDirNotFoundError(GlusterfsException):
pass
+
class NotDirectoryError(GlusterfsException):
pass
+
class AlreadyExistsAsDir(GlusterfsException):
pass
-
diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py
index 0613a26..de450a5 100644
--- a/gluster/swift/common/fs_utils.py
+++ b/gluster/swift/common/fs_utils.py
@@ -16,14 +16,16 @@
import logging
import os
import errno
-import os.path as os_path
+import os.path as os_path # noqa
from eventlet import tpool
from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
NotDirectoryError
+
def do_walk(*args, **kwargs):
return os.walk(*args, **kwargs)
+
def do_write(fd, msg):
try:
cnt = os.write(fd, msg)
@@ -32,6 +34,7 @@ def do_write(fd, msg):
raise
return cnt
+
def do_mkdir(path):
try:
os.mkdir(path)
@@ -41,15 +44,18 @@ def do_mkdir(path):
raise
return True
+
def do_makedirs(path):
try:
os.makedirs(path)
except OSError as err:
if err.errno != errno.EEXIST:
- logging.exception("Makedirs failed on %s err: %s", path, err.strerror)
+ logging.exception("Makedirs failed on %s err: %s",
+ path, err.strerror)
raise
return True
+
def do_listdir(path):
try:
buf = os.listdir(path)
@@ -58,6 +64,7 @@ def do_listdir(path):
raise
return buf
+
def do_chown(path, uid, gid):
try:
os.chown(path, uid, gid)
@@ -66,6 +73,7 @@ def do_chown(path, uid, gid):
raise
return True
+
def do_stat(path):
try:
#Check for fd.
@@ -78,6 +86,7 @@ def do_stat(path):
raise
return buf
+
def do_open(path, mode):
if isinstance(mode, int):
try:
@@ -93,6 +102,7 @@ def do_open(path, mode):
raise
return fd
+
def do_close(fd):
#fd could be file or int type.
try:
@@ -105,16 +115,19 @@ def do_close(fd):
raise
return True
-def do_unlink(path, log = True):
+
+def do_unlink(path, log=True):
try:
os.unlink(path)
except OSError as err:
if err.errno != errno.ENOENT:
if log:
- logging.exception("Unlink failed on %s err: %s", path, err.strerror)
+ logging.exception("Unlink failed on %s err: %s",
+ path, err.strerror)
raise
return True
+
def do_rmdir(path):
try:
os.rmdir(path)
@@ -127,15 +140,17 @@ def do_rmdir(path):
res = True
return res
+
def do_rename(old_path, new_path):
try:
os.rename(old_path, new_path)
except OSError as err:
- logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \
- err.strerror)
+ logging.exception("Rename failed on %s to %s err: %s",
+ old_path, new_path, err.strerror)
raise
return True
+
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path
@@ -146,6 +161,7 @@ def mkdirs(path):
if not os.path.isdir(path):
do_makedirs(path)
+
def dir_empty(path):
"""
Return true if directory/container is empty.
@@ -159,6 +175,7 @@ def dir_empty(path):
raise FileOrDirNotFoundError()
raise NotDirectoryError()
+
def rmdirs(path):
if not os.path.isdir(path):
return False
@@ -170,6 +187,7 @@ def rmdirs(path):
return False
return True
+
def do_fsync(fd):
try:
tpool.execute(os.fsync, fd)
diff --git a/gluster/swift/common/ring.py b/gluster/swift/common/ring.py
index 06aab8d..f88af4e 100644
--- a/gluster/swift/common/ring.py
+++ b/gluster/swift/common/ring.py
@@ -36,13 +36,12 @@ if conf_files and _conf.read(conf_file):
if not reseller_prefix.endswith('_'):
reseller_prefix = reseller_prefix + '_'
+
class Ring(ring.Ring):
def _get_part_nodes(self, part):
seen_ids = set()
- nodes = [dev for dev in self._devs \
- if dev['device'] == self.acc_name \
- and not (dev['id'] in seen_ids \
- or seen_ids.add(dev['id']))]
+ nodes = [dev for dev in self._devs if dev['device'] == self.acc_name
+ and not (dev['id'] in seen_ids or seen_ids.add(dev['id']))]
if not nodes:
nodes = [self.false_node]
return nodes
@@ -86,8 +85,8 @@ class Ring(ring.Ring):
hardware description
====== ===============================================================
"""
- self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \
- 'meta': '', 'device': 'volume_not_in_ring', \
+ self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1',
+ 'id': 0, 'meta': '', 'device': 'volume_not_in_ring',
'port': 6012}
if account.startswith(reseller_prefix):
self.acc_name = account.replace(reseller_prefix, '', 1)
@@ -97,7 +96,6 @@ class Ring(ring.Ring):
part = 0
return part, self._get_part_nodes(part)
-
def get_more_nodes(self, part):
"""
Generator to get extra nodes for a partition for hinted handoff.
diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index f2cd8de..2cbb7f3 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -21,9 +21,9 @@ import random
from hashlib import md5
from eventlet import sleep
import cPickle as pickle
-from ConfigParser import ConfigParser, NoSectionError, NoOptionError
-from swift.common.utils import normalize_timestamp, TRUE_VALUES
-from gluster.swift.common.fs_utils import *
+from swift.common.utils import normalize_timestamp
+from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \
+ do_stat, do_listdir, do_walk
from gluster.swift.common import Glusterfs
X_CONTENT_TYPE = 'Content-Type'
@@ -54,8 +54,11 @@ DEFAULT_GID = -1
PICKLE_PROTOCOL = 2
CHUNK_SIZE = 65536
MEMCACHE_KEY_PREFIX = 'gluster.swift.'
-MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.'
-MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.'
+MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \
+ 'account.details.'
+MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + \
+ 'container.details.'
+
def read_metadata(path):
"""
@@ -70,7 +73,8 @@ def read_metadata(path):
key = 0
while metadata is None:
try:
- metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or '')))
+ metadata_s += xattr.getxattr(path,
+ '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
if key > 0:
@@ -108,6 +112,7 @@ def read_metadata(path):
key += 1
return metadata
+
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
@@ -120,13 +125,17 @@ def write_metadata(path, metadata):
key = 0
while metastr:
try:
- xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE])
+ xattr.setxattr(path,
+ '%s%s' % (METADATA_KEY, key or ''),
+ metastr[:MAX_XATTR_SIZE])
except IOError as err:
- logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err))
+ logging.exception("setxattr failed on %s key %s err: %s",
+ path, key, str(err))
raise
metastr = metastr[MAX_XATTR_SIZE:]
key += 1
+
def clean_metadata(path):
key = 0
while True:
@@ -138,21 +147,25 @@ def clean_metadata(path):
raise
key += 1
+
def check_user_xattr(path):
if not os_path.exists(path):
return False
try:
xattr.setxattr(path, 'user.test.key1', 'value1')
except IOError as err:
- logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
+ logging.exception("check_user_xattr: set failed on %s err: %s",
+ path, str(err))
raise
try:
xattr.removexattr(path, 'user.test.key1')
except IOError as err:
- logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err))
+ logging.exception("check_user_xattr: remove failed on %s err: %s",
+ path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
+
def validate_container(metadata):
if not metadata:
logging.warn('validate_container: No metadata')
@@ -163,16 +176,17 @@ def validate_container(metadata):
X_PUT_TIMESTAMP not in metadata.keys() or \
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys():
- #logging.warn('validate_container: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == CONTAINER:
return True
- logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,))
+ logging.warn('validate_container: metadata type is not CONTAINER (%r)',
+ value)
return False
+
def validate_account(metadata):
if not metadata:
logging.warn('validate_account: No metadata')
@@ -184,16 +198,17 @@ def validate_account(metadata):
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys() or \
X_CONTAINER_COUNT not in metadata.keys():
- #logging.warn('validate_account: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == ACCOUNT:
return True
- logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,))
+ logging.warn('validate_account: metadata type is not ACCOUNT (%r)',
+ value)
return False
+
def validate_object(metadata):
if not metadata:
logging.warn('validate_object: No metadata')
@@ -205,22 +220,24 @@ def validate_object(metadata):
X_CONTENT_LENGTH not in metadata.keys() or \
X_TYPE not in metadata.keys() or \
X_OBJECT_TYPE not in metadata.keys():
- #logging.warn('validate_object: Metadata missing entries: %s' % metadata)
return False
if metadata[X_TYPE] == OBJECT:
return True
- logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],))
+ logging.warn('validate_object: metadata type is not OBJECT (%r)',
+ metadata[X_TYPE])
return False
+
def is_marker(metadata):
if not metadata:
logging.warn('is_marker: No metadata')
return False
if X_OBJECT_TYPE not in metadata.keys():
- logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata)
+ logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s',
+ metadata)
return False
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
@@ -228,6 +245,7 @@ def is_marker(metadata):
else:
return False
+
def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
bytes_used=0, obj_list=[]):
# strip the prefix off, also stripping the leading and trailing slashes
@@ -247,6 +265,7 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
return object_count, bytes_used
+
def update_list(path, cont_path, dirs=[], files=[], object_count=0,
bytes_used=0, obj_list=[]):
if files:
@@ -279,15 +298,16 @@ def _get_container_details_from_fs(cont_path):
if os_path.isdir(cont_path):
for (path, dirs, files) in do_walk(cont_path):
- object_count, bytes_used = update_list(path, cont_path, dirs, files,
- object_count, bytes_used,
- obj_list)
+ object_count, bytes_used = update_list(path, cont_path, dirs,
+ files, object_count,
+ bytes_used, obj_list)
dir_list.append((path, do_stat(path).st_mtime))
sleep()
return ContainerDetails(bytes_used, object_count, obj_list, dir_list)
+
def get_container_details(cont_path, memcache=None):
"""
Return object_list, object_count and bytes_used.
@@ -344,6 +364,7 @@ def _get_account_details_from_fs(acc_path, acc_stats):
return AccountDetails(acc_stats.st_mtime, container_count, container_list)
+
def get_account_details(acc_path, memcache=None):
"""
Return container_list and container_count.
@@ -369,6 +390,7 @@ def get_account_details(acc_path, memcache=None):
memcache.set(mkey, ad)
return ad.container_list, ad.container_count
+
def _get_etag(path):
etag = md5()
with open(path, 'rb') as fp:
@@ -380,6 +402,7 @@ def _get_etag(path):
break
return etag.hexdigest()
+
def get_object_metadata(obj_path):
"""
Return metadata of object.
@@ -398,10 +421,10 @@ def get_object_metadata(obj_path):
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
- X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path),
- }
+ X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path)}
return metadata
+
def _add_timestamp(metadata_i):
# At this point we have a simple key/value dictionary, turn it into
# key/(value,timestamp) pairs.
@@ -414,30 +437,38 @@ def _add_timestamp(metadata_i):
metadata[key] = value_i
return metadata
+
def get_container_metadata(cont_path, memcache=None):
objects = []
object_count = 0
bytes_used = 0
- objects, object_count, bytes_used = get_container_details(cont_path, memcache)
+ objects, object_count, bytes_used = get_container_details(cont_path,
+ memcache)
metadata = {X_TYPE: CONTAINER,
- X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)),
- X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)),
+ X_TIMESTAMP: normalize_timestamp(
+ os_path.getctime(cont_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(
+ os_path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
return _add_timestamp(metadata)
+
def get_account_metadata(acc_path, memcache=None):
containers = []
container_count = 0
containers, container_count = get_account_details(acc_path, memcache)
metadata = {X_TYPE: ACCOUNT,
- X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)),
- X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)),
+ X_TIMESTAMP: normalize_timestamp(
+ os_path.getctime(acc_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(
+ os_path.getmtime(acc_path)),
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
return _add_timestamp(metadata)
+
def restore_metadata(path, metadata):
meta_orig = read_metadata(path)
if meta_orig:
@@ -449,18 +480,22 @@ def restore_metadata(path, metadata):
write_metadata(path, meta_new)
return meta_new
+
def create_object_metadata(obj_path):
metadata = get_object_metadata(obj_path)
return restore_metadata(obj_path, metadata)
+
def create_container_metadata(cont_path, memcache=None):
metadata = get_container_metadata(cont_path, memcache)
return restore_metadata(cont_path, metadata)
+
def create_account_metadata(acc_path, memcache=None):
metadata = get_account_metadata(acc_path, memcache)
return restore_metadata(acc_path, metadata)
+
def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
"""
Ensure that a pickle file gets written to disk. The file is first written
@@ -479,7 +514,8 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
"""
dirname = os.path.dirname(dest)
basename = os.path.basename(dest)
- tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest()
+ tmpname = '.' + basename + '.' + \
+ md5(basename + str(random.random())).hexdigest()
tmppath = os.path.join(dirname, tmpname)
with open(tmppath, 'wb') as fo:
pickle.dump(obj, fo, pickle_protocol)
@@ -491,6 +527,7 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
do_fsync(fo)
do_rename(tmppath, dest)
+
# Over-ride Swift's utils.write_pickle with ours
import swift.common.utils
swift.common.utils.write_pickle = write_pickle
diff --git a/gluster/swift/container/server.py b/gluster/swift/container/server.py
index c5792aa..ee284c9 100644
--- a/gluster/swift/container/server.py
+++ b/gluster/swift/container/server.py
@@ -17,7 +17,7 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.container import server
from gluster.swift.common.DiskDir import DiskDir
diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py
index 1c2b6cb..1223036 100644
--- a/gluster/swift/obj/server.py
+++ b/gluster/swift/obj/server.py
@@ -17,8 +17,8 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
-import gluster.swift.common.utils
+import gluster.swift.common.constraints # noqa
+import gluster.swift.common.utils # noqa
from swift.obj import server
from gluster.swift.common.DiskFile import Gluster_DiskFile
diff --git a/gluster/swift/proxy/server.py b/gluster/swift/proxy/server.py
index 792a97d..df9b245 100644
--- a/gluster/swift/proxy/server.py
+++ b/gluster/swift/proxy/server.py
@@ -16,10 +16,11 @@
# Simply importing this monkey patches the constraint handling to fit our
# needs
-import gluster.swift.common.constraints
+import gluster.swift.common.constraints # noqa
from swift.proxy import server
+
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()
generated by cgit (git 2.34.1) at 2024-04-24 21:21:46 +0000