From 3a75da67cbcf09c19df2a9dca2f576d51ed2191d Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 29 Apr 2013 23:04:06 -0400 Subject: Remove "ufo" directory, promoting contents to top-level Additionally, we drop the "ufo" references from setup.py, spec file and README, and add the HISTORY file describing how the repo was initially created. We also update the RPM spec file to use the name "gluster-for-swift" to avoid colliding with existing RPM names from RHS 2.0 (the spec file's description was also updated, along with the version number). Change-Id: If804224a94208d57896e4189c63736ffc9e01d5e Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/4966 Reviewed-by: Luis Pabon Reviewed-by: Mohammed Junaid Reviewed-by: Kaleb KEITHLEY Reviewed-by: Peter Portante Tested-by: Peter Portante --- .gitignore | 4 + HISTORY | 160 +++ README | 22 + bin/gluster-swift-gen-builders | 44 + etc/account-server/1.conf-gluster | 19 + etc/container-server/1.conf-gluster | 21 + etc/fs.conf-gluster | 17 + etc/object-server/1.conf-gluster | 36 + etc/proxy-server.conf-gluster | 69 ++ etc/swift.conf-gluster | 91 ++ gluster-swift.spec | 78 ++ gluster/__init__.py | 0 gluster/swift/__init__.py | 18 + gluster/swift/account/__init__.py | 0 gluster/swift/account/server.py | 45 + gluster/swift/common/DiskDir.py | 496 ++++++++++ gluster/swift/common/DiskFile.py | 338 +++++++ gluster/swift/common/Glusterfs.py | 144 +++ gluster/swift/common/__init__.py | 0 gluster/swift/common/constraints.py | 94 ++ gluster/swift/common/exceptions.py | 27 + gluster/swift/common/fs_utils.py | 179 ++++ gluster/swift/common/middleware/__init__.py | 0 gluster/swift/common/middleware/gluster.py | 40 + gluster/swift/common/ring.py | 111 +++ gluster/swift/common/utils.py | 496 ++++++++++ gluster/swift/container/__init__.py | 0 gluster/swift/container/server.py | 46 + gluster/swift/obj/__init__.py | 0 gluster/swift/obj/server.py | 34 + gluster/swift/proxy/__init__.py | 0 gluster/swift/proxy/server.py | 27 + setup.py | 57 ++ test/__init__.py | 49 + test/unit/__init__.py | 95 ++ test/unit/common/__init__.py | 0 test/unit/common/data/README.rings | 3 + test/unit/common/data/account.builder | Bin 0 -> 537 bytes test/unit/common/data/account.ring.gz | Bin 0 -> 183 bytes test/unit/common/data/account_tree.tar.bz2 | Bin 0 -> 228 bytes .../common/data/backups/1365124498.account.builder | Bin 0 -> 537 bytes .../data/backups/1365124498.container.builder | Bin 0 -> 537 bytes .../common/data/backups/1365124498.object.builder | Bin 0 -> 228 bytes .../common/data/backups/1365124499.object.builder | Bin 0 -> 537 bytes test/unit/common/data/container.builder | Bin 0 -> 537 bytes test/unit/common/data/container.ring.gz | Bin 0 -> 185 bytes test/unit/common/data/container_tree.tar.bz2 | Bin 0 -> 282 bytes test/unit/common/data/object.builder | Bin 0 -> 537 bytes test/unit/common/data/object.ring.gz | Bin 0 -> 182 bytes test/unit/common/test_Glusterfs.py | 95 ++ test/unit/common/test_diskfile.py | 932 ++++++++++++++++++ test/unit/common/test_fs_utils.py | 277 ++++++ test/unit/common/test_ring.py | 55 ++ test/unit/common/test_utils.py | 1020 ++++++++++++++++++++ tools/test-requires | 6 + tox.ini | 25 + ufo/README | 16 - ufo/bin/gluster-swift-gen-builders | 44 - ufo/etc/account-server/1.conf-gluster | 19 - ufo/etc/container-server/1.conf-gluster | 21 - ufo/etc/fs.conf-gluster | 17 - ufo/etc/object-server/1.conf-gluster | 36 - ufo/etc/proxy-server.conf-gluster | 69 -- ufo/etc/swift.conf-gluster | 91 -- ufo/gluster-swift-ufo.spec | 79 -- ufo/gluster/__init__.py | 0 ufo/gluster/swift/__init__.py | 18 - ufo/gluster/swift/account/__init__.py | 0 ufo/gluster/swift/account/server.py | 45 - ufo/gluster/swift/common/DiskDir.py | 496 ---------- ufo/gluster/swift/common/DiskFile.py | 338 ------- ufo/gluster/swift/common/Glusterfs.py | 144 --- ufo/gluster/swift/common/__init__.py | 0 ufo/gluster/swift/common/constraints.py | 94 -- ufo/gluster/swift/common/exceptions.py | 27 - ufo/gluster/swift/common/fs_utils.py | 179 ---- ufo/gluster/swift/common/middleware/__init__.py | 0 ufo/gluster/swift/common/middleware/gluster.py | 40 - ufo/gluster/swift/common/ring.py | 111 --- ufo/gluster/swift/common/utils.py | 496 ---------- ufo/gluster/swift/container/__init__.py | 0 ufo/gluster/swift/container/server.py | 46 - ufo/gluster/swift/obj/__init__.py | 0 ufo/gluster/swift/obj/server.py | 34 - ufo/gluster/swift/proxy/__init__.py | 0 ufo/gluster/swift/proxy/server.py | 27 - ufo/setup.py | 57 -- ufo/test/__init__.py | 49 - ufo/test/unit/__init__.py | 95 -- ufo/test/unit/common/__init__.py | 0 ufo/test/unit/common/data/README.rings | 3 - ufo/test/unit/common/data/account.builder | Bin 537 -> 0 bytes ufo/test/unit/common/data/account.ring.gz | Bin 183 -> 0 bytes ufo/test/unit/common/data/account_tree.tar.bz2 | Bin 228 -> 0 bytes .../common/data/backups/1365124498.account.builder | Bin 537 -> 0 bytes .../data/backups/1365124498.container.builder | Bin 537 -> 0 bytes .../common/data/backups/1365124498.object.builder | Bin 228 -> 0 bytes .../common/data/backups/1365124499.object.builder | Bin 537 -> 0 bytes ufo/test/unit/common/data/container.builder | Bin 537 -> 0 bytes ufo/test/unit/common/data/container.ring.gz | Bin 185 -> 0 bytes ufo/test/unit/common/data/container_tree.tar.bz2 | Bin 282 -> 0 bytes ufo/test/unit/common/data/object.builder | Bin 537 -> 0 bytes ufo/test/unit/common/data/object.ring.gz | Bin 182 -> 0 bytes ufo/test/unit/common/test_Glusterfs.py | 95 -- ufo/test/unit/common/test_diskfile.py | 932 ------------------ ufo/test/unit/common/test_fs_utils.py | 277 ------ ufo/test/unit/common/test_ring.py | 55 -- ufo/test/unit/common/test_utils.py | 1020 -------------------- ufo/tools/test-requires | 6 - ufo/tox.ini | 25 - ufo/unittests.sh | 7 - unittests.sh | 7 + 112 files changed, 5277 insertions(+), 5108 deletions(-) create mode 100644 .gitignore create mode 100644 HISTORY create mode 100644 README create mode 100755 bin/gluster-swift-gen-builders create mode 100644 etc/account-server/1.conf-gluster create mode 100644 etc/container-server/1.conf-gluster create mode 100644 etc/fs.conf-gluster create mode 100644 etc/object-server/1.conf-gluster create mode 100644 etc/proxy-server.conf-gluster create mode 100644 etc/swift.conf-gluster create mode 100644 gluster-swift.spec create mode 100644 gluster/__init__.py create mode 100644 gluster/swift/__init__.py create mode 100644 gluster/swift/account/__init__.py create mode 100644 gluster/swift/account/server.py create mode 100644 gluster/swift/common/DiskDir.py create mode 100644 gluster/swift/common/DiskFile.py create mode 100644 gluster/swift/common/Glusterfs.py create mode 100644 gluster/swift/common/__init__.py create mode 100644 gluster/swift/common/constraints.py create mode 100644 gluster/swift/common/exceptions.py create mode 100644 gluster/swift/common/fs_utils.py create mode 100644 gluster/swift/common/middleware/__init__.py create mode 100644 gluster/swift/common/middleware/gluster.py create mode 100644 gluster/swift/common/ring.py create mode 100644 gluster/swift/common/utils.py create mode 100644 gluster/swift/container/__init__.py create mode 100644 gluster/swift/container/server.py create mode 100644 gluster/swift/obj/__init__.py create mode 100644 gluster/swift/obj/server.py create mode 100644 gluster/swift/proxy/__init__.py create mode 100644 gluster/swift/proxy/server.py create mode 100644 setup.py create mode 100644 test/__init__.py create mode 100644 test/unit/__init__.py create mode 100644 test/unit/common/__init__.py create mode 100644 test/unit/common/data/README.rings create mode 100644 test/unit/common/data/account.builder create mode 100644 test/unit/common/data/account.ring.gz create mode 100644 test/unit/common/data/account_tree.tar.bz2 create mode 100644 test/unit/common/data/backups/1365124498.account.builder create mode 100644 test/unit/common/data/backups/1365124498.container.builder create mode 100644 test/unit/common/data/backups/1365124498.object.builder create mode 100644 test/unit/common/data/backups/1365124499.object.builder create mode 100644 test/unit/common/data/container.builder create mode 100644 test/unit/common/data/container.ring.gz create mode 100644 test/unit/common/data/container_tree.tar.bz2 create mode 100644 test/unit/common/data/object.builder create mode 100644 test/unit/common/data/object.ring.gz create mode 100644 test/unit/common/test_Glusterfs.py create mode 100644 test/unit/common/test_diskfile.py create mode 100644 test/unit/common/test_fs_utils.py create mode 100644 test/unit/common/test_ring.py create mode 100644 test/unit/common/test_utils.py create mode 100644 tools/test-requires create mode 100644 tox.ini delete mode 100644 ufo/README delete mode 100755 ufo/bin/gluster-swift-gen-builders delete mode 100644 ufo/etc/account-server/1.conf-gluster delete mode 100644 ufo/etc/container-server/1.conf-gluster delete mode 100644 ufo/etc/fs.conf-gluster delete mode 100644 ufo/etc/object-server/1.conf-gluster delete mode 100644 ufo/etc/proxy-server.conf-gluster delete mode 100644 ufo/etc/swift.conf-gluster delete mode 100644 ufo/gluster-swift-ufo.spec delete mode 100644 ufo/gluster/__init__.py delete mode 100644 ufo/gluster/swift/__init__.py delete mode 100644 ufo/gluster/swift/account/__init__.py delete mode 100644 ufo/gluster/swift/account/server.py delete mode 100644 ufo/gluster/swift/common/DiskDir.py delete mode 100644 ufo/gluster/swift/common/DiskFile.py delete mode 100644 ufo/gluster/swift/common/Glusterfs.py delete mode 100644 ufo/gluster/swift/common/__init__.py delete mode 100644 ufo/gluster/swift/common/constraints.py delete mode 100644 ufo/gluster/swift/common/exceptions.py delete mode 100644 ufo/gluster/swift/common/fs_utils.py delete mode 100644 ufo/gluster/swift/common/middleware/__init__.py delete mode 100644 ufo/gluster/swift/common/middleware/gluster.py delete mode 100644 ufo/gluster/swift/common/ring.py delete mode 100644 ufo/gluster/swift/common/utils.py delete mode 100644 ufo/gluster/swift/container/__init__.py delete mode 100644 ufo/gluster/swift/container/server.py delete mode 100644 ufo/gluster/swift/obj/__init__.py delete mode 100644 ufo/gluster/swift/obj/server.py delete mode 100644 ufo/gluster/swift/proxy/__init__.py delete mode 100644 ufo/gluster/swift/proxy/server.py delete mode 100644 ufo/setup.py delete mode 100644 ufo/test/__init__.py delete mode 100644 ufo/test/unit/__init__.py delete mode 100644 ufo/test/unit/common/__init__.py delete mode 100644 ufo/test/unit/common/data/README.rings delete mode 100644 ufo/test/unit/common/data/account.builder delete mode 100644 ufo/test/unit/common/data/account.ring.gz delete mode 100644 ufo/test/unit/common/data/account_tree.tar.bz2 delete mode 100644 ufo/test/unit/common/data/backups/1365124498.account.builder delete mode 100644 ufo/test/unit/common/data/backups/1365124498.container.builder delete mode 100644 ufo/test/unit/common/data/backups/1365124498.object.builder delete mode 100644 ufo/test/unit/common/data/backups/1365124499.object.builder delete mode 100644 ufo/test/unit/common/data/container.builder delete mode 100644 ufo/test/unit/common/data/container.ring.gz delete mode 100644 ufo/test/unit/common/data/container_tree.tar.bz2 delete mode 100644 ufo/test/unit/common/data/object.builder delete mode 100644 ufo/test/unit/common/data/object.ring.gz delete mode 100644 ufo/test/unit/common/test_Glusterfs.py delete mode 100644 ufo/test/unit/common/test_diskfile.py delete mode 100644 ufo/test/unit/common/test_fs_utils.py delete mode 100644 ufo/test/unit/common/test_ring.py delete mode 100644 ufo/test/unit/common/test_utils.py delete mode 100644 ufo/tools/test-requires delete mode 100644 ufo/tox.ini delete mode 100755 ufo/unittests.sh create mode 100755 unittests.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b5957ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.tox +gluster_swift.egg-info +test/unit/.coverage +build diff --git a/HISTORY b/HISTORY new file mode 100644 index 0000000..dcc4ce3 --- /dev/null +++ b/HISTORY @@ -0,0 +1,160 @@ +This git repository is a child (of sorts) of the git.gluster.com/glusterfs.git +repository, where origins of its contents come from the "swift" and "ufo" +directories of its parent repository. The goal was to acknowledge that the +gluster-swift integration work is in fact a separate and independent coding +effort intersecting at the packaging level with GlusterFS upstream and its +derivative products, for example, Red Hat's RHS. + +To create this repository, we followed the steps below (in pseudo script +form), which assumes that one is using an up-to-date glusterfs.git +locally-cloned repository: + +$ # +$ # First list all of the changes to the "ufo" directory: +$ # +$ git checkout master +$ git log --oneline ufo +6d6205e object-storage: provide more common cfg defaults +bf7b83d object-storage: cleanup err handling in Glusterfs. +bbaa273 object-storage: turn off stat() for container list +6a7d28c object-storage: use tox for unit tests; fix em too +a56dca9 object-storage: Import missing sys and errno modules. +93175bd Adds missing functions to ring.py, and more thorough tests. +790cbb8 object-storage: Removed the redundant REMOTE_CLUSTER option. +ee5df68 object-storage: Fixing the errors and warnings in unittest. +598ca6b object-storage: Restoring multi volume support in UFO. +311a5df object-storage: Use the wrapper functions provided by fs_utils.py to make system calls. +6a6ec98 object-storage: Store the lock file in /var/run/swift. +189eaec object-storage: yield during directory tree walks +16d2801 object-storage: use temp file optimization +8b87819 object-storage: add UFO unit tests +cdcbc5b object-storage: only open file when requested +a1f4a41 object-storage: don't sort object list, not req'd +10ef8ef object-storage: do not list directory for delete +1338ad1 object-storage: apply upstream DiskFile refactor +2fd8af2 object-storage: a set of cleanups to DiskFile +47e21af object-storage: Initial unittest of DiskFile class +0c9f4d1 object-storage: unit tests to 100% cover utils +29c23f2 object-storage: use temp file optimization for pkl +c399ca2 object-storage: Use "-dev" in the version string +7f9adbe object-storage: enforce RPM package dependencies +a8d5907 object-storage: fix account mapping +3d83e6c Replace "ssh foo gluster" with "gluster --remote-host=foo" +f35d192 object-storage: Add a sorted list comparison to test +a64a0d8 object-storage: del 1.4.8 ref from dir hierarchy +$ # +$ # Notice that it only goes back to the point where the "swift" +$ # directory was renamed to "ufo", so we need to checkout the +$ # previous revision by: +$ # +$ git log --oneline | grep -C 1 -F "del 1.4.8" +1f64e5e gsyncd / geo-rep: include hostname in status cmd +a64a0d8 object-storage: del 1.4.8 ref from dir hierarchy +94ddaec Don't ignore all files that match (*o|*la|*lo) +$ git checkout 94ddaec +$ git log --oneline swift +1129698 object-storage: final changes to remove swift diff +e8d9565 object-storage: refactor to use swift devices +05cc14c object-storage: make proper refs to module vars +b696fc0 object-storage: reduce diffs, bury metadata update +ffc7618 object-storage: swift-diff removal: hide obj only +89aff74 object-storage: fix trailing whitespace +d2040d1 object-storage: remove the device and part params +6bd81f2 object-storage: only update metadata that changes +7066b49 object-storage: DiskAccount subclassing of DiskDir +92fed12 object-storage: fix bad refs to check_acct_exists +f3daa0b object-storage: fix bad refs to REMOTE_CLUSTER +ee28eae object-storage: rename self.name to self.container +7fa7029 object-storage: unused parameter cleanup & handl'n +a1fd8c8 object-storage: add missing docs for params +a6b51d6 object-storage: minor code cleanups +b0cb7aa object-storage: remove glusterfs filter requirement +cae3216 object-storage: use constants for directory names +6478b56 object-storeage: refactor to use one memcache key +5c272f4 Minor refactorings to mark an internal routine +8534cd5 Reduce the number of stat/fstat system calls made +4f6aeb6 Further reduce extended attribute reads/writes +3ab5404 Move _add_timestamp() closer to where it is used +1a013e3 object-storage: reduce the number of getxattr system calls by one +5110fc0 object-storage: Bump size of metadata stored per xattr key +2b80439 object-storage: Don't catch generic exceptions around xattr calls +dbe793b object-storage: Refactor code to reduce Swift diffs carried +22050e5 object-storage: change logging of some errors to warnings +ddb4652 Remove the test subdirectory diffs ahead of initial refactoring. +49a3d88 Reduce the number of gratuitous differences in constraints. +3c05477 Fix BZ 865858: remove unnecessary system calls around xattr ops +49fad94 Refactor to use pyxattr's get/set/remove methods +a9c613d Fix a small typ-o in the Swift README +72301d4 swift: Donot display async_pending container. +bf7634c swift: Fix for multiple mounts on the same mount point. +93893b9 swift: Passing account name in container_update. +530a44a swift: Gluster object storage plugin for Openstack Swift. +$ # +$ # Now we see that we have the entire history above. +$ # +$ # Next we need to generate patch files to apply to a clean tree. +$ # To do that, we start by listing the patches for the swift tree +$ # in reverse order and explicitly requesting a numbered format- +$ # patch: +$ let idx=1 +$ for i in $(git log --oneline --reverse swift | awk '{ print $1 }') ; do +$ git checkout $i +$ git format-patch --start-number $idx -1 +$ ((idx++)) +$ done +$ # +$ # Then we need to switch back to top-of-tree and do the same for +$ # the "ufo" directory: +$ # +$ git checkout master +$ for i in $(git log --oneline --reverse ufo | awk '{ print $1 }') ; do +$ git checkout $i +$ git format-patch --start-number $idx -1 +$ ((idx++)) +$ done +$ # +$ # At this point we have a set of sequentially numbered diffs that +$ # can be applied to any git repo ... with two exceptions: patches +$ # 0056-*.patch and 0061-*.patch have references to files that do not +$ # exist ahead of time, so one has to edit out the cli* and .gitignore +$ # diffs from those two patches, respectively, and then one can run +$ # the following command to create a new repo: +$ # +$ cd /tmp # Or what ever directory you want to store the new repo in +$ mkdir newdir +$ cd newdir +$ git init +$ git am /path/to/*.patch # ... generated above +$ git remote add -f origin ssh://@git.gluster.com/gluster-swift.git +$ git pull --no-commit origin master +$ # +$ # By default, a git pull will provide its own commit message, but +$ # we list one below to be explicit in case some environments don't +$ # have the proper "editor" set. +$ # +$ git commit -s -m "Merge up to master before committing initial changes" +$ # +$ # Once we have "Push" commit rights, the following updates the tree: +$ # +$ git push origin master +Counting objects: 660, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (584/584), done. +Writing objects: 100% (659/659), 158.15 KiB, done. +Total 659 (delta 316), reused 0 (delta 0) +remote: Resolving deltas: 100% (316/316) +remote: Processing changes: refs: 1, done +remote: (W) ce1fd47: commit message lines >70 characters; manually wrap lines +remote: (W) cfb17e8: commit subject >65 characters; use shorter first paragraph +remote: (W) c48dcdc: commit message lines >70 characters; manually wrap lines +remote: (W) dd826e2: commit message lines >70 characters; manually wrap lines +remote: (W) b85ce6f: commit message lines >70 characters; manually wrap lines +remote: (W) c565fc2: commit message lines >70 characters; manually wrap lines +remote: (W) 6237313: commit message lines >70 characters; manually wrap lines +remote: (W) 29d922d: commit message lines >70 characters; manually wrap lines +remote: (W) ea1077a: commit message lines >70 characters; manually wrap lines +remote: (W) c61aea1: commit message lines >70 characters; manually wrap lines +remote: (W) a3a03e0: commit message lines >70 characters; manually wrap lines +remote: (W) 3ad7809: commit message lines >70 characters; manually wrap lines +To ssh://Portante@git.gluster.com/gluster-swift.git + 3303443..f534d66 master -> master diff --git a/README b/README new file mode 100644 index 0000000..1dd12e3 --- /dev/null +++ b/README @@ -0,0 +1,22 @@ +Gluster For Swift - Integrating GlusterFS with OpenStack Swift Implementation +of the Swift API. This is a component of the Unified File and Object (UFO) +story for GlusterFS, which allows files and directories created via +gluster-native/nfs/samba mount(s) to be accessed as accounts, containers and +objects. + +Install + * yum install openstack-swift*.noarch + * yum install gluster-swift-1.3*.noarch + +Once this is done, you can access GlusterFS volumes via the Swift API where +accounts are mounted volumes (today), containers are top-level directories, +and objects are files and sub-directories of container directories. + +Command to start the servers (TBD) + swift-init main start + +Command to stop the servers (TBD) + swift-init main stop + +Command to gracefully reload the servers + swift-init main reload \ No newline at end of file diff --git a/bin/gluster-swift-gen-builders b/bin/gluster-swift-gen-builders new file mode 100755 index 0000000..bef5d14 --- /dev/null +++ b/bin/gluster-swift-gen-builders @@ -0,0 +1,44 @@ +#!/bin/bash + +# Note that these port numbers must match the configured values for the +# various servers in their configuration files. +declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \ + ["object.builder"]=6010) + +builder_files="account.builder container.builder object.builder" + +function create { + swift-ring-builder $1 create 1 1 1 >> /tmp/out +} + +function add { + swift-ring-builder $1 add z$2-127.0.0.1:$3/$4_ 100.0 +} + +function rebalance { + swift-ring-builder $1 rebalance +} + +function build { + swift-ring-builder $1 +} + +if [ "$1x" = "x" ]; then + echo "Please specify the gluster volume name to use." + exit 1 +fi + +for builder_file in $builder_files +do + create $builder_file + + zone=1 + for volname in $@ + do + add $builder_file $zone ${port[$builder_file]} $volname + zone=$(expr $zone + 1) + done + + rebalance $builder_file + build $builder_file +done diff --git a/etc/account-server/1.conf-gluster b/etc/account-server/1.conf-gluster new file mode 100644 index 0000000..da8f317 --- /dev/null +++ b/etc/account-server/1.conf-gluster @@ -0,0 +1,19 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6012 +user = root +log_facility = LOG_LOCAL2 + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:gluster_swift_ufo#account + +[account-replicator] +vm_test_mode = yes + +[account-auditor] + +[account-reaper] diff --git a/etc/container-server/1.conf-gluster b/etc/container-server/1.conf-gluster new file mode 100644 index 0000000..acad621 --- /dev/null +++ b/etc/container-server/1.conf-gluster @@ -0,0 +1,21 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6011 +user = root +log_facility = LOG_LOCAL2 + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:gluster_swift_ufo#container + +[container-replicator] +vm_test_mode = yes + +[container-updater] + +[container-auditor] + +[container-sync] diff --git a/etc/fs.conf-gluster b/etc/fs.conf-gluster new file mode 100644 index 0000000..71a9b03 --- /dev/null +++ b/etc/fs.conf-gluster @@ -0,0 +1,17 @@ +[DEFAULT] +# IP address of a GlusterFS volume server member. By default, we assume the +# local host. +mount_ip = localhost + +# By default it is assumed the Gluster volumes can be accessed using other +# methods besides UFO (not object only), which disables a caching +# optimizations in order to keep in sync with file system changes. +object_only = no + +# Performance optimization parameter. When turned off, the filesystem will +# see a reduced number of stat calls, resulting in substantially faster +# response time for GET and HEAD container requests on containers with large +# numbers of objects, at the expense of an accurate count of combined bytes +# used by all objects in the container. For most installations "off" works +# fine. +accurate_size_in_listing = off \ No newline at end of file diff --git a/etc/object-server/1.conf-gluster b/etc/object-server/1.conf-gluster new file mode 100644 index 0000000..0d85546 --- /dev/null +++ b/etc/object-server/1.conf-gluster @@ -0,0 +1,36 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6010 +# If not doing the above, setting this value initially to match the number of +# CPUs is a good starting point for determining the right value. +workers = 1 + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:gluster_swift_ufo#object +user = root +log_facility = LOG_LOCAL2 +# Timeout clients that don't read or write to the proxy server after 5 +# seconds. +conn_timeout = 5 +# For high load situations, once connected to a container server, allow for +# delays communicating with it. +node_timeout = 60 +# Adjust this value to match the stripe width of the underlying storage array +# (not the stripe element size). This will provide a reasonable starting point +# for tuning this value. +disk_chunk_size = 65536 +# Adjust this value match whatever is set for the disk_chunk_size +# initially. This will provide a reasonable starting point for tuning this +# value. +network_chunk_size = 65556 + +[object-replicator] +vm_test_mode = yes + +[object-updater] + +[object-auditor] diff --git a/etc/proxy-server.conf-gluster b/etc/proxy-server.conf-gluster new file mode 100644 index 0000000..e04efec --- /dev/null +++ b/etc/proxy-server.conf-gluster @@ -0,0 +1,69 @@ +[DEFAULT] +bind_port = 8080 +user = root +log_facility = LOG_LOCAL1 +# Consider using 1 worker per CPU +workers = 1 + +[pipeline:main] +pipeline = healthcheck cache tempauth proxy-server + +[app:proxy-server] +use = egg:gluster_swift_ufo#proxy +log_facility = LOG_LOCAL1 +# The API allows for account creation and deletion, but since Gluster/Swift +# automounts a Gluster volume for a given account, there is no way to create +# or delete an account. So leave this off. +allow_account_management = false +account_autocreate = true +# Only need to recheck the account exists once a day +recheck_account_existence = 86400 +# May want to consider bumping this up if containers are created and destroyed +# infrequently. +recheck_container_existence = 60 +# Timeout clients that don't read or write to the proxy server after 5 +# seconds. +client_timeout = 5 +# Give more time to connect to the object, container or account servers in +# cases of high load. +conn_timeout = 5 +# For high load situations, once connected to an object, container or account +# server, allow for delays communicating with them. +node_timeout = 60 +# May want to consider bumping up this value to 1 - 4 MB depending on how much +# traffic is for multi-megabyte or gigabyte requests; perhaps matching the +# stripe width (not stripe element size) of your storage volume is a good +# starting point. See below for sizing information. +object_chunk_size = 65536 +# If you do decide to increase the object_chunk_size, then consider lowering +# this value to one. Up to "put_queue_length" object_chunk_size'd buffers can +# be queued to the object server for processing. Given one proxy server worker +# can handle up to 1,024 connections, by default, it will consume 10 * 65,536 +# * 1,024 bytes of memory in the worse case (default values). Be sure the +# amount of memory available on the system can accommodate increased values +# for object_chunk_size. +put_queue_depth = 10 + +[filter:tempauth] +use = egg:swift#tempauth +# Here you need to add users explicitly. See the OpenStack Swift Deployment +# Guide for more information. The user and user64 directives take the +# following form: +# user__ = [group] [group] [...] [storage_url] +# user64__ = [group] [group] [...] [storage_url] +# Where you use user64 for accounts and/or usernames that include underscores. +# +# NOTE (and WARNING): The account name must match the device name specified +# when generating the account, container, and object build rings. +# +# E.g. +# user_ufo0_admin = abc123 .admin + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 diff --git a/etc/swift.conf-gluster b/etc/swift.conf-gluster new file mode 100644 index 0000000..25c3ca1 --- /dev/null +++ b/etc/swift.conf-gluster @@ -0,0 +1,91 @@ +[DEFAULT] + + +[swift-hash] +# random unique string that can never change (DO NOT LOSE) +swift_hash_path_suffix = gluster + + +# The swift-constraints section sets the basic constraints on data +# saved in the swift cluster. + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. It is STRONGLY recommended to leave this value at +# the default (5 * 2**30 + 2). + +# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting +# web service handle such a size? I think with UFO, we need to keep with the +# default size from Swift and encourage users to research what size their web +# services infrastructure can handle. + +max_file_size = 18446744073709551616 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 + + +# max_object_name_length is the max number of bytes in the utf8 encoding of an +# object name: Gluster FS can handle much longer file names, but the length +# between the slashes of the URL is handled below. Remember that most web +# clients can't handle anything greater than 2048, and those that do are +# rather clumsy. + +max_object_name_length = 2048 + +# max_object_name_component_length (GlusterFS) is the max number of bytes in +# the utf8 encoding of an object name component (the part between the +# slashes); this is a limit imposed by the underlying file system (for XFS it +# is 255 bytes). + +max_object_name_component_length = 255 + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request + +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding of +# an account name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_account_name_length = 255 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_container_name_length = 255 diff --git a/gluster-swift.spec b/gluster-swift.spec new file mode 100644 index 0000000..af80e40 --- /dev/null +++ b/gluster-swift.spec @@ -0,0 +1,78 @@ +############################################################################################################ +# Command to build rpms.# +# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz # +############################################################################################################ +# Setting up the environment. # +# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES # +# * Copy the contents of gluster directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} # +# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec # +# For more information refer # +# http://fedoraproject.org/wiki/How_to_create_an_RPM_package # +############################################################################################################ + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%endif + +%define _confdir %{_sysconfdir}/swift +%define _version 1.3 +%define _release 0 + +Summary : GlusterFS Integration with OpenStack Object Storage (Swift). +Name : gluster-for-swift +Version : %{_version} +Release : %{_release} +Group : Application/File +Vendor : Red Hat Inc. +Source0 : %{name}-%{version}-%{release}.tar.gz +Packager : gluster-users@gluster.org +License : Apache +BuildArch: noarch +Requires : memcached +Requires : openssl +Requires : python +Requires : openstack-swift >= 1.8.0 +Requires : openstack-swift-account >= 1.8.0 +Requires : openstack-swift-container >= 1.8.0 +Requires : openstack-swift-object >= 1.8.0 +Requires : openstack-swift-proxy >= 1.8.0 + +%description +Gluster-For-Swift (G4S, pronounced "gee-force") integrates GlusterFS as an +alternative back end for OpenStack Object Storage (Swift) leveraging the +existing front end OpenStack Swift code. Gluster volumes are used to store +objects in files, containers are maintained as top-level directories of volumes, +where accounts are mapped one-to-one to gluster volumes. + +%prep +%setup -q + +%build +%{__python} setup.py build + +%install +rm -rf %{buildroot} + +%{__python} setup.py install -O1 --skip-build --root %{buildroot} + +mkdir -p %{buildroot}/%{_confdir}/ +cp -r etc/* %{buildroot}/%{_confdir}/ + +mkdir -p %{buildroot}/%{_bindir}/ +cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/ + +%clean +rm -rf %{buildroot} + +%files +%defattr(-,root,root) +%{python_sitelib}/gluster +%{python_sitelib}/gluster_swift-%{version}-*.egg-info +%{_bindir}/gluster-swift-gen-builders +%dir %{_confdir} +%config %{_confdir}/account-server/1.conf-gluster +%config %{_confdir}/container-server/1.conf-gluster +%config %{_confdir}/object-server/1.conf-gluster +%config %{_confdir}/swift.conf-gluster +%config %{_confdir}/proxy-server.conf-gluster +%config %{_confdir}/fs.conf-gluster diff --git a/gluster/__init__.py b/gluster/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/__init__.py b/gluster/swift/__init__.py new file mode 100644 index 0000000..ef350fd --- /dev/null +++ b/gluster/swift/__init__.py @@ -0,0 +1,18 @@ +""" Gluster Swift UFO """ + +class Version(object): + def __init__(self, canonical_version, final): + self.canonical_version = canonical_version + self.final = final + + @property + def pretty_version(self): + if self.final: + return self.canonical_version + else: + return '%s-dev' % (self.canonical_version,) + + +_version = Version('1.1', False) +__version__ = _version.pretty_version +__canonical_version__ = _version.canonical_version diff --git a/gluster/swift/account/__init__.py b/gluster/swift/account/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/account/server.py b/gluster/swift/account/server.py new file mode 100644 index 0000000..aeaabc9 --- /dev/null +++ b/gluster/swift/account/server.py @@ -0,0 +1,45 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Account Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.account import server +from gluster.swift.common.DiskDir import DiskAccount + + +class AccountController(server.AccountController): + def _get_account_broker(self, drive, part, account): + """ + Overriden to provide the GlusterFS specific broker that talks to + Gluster for the information related to servicing a given request + instead of talking to a database. + + :param drive: drive that holds the container + :param part: partition the container is in + :param account: account name + :returns: DiskDir object + """ + return DiskAccount(self.root, drive, account, self.logger) + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI account server apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return AccountController(conf) diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py new file mode 100644 index 0000000..18d08cc --- /dev/null +++ b/gluster/swift/common/DiskDir.py @@ -0,0 +1,496 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, errno + +from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \ + mkdirs, validate_account, validate_container, is_marker, \ + get_container_details, get_account_details, get_container_metadata, \ + create_container_metadata, create_account_metadata, DEFAULT_GID, \ + DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ + write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ + X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ + X_CONTAINER_COUNT, CONTAINER, os_path +from gluster.swift.common import Glusterfs + +from swift.common.constraints import CONTAINER_LISTING_LIMIT +from swift.common.utils import normalize_timestamp, TRUE_VALUES + + +DATADIR = 'containers' + +# Create a dummy db_file in /etc/swift +_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') +if _unittests_enabled in TRUE_VALUES: + _tmp_dir = '/tmp/gluster_unit_tests' + try: + os.mkdir(_tmp_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + _db_file = os.path.join(_tmp_dir, 'db_file.db') +else: + _db_file = '/etc/swift/db_file.db' +if not os.path.exists(_db_file): + file(_db_file, 'w+') + + +def _read_metadata(dd): + """ Filter read metadata so that it always returns a tuple that includes + some kind of timestamp. With 1.4.8 of the Swift integration the + timestamps were not stored. Here we fabricate timestamps for volumes + where the existing data has no timestamp (that is, stored data is not + a tuple), allowing us a measure of backward compatibility. + + FIXME: At this time it does not appear that the timestamps on each + metadata are used for much, so this should not hurt anything. + """ + metadata_i = read_metadata(dd) + metadata = {} + timestamp = 0 + for key, value in metadata_i.iteritems(): + if not isinstance(value, tuple): + value = (value, timestamp) + metadata[key] = value + return metadata + + +class DiskCommon(object): + def is_deleted(self): + return not os_path.exists(self.datadir) + + def filter_prefix(self, objects, prefix): + """ + Accept sorted list. + """ + found = 0 + filtered_objs = [] + for object_name in objects: + if object_name.startswith(prefix): + filtered_objs.append(object_name) + found = 1 + else: + if found: + break + return filtered_objs + + def filter_delimiter(self, objects, delimiter, prefix): + """ + Accept sorted list. + Objects should start with prefix. + """ + filtered_objs=[] + for object_name in objects: + tmp_obj = object_name.replace(prefix, '', 1) + sufix = tmp_obj.split(delimiter, 1) + new_obj = prefix + sufix[0] + if new_obj and new_obj not in filtered_objs: + filtered_objs.append(new_obj) + + return filtered_objs + + def filter_marker(self, objects, marker): + """ + TODO: We can traverse in reverse order to optimize. + Accept sorted list. + """ + filtered_objs=[] + found = 0 + if objects[-1] < marker: + return filtered_objs + for object_name in objects: + if object_name > marker: + filtered_objs.append(object_name) + + return filtered_objs + + def filter_end_marker(self, objects, end_marker): + """ + Accept sorted list. + """ + filtered_objs=[] + for object_name in objects: + if object_name < end_marker: + filtered_objs.append(object_name) + else: + break + + return filtered_objs + + def filter_limit(self, objects, limit): + filtered_objs=[] + for i in range(0, limit): + filtered_objs.append(objects[i]) + + return filtered_objs + + +class DiskDir(DiskCommon): + """ + Manage object files on disk. + + :param path: path to devices on the node + :param drive: gluster volume drive name + :param account: account name for the object + :param container: container name for the object + :param logger: account or container server logging object + :param uid: user ID container object should assume + :param gid: group ID container object should assume + """ + + def __init__(self, path, drive, account, container, logger, + uid=DEFAULT_UID, gid=DEFAULT_GID): + self.root = path + if container: + self.container = container + else: + self.container = None + if self.container: + self.datadir = os.path.join(path, drive, self.container) + else: + self.datadir = os.path.join(path, drive) + self.account = account + assert logger is not None + self.logger = logger + self.metadata = {} + self.container_info = None + self.object_info = None + self.uid = int(uid) + self.gid = int(gid) + self.db_file = _db_file + self.dir_exists = os_path.exists(self.datadir) + if self.dir_exists: + try: + self.metadata = _read_metadata(self.datadir) + except EOFError: + create_container_metadata(self.datadir) + else: + return + if self.container: + if not self.metadata: + create_container_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not validate_container(self.metadata): + create_container_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not self.metadata: + create_account_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not validate_account(self.metadata): + create_account_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + + def empty(self): + return dir_empty(self.datadir) + + def delete(self): + if self.empty(): + #For delete account. + if os_path.ismount(self.datadir): + clean_metadata(self.datadir) + else: + rmdirs(self.datadir) + self.dir_exists = False + + def put_metadata(self, metadata): + """ + Write metadata to directory/container. + """ + write_metadata(self.datadir, metadata) + self.metadata = metadata + + def put(self, metadata): + """ + Create and write metatdata to directory/container. + :param metadata: Metadata to write. + """ + if not self.dir_exists: + mkdirs(self.datadir) + + os.chown(self.datadir, self.uid, self.gid) + write_metadata(self.datadir, metadata) + self.metadata = metadata + self.dir_exists = True + + def put_obj(self, content_length, timestamp): + ocnt = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) + self.metadata[X_PUT_TIMESTAMP] = timestamp + bused = self.metadata[X_BYTES_USED][0] + self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) + #TODO: define update_metadata instad of writing whole metadata again. + self.put_metadata(self.metadata) + + def delete_obj(self, content_length): + ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) + bused, timestamp = self.metadata[X_BYTES_USED] + self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) + self.put_metadata(self.metadata) + + def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): + """ + For account server. + """ + self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) + self.metadata[X_BYTES_USED] = (0, put_timestamp) + ccnt = self.metadata[X_CONTAINER_COUNT][0] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) + self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) + self.put_metadata(self.metadata) + + def delete_container(self, object_count, bytes_used): + """ + For account server. + """ + self.metadata[X_OBJECTS_COUNT] = (0, 0) + self.metadata[X_BYTES_USED] = (0, 0) + ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) + self.put_metadata(self.metadata) + + def unlink(self): + """ + Remove directory/container if empty. + """ + if dir_empty(self.datadir): + rmdirs(self.datadir) + + def list_objects_iter(self, limit, marker, end_marker, + prefix, delimiter, path): + """ + Returns tuple of name, created_at, size, content_type, etag. + """ + if path: + prefix = path = path.rstrip('/') + '/' + delimiter = '/' + if delimiter and not prefix: + prefix = '' + + self.update_object_count() + + objects, object_count, bytes_used = self.object_info + + if objects and prefix: + objects = self.filter_prefix(objects, prefix) + + if objects and delimiter: + objects = self.filter_delimiter(objects, delimiter, prefix) + + if objects and marker: + objects = self.filter_marker(objects, marker) + + if objects and end_marker: + objects = self.filter_end_marker(objects, end_marker) + + if objects and limit: + if len(objects) > limit: + objects = self.filter_limit(objects, limit) + + container_list = [] + if objects: + for obj in objects: + list_item = [] + list_item.append(obj) + obj_path = os.path.join(self.datadir, obj) + metadata = read_metadata(obj_path) + if not metadata or not validate_object(metadata): + metadata = create_object_metadata(obj_path) + if metadata: + list_item.append(metadata[X_TIMESTAMP]) + list_item.append(int(metadata[X_CONTENT_LENGTH])) + list_item.append(metadata[X_CONTENT_TYPE]) + list_item.append(metadata[X_ETAG]) + container_list.append(list_item) + + return container_list + + def update_object_count(self): + if not self.object_info: + self.object_info = get_container_details(self.datadir) + + objects, object_count, bytes_used = self.object_info + + if X_OBJECTS_COUNT not in self.metadata \ + or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ + or X_BYTES_USED not in self.metadata \ + or int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) + write_metadata(self.datadir, self.metadata) + + def update_container_count(self): + if not self.container_info: + self.container_info = get_account_details(self.datadir) + + containers, container_count = self.container_info + + if X_CONTAINER_COUNT not in self.metadata \ + or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) + write_metadata(self.datadir, self.metadata) + + def get_info(self, include_metadata=False): + """ + Get global data for the container. + :returns: dict with keys: account, container, object_count, bytes_used, + hash, id, created_at, put_timestamp, delete_timestamp, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, and reported_bytes_used. + If include_metadata is set, metadata is included as a key + pointing to a dict of tuples of the metadata + """ + # TODO: delete_timestamp, reported_put_timestamp + # reported_delete_timestamp, reported_object_count, + # reported_bytes_used, created_at + if not Glusterfs.OBJECT_ONLY: + # If we are not configured for object only environments, we should + # update the object counts in case they changed behind our back. + self.update_object_count() + + data = {'account' : self.account, 'container' : self.container, + 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], + 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], + 'hash': '', 'id' : '', 'created_at' : '1', + 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], + 'delete_timestamp' : '1', + 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', + 'reported_object_count' : '1', 'reported_bytes_used' : '1'} + if include_metadata: + data['metadata'] = self.metadata + return data + + def put_object(self, name, timestamp, size, content_type, + etag, deleted=0): + # TODO: Implement the specifics of this func. + pass + + def initialize(self, timestamp): + pass + + def update_put_timestamp(self, timestamp): + """ + Create the container if it doesn't exist and update the timestamp + """ + if not os_path.exists(self.datadir): + self.put(self.metadata) + + def delete_object(self, name, timestamp): + # TODO: Implement the delete object + pass + + def delete_db(self, timestamp): + """ + Delete the container + """ + self.unlink() + + def update_metadata(self, metadata): + assert self.metadata, "Valid container/account metadata should have been created by now" + if metadata: + new_metadata = self.metadata.copy() + new_metadata.update(metadata) + if new_metadata != self.metadata: + write_metadata(self.datadir, new_metadata) + self.metadata = new_metadata + + +class DiskAccount(DiskDir): + def __init__(self, root, drive, account, logger): + super(DiskAccount, self).__init__(root, drive, account, None, logger) + assert self.dir_exists + + def list_containers_iter(self, limit, marker, end_marker, + prefix, delimiter): + """ + Return tuple of name, object_count, bytes_used, 0(is_subdir). + Used by account server. + """ + if delimiter and not prefix: + prefix = '' + + self.update_container_count() + + containers, container_count = self.container_info + + if containers: + containers.sort() + + if containers and prefix: + containers = self.filter_prefix(containers, prefix) + + if containers and delimiter: + containers = self.filter_delimiter(containers, delimiter, prefix) + + if containers and marker: + containers = self.filter_marker(containers, marker) + + if containers and end_marker: + containers = self.filter_end_marker(containers, end_marker) + + if containers and limit: + if len(containers) > limit: + containers = self.filter_limit(containers, limit) + + account_list = [] + if containers: + for cont in containers: + list_item = [] + metadata = None + list_item.append(cont) + cont_path = os.path.join(self.datadir, cont) + metadata = _read_metadata(cont_path) + if not metadata or not validate_container(metadata): + metadata = create_container_metadata(cont_path) + + if metadata: + list_item.append(metadata[X_OBJECTS_COUNT][0]) + list_item.append(metadata[X_BYTES_USED][0]) + list_item.append(0) + account_list.append(list_item) + + return account_list + + def get_info(self, include_metadata=False): + """ + Get global data for the account. + :returns: dict with keys: account, created_at, put_timestamp, + delete_timestamp, container_count, object_count, + bytes_used, hash, id + """ + if not Glusterfs.OBJECT_ONLY: + # If we are not configured for object only environments, we should + # update the container counts in case they changed behind our back. + self.update_container_count() + + data = {'account' : self.account, 'created_at' : '1', + 'put_timestamp' : '1', 'delete_timestamp' : '1', + 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], + 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], + 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], + 'hash' : '', 'id' : ''} + + if include_metadata: + data['metadata'] = self.metadata + return data + + def get_container_timestamp(self, container): + cont_path = os.path.join(self.datadir, container) + metadata = read_metadata(cont_path) + + return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None diff --git a/gluster/swift/common/DiskFile.py b/gluster/swift/common/DiskFile.py new file mode 100644 index 0000000..900bd49 --- /dev/null +++ b/gluster/swift/common/DiskFile.py @@ -0,0 +1,338 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import errno +import random +from hashlib import md5 +from contextlib import contextmanager +from swift.common.utils import normalize_timestamp, renamer +from swift.common.exceptions import DiskFileNotExist +from gluster.swift.common.exceptions import AlreadyExistsAsDir +from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \ + create_object_metadata, do_open, do_close, do_unlink, do_chown, \ + do_listdir, read_metadata, write_metadata, os_path, do_fsync +from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \ + X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \ + X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \ + FILE_TYPE, DEFAULT_UID, DEFAULT_GID + +import logging +from swift.obj.server import DiskFile + + +DEFAULT_DISK_CHUNK_SIZE = 65536 +# keep these lower-case +DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) + + +def _adjust_metadata(metadata): + # Fix up the metadata to ensure it has a proper value for the + # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE + # metadata values. + content_type = metadata['Content-Type'] + if not content_type: + # FIXME: How can this be that our caller supplied us with metadata + # that has a content type that evaluates to False? + # + # FIXME: If the file exists, we would already know it is a + # directory. So why are we assuming it is a file object? + metadata['Content-Type'] = FILE_TYPE + x_object_type = FILE + else: + x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + metadata[X_TYPE] = OBJECT + metadata[X_OBJECT_TYPE] = x_object_type + return metadata + + +class Gluster_DiskFile(DiskFile): + """ + Manage object files on disk. + + :param path: path to devices on the node/mount path for UFO. + :param device: device name/account_name for UFO. + :param partition: partition on the device the object lives in + :param account: account name for the object + :param container: container name for the object + :param obj: object name for the object + :param logger: logger object for writing out log file messages + :param keep_data_fp: if True, don't close the fp, otherwise close it + :param disk_chunk_Size: size of chunks on file reads + :param uid: user ID disk object should assume (file or directory) + :param gid: group ID disk object should assume (file or directory) + """ + + def __init__(self, path, device, partition, account, container, obj, + logger, keep_data_fp=False, + disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, + uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): + self.disk_chunk_size = disk_chunk_size + self.iter_hook = iter_hook + # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/, + # etc. + obj = obj.strip(os.path.sep) + if os.path.sep in obj: + self._obj_path, self._obj = os.path.split(obj) + else: + self._obj_path = '' + self._obj = obj + + if self._obj_path: + self.name = os.path.join(container, self._obj_path) + else: + self.name = container + # Absolute path for object directory. + self.datadir = os.path.join(path, device, self.name) + self.device_path = os.path.join(path, device) + self._container_path = os.path.join(path, device, container) + self._is_dir = False + self.tmppath = None + self.logger = logger + self.metadata = {} + self.meta_file = None + self.fp = None + self.iter_etag = None + self.started_at_0 = False + self.read_to_eof = False + self.quarantined_dir = None + self.keep_cache = False + self.uid = int(uid) + self.gid = int(gid) + + # Don't store a value for data_file until we know it exists. + self.data_file = None + data_file = os.path.join(self.datadir, self._obj) + if not os_path.exists(data_file): + return + + self.data_file = os.path.join(data_file) + self.metadata = read_metadata(data_file) + if not self.metadata: + create_object_metadata(data_file) + self.metadata = read_metadata(data_file) + + if not validate_object(self.metadata): + create_object_metadata(data_file) + self.metadata = read_metadata(data_file) + + self.filter_metadata() + + if os_path.isdir(data_file): + self._is_dir = True + else: + if keep_data_fp: + # The caller has an assumption that the "fp" field of this + # object is an file object if keep_data_fp is set. However, + # this implementation of the DiskFile object does not need to + # open the file for internal operations. So if the caller + # requests it, we'll just open the file for them. + self.fp = do_open(data_file, 'rb') + + def close(self, verify_file=True): + """ + Close the file. Will handle quarantining file if necessary. + + :param verify_file: Defaults to True. If false, will not check + file to see if it needs quarantining. + """ + #Marker directory + if self._is_dir: + return + if self.fp: + do_close(self.fp) + self.fp = None + + def is_deleted(self): + """ + Check if the file is deleted. + + :returns: True if the file doesn't exist or has been flagged as + deleted. + """ + return not self.data_file + + def _create_dir_object(self, dir_path): + #TODO: if object already exists??? + if os_path.exists(dir_path) and not os_path.isdir(dir_path): + self.logger.error("Deleting file %s", dir_path) + do_unlink(dir_path) + #If dir aleady exist just override metadata. + mkdirs(dir_path) + do_chown(dir_path, self.uid, self.gid) + create_object_metadata(dir_path) + + def put_metadata(self, metadata, tombstone=False): + """ + Short hand for putting metadata to .meta and .ts files. + + :param metadata: dictionary of metadata to be written + :param tombstone: whether or not we are writing a tombstone + """ + if tombstone: + # We don't write tombstone files. So do nothing. + return + assert self.data_file is not None, "put_metadata: no file to put metadata into" + metadata = _adjust_metadata(metadata) + write_metadata(self.data_file, metadata) + self.metadata = metadata + self.filter_metadata() + + def put(self, fd, metadata, extension='.data'): + """ + Finalize writing the file on disk, and renames it from the temp file to + the real location. This should be called after the data has been + written to the temp file. + + :param fd: file descriptor of the temp file + :param metadata: dictionary of metadata to be written + :param extension: extension to be used when making the file + """ + # Our caller will use '.data' here; we just ignore it since we map the + # URL directly to the file system. + extension = '' + + metadata = _adjust_metadata(metadata) + + if metadata[X_OBJECT_TYPE] == MARKER_DIR: + if not self.data_file: + self.data_file = os.path.join(self.datadir, self._obj) + self._create_dir_object(self.data_file) + self.put_metadata(metadata) + return + + # Check if directory already exists. + if self._is_dir: + # FIXME: How can we have a directory and it not be marked as a + # MARKER_DIR (see above)? + msg = 'File object exists as a directory: %s' % self.data_file + raise AlreadyExistsAsDir(msg) + + timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) + write_metadata(self.tmppath, metadata) + if X_CONTENT_LENGTH in metadata: + self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) + do_fsync(fd) + if self._obj_path: + dir_objs = self._obj_path.split('/') + assert len(dir_objs) >= 1 + tmp_path = self._container_path + for dir_name in dir_objs: + tmp_path = os.path.join(tmp_path, dir_name) + self._create_dir_object(tmp_path) + + newpath = os.path.join(self.datadir, self._obj) + renamer(self.tmppath, newpath) + do_chown(newpath, self.uid, self.gid) + self.metadata = metadata + self.data_file = newpath + self.filter_metadata() + return + + def unlinkold(self, timestamp): + """ + Remove any older versions of the object file. Any file that has an + older timestamp than timestamp will be deleted. + + :param timestamp: timestamp to compare with each file + """ + if not self.metadata or self.metadata['X-Timestamp'] >= timestamp: + return + + assert self.data_file, \ + "Have metadata, %r, but no data_file" % self.metadata + + if self._is_dir: + # Marker directory object + if not rmdirs(self.data_file): + logging.error('Unable to delete dir object: %s', self.data_file) + return + else: + # File object + do_unlink(self.data_file) + + self.metadata = {} + self.data_file = None + + def get_data_file_size(self): + """ + Returns the os_path.getsize for the file. Raises an exception if this + file does not match the Content-Length stored in the metadata. Or if + self.data_file does not exist. + + :returns: file size as an int + :raises DiskFileError: on file size mismatch. + :raises DiskFileNotExist: on file not existing (including deleted) + """ + #Marker directory. + if self._is_dir: + return 0 + try: + file_size = 0 + if self.data_file: + file_size = os_path.getsize(self.data_file) + if X_CONTENT_LENGTH in self.metadata: + metadata_size = int(self.metadata[X_CONTENT_LENGTH]) + if file_size != metadata_size: + self.metadata[X_CONTENT_LENGTH] = file_size + write_metadata(self.data_file, self.metadata) + + return file_size + except OSError as err: + if err.errno != errno.ENOENT: + raise + raise DiskFileNotExist('Data File does not exist.') + + def filter_metadata(self): + if X_TYPE in self.metadata: + self.metadata.pop(X_TYPE) + if X_OBJECT_TYPE in self.metadata: + self.metadata.pop(X_OBJECT_TYPE) + + @contextmanager + def mkstemp(self): + """Contextmanager to make a temporary file.""" + + # Creating intermidiate directories and corresponding metadata. + # For optimization, check if the subdirectory already exists, + # if exists, then it means that it also has its metadata. + # Not checking for container, since the container should already + # exist for the call to come here. + if not os_path.exists(self.datadir): + path = self._container_path + subdir_list = self._obj_path.split(os.path.sep) + for i in range(len(subdir_list)): + path = os.path.join(path, subdir_list[i]); + if not os_path.exists(path): + self._create_dir_object(path) + + tmpfile = '.' + self._obj + '.' + md5(self._obj + \ + str(random.random())).hexdigest() + + self.tmppath = os.path.join(self.datadir, tmpfile) + fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL) + try: + yield fd + finally: + try: + do_close(fd) + except OSError: + pass + tmppath, self.tmppath = self.tmppath, None + try: + do_unlink(tmppath) + except OSError as err: + if err.errno != errno.ENOENT: + raise diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py new file mode 100644 index 0000000..a0c8126 --- /dev/null +++ b/gluster/swift/common/Glusterfs.py @@ -0,0 +1,144 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os, sys, fcntl, time, errno +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.utils import TRUE_VALUES, search_tree +from gluster.swift.common.fs_utils import mkdirs + +# +# Read the fs.conf file once at startup (module load) +# +_fs_conf = ConfigParser() +MOUNT_IP = 'localhost' +OBJECT_ONLY = False +RUN_DIR='/var/run/swift' +SWIFT_DIR = '/etc/swift' +_do_getsize = False +if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): + try: + MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') + except (NoSectionError, NoOptionError): + pass + try: + OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + try: + RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift') + except (NoSectionError, NoOptionError): + pass + + try: + _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \ + "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + +NAME = 'glusterfs' + + +def _busy_wait(full_mount_path): + # Iterate for definite number of time over a given + # interval for successful mount + for i in range(0, 5): + if os.path.ismount(os.path.join(full_mount_path)): + return True + time.sleep(2) + logging.error('Busy wait for mount timed out for mount %s', full_mount_path) + return False + +def mount(root, drive): + # FIXME: Possible thundering herd problem here + + el = _get_export_list() + for export in el: + if drive == export: + break + else: + logging.error('No export found in %r matching drive, %s', el, drive) + return False + + # NOTE: root is typically the default value of /mnt/gluster-object + full_mount_path = os.path.join(root, drive) + if not os.path.isdir(full_mount_path): + mkdirs(full_mount_path) + + lck_file = os.path.join(RUN_DIR, '%s.lock' %drive); + + if not os.path.exists(RUN_DIR): + mkdirs(RUN_DIR) + + fd = os.open(lck_file, os.O_CREAT|os.O_RDWR) + with os.fdopen(fd, 'r+b') as f: + try: + fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) + except IOError as ex: + if ex.errno in (errno.EACCES, errno.EAGAIN): + # This means that some other process is mounting the + # filesystem, so wait for the mount process to complete + return _busy_wait(full_mount_path) + else: + raise ex + mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ + full_mount_path) + if os.system(mnt_cmd) or not _busy_wait(full_mount_path): + logging.error('Mount failed %s: %s', NAME, mnt_cmd) + return False + return True + +def unmount(full_mount_path): + # FIXME: Possible thundering herd problem here + + umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path + if os.system(umnt_cmd): + logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) + +def _get_export_list(): + cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP + + export_list = [] + + if os.system(cmnd + ' >> /dev/null'): + logging.error('Getting volume info failed for %s', NAME) + else: + fp = os.popen(cmnd) + while True: + item = fp.readline() + if not item: + break + item = item.strip('\n').strip(' ') + if item.lower().startswith('volume name:'): + export_list.append(item.split(':')[1].strip(' ')) + + return export_list + +def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"): + """Read the object-server's configuration file and return + the device value""" + + mnt_dir = '' + conf_files = search_tree(conf_dir, conf_file, '.conf') + if not conf_files: + raise Exception("Config file not found") + + _conf = ConfigParser() + if _conf.read(conf_files[0]): + try: + mnt_dir = _conf.get('DEFAULT', 'devices', '') + except (NoSectionError, NoOptionError): + raise + return os.path.join(mnt_dir, vol_name) diff --git a/gluster/swift/common/__init__.py b/gluster/swift/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/common/constraints.py b/gluster/swift/common/constraints.py new file mode 100644 index 0000000..11f626b --- /dev/null +++ b/gluster/swift/common/constraints.py @@ -0,0 +1,94 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from webob.exc import HTTPBadRequest +except ImportError: + from swift.common.swob import HTTPBadRequest +import swift.common.constraints +import swift.common.ring as _ring +from gluster.swift.common import Glusterfs, ring + +if hasattr(swift.common.constraints, 'constraints_conf_int'): + MAX_OBJECT_NAME_COMPONENT_LENGTH = \ + swift.common.constraints.constraints_conf_int( + 'max_object_name_component_length', 255) +else: + MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 + +def validate_obj_name_component(obj): + if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: + return 'too long (%d)' % len(obj) + if obj == '.' or obj == '..': + return 'cannot be . or ..' + return '' + +# Save the original check object creation +__check_object_creation = swift.common.constraints.check_object_creation + +# Define our new one which invokes the original +def gluster_check_object_creation(req, object_name): + """ + Check to ensure that everything is alright about an object to be created. + Monkey patches swift.common.constraints.check_object_creation, invoking + the original, and then adding an additional check for individual object + name components. + + :param req: HTTP request object + :param object_name: name of object to be created + :raises HTTPRequestEntityTooLarge: the object is too large + :raises HTTPLengthRequered: missing content-length header and not + a chunked request + :raises HTTPBadRequest: missing or bad content-type header, or + bad metadata + """ + ret = __check_object_creation(req, object_name) + + if ret is None: + for obj in object_name.split('/'): + reason = validate_obj_name_component(obj) + if reason: + bdy = 'Invalid object name "%s", component "%s" %s' \ + % (object_name, obj, reason) + ret = HTTPBadRequest(body=bdy, + request=req, + content_type='text/plain') + + return ret + +# Replace the original check object creation with ours +swift.common.constraints.check_object_creation = gluster_check_object_creation + +# Save the original check mount +__check_mount = swift.common.constraints.check_mount + +# Define our new one which invokes the original +def gluster_check_mount(root, drive): + # FIXME: Potential performance optimization here to not call the original + # check mount which makes two stat calls. We could do what they do with + # just one. + if __check_mount(root, drive): + return True + + return Glusterfs.mount(root, drive) + +# Replace the original check mount with ours +swift.common.constraints.check_mount = gluster_check_mount + +# Save the original Ring class +__Ring = _ring.Ring + +# Replace the original Ring class +_ring.Ring = ring.Ring diff --git a/gluster/swift/common/exceptions.py b/gluster/swift/common/exceptions.py new file mode 100644 index 0000000..d9357db --- /dev/null +++ b/gluster/swift/common/exceptions.py @@ -0,0 +1,27 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class GlusterfsException(Exception): + pass + +class FileOrDirNotFoundError(GlusterfsException): + pass + +class NotDirectoryError(GlusterfsException): + pass + +class AlreadyExistsAsDir(GlusterfsException): + pass + diff --git a/gluster/swift/common/fs_utils.py b/gluster/swift/common/fs_utils.py new file mode 100644 index 0000000..0613a26 --- /dev/null +++ b/gluster/swift/common/fs_utils.py @@ -0,0 +1,179 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno +import os.path as os_path +from eventlet import tpool +from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ + NotDirectoryError + +def do_walk(*args, **kwargs): + return os.walk(*args, **kwargs) + +def do_write(fd, msg): + try: + cnt = os.write(fd, msg) + except OSError as err: + logging.exception("Write failed, err: %s", str(err)) + raise + return cnt + +def do_mkdir(path): + try: + os.mkdir(path) + except OSError as err: + if err.errno != errno.EEXIST: + logging.exception("Mkdir failed on %s err: %s", path, err.strerror) + raise + return True + +def do_makedirs(path): + try: + os.makedirs(path) + except OSError as err: + if err.errno != errno.EEXIST: + logging.exception("Makedirs failed on %s err: %s", path, err.strerror) + raise + return True + +def do_listdir(path): + try: + buf = os.listdir(path) + except OSError as err: + logging.exception("Listdir failed on %s err: %s", path, err.strerror) + raise + return buf + +def do_chown(path, uid, gid): + try: + os.chown(path, uid, gid) + except OSError as err: + logging.exception("Chown failed on %s err: %s", path, err.strerror) + raise + return True + +def do_stat(path): + try: + #Check for fd. + if isinstance(path, int): + buf = os.fstat(path) + else: + buf = os.stat(path) + except OSError as err: + logging.exception("Stat failed on %s err: %s", path, err.strerror) + raise + return buf + +def do_open(path, mode): + if isinstance(mode, int): + try: + fd = os.open(path, mode) + except OSError as err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + else: + try: + fd = open(path, mode) + except IOError as err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + return fd + +def do_close(fd): + #fd could be file or int type. + try: + if isinstance(fd, int): + os.close(fd) + else: + fd.close() + except OSError as err: + logging.exception("Close failed on %s err: %s", fd, err.strerror) + raise + return True + +def do_unlink(path, log = True): + try: + os.unlink(path) + except OSError as err: + if err.errno != errno.ENOENT: + if log: + logging.exception("Unlink failed on %s err: %s", path, err.strerror) + raise + return True + +def do_rmdir(path): + try: + os.rmdir(path) + except OSError as err: + if err.errno != errno.ENOENT: + logging.exception("Rmdir failed on %s err: %s", path, err.strerror) + raise + res = False + else: + res = True + return res + +def do_rename(old_path, new_path): + try: + os.rename(old_path, new_path) + except OSError as err: + logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ + err.strerror) + raise + return True + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + do_makedirs(path) + +def dir_empty(path): + """ + Return true if directory/container is empty. + :param path: Directory path. + :returns: True/False. + """ + if os.path.isdir(path): + files = do_listdir(path) + return not files + elif not os.path.exists(path): + raise FileOrDirNotFoundError() + raise NotDirectoryError() + +def rmdirs(path): + if not os.path.isdir(path): + return False + try: + os.rmdir(path) + except OSError as err: + if err.errno != errno.ENOENT: + logging.error("rmdirs failed on %s, err: %s", path, err.strerror) + return False + return True + +def do_fsync(fd): + try: + tpool.execute(os.fsync, fd) + except OSError as err: + logging.exception("fsync failed with err: %s", err.strerror) + raise + return True diff --git a/gluster/swift/common/middleware/__init__.py b/gluster/swift/common/middleware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/common/middleware/gluster.py b/gluster/swift/common/middleware/gluster.py new file mode 100644 index 0000000..ab63c51 --- /dev/null +++ b/gluster/swift/common/middleware/gluster.py @@ -0,0 +1,40 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Noop Middleware that simply allows us to monkey patch the constraints +import gluster.swift.common.constraints + +class Gluster(object): + """ + Noop middleware for use with the proxy server to get paste.deploy to load + this middleware such that the plugin constraints monkey patch the common + constraints ahead of their use. + """ + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, env, start_response): + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def gluster_filter(app): + return Gluster(app, conf) + return gluster_filter diff --git a/gluster/swift/common/ring.py b/gluster/swift/common/ring.py new file mode 100644 index 0000000..06aab8d --- /dev/null +++ b/gluster/swift/common/ring.py @@ -0,0 +1,111 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +from swift.common.ring import ring +from swift.common.utils import search_tree +from gluster.swift.common.Glusterfs import SWIFT_DIR + +reseller_prefix = "AUTH_" +conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf') +if conf_files: + conf_file = conf_files[0] + +_conf = ConfigParser() +if conf_files and _conf.read(conf_file): + if _conf.defaults().get("reseller_prefix", None): + reseller_prefix = _conf.defaults().get("reseller_prefix") + else: + for key, value in _conf._sections.items(): + if value.get("reseller_prefix", None): + reseller_prefix = value["reseller_prefix"] + break + +if not reseller_prefix.endswith('_'): + reseller_prefix = reseller_prefix + '_' + +class Ring(ring.Ring): + def _get_part_nodes(self, part): + seen_ids = set() + nodes = [dev for dev in self._devs \ + if dev['device'] == self.acc_name \ + and not (dev['id'] in seen_ids \ + or seen_ids.add(dev['id']))] + if not nodes: + nodes = [self.false_node] + return nodes + + def get_part_nodes(self, part): + """ + Get the nodes that are responsible for the partition. If one + node is responsible for more than one replica of the same + partition, it will only appear in the output once. + + :param part: partition to get nodes for + :returns: list of node dicts + + See :func:`get_nodes` for a description of the node dicts. + """ + return self._get_part_nodes(part) + + def get_nodes(self, account, container=None, obj=None): + """ + Get the partition and nodes for an account/container/object. + If a node is responsible for more than one replica, it will + only appear in the output once. + :param account: account name + :param container: container name + :param obj: object name + :returns: a tuple of (partition, list of node dicts) + + Each node dict will have at least the following keys: + ====== =============================================================== + id unique integer identifier amongst devices + weight a float of the relative weight of this device as compared to + others; this indicates how many partitions the builder will try + to assign to this device + zone integer indicating which zone the device is in; a given + partition will not be assigned to multiple devices within the + same zone + ip the ip address of the device + port the tcp port of the device + device the device's name on disk (sdb1, for example) + meta general use 'extra' field; for example: the online date, the + hardware description + ====== =============================================================== + """ + self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \ + 'meta': '', 'device': 'volume_not_in_ring', \ + 'port': 6012} + if account.startswith(reseller_prefix): + self.acc_name = account.replace(reseller_prefix, '', 1) + else: + self.acc_name = account + + part = 0 + return part, self._get_part_nodes(part) + + + def get_more_nodes(self, part): + """ + Generator to get extra nodes for a partition for hinted handoff. + + :param part: partition to get handoff nodes for + :returns: generator of node dicts + + See :func:`get_nodes` for a description of the node dicts. + Should never be called in the swift UFO environment, so yield nothing + """ + yield self.false_node diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py new file mode 100644 index 0000000..f2cd8de --- /dev/null +++ b/gluster/swift/common/utils.py @@ -0,0 +1,496 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno +import xattr +import random +from hashlib import md5 +from eventlet import sleep +import cPickle as pickle +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.utils import normalize_timestamp, TRUE_VALUES +from gluster.swift.common.fs_utils import * +from gluster.swift.common import Glusterfs + +X_CONTENT_TYPE = 'Content-Type' +X_CONTENT_LENGTH = 'Content-Length' +X_TIMESTAMP = 'X-Timestamp' +X_PUT_TIMESTAMP = 'X-PUT-Timestamp' +X_TYPE = 'X-Type' +X_ETAG = 'ETag' +X_OBJECTS_COUNT = 'X-Object-Count' +X_BYTES_USED = 'X-Bytes-Used' +X_CONTAINER_COUNT = 'X-Container-Count' +X_OBJECT_TYPE = 'X-Object-Type' +DIR_TYPE = 'application/directory' +ACCOUNT = 'Account' +METADATA_KEY = 'user.swift.metadata' +MAX_XATTR_SIZE = 65536 +CONTAINER = 'container' +DIR = 'dir' +MARKER_DIR = 'marker_dir' +TEMP_DIR = 'tmp' +ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR +FILE = 'file' +FILE_TYPE = 'application/octet-stream' +OBJECT = 'Object' +OBJECT_TYPE = 'application/octet-stream' +DEFAULT_UID = -1 +DEFAULT_GID = -1 +PICKLE_PROTOCOL = 2 +CHUNK_SIZE = 65536 +MEMCACHE_KEY_PREFIX = 'gluster.swift.' +MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' +MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' + +def read_metadata(path): + """ + Helper function to read the pickled metadata from a File/Directory. + + :param path: File/Directory to read metadata from. + + :returns: dictionary of metadata + """ + metadata = None + metadata_s = '' + key = 0 + while metadata is None: + try: + metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or ''))) + except IOError as err: + if err.errno == errno.ENODATA: + if key > 0: + # No errors reading the xattr keys, but since we have not + # been able to find enough chunks to get a successful + # unpickle operation, we consider the metadata lost, and + # drop the existing data so that the internal state can be + # recreated. + clean_metadata(path) + # We either could not find any metadata key, or we could find + # some keys, but were not successful in performing the + # unpickling (missing keys perhaps)? Either way, just report + # to the caller we have no metadata. + metadata = {} + else: + logging.exception("xattr.getxattr failed on %s key %s err: %s", + path, key, str(err)) + # Note that we don't touch the keys on errors fetching the + # data since it could be a transient state. + raise + else: + try: + # If this key provides all or the remaining part of the pickle + # data, we don't need to keep searching for more keys. This + # means if we only need to store data in N xattr key/value + # pair, we only need to invoke xattr get N times. With large + # keys sizes we are shooting for N = 1. + metadata = pickle.loads(metadata_s) + assert isinstance(metadata, dict) + except EOFError, pickle.UnpicklingError: + # We still are not able recognize this existing data collected + # as a pickled object. Make sure we loop around to try to get + # more from another xattr key. + metadata = None + key += 1 + return metadata + +def write_metadata(path, metadata): + """ + Helper function to write pickled metadata for a File/Directory. + + :param path: File/Directory path to write the metadata + :param metadata: dictionary to metadata write + """ + assert isinstance(metadata, dict) + metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) + key = 0 + while metastr: + try: + xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) + except IOError as err: + logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err)) + raise + metastr = metastr[MAX_XATTR_SIZE:] + key += 1 + +def clean_metadata(path): + key = 0 + while True: + try: + xattr.removexattr(path, '%s%s' % (METADATA_KEY, (key or ''))) + except IOError as err: + if err.errno == errno.ENODATA: + break + raise + key += 1 + +def check_user_xattr(path): + if not os_path.exists(path): + return False + try: + xattr.setxattr(path, 'user.test.key1', 'value1') + except IOError as err: + logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) + raise + try: + xattr.removexattr(path, 'user.test.key1') + except IOError as err: + logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) + #Remove xattr may fail in case of concurrent remove. + return True + +def validate_container(metadata): + if not metadata: + logging.warn('validate_container: No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys(): + #logging.warn('validate_container: Metadata missing entries: %s' % metadata) + return False + + (value, timestamp) = metadata[X_TYPE] + if value == CONTAINER: + return True + + logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) + return False + +def validate_account(metadata): + if not metadata: + logging.warn('validate_account: No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys() or \ + X_CONTAINER_COUNT not in metadata.keys(): + #logging.warn('validate_account: Metadata missing entries: %s' % metadata) + return False + + (value, timestamp) = metadata[X_TYPE] + if value == ACCOUNT: + return True + + logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) + return False + +def validate_object(metadata): + if not metadata: + logging.warn('validate_object: No metadata') + return False + + if X_TIMESTAMP not in metadata.keys() or \ + X_CONTENT_TYPE not in metadata.keys() or \ + X_ETAG not in metadata.keys() or \ + X_CONTENT_LENGTH not in metadata.keys() or \ + X_TYPE not in metadata.keys() or \ + X_OBJECT_TYPE not in metadata.keys(): + #logging.warn('validate_object: Metadata missing entries: %s' % metadata) + return False + + if metadata[X_TYPE] == OBJECT: + return True + + logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) + return False + +def is_marker(metadata): + if not metadata: + logging.warn('is_marker: No metadata') + return False + + if X_OBJECT_TYPE not in metadata.keys(): + logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) + return False + + if metadata[X_OBJECT_TYPE] == MARKER_DIR: + return True + else: + return False + +def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, + bytes_used=0, obj_list=[]): + # strip the prefix off, also stripping the leading and trailing slashes + obj_path = path.replace(cont_path, '').strip(os.path.sep) + + for obj_name in src_list: + if obj_path: + obj_list.append(os.path.join(obj_path, obj_name)) + else: + obj_list.append(obj_name) + + object_count += 1 + + if Glusterfs._do_getsize and reg_file: + bytes_used += os_path.getsize(os.path.join(path, obj_name)) + sleep() + + return object_count, bytes_used + +def update_list(path, cont_path, dirs=[], files=[], object_count=0, + bytes_used=0, obj_list=[]): + if files: + object_count, bytes_used = _update_list(path, cont_path, files, True, + object_count, bytes_used, + obj_list) + if dirs: + object_count, bytes_used = _update_list(path, cont_path, dirs, False, + object_count, bytes_used, + obj_list) + return object_count, bytes_used + + +class ContainerDetails(object): + def __init__(self, bytes_used, object_count, obj_list, dir_list): + self.bytes_used = bytes_used + self.object_count = object_count + self.obj_list = obj_list + self.dir_list = dir_list + + +def _get_container_details_from_fs(cont_path): + """ + get container details by traversing the filesystem + """ + bytes_used = 0 + object_count = 0 + obj_list = [] + dir_list = [] + + if os_path.isdir(cont_path): + for (path, dirs, files) in do_walk(cont_path): + object_count, bytes_used = update_list(path, cont_path, dirs, files, + object_count, bytes_used, + obj_list) + + dir_list.append((path, do_stat(path).st_mtime)) + sleep() + + return ContainerDetails(bytes_used, object_count, obj_list, dir_list) + +def get_container_details(cont_path, memcache=None): + """ + Return object_list, object_count and bytes_used. + """ + mkey = '' + if memcache: + mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path + cd = memcache.get(mkey) + if cd: + if not cd.dir_list: + cd = None + else: + for (path, mtime) in cd.dir_list: + if mtime != do_stat(path).st_mtime: + cd = None + else: + cd = None + if not cd: + cd = _get_container_details_from_fs(cont_path) + if memcache: + memcache.set(mkey, cd) + return cd.obj_list, cd.object_count, cd.bytes_used + + +class AccountDetails(object): + """ A simple class to store the three pieces of information associated + with an account: + + 1. The last known modification time + 2. The count of containers in the following list + 3. The list of containers + """ + def __init__(self, mtime, container_count, container_list): + self.mtime = mtime + self.container_count = container_count + self.container_list = container_list + + +def _get_account_details_from_fs(acc_path, acc_stats): + container_list = [] + container_count = 0 + + if not acc_stats: + acc_stats = do_stat(acc_path) + is_dir = (acc_stats.st_mode & 0040000) != 0 + if is_dir: + for name in do_listdir(acc_path): + if name.lower() == TEMP_DIR \ + or name.lower() == ASYNCDIR \ + or not os_path.isdir(os.path.join(acc_path, name)): + continue + container_count += 1 + container_list.append(name) + + return AccountDetails(acc_stats.st_mtime, container_count, container_list) + +def get_account_details(acc_path, memcache=None): + """ + Return container_list and container_count. + """ + acc_stats = None + mkey = '' + if memcache: + mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path + ad = memcache.get(mkey) + if ad: + # FIXME: Do we really need to stat the file? If we are object + # only, then we can track the other Swift HTTP APIs that would + # modify the account and invalidate the cached entry there. If we + # are not object only, are we even called on this path? + acc_stats = do_stat(acc_path) + if ad.mtime != acc_stats.st_mtime: + ad = None + else: + ad = None + if not ad: + ad = _get_account_details_from_fs(acc_path, acc_stats) + if memcache: + memcache.set(mkey, ad) + return ad.container_list, ad.container_count + +def _get_etag(path): + etag = md5() + with open(path, 'rb') as fp: + while True: + chunk = fp.read(CHUNK_SIZE) + if chunk: + etag.update(chunk) + else: + break + return etag.hexdigest() + +def get_object_metadata(obj_path): + """ + Return metadata of object. + """ + try: + stats = do_stat(obj_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + metadata = {} + else: + is_dir = (stats.st_mode & 0040000) != 0 + metadata = { + X_TYPE: OBJECT, + X_TIMESTAMP: normalize_timestamp(stats.st_ctime), + X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, + X_OBJECT_TYPE: DIR if is_dir else FILE, + X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, + X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), + } + return metadata + +def _add_timestamp(metadata_i): + # At this point we have a simple key/value dictionary, turn it into + # key/(value,timestamp) pairs. + timestamp = 0 + metadata = {} + for key, value_i in metadata_i.iteritems(): + if not isinstance(value_i, tuple): + metadata[key] = (value_i, timestamp) + else: + metadata[key] = value_i + return metadata + +def get_container_metadata(cont_path, memcache=None): + objects = [] + object_count = 0 + bytes_used = 0 + objects, object_count, bytes_used = get_container_details(cont_path, memcache) + metadata = {X_TYPE: CONTAINER, + X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)), + X_OBJECTS_COUNT: object_count, + X_BYTES_USED: bytes_used} + return _add_timestamp(metadata) + +def get_account_metadata(acc_path, memcache=None): + containers = [] + container_count = 0 + containers, container_count = get_account_details(acc_path, memcache) + metadata = {X_TYPE: ACCOUNT, + X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)), + X_OBJECTS_COUNT: 0, + X_BYTES_USED: 0, + X_CONTAINER_COUNT: container_count} + return _add_timestamp(metadata) + +def restore_metadata(path, metadata): + meta_orig = read_metadata(path) + if meta_orig: + meta_new = meta_orig.copy() + meta_new.update(metadata) + else: + meta_new = metadata + if meta_orig != meta_new: + write_metadata(path, meta_new) + return meta_new + +def create_object_metadata(obj_path): + metadata = get_object_metadata(obj_path) + return restore_metadata(obj_path, metadata) + +def create_container_metadata(cont_path, memcache=None): + metadata = get_container_metadata(cont_path, memcache) + return restore_metadata(cont_path, metadata) + +def create_account_metadata(acc_path, memcache=None): + metadata = get_account_metadata(acc_path, memcache) + return restore_metadata(acc_path, metadata) + +def write_pickle(obj, dest, tmp=None, pickle_protocol=0): + """ + Ensure that a pickle file gets written to disk. The file is first written + to a tmp file location in the destination directory path, ensured it is + synced to disk, then moved to its final destination name. + + This version takes advantage of Gluster's dot-prefix-dot-suffix naming + where the a file named ".thefile.name.9a7aasv" is hashed to the same + Gluster node as "thefile.name". This ensures the renaming of a temp file + once written does not move it to another Gluster node. + + :param obj: python object to be pickled + :param dest: path of final destination file + :param tmp: path to tmp to use, defaults to None (ignored) + :param pickle_protocol: protocol to pickle the obj with, defaults to 0 + """ + dirname = os.path.dirname(dest) + basename = os.path.basename(dest) + tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest() + tmppath = os.path.join(dirname, tmpname) + with open(tmppath, 'wb') as fo: + pickle.dump(obj, fo, pickle_protocol) + # TODO: This flush() method call turns into a flush() system call + # We'll need to wrap this as well, but we would do this by writing + #a context manager for our own open() method which returns an object + # in fo which makes the gluster API call. + fo.flush() + do_fsync(fo) + do_rename(tmppath, dest) + +# Over-ride Swift's utils.write_pickle with ours +import swift.common.utils +swift.common.utils.write_pickle = write_pickle diff --git a/gluster/swift/container/__init__.py b/gluster/swift/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/container/server.py b/gluster/swift/container/server.py new file mode 100644 index 0000000..c5792aa --- /dev/null +++ b/gluster/swift/container/server.py @@ -0,0 +1,46 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Container Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.container import server +from gluster.swift.common.DiskDir import DiskDir + + +class ContainerController(server.ContainerController): + def _get_container_broker(self, drive, part, account, container): + """ + Overriden to provide the GlusterFS specific broker that talks to + Gluster for the information related to servicing a given request + instead of talking to a database. + + :param drive: drive that holds the container + :param part: partition the container is in + :param account: account name + :param container: container name + :returns: DiskDir object + """ + return DiskDir(self.root, drive, account, container, self.logger) + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI container server apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return ContainerController(conf) diff --git a/gluster/swift/obj/__init__.py b/gluster/swift/obj/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py new file mode 100644 index 0000000..1c2b6cb --- /dev/null +++ b/gluster/swift/obj/server.py @@ -0,0 +1,34 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Object Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints +import gluster.swift.common.utils + +from swift.obj import server +from gluster.swift.common.DiskFile import Gluster_DiskFile + +# Monkey patch the object server module to use Gluster's DiskFile definition +server.DiskFile = Gluster_DiskFile + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI object server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return server.ObjectController(conf) diff --git a/gluster/swift/proxy/__init__.py b/gluster/swift/proxy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gluster/swift/proxy/server.py b/gluster/swift/proxy/server.py new file mode 100644 index 0000000..792a97d --- /dev/null +++ b/gluster/swift/proxy/server.py @@ -0,0 +1,27 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.proxy import server + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI proxy apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return server.Application(conf) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2da9353 --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages + +from gluster.swift import __canonical_version__ as version + + +name = 'gluster_swift' + + +setup( + name=name, + version=version, + description='Gluster For Swift', + license='Apache License (2.0)', + author='Red Hat, Inc.', + author_email='gluster-users@gluster.org', + url='https://gluster.org/', + packages=find_packages(exclude=['test', 'bin']), + test_suite='nose.collector', + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Environment :: No Input/Output (Daemon)', + ], + install_requires=[], # removed for better compat + scripts=[ + 'bin/gluster-swift-gen-builders', + ], + entry_points={ + 'paste.app_factory': [ + 'proxy=gluster.swift.proxy.server:app_factory', + 'object=gluster.swift.obj.server:app_factory', + 'container=gluster.swift.container.server:app_factory', + 'account=gluster.swift.account.server:app_factory', + ], + 'paste.filter_factory': [ + 'gluster=gluster.swift.common.middleware.gluster:filter_factory', + ], + }, + ) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..50b24ed --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,49 @@ +# See http://code.google.com/p/python-nose/issues/detail?id=373 +# The code below enables nosetests to work with i18n _() blocks + +import __builtin__ +import sys +import os +from ConfigParser import MissingSectionHeaderError +from StringIO import StringIO + +from swift.common.utils import readconf + +setattr(__builtin__, '_', lambda x: x) + + +# Work around what seems to be a Python bug. +# c.f. https://bugs.launchpad.net/swift/+bug/820185. +import logging +logging.raiseExceptions = False + + +def get_config(section_name=None, defaults=None): + """ + Attempt to get a test config dictionary. + + :param section_name: the section to read (all sections if not defined) + :param defaults: an optional dictionary namespace of defaults + """ + config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', + '/etc/swift/test.conf') + config = {} + if defaults is not None: + config.update(defaults) + + try: + config = readconf(config_file, section_name) + except SystemExit: + if not os.path.exists(config_file): + print >>sys.stderr, \ + 'Unable to read test config %s - file not found' \ + % config_file + elif not os.access(config_file, os.R_OK): + print >>sys.stderr, \ + 'Unable to read test config %s - permission denied' \ + % config_file + else: + print >>sys.stderr, \ + 'Unable to read test config %s - section %s not found' \ + % (config_file, section_name) + return config diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000..cb24764 --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,95 @@ +""" Gluster Swift Unit Tests """ + +import logging +from collections import defaultdict +from test import get_config +from swift.common.utils import TRUE_VALUES + + +class NullLoggingHandler(logging.Handler): + + def emit(self, record): + pass + + +class FakeLogger(object): + # a thread safe logger + + def __init__(self, *args, **kwargs): + self._clear() + self.level = logging.NOTSET + if 'facility' in kwargs: + self.facility = kwargs['facility'] + + def _clear(self): + self.log_dict = defaultdict(list) + + def _store_in(store_name): + def stub_fn(self, *args, **kwargs): + self.log_dict[store_name].append((args, kwargs)) + return stub_fn + + error = _store_in('error') + info = _store_in('info') + warning = _store_in('warning') + debug = _store_in('debug') + + def exception(self, *args, **kwargs): + self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) + + # mock out the StatsD logging methods: + increment = _store_in('increment') + decrement = _store_in('decrement') + timing = _store_in('timing') + timing_since = _store_in('timing_since') + update_stats = _store_in('update_stats') + set_statsd_prefix = _store_in('set_statsd_prefix') + + def setFormatter(self, obj): + self.formatter = obj + + def close(self): + self._clear() + + def set_name(self, name): + # don't touch _handlers + self._name = name + + def acquire(self): + pass + + def release(self): + pass + + def createLock(self): + pass + + def emit(self, record): + pass + + def handle(self, record): + pass + + def flush(self): + pass + + def handleError(self, record): + pass + + +original_syslog_handler = logging.handlers.SysLogHandler + + +def fake_syslog_handler(): + for attr in dir(original_syslog_handler): + if attr.startswith('LOG'): + setattr(FakeLogger, attr, + copy.copy(getattr(logging.handlers.SysLogHandler, attr))) + FakeLogger.priority_map = \ + copy.deepcopy(logging.handlers.SysLogHandler.priority_map) + + logging.handlers.SysLogHandler = FakeLogger + + +if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES: + fake_syslog_handler() diff --git a/test/unit/common/__init__.py b/test/unit/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/common/data/README.rings b/test/unit/common/data/README.rings new file mode 100644 index 0000000..6457501 --- /dev/null +++ b/test/unit/common/data/README.rings @@ -0,0 +1,3 @@ +The unit tests expect certain ring data built using the following command: + + ../../../../bin/gluster-swift-gen-builders test iops \ No newline at end of file diff --git a/test/unit/common/data/account.builder b/test/unit/common/data/account.builder new file mode 100644 index 0000000..090ba4b Binary files /dev/null and b/test/unit/common/data/account.builder differ diff --git a/test/unit/common/data/account.ring.gz b/test/unit/common/data/account.ring.gz new file mode 100644 index 0000000..6d4c854 Binary files /dev/null and b/test/unit/common/data/account.ring.gz differ diff --git a/test/unit/common/data/account_tree.tar.bz2 b/test/unit/common/data/account_tree.tar.bz2 new file mode 100644 index 0000000..cb23e4d Binary files /dev/null and b/test/unit/common/data/account_tree.tar.bz2 differ diff --git a/test/unit/common/data/backups/1365124498.account.builder b/test/unit/common/data/backups/1365124498.account.builder new file mode 100644 index 0000000..090ba4b Binary files /dev/null and b/test/unit/common/data/backups/1365124498.account.builder differ diff --git a/test/unit/common/data/backups/1365124498.container.builder b/test/unit/common/data/backups/1365124498.container.builder new file mode 100644 index 0000000..733d27d Binary files /dev/null and b/test/unit/common/data/backups/1365124498.container.builder differ diff --git a/test/unit/common/data/backups/1365124498.object.builder b/test/unit/common/data/backups/1365124498.object.builder new file mode 100644 index 0000000..ff877ec Binary files /dev/null and b/test/unit/common/data/backups/1365124498.object.builder differ diff --git a/test/unit/common/data/backups/1365124499.object.builder b/test/unit/common/data/backups/1365124499.object.builder new file mode 100644 index 0000000..8b8cd6c Binary files /dev/null and b/test/unit/common/data/backups/1365124499.object.builder differ diff --git a/test/unit/common/data/container.builder b/test/unit/common/data/container.builder new file mode 100644 index 0000000..733d27d Binary files /dev/null and b/test/unit/common/data/container.builder differ diff --git a/test/unit/common/data/container.ring.gz b/test/unit/common/data/container.ring.gz new file mode 100644 index 0000000..592b84b Binary files /dev/null and b/test/unit/common/data/container.ring.gz differ diff --git a/test/unit/common/data/container_tree.tar.bz2 b/test/unit/common/data/container_tree.tar.bz2 new file mode 100644 index 0000000..b4a1492 Binary files /dev/null and b/test/unit/common/data/container_tree.tar.bz2 differ diff --git a/test/unit/common/data/object.builder b/test/unit/common/data/object.builder new file mode 100644 index 0000000..8b8cd6c Binary files /dev/null and b/test/unit/common/data/object.builder differ diff --git a/test/unit/common/data/object.ring.gz b/test/unit/common/data/object.ring.gz new file mode 100644 index 0000000..d2f7192 Binary files /dev/null and b/test/unit/common/data/object.ring.gz differ diff --git a/test/unit/common/test_Glusterfs.py b/test/unit/common/test_Glusterfs.py new file mode 100644 index 0000000..7de060a --- /dev/null +++ b/test/unit/common/test_Glusterfs.py @@ -0,0 +1,95 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import os, fcntl, errno, shutil +from tempfile import mkdtemp +import gluster.swift.common.Glusterfs as gfs + +def mock_os_path_ismount(path): + return True + +def mock_get_export_list(): + return ['test', 'test2'] + +def mock_os_system(cmd): + return False + +def mock_fcntl_lockf(f, *a, **kw): + raise IOError(errno.EAGAIN) + +def _init(): + global _RUN_DIR, _OS_SYSTEM, _FCNTL_LOCKF + global _OS_PATH_ISMOUNT, __GET_EXPORT_LIST + + _RUN_DIR = gfs.RUN_DIR + _OS_SYSTEM = os.system + _FCNTL_LOCKF = fcntl.lockf + _OS_PATH_ISMOUNT = os.path.ismount + __GET_EXPORT_LIST = gfs._get_export_list + +def _init_mock_variables(tmpdir): + os.system = mock_os_system + os.path.ismount = mock_os_path_ismount + gfs.RUN_DIR = os.path.join(tmpdir, 'var/run/swift') + gfs._get_export_list = mock_get_export_list + +def _reset_mock_variables(): + gfs.RUN_DIR = _RUN_DIR + gfs._get_export_list = __GET_EXPORT_LIST + + os.system = _OS_SYSTEM + fcntl.lockf = _FCNTL_LOCKF + os.path.ismount = _OS_PATH_ISMOUNT + +class TestGlusterfs(unittest.TestCase): + """ Tests for common.GlusterFS """ + + def setUp(self): + _init() + + def test_mount(self): + try: + tmpdir = mkdtemp() + root = os.path.join(tmpdir, 'mnt/gluster-object') + drive = 'test' + + _init_mock_variables(tmpdir) + assert gfs.mount(root, drive) + finally: + _reset_mock_variables() + shutil.rmtree(tmpdir) + + def test_mount_egain(self): + try: + tmpdir = mkdtemp() + root = os.path.join(tmpdir, 'mnt/gluster-object') + drive = 'test' + + _init_mock_variables(tmpdir) + assert gfs.mount(root, drive) + fcntl.lockf = mock_fcntl_lockf + assert gfs.mount(root, drive) + finally: + _reset_mock_variables() + shutil.rmtree(tmpdir) + + def test_mount_get_export_list_err(self): + gfs._get_export_list = mock_get_export_list + assert not gfs.mount(None, 'drive') + _reset_mock_variables() + + def tearDown(self): + _reset_mock_variables() diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py new file mode 100644 index 0000000..85d539a --- /dev/null +++ b/test/unit/common/test_diskfile.py @@ -0,0 +1,932 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for gluster.swift.common.DiskFile """ + +import os +import stat +import errno +import unittest +import tempfile +import shutil +from hashlib import md5 +from swift.common.utils import normalize_timestamp +from swift.common.exceptions import DiskFileNotExist +import gluster.swift.common.DiskFile +import gluster.swift.common.utils +from gluster.swift.common.DiskFile import Gluster_DiskFile, \ + AlreadyExistsAsDir +from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \ + X_OBJECT_TYPE +from test_utils import _initxattr, _destroyxattr +from test.unit import FakeLogger + + +_metadata = {} + +def _mock_read_metadata(filename): + if filename in _metadata: + md = _metadata[filename] + else: + md = {} + return md + +def _mock_write_metadata(filename, metadata): + _metadata[filename] = metadata + +def _mock_clear_metadata(): + _metadata = {} + + +class MockException(Exception): + pass + + +def _mock_rmdirs(p): + raise MockException("gluster.swift.common.DiskFile.rmdirs() called") + +def _mock_do_listdir(p): + raise MockException("gluster.swift.common.DiskFile.do_listdir() called") + +def _mock_do_unlink(f): + ose = OSError() + ose.errno = errno.ENOENT + raise ose + + +def _mock_do_unlink_eacces_err(f): + ose = OSError() + ose.errno = errno.EACCES + raise ose + +def _mock_getsize_eaccess_err(f): + ose = OSError() + ose.errno = errno.EACCES + raise ose + +def _mock_do_rmdir_eacces_err(f): + ose = OSError() + ose.errno = errno.EACCES + raise ose + +class MockRenamerCalled(Exception): + pass + + +def _mock_renamer(a, b): + raise MockRenamerCalled() + + +class TestDiskFile(unittest.TestCase): + """ Tests for gluster.swift.common.DiskFile """ + + def setUp(self): + self.lg = FakeLogger() + _initxattr() + _mock_clear_metadata() + self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata + self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata + gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata + gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata + self._saved_ut_wm = gluster.swift.common.utils.write_metadata + self._saved_ut_rm = gluster.swift.common.utils.read_metadata + gluster.swift.common.utils.write_metadata = _mock_write_metadata + gluster.swift.common.utils.read_metadata = _mock_read_metadata + + def tearDown(self): + self.lg = None + _destroyxattr() + gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm + gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm + gluster.swift.common.utils.write_metadata = self._saved_ut_wm + gluster.swift.common.utils.read_metadata = self._saved_ut_rm + + def test_constructor_no_slash(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf._obj_path == "" + assert gdf.name == "bar" + assert gdf.datadir == "/tmp/foo/vol0/bar" + assert gdf.device_path == "/tmp/foo/vol0" + assert gdf._container_path == "/tmp/foo/vol0/bar" + assert gdf.disk_chunk_size == 65536 + assert gdf.iter_hook == None + assert gdf.logger == self.lg + assert gdf.uid == DEFAULT_UID + assert gdf.gid == DEFAULT_GID + assert gdf.metadata == {} + assert gdf.meta_file == None + assert gdf.data_file == None + assert gdf.fp == None + assert gdf.iter_etag == None + assert not gdf.started_at_0 + assert not gdf.read_to_eof + assert gdf.quarantined_dir == None + assert not gdf.keep_cache + assert not gdf._is_dir + + def test_constructor_leadtrail_slash(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "/b/a/z/", self.lg) + assert gdf._obj == "z" + assert gdf._obj_path == "b/a" + assert gdf.name == "bar/b/a" + assert gdf.datadir == "/tmp/foo/vol0/bar/b/a" + assert gdf.device_path == "/tmp/foo/vol0" + + def test_constructor_no_metadata(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + stats = os.stat(the_file) + ts = normalize_timestamp(stats.st_ctime) + etag = md5() + etag.update("1234") + etag = etag.hexdigest() + exp_md = { + 'Content-Length': 4, + 'ETag': etag, + 'X-Timestamp': ts, + 'Content-Type': 'application/octet-stream'} + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is None + assert gdf.metadata == exp_md + finally: + shutil.rmtree(td) + + def test_constructor_existing_metadata(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + ini_md = { + 'X-Type': 'Object', + 'X-Object-Type': 'file', + 'Content-Length': 5, + 'ETag': 'etag', + 'X-Timestamp': 'ts', + 'Content-Type': 'application/loctet-stream'} + _metadata[the_file] = ini_md + exp_md = ini_md.copy() + del exp_md['X-Type'] + del exp_md['X-Object-Type'] + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is None + assert gdf.metadata == exp_md + finally: + shutil.rmtree(td) + + def test_constructor_invalid_existing_metadata(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + inv_md = { + 'Content-Length': 5, + 'ETag': 'etag', + 'X-Timestamp': 'ts', + 'Content-Type': 'application/loctet-stream'} + _metadata[the_file] = inv_md + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is None + assert gdf.metadata != inv_md + finally: + shutil.rmtree(td) + + def test_constructor_isdir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "d") + try: + os.makedirs(the_dir) + ini_md = { + 'X-Type': 'Object', + 'X-Object-Type': 'dir', + 'Content-Length': 5, + 'ETag': 'etag', + 'X-Timestamp': 'ts', + 'Content-Type': 'application/loctet-stream'} + _metadata[the_dir] = ini_md + exp_md = ini_md.copy() + del exp_md['X-Type'] + del exp_md['X-Object-Type'] + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "d", self.lg, keep_data_fp=True) + assert gdf._obj == "d" + assert gdf.data_file == the_dir + assert gdf._is_dir + assert gdf.fp is None + assert gdf.metadata == exp_md + finally: + shutil.rmtree(td) + + def test_constructor_keep_data_fp(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg, keep_data_fp=True) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is not None + finally: + shutil.rmtree(td) + + def test_constructor_chunk_size(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg, disk_chunk_size=8192) + assert gdf.disk_chunk_size == 8192 + + def test_constructor_iter_hook(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg, iter_hook='hook') + assert gdf.iter_hook == 'hook' + + def test_close(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + # Should be a no-op, as by default is_dir is False, but fp is None + gdf.close() + + gdf._is_dir = True + gdf.fp = "123" + # Should still be a no-op as is_dir is True (marker directory) + gdf.close() + assert gdf.fp == "123" + + gdf._is_dir = False + saved_dc = gluster.swift.common.DiskFile.do_close + self.called = False + def our_do_close(fp): + self.called = True + gluster.swift.common.DiskFile.do_close = our_do_close + try: + gdf.close() + assert self.called + assert gdf.fp is None + finally: + gluster.swift.common.DiskFile.do_close = saved_dc + + def test_is_deleted(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf.is_deleted() + gdf.data_file = "/tmp/foo/bar" + assert not gdf.is_deleted() + + def test_create_dir_object(self): + td = tempfile.mkdtemp() + the_dir = os.path.join(td, "vol0", "bar", "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir/z", self.lg) + # Not created, dir object path is different, just checking + assert gdf._obj == "z" + gdf._create_dir_object(the_dir) + assert os.path.isdir(the_dir) + assert the_dir in _metadata + finally: + shutil.rmtree(td) + + def test_create_dir_object_exists(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + os.makedirs(the_path) + with open(the_dir, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir/z", self.lg) + # Not created, dir object path is different, just checking + assert gdf._obj == "z" + def _mock_do_chown(p, u, g): + assert u == DEFAULT_UID + assert g == DEFAULT_GID + dc = gluster.swift.common.DiskFile.do_chown + gluster.swift.common.DiskFile.do_chown = _mock_do_chown + try: + gdf._create_dir_object(the_dir) + finally: + gluster.swift.common.DiskFile.do_chown = dc + assert os.path.isdir(the_dir) + assert the_dir in _metadata + finally: + shutil.rmtree(td) + + def test_put_metadata(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "z") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + md = { 'Content-Type': 'application/octet-stream', 'a': 'b' } + gdf.put_metadata(md.copy()) + assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md) + assert _metadata[the_dir] == md + finally: + shutil.rmtree(td) + + def test_put_w_tombstone(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf.metadata == {} + + gdf.put_metadata({'x': '1'}, tombstone=True) + assert gdf.metadata == {} + + def test_put_w_meta_file(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + newmd = gdf.metadata.copy() + newmd['X-Object-Meta-test'] = '1234' + gdf.put_metadata(newmd) + assert gdf.metadata == newmd + assert _metadata[the_file] == newmd + finally: + shutil.rmtree(td) + + def test_put_w_meta_file_no_content_type(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + newmd = gdf.metadata.copy() + newmd['Content-Type'] = '' + newmd['X-Object-Meta-test'] = '1234' + gdf.put_metadata(newmd) + assert gdf.metadata == newmd + assert _metadata[the_file] == newmd + finally: + shutil.rmtree(td) + + def test_put_w_meta_dir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir", self.lg) + newmd = gdf.metadata.copy() + newmd['X-Object-Meta-test'] = '1234' + gdf.put_metadata(newmd) + assert gdf.metadata == newmd + assert _metadata[the_dir] == newmd + finally: + shutil.rmtree(td) + + def test_put_w_marker_dir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir", self.lg) + newmd = gdf.metadata.copy() + newmd['X-Object-Meta-test'] = '1234' + gdf.put_metadata(newmd) + assert gdf.metadata == newmd + assert _metadata[the_dir] == newmd + finally: + shutil.rmtree(td) + + def test_put_w_marker_dir_create(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir", self.lg) + assert gdf.metadata == {} + newmd = { + 'Content-Length': 0, + 'ETag': 'etag', + 'X-Timestamp': 'ts', + 'Content-Type': 'application/directory'} + gdf.put(None, newmd, extension='.dir') + assert gdf.data_file == the_dir + assert gdf.metadata == newmd + assert _metadata[the_dir] == newmd + finally: + shutil.rmtree(td) + + def test_put_is_dir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir", self.lg) + origmd = gdf.metadata.copy() + origfmd = _metadata[the_dir] + newmd = gdf.metadata.copy() + # FIXME: This is a hack to get to the code-path; it is not clear + # how this can happen normally. + newmd['Content-Type'] = '' + newmd['X-Object-Meta-test'] = '1234' + try: + gdf.put(None, newmd, extension='.data') + except AlreadyExistsAsDir: + pass + else: + self.fail("Expected to encounter 'already-exists-as-dir' exception") + assert gdf.metadata == origmd + assert _metadata[the_dir] == origfmd + finally: + shutil.rmtree(td) + + def test_put(self): + td = tempfile.mkdtemp() + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf._obj_path == "" + assert gdf.name == "bar" + assert gdf.datadir == os.path.join(td, "vol0", "bar") + assert gdf.data_file is None + + body = '1234\n' + etag = md5() + etag.update(body) + etag = etag.hexdigest() + metadata = { + 'X-Timestamp': '1234', + 'Content-Type': 'file', + 'ETag': etag, + 'Content-Length': '5', + } + + with gdf.mkstemp() as fd: + assert gdf.tmppath is not None + tmppath = gdf.tmppath + os.write(fd, body) + gdf.put(fd, metadata) + + assert gdf.data_file == os.path.join(td, "vol0", "bar", "z") + assert os.path.exists(gdf.data_file) + assert not os.path.exists(tmppath) + finally: + shutil.rmtree(td) + + def test_put_obj_path(self): + the_obj_path = os.path.join("b", "a") + the_file = os.path.join(the_obj_path, "z") + td = tempfile.mkdtemp() + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + the_file, self.lg) + assert gdf._obj == "z" + assert gdf._obj_path == the_obj_path + assert gdf.name == os.path.join("bar", "b", "a") + assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a") + assert gdf.data_file is None + + body = '1234\n' + etag = md5() + etag.update(body) + etag = etag.hexdigest() + metadata = { + 'X-Timestamp': '1234', + 'Content-Type': 'file', + 'ETag': etag, + 'Content-Length': '5', + } + + with gdf.mkstemp() as fd: + assert gdf.tmppath is not None + tmppath = gdf.tmppath + os.write(fd, body) + gdf.put(fd, metadata) + + assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z") + assert os.path.exists(gdf.data_file) + assert not os.path.exists(tmppath) + finally: + shutil.rmtree(td) + + def test_unlinkold_no_metadata(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf.metadata == {} + _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs + _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir + gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs + gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir + try: + gdf.unlinkold(None) + except MockException as exp: + self.fail(str(exp)) + finally: + gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs + gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir + + def test_unlinkold_same_timestamp(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf.metadata == {} + gdf.metadata['X-Timestamp'] = 1 + _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs + _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir + gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs + gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir + try: + gdf.unlinkold(1) + except MockException as exp: + self.fail(str(exp)) + finally: + gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs + gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir + + def test_unlinkold_file(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + + later = float(gdf.metadata['X-Timestamp']) + 1 + gdf.unlinkold(normalize_timestamp(later)) + assert os.path.isdir(gdf.datadir) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) + finally: + shutil.rmtree(td) + + def test_unlinkold_file_not_found(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + + # Handle the case the file is not in the directory listing. + os.unlink(the_file) + + later = float(gdf.metadata['X-Timestamp']) + 1 + gdf.unlinkold(normalize_timestamp(later)) + assert os.path.isdir(gdf.datadir) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) + finally: + shutil.rmtree(td) + + def test_unlinkold_file_unlink_error(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + + later = float(gdf.metadata['X-Timestamp']) + 1 + + stats = os.stat(the_path) + os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) + + # Handle the case do_unlink() raises an OSError + __os_unlink = os.unlink + os.unlink = _mock_do_unlink_eacces_err + try: + gdf.unlinkold(normalize_timestamp(later)) + except OSError as e: + assert e.errno == errno.EACCES + else: + self.fail("Excepted an OSError when unlinking file") + finally: + os.unlink = __os_unlink + os.chmod(the_path, stats.st_mode) + + assert os.path.isdir(gdf.datadir) + assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) + finally: + shutil.rmtree(td) + + def test_unlinkold_is_dir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "d") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "d", self.lg, keep_data_fp=True) + assert gdf.data_file == the_dir + assert gdf._is_dir + + later = float(gdf.metadata['X-Timestamp']) + 1 + gdf.unlinkold(normalize_timestamp(later)) + assert os.path.isdir(gdf.datadir) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) + finally: + shutil.rmtree(td) + + def test_unlinkold_is_dir_failure(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "d") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "d", self.lg, keep_data_fp=True) + assert gdf.data_file == the_dir + assert gdf._is_dir + + stats = os.stat(gdf.datadir) + os.chmod(gdf.datadir, 0) + __os_rmdir = os.rmdir + os.rmdir = _mock_do_rmdir_eacces_err + try: + later = float(gdf.metadata['X-Timestamp']) + 1 + gdf.unlinkold(normalize_timestamp(later)) + finally: + os.chmod(gdf.datadir, stats.st_mode) + os.rmdir = __os_rmdir + assert os.path.isdir(gdf.datadir) + assert os.path.isdir(gdf.data_file) + finally: + shutil.rmtree(td) + + def test_get_data_file_size(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert 4 == gdf.get_data_file_size() + finally: + shutil.rmtree(td) + + def test_get_data_file_size(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + assert 4 == gdf.metadata['Content-Length'] + gdf.metadata['Content-Length'] = 3 + assert 4 == gdf.get_data_file_size() + assert 4 == gdf.metadata['Content-Length'] + finally: + shutil.rmtree(td) + + def test_get_data_file_size_dne(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "/b/a/z/", self.lg) + try: + s = gdf.get_data_file_size() + except DiskFileNotExist: + pass + else: + self.fail("Expected DiskFileNotExist exception") + + def test_get_data_file_size_dne_os_err(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + gdf.data_file = gdf.data_file + ".dne" + try: + s = gdf.get_data_file_size() + except DiskFileNotExist: + pass + else: + self.fail("Expected DiskFileNotExist exception") + finally: + shutil.rmtree(td) + + def test_get_data_file_size_os_err(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_file = os.path.join(the_path, "z") + try: + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf._obj == "z" + assert gdf.data_file == the_file + assert not gdf._is_dir + stats = os.stat(the_path) + os.chmod(the_path, 0) + __os_path_getsize = os.path.getsize + os.path.getsize = _mock_getsize_eaccess_err + try: + s = gdf.get_data_file_size() + except OSError as err: + assert err.errno == errno.EACCES + else: + self.fail("Expected OSError exception") + finally: + os.path.getsize = __os_path_getsize + os.chmod(the_path, stats.st_mode) + finally: + shutil.rmtree(td) + + def test_get_data_file_size_dir(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "d") + try: + os.makedirs(the_dir) + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "d", self.lg, keep_data_fp=True) + assert gdf._obj == "d" + assert gdf.data_file == the_dir + assert gdf._is_dir + assert 0 == gdf.get_data_file_size() + finally: + shutil.rmtree(td) + + def test_filter_metadata(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg) + assert gdf.metadata == {} + gdf.filter_metadata() + assert gdf.metadata == {} + + gdf.metadata[X_TYPE] = 'a' + gdf.metadata[X_OBJECT_TYPE] = 'b' + gdf.metadata['foobar'] = 'c' + gdf.filter_metadata() + assert X_TYPE not in gdf.metadata + assert X_OBJECT_TYPE not in gdf.metadata + assert 'foobar' in gdf.metadata + + def test_mkstemp(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir/z", self.lg) + saved_tmppath = '' + with gdf.mkstemp() as fd: + assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") + assert os.path.isdir(gdf.datadir) + saved_tmppath = gdf.tmppath + assert os.path.dirname(saved_tmppath) == gdf.datadir + assert os.path.basename(saved_tmppath)[:3] == '.z.' + assert os.path.exists(saved_tmppath) + os.write(fd, "123") + assert not os.path.exists(saved_tmppath) + finally: + shutil.rmtree(td) + + def test_mkstemp_err_on_close(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir/z", self.lg) + saved_tmppath = '' + with gdf.mkstemp() as fd: + assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") + assert os.path.isdir(gdf.datadir) + saved_tmppath = gdf.tmppath + assert os.path.dirname(saved_tmppath) == gdf.datadir + assert os.path.basename(saved_tmppath)[:3] == '.z.' + assert os.path.exists(saved_tmppath) + os.write(fd, "123") + # At the end of previous with block a close on fd is called. + # Calling os.close on the same fd will raise an OSError + # exception and we must catch it. + try: + os.close(fd) + except OSError as err: + pass + else: + self.fail("Exception expected") + assert not os.path.exists(saved_tmppath) + finally: + shutil.rmtree(td) + + def test_mkstemp_err_on_unlink(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir/z", self.lg) + saved_tmppath = '' + with gdf.mkstemp() as fd: + assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") + assert os.path.isdir(gdf.datadir) + saved_tmppath = gdf.tmppath + assert os.path.dirname(saved_tmppath) == gdf.datadir + assert os.path.basename(saved_tmppath)[:3] == '.z.' + assert os.path.exists(saved_tmppath) + os.write(fd, "123") + os.unlink(saved_tmppath) + assert not os.path.exists(saved_tmppath) + finally: + shutil.rmtree(td) diff --git a/test/unit/common/test_fs_utils.py b/test/unit/common/test_fs_utils.py new file mode 100644 index 0000000..186e07d --- /dev/null +++ b/test/unit/common/test_fs_utils.py @@ -0,0 +1,277 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import random +import unittest +from tempfile import mkdtemp, mkstemp +from gluster.swift.common import fs_utils as fs +from gluster.swift.common.exceptions import NotDirectoryError, \ + FileOrDirNotFoundError + +class TestUtils(unittest.TestCase): + """ Tests for common.utils """ + + def test_do_walk(self): + try: + # create directory structure + tmpparent = mkdtemp() + tmpdirs = [] + tmpfiles = [] + for i in range(5): + tmpdirs.append(mkdtemp(dir=tmpparent).rsplit(os.path.sep, 1)[1]) + tmpfiles.append(mkstemp(dir=tmpparent)[1].rsplit(os.path.sep, \ + 1)[1]) + + for path, dirnames, filenames in fs.do_walk(tmpparent): + assert path == tmpparent + assert dirnames.sort() == tmpdirs.sort() + assert filenames.sort() == tmpfiles.sort() + break + finally: + shutil.rmtree(tmpparent) + + def test_do_open(self): + try: + fd, tmpfile = mkstemp() + f = fs.do_open(tmpfile, 'r') + try: + f.write('test') + except IOError as err: + pass + else: + self.fail("IOError expected") + finally: + f.close() + os.close(fd) + os.remove(tmpfile) + + def test_do_open_err(self): + try: + fs.do_open(os.path.join('/tmp', str(random.random())), 'r') + except IOError: + pass + else: + self.fail("IOError expected") + + def test_do_write(self): + try: + fd, tmpfile = mkstemp() + cnt = fs.do_write(fd, "test") + assert cnt == len("test") + finally: + os.close(fd) + os.remove(tmpfile) + + def test_do_write_err(self): + try: + fd, tmpfile = mkstemp() + fd1 = os.open(tmpfile, os.O_RDONLY) + fs.do_write(fd1, "test") + except OSError: + pass + else: + self.fail("OSError expected") + finally: + os.close(fd) + os.close(fd1) + + def test_do_mkdir(self): + try: + path = os.path.join('/tmp', str(random.random())) + fs.do_mkdir(path) + assert os.path.exists(path) + assert fs.do_mkdir(path) + finally: + os.rmdir(path) + + def test_do_mkdir_err(self): + try: + path = os.path.join('/tmp', str(random.random()), str(random.random())) + fs.do_mkdir(path) + except OSError: + pass + else: + self.fail("OSError expected") + + + def test_do_makedirs(self): + try: + subdir = os.path.join('/tmp', str(random.random())) + path = os.path.join(subdir, str(random.random())) + fs.do_makedirs(path) + assert os.path.exists(path) + assert fs.do_makedirs(path) + finally: + shutil.rmtree(subdir) + + def test_do_listdir(self): + try: + tmpdir = mkdtemp() + subdir = [] + for i in range(5): + subdir.append(mkdtemp(dir=tmpdir).rsplit(os.path.sep, 1)[1]) + + assert subdir.sort() == fs.do_listdir(tmpdir).sort() + finally: + shutil.rmtree(tmpdir) + + def test_do_listdir_err(self): + try: + path = os.path.join('/tmp', str(random.random())) + fs.do_listdir(path) + except OSError: + pass + else: + self.fail("OSError expected") + + def test_do_stat(self): + try: + tmpdir = mkdtemp() + fd, tmpfile = mkstemp(dir=tmpdir) + buf1 = os.stat(tmpfile) + buf2 = fs.do_stat(fd) + buf3 = fs.do_stat(tmpfile) + + assert buf1 == buf2 + assert buf1 == buf3 + finally: + os.close(fd) + os.remove(tmpfile) + os.rmdir(tmpdir) + + def test_do_stat_err(self): + try: + fs.do_stat(os.path.join('/tmp', str(random.random()))) + except OSError: + pass + else: + self.fail("OSError expected") + + def test_do_close(self): + try: + fd, tmpfile = mkstemp() + fs.do_close(fd); + try: + os.write(fd, "test") + except OSError: + pass + else: + self.fail("OSError expected") + fp = open(tmpfile) + fs.do_close(fp) + finally: + os.remove(tmpfile) + + def test_do_unlink(self): + try: + fd, tmpfile = mkstemp() + fs.do_unlink(tmpfile) + assert not os.path.exists(tmpfile) + assert fs.do_unlink(os.path.join('/tmp', str(random.random()))) + finally: + os.close(fd) + + def test_do_unlink_err(self): + try: + tmpdir = mkdtemp() + fs.do_unlink(tmpdir) + except OSError: + pass + else: + self.fail('OSError expected') + finally: + os.rmdir(tmpdir) + + def test_do_rmdir(self): + tmpdir = mkdtemp() + fs.do_rmdir(tmpdir) + assert not os.path.exists(tmpdir) + assert not fs.do_rmdir(os.path.join('/tmp', str(random.random()))) + + def test_do_rmdir_err(self): + try: + fd, tmpfile = mkstemp() + fs.do_rmdir(tmpfile) + except OSError: + pass + else: + self.fail('OSError expected') + finally: + os.close(fd) + os.remove(tmpfile) + + def test_do_rename(self): + try: + srcpath = mkdtemp() + destpath = os.path.join('/tmp', str(random.random())) + fs.do_rename(srcpath, destpath) + assert not os.path.exists(srcpath) + assert os.path.exists(destpath) + finally: + os.rmdir(destpath) + + def test_do_rename_err(self): + try: + srcpath = os.path.join('/tmp', str(random.random())) + destpath = os.path.join('/tmp', str(random.random())) + fs.do_rename(srcpath, destpath) + except OSError: + pass + else: + self.fail("OSError expected") + + def test_dir_empty(self): + try: + tmpdir = mkdtemp() + subdir = mkdtemp(dir=tmpdir) + assert not fs.dir_empty(tmpdir) + assert fs.dir_empty(subdir) + finally: + shutil.rmtree(tmpdir) + + def test_dir_empty_err(self): + try: + try: + assert fs.dir_empty(os.path.join('/tmp', str(random.random()))) + except FileOrDirNotFoundError: + pass + else: + self.fail("FileOrDirNotFoundError exception expected") + + fd, tmpfile = mkstemp() + try: + fs.dir_empty(tmpfile) + except NotDirectoryError: + pass + else: + self.fail("NotDirectoryError exception expected") + finally: + os.close(fd) + os.unlink(tmpfile) + + def test_rmdirs(self): + try: + tmpdir = mkdtemp() + subdir = mkdtemp(dir=tmpdir) + fd, tmpfile = mkstemp(dir=tmpdir) + assert not fs.rmdirs(tmpfile) + assert not fs.rmdirs(tmpdir) + assert fs.rmdirs(subdir) + assert not os.path.exists(subdir) + finally: + os.close(fd) + shutil.rmtree(tmpdir) diff --git a/test/unit/common/test_ring.py b/test/unit/common/test_ring.py new file mode 100644 index 0000000..8b7509c --- /dev/null +++ b/test/unit/common/test_ring.py @@ -0,0 +1,55 @@ +# Copyright (c) 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +import gluster.swift.common.constraints +import swift.common.utils +from gluster.swift.common.ring import Ring + + +class TestRing(unittest.TestCase): + """ Tests for common.utils """ + + def setUp(self): + swift.common.utils.HASH_PATH_SUFFIX = 'endcap' + swiftdir = os.path.join(os.getcwd(), "common", "data") + self.ring = Ring(swiftdir, ring_name='object') + + def test_first_device(self): + part, node = self.ring.get_nodes('test') + assert node[0]['device'] == 'test' + node = self.ring.get_part_nodes(0) + assert node[0]['device'] == 'test' + for node in self.ring.get_more_nodes(0): + assert node['device'] == 'volume_not_in_ring' + + def test_invalid_device(self): + part, node = self.ring.get_nodes('test2') + assert node[0]['device'] == 'volume_not_in_ring' + node = self.ring.get_part_nodes(0) + assert node[0]['device'] == 'volume_not_in_ring' + + def test_second_device(self): + part, node = self.ring.get_nodes('iops') + assert node[0]['device'] == 'iops' + node = self.ring.get_part_nodes(0) + assert node[0]['device'] == 'iops' + for node in self.ring.get_more_nodes(0): + assert node['device'] == 'volume_not_in_ring' + + def test_second_device_with_reseller_prefix(self): + part, node = self.ring.get_nodes('AUTH_iops') + assert node[0]['device'] == 'iops' diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py new file mode 100644 index 0000000..c645509 --- /dev/null +++ b/test/unit/common/test_utils.py @@ -0,0 +1,1020 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for common.utils """ + +import os +import unittest +import errno +import xattr +import cPickle as pickle +import tempfile +import hashlib +import tarfile +import shutil +from collections import defaultdict +from swift.common.utils import normalize_timestamp +from gluster.swift.common import utils, Glusterfs + +# +# Somewhat hacky way of emulating the operation of xattr calls. They are made +# against a dictionary that stores the xattr key/value pairs. +# +_xattrs = {} +_xattr_op_cnt = defaultdict(int) +_xattr_set_err = {} +_xattr_get_err = {} +_xattr_rem_err = {} + +def _xkey(path, key): + return "%s:%s" % (path, key) + +def _setxattr(path, key, value, *args, **kwargs): + _xattr_op_cnt['set'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_set_err: + e = IOError() + e.errno = _xattr_set_err[xkey] + raise e + global _xattrs + _xattrs[xkey] = value + +def _getxattr(path, key, *args, **kwargs): + _xattr_op_cnt['get'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_get_err: + e = IOError() + e.errno = _xattr_get_err[xkey] + raise e + global _xattrs + if xkey in _xattrs: + ret_val = _xattrs[xkey] + else: + e = IOError("Fake IOError") + e.errno = errno.ENODATA + raise e + return ret_val + +def _removexattr(path, key, *args, **kwargs): + _xattr_op_cnt['remove'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_rem_err: + e = IOError() + e.errno = _xattr_rem_err[xkey] + raise e + global _xattrs + if xkey in _xattrs: + del _xattrs[xkey] + else: + e = IOError("Fake IOError") + e.errno = errno.ENODATA + raise e + +def _initxattr(): + global _xattrs + _xattrs = {} + global _xattr_op_cnt + _xattr_op_cnt = defaultdict(int) + global _xattr_set_err, _xattr_get_err, _xattr_rem_err + _xattr_set_err = {} + _xattr_get_err = {} + _xattr_rem_err = {} + + # Save the current methods + global _xattr_set; _xattr_set = xattr.setxattr + global _xattr_get; _xattr_get = xattr.getxattr + global _xattr_remove; _xattr_remove = xattr.removexattr + + # Monkey patch the calls we use with our internal unit test versions + xattr.setxattr = _setxattr + xattr.getxattr = _getxattr + xattr.removexattr = _removexattr + +def _destroyxattr(): + # Restore the current methods just in case + global _xattr_set; xattr.setxattr = _xattr_set + global _xattr_get; xattr.getxattr = _xattr_get + global _xattr_remove; xattr.removexattr = _xattr_remove + # Destroy the stored values and + global _xattrs; _xattrs = None + + +class SimMemcache(object): + def __init__(self): + self._d = {} + + def get(self, key): + return self._d.get(key, None) + + def set(self, key, value): + self._d[key] = value + + +class TestUtils(unittest.TestCase): + """ Tests for common.utils """ + + def setUp(self): + _initxattr() + + def tearDown(self): + _destroyxattr() + + def test_write_metadata(self): + path = "/tmp/foo/w" + orig_d = { 'bar' : 'foo' } + utils.write_metadata(path, orig_d) + xkey = _xkey(path, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert orig_d == pickle.loads(_xattrs[xkey]) + assert _xattr_op_cnt['set'] == 1 + + def test_write_metadata_err(self): + path = "/tmp/foo/w" + orig_d = { 'bar' : 'foo' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattr_set_err[xkey] = errno.EOPNOTSUPP + try: + utils.write_metadata(path, orig_d) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert len(_xattrs.keys()) == 0 + assert _xattr_op_cnt['set'] == 1 + else: + self.fail("Expected an IOError exception on write") + + def test_write_metadata_multiple(self): + # At 64 KB an xattr key/value pair, this should generate three keys. + path = "/tmp/foo/w" + orig_d = { 'bar' : 'x' * 150000 } + utils.write_metadata(path, orig_d) + assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys()) + payload = '' + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + assert xkey in _xattrs + assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE + payload += _xattrs[xkey] + assert orig_d == pickle.loads(payload) + assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt + + def test_clean_metadata(self): + path = "/tmp/foo/c" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert not expected_p + utils.clean_metadata(path) + assert _xattr_op_cnt['remove'] == 4, "%r" % _xattr_op_cnt + + def test_clean_metadata_err(self): + path = "/tmp/foo/c" + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps({ 'a': 'y' }, utils.PICKLE_PROTOCOL) + _xattr_rem_err[xkey] = errno.EOPNOTSUPP + try: + utils.clean_metadata(path) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert _xattr_op_cnt['remove'] == 1, "%r" % _xattr_op_cnt + else: + self.fail("Expected an IOError exception on remove") + + def test_read_metadata(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + res_d = utils.read_metadata(path) + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + + def test_read_metadata_notfound(self): + path = "/tmp/foo/r" + res_d = utils.read_metadata(path) + assert res_d == {} + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + + def test_read_metadata_err(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + _xattr_get_err[xkey] = errno.EOPNOTSUPP + try: + res_d = utils.read_metadata(path) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt + else: + self.fail("Expected an IOError exception on get") + + def test_read_metadata_multiple(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert not expected_p + res_d = utils.read_metadata(path) + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt + + def test_read_metadata_multiple_one_missing(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,2): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert len(expected_p) <= utils.MAX_XATTR_SIZE + res_d = utils.read_metadata(path) + assert res_d == {} + assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt + assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) + + def test_restore_metadata_none(self): + # No initial metadata + path = "/tmp/foo/i" + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata(self): + # Initial metadata + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'a': 'z', 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata_nochange(self): + # Initial metadata but no changes + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, {}) + expected_d = { 'a': 'z' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt + + def test_add_timestamp_empty(self): + orig = {} + res = utils._add_timestamp(orig) + assert res == {} + + def test_add_timestamp_none(self): + orig = { 'a': 1, 'b': 2, 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_mixed(self): + orig = { 'a': 1, 'b': (2, 1), 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_all(self): + orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == orig + + def test_get_etag_empty(self): + tf = tempfile.NamedTemporaryFile() + hd = utils._get_etag(tf.name) + assert hd == hashlib.md5().hexdigest() + + def test_get_etag(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123' * utils.CHUNK_SIZE) + tf.file.flush() + hd = utils._get_etag(tf.name) + tf.file.seek(0) + md5 = hashlib.md5() + while True: + chunk = tf.file.read(utils.CHUNK_SIZE) + if not chunk: + break + md5.update(chunk) + assert hd == md5.hexdigest() + + def test_get_object_metadata_dne(self): + md = utils.get_object_metadata("/tmp/doesNotEx1st") + assert md == {} + + def test_get_object_metadata_err(self): + tf = tempfile.NamedTemporaryFile() + try: + md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) + except OSError as e: + assert e.errno != errno.ENOENT + else: + self.fail("Expected exception") + + obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, + utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) + + def test_get_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123'); tf.file.flush() + md = utils.get_object_metadata(tf.name) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils._get_etag(tf.name) + + def test_get_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + md = utils.get_object_metadata(td) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_create_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('4567'); tf.file.flush() + r_md = utils.create_object_metadata(tf.name) + + xkey = _xkey(tf.name, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils._get_etag(tf.name) + + def test_create_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_object_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_get_container_metadata(self): + def _mock_get_container_details(path, memcache=None): + o_list = [ 'a', 'b', 'c' ] + o_count = 3 + b_used = 47 + return o_list, o_count, b_used + orig_gcd = utils.get_container_details + utils.get_container_details = _mock_get_container_details + td = tempfile.mkdtemp() + try: + exp_md = { + utils.X_TYPE: (utils.CONTAINER, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (3, 0), + utils.X_BYTES_USED: (47, 0), + } + md = utils.get_container_metadata(td) + assert md == exp_md + finally: + utils.get_container_details = orig_gcd + os.rmdir(td) + + def test_get_account_metadata(self): + def _mock_get_account_details(path, memcache=None): + c_list = [ '123', 'abc' ] + c_count = 2 + return c_list, c_count + orig_gad = utils.get_account_details + utils.get_account_details = _mock_get_account_details + td = tempfile.mkdtemp() + try: + exp_md = { + utils.X_TYPE: (utils.ACCOUNT, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (0, 0), + utils.X_BYTES_USED: (0, 0), + utils.X_CONTAINER_COUNT: (2, 0), + } + md = utils.get_account_metadata(td) + assert md == exp_md + finally: + utils.get_account_details = orig_gad + os.rmdir(td) + + cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, + utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] + + def test_create_container_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_container_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.cont_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.CONTAINER, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + finally: + os.rmdir(td) + + acct_keys = [val for val in cont_keys] + acct_keys.append(utils.X_CONTAINER_COUNT) + + def test_create_account_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_account_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.acct_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + assert md[utils.X_CONTAINER_COUNT] == (0, 0) + finally: + os.rmdir(td) + + def test_container_details_uncached(self): + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path): + bu = 5 + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + try: + retval = utils.get_container_details(the_path) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + finally: + utils._get_container_details_from_fs = orig_gcdff + + def test_container_details_cached_hit(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + # If it did not properly use memcache, the default mocked version + # of get details from fs would return 5 bytes used instead of the + # 6 we specified above. + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_key(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we don't miss due to mtimes not matching + self.fail("do_stat should not have been called") + cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + "u", cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_dir_list(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we don't miss due to mtimes not matching + self.fail("do_stat should not have been called") + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + assert 5 == mc._d[mkey].bytes_used + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_mtime(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we miss due to mtimes not matching + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(200) + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + assert 5 == mc._d[mkey].bytes_used + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_uncached(self): + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + orig_gcdff = utils._get_account_details_from_fs + utils._get_account_details_from_fs = mock_get_account_details_from_fs + try: + retval = utils.get_account_details(the_path) + ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (ad.container_list, ad.container_count) + finally: + utils._get_account_details_from_fs = orig_gcdff + + def test_account_details_cached_hit(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + assert retval == (ad.container_list, ad.container_count) + wrong_ad = mock_get_account_details_from_fs(the_path, None) + assert wrong_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_cached_miss(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path + 'u', ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + correct_ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (correct_ad.container_list, correct_ad.container_count) + assert correct_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_cached_miss_mtime(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + ad.mtime = 200 + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + correct_ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (correct_ad.container_list, correct_ad.container_count) + assert correct_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_get_account_details_from_fs(self): + orig_cwd = os.getcwd() + td = tempfile.mkdtemp() + try: + tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") + os.chdir(td) + tf.extractall() + + ad = utils._get_account_details_from_fs(td, None) + assert ad.mtime == os.path.getmtime(td) + assert ad.container_count == 3 + assert set(ad.container_list) == set(['c1', 'c2', 'c3']) + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_container_details_from_fs_notadir(self): + tf = tempfile.NamedTemporaryFile() + cd = utils._get_container_details_from_fs(tf.name) + assert cd.bytes_used == 0 + assert cd.object_count == 0 + assert cd.obj_list == [] + assert cd.dir_list == [] + + def test_get_container_details_from_fs(self): + orig_cwd = os.getcwd() + td = tempfile.mkdtemp() + try: + tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") + os.chdir(td) + tf.extractall() + + cd = utils._get_container_details_from_fs(td) + assert cd.bytes_used == 0, repr(cd.bytes_used) + assert cd.object_count == 8, repr(cd.object_count) + assert set(cd.obj_list) == set(['file1', 'file3', 'file2', + 'dir3', 'dir1', 'dir2', + 'dir1/file1', 'dir1/file2' + ]), repr(cd.obj_list) + + full_dir1 = os.path.join(td, 'dir1') + full_dir2 = os.path.join(td, 'dir2') + full_dir3 = os.path.join(td, 'dir3') + exp_dir_dict = { td: os.path.getmtime(td), + full_dir1: os.path.getmtime(full_dir1), + full_dir2: os.path.getmtime(full_dir2), + full_dir3: os.path.getmtime(full_dir3), + } + for d,m in cd.dir_list: + assert d in exp_dir_dict + assert exp_dir_dict[d] == m + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + + def test_get_container_details_from_fs_do_getsize_true(self): + orig_cwd = os.getcwd() + td = tempfile.mkdtemp() + try: + tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") + os.chdir(td) + tf.extractall() + + __do_getsize = Glusterfs._do_getsize + Glusterfs._do_getsize = True + + cd = utils._get_container_details_from_fs(td) + assert cd.bytes_used == 30, repr(cd.bytes_used) + assert cd.object_count == 8, repr(cd.object_count) + assert set(cd.obj_list) == set(['file1', 'file3', 'file2', + 'dir3', 'dir1', 'dir2', + 'dir1/file1', 'dir1/file2' + ]), repr(cd.obj_list) + + full_dir1 = os.path.join(td, 'dir1') + full_dir2 = os.path.join(td, 'dir2') + full_dir3 = os.path.join(td, 'dir3') + exp_dir_dict = { td: os.path.getmtime(td), + full_dir1: os.path.getmtime(full_dir1), + full_dir2: os.path.getmtime(full_dir2), + full_dir3: os.path.getmtime(full_dir3), + } + for d,m in cd.dir_list: + assert d in exp_dir_dict + assert exp_dir_dict[d] == m + finally: + Glusterfs._do_getsize = __do_getsize + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_account_details_from_fs_notadir_w_stats(self): + tf = tempfile.NamedTemporaryFile() + ad = utils._get_account_details_from_fs(tf.name, os.stat(tf.name)) + assert ad.mtime == os.path.getmtime(tf.name) + assert ad.container_count == 0 + assert ad.container_list == [] + + def test_get_account_details_from_fs_notadir(self): + tf = tempfile.NamedTemporaryFile() + ad = utils._get_account_details_from_fs(tf.name, None) + assert ad.mtime == os.path.getmtime(tf.name) + assert ad.container_count == 0 + assert ad.container_list == [] + + def test_write_pickle(self): + td = tempfile.mkdtemp() + try: + fpp = os.path.join(td, 'pp') + utils.write_pickle('pickled peppers', fpp) + with open(fpp, "rb") as f: + contents = f.read() + s = pickle.loads(contents) + assert s == 'pickled peppers', repr(s) + finally: + shutil.rmtree(td) + + def test_write_pickle_ignore_tmp(self): + tf = tempfile.NamedTemporaryFile() + td = tempfile.mkdtemp() + try: + fpp = os.path.join(td, 'pp') + # Also test an explicity pickle protocol + utils.write_pickle('pickled peppers', fpp, tmp=tf.name, pickle_protocol=2) + with open(fpp, "rb") as f: + contents = f.read() + s = pickle.loads(contents) + assert s == 'pickled peppers', repr(s) + with open(tf.name, "rb") as f: + contents = f.read() + assert contents == '' + finally: + shutil.rmtree(td) + + def test_check_user_xattr_bad_path(self): + assert False == utils.check_user_xattr("/tmp/foo/bar/check/user/xattr") + + def test_check_user_xattr_bad_set(self): + td = tempfile.mkdtemp() + xkey = _xkey(td, 'user.test.key1') + _xattr_set_err[xkey] = errno.EOPNOTSUPP + try: + assert False == utils.check_user_xattr(td) + except IOError: + pass + else: + self.fail("Expected IOError") + finally: + shutil.rmtree(td) + + def test_check_user_xattr_bad_remove(self): + td = tempfile.mkdtemp() + xkey = _xkey(td, 'user.test.key1') + _xattr_rem_err[xkey] = errno.EOPNOTSUPP + try: + utils.check_user_xattr(td) + except IOError: + self.fail("Unexpected IOError") + finally: + shutil.rmtree(td) + + def test_check_user_xattr(self): + td = tempfile.mkdtemp() + try: + assert utils.check_user_xattr(td) + finally: + shutil.rmtree(td) + + def test_validate_container_empty(self): + ret = utils.validate_container({}) + assert ret == False + + def test_validate_container_missing_keys(self): + ret = utils.validate_container({ 'foo': 'bar' }) + assert ret == False + + def test_validate_container_bad_type(self): + md = { utils.X_TYPE: ('bad', 0), + utils.X_TIMESTAMP: ('na', 0), + utils.X_PUT_TIMESTAMP: ('na', 0), + utils.X_OBJECTS_COUNT: ('na', 0), + utils.X_BYTES_USED: ('na', 0) } + ret = utils.validate_container(md) + assert ret == False + + def test_validate_container_good_type(self): + md = { utils.X_TYPE: (utils.CONTAINER, 0), + utils.X_TIMESTAMP: ('na', 0), + utils.X_PUT_TIMESTAMP: ('na', 0), + utils.X_OBJECTS_COUNT: ('na', 0), + utils.X_BYTES_USED: ('na', 0) } + ret = utils.validate_container(md) + assert ret + + def test_validate_account_empty(self): + ret = utils.validate_account({}) + assert ret == False + + def test_validate_account_missing_keys(self): + ret = utils.validate_account({ 'foo': 'bar' }) + assert ret == False + + def test_validate_account_bad_type(self): + md = { utils.X_TYPE: ('bad', 0), + utils.X_TIMESTAMP: ('na', 0), + utils.X_PUT_TIMESTAMP: ('na', 0), + utils.X_OBJECTS_COUNT: ('na', 0), + utils.X_BYTES_USED: ('na', 0), + utils.X_CONTAINER_COUNT: ('na', 0) } + ret = utils.validate_account(md) + assert ret == False + + def test_validate_account_good_type(self): + md = { utils.X_TYPE: (utils.ACCOUNT, 0), + utils.X_TIMESTAMP: ('na', 0), + utils.X_PUT_TIMESTAMP: ('na', 0), + utils.X_OBJECTS_COUNT: ('na', 0), + utils.X_BYTES_USED: ('na', 0), + utils.X_CONTAINER_COUNT: ('na', 0) } + ret = utils.validate_account(md) + assert ret + + def test_validate_object_empty(self): + ret = utils.validate_object({}) + assert ret == False + + def test_validate_object_missing_keys(self): + ret = utils.validate_object({ 'foo': 'bar' }) + assert ret == False + + def test_validate_object_bad_type(self): + md = { utils.X_TIMESTAMP: 'na', + utils.X_CONTENT_TYPE: 'na', + utils.X_ETAG: 'bad', + utils.X_CONTENT_LENGTH: 'na', + utils.X_TYPE: 'bad', + utils.X_OBJECT_TYPE: 'na' } + ret = utils.validate_object(md) + assert ret == False + + def test_validate_object_good_type(self): + md = { utils.X_TIMESTAMP: 'na', + utils.X_CONTENT_TYPE: 'na', + utils.X_ETAG: 'bad', + utils.X_CONTENT_LENGTH: 'na', + utils.X_TYPE: utils.OBJECT, + utils.X_OBJECT_TYPE: 'na' } + ret = utils.validate_object(md) + assert ret + + def test_is_marker_empty(self): + assert False == utils.is_marker(None) + + def test_is_marker_missing(self): + assert False == utils.is_marker( { 'foo': 'bar' } ) + + def test_is_marker_not_marker(self): + md = { utils.X_OBJECT_TYPE: utils.DIR } + assert False == utils.is_marker(md) + + def test_is_marker(self): + md = { utils.X_OBJECT_TYPE: utils.MARKER_DIR } + assert utils.is_marker(md) diff --git a/tools/test-requires b/tools/test-requires new file mode 100644 index 0000000..ef8bf14 --- /dev/null +++ b/tools/test-requires @@ -0,0 +1,6 @@ +coverage +nose +nosexcover +openstack.nose_plugin +nosehtmloutput +mock>=0.8.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..792bb41 --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +envlist = py26,py27 + +[testenv] +setenv = VIRTUAL_ENV={envdir} + NOSE_WITH_OPENSTACK=1 + NOSE_OPENSTACK_COLOR=1 + NOSE_OPENSTACK_RED=0.05 + NOSE_OPENSTACK_YELLOW=0.025 + NOSE_OPENSTACK_SHOW_ELAPSED=1 + NOSE_OPENSTACK_STDOUT=1 +deps = + https://launchpad.net/swift/grizzly/1.8.0/+download/swift-1.8.0.tar.gz + -r{toxinidir}/tools/test-requires +changedir = {toxinidir}/test/unit +commands = nosetests -v --exe --with-coverage --cover-package gluster --cover-erase {posargs} + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 + +[testenv:venv] +commands = {posargs} diff --git a/ufo/README b/ufo/README deleted file mode 100644 index 9efd918..0000000 --- a/ufo/README +++ /dev/null @@ -1,16 +0,0 @@ -Gluster Unified File and Object Storage allows files and directories created -via gluster-native/nfs mount to be accessed as containers and objects. It is -a plugin for OpenStack Swift project. - -Install - * TBD - -Once this is done, you can access the GlusterFS volumes as Swift accounts. -Add the Volume names with the user-name and its corresponding password to the -/etc/swift/proxy-server.conf (follow the syntax used in the sample conf file). - -Command to start the servers (TBD) - swift-init main start - -Command to stop the servers (TBD) - swift-init main stop diff --git a/ufo/bin/gluster-swift-gen-builders b/ufo/bin/gluster-swift-gen-builders deleted file mode 100755 index bef5d14..0000000 --- a/ufo/bin/gluster-swift-gen-builders +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Note that these port numbers must match the configured values for the -# various servers in their configuration files. -declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \ - ["object.builder"]=6010) - -builder_files="account.builder container.builder object.builder" - -function create { - swift-ring-builder $1 create 1 1 1 >> /tmp/out -} - -function add { - swift-ring-builder $1 add z$2-127.0.0.1:$3/$4_ 100.0 -} - -function rebalance { - swift-ring-builder $1 rebalance -} - -function build { - swift-ring-builder $1 -} - -if [ "$1x" = "x" ]; then - echo "Please specify the gluster volume name to use." - exit 1 -fi - -for builder_file in $builder_files -do - create $builder_file - - zone=1 - for volname in $@ - do - add $builder_file $zone ${port[$builder_file]} $volname - zone=$(expr $zone + 1) - done - - rebalance $builder_file - build $builder_file -done diff --git a/ufo/etc/account-server/1.conf-gluster b/ufo/etc/account-server/1.conf-gluster deleted file mode 100644 index da8f317..0000000 --- a/ufo/etc/account-server/1.conf-gluster +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6012 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift_ufo#account - -[account-replicator] -vm_test_mode = yes - -[account-auditor] - -[account-reaper] diff --git a/ufo/etc/container-server/1.conf-gluster b/ufo/etc/container-server/1.conf-gluster deleted file mode 100644 index acad621..0000000 --- a/ufo/etc/container-server/1.conf-gluster +++ /dev/null @@ -1,21 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6011 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift_ufo#container - -[container-replicator] -vm_test_mode = yes - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/ufo/etc/fs.conf-gluster b/ufo/etc/fs.conf-gluster deleted file mode 100644 index 71a9b03..0000000 --- a/ufo/etc/fs.conf-gluster +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] -# IP address of a GlusterFS volume server member. By default, we assume the -# local host. -mount_ip = localhost - -# By default it is assumed the Gluster volumes can be accessed using other -# methods besides UFO (not object only), which disables a caching -# optimizations in order to keep in sync with file system changes. -object_only = no - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -accurate_size_in_listing = off \ No newline at end of file diff --git a/ufo/etc/object-server/1.conf-gluster b/ufo/etc/object-server/1.conf-gluster deleted file mode 100644 index 0d85546..0000000 --- a/ufo/etc/object-server/1.conf-gluster +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6010 -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift_ufo#object -user = root -log_facility = LOG_LOCAL2 -# Timeout clients that don't read or write to the proxy server after 5 -# seconds. -conn_timeout = 5 -# For high load situations, once connected to a container server, allow for -# delays communicating with it. -node_timeout = 60 -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# Adjust this value match whatever is set for the disk_chunk_size -# initially. This will provide a reasonable starting point for tuning this -# value. -network_chunk_size = 65556 - -[object-replicator] -vm_test_mode = yes - -[object-updater] - -[object-auditor] diff --git a/ufo/etc/proxy-server.conf-gluster b/ufo/etc/proxy-server.conf-gluster deleted file mode 100644 index e04efec..0000000 --- a/ufo/etc/proxy-server.conf-gluster +++ /dev/null @@ -1,69 +0,0 @@ -[DEFAULT] -bind_port = 8080 -user = root -log_facility = LOG_LOCAL1 -# Consider using 1 worker per CPU -workers = 1 - -[pipeline:main] -pipeline = healthcheck cache tempauth proxy-server - -[app:proxy-server] -use = egg:gluster_swift_ufo#proxy -log_facility = LOG_LOCAL1 -# The API allows for account creation and deletion, but since Gluster/Swift -# automounts a Gluster volume for a given account, there is no way to create -# or delete an account. So leave this off. -allow_account_management = false -account_autocreate = true -# Only need to recheck the account exists once a day -recheck_account_existence = 86400 -# May want to consider bumping this up if containers are created and destroyed -# infrequently. -recheck_container_existence = 60 -# Timeout clients that don't read or write to the proxy server after 5 -# seconds. -client_timeout = 5 -# Give more time to connect to the object, container or account servers in -# cases of high load. -conn_timeout = 5 -# For high load situations, once connected to an object, container or account -# server, allow for delays communicating with them. -node_timeout = 60 -# May want to consider bumping up this value to 1 - 4 MB depending on how much -# traffic is for multi-megabyte or gigabyte requests; perhaps matching the -# stripe width (not stripe element size) of your storage volume is a good -# starting point. See below for sizing information. -object_chunk_size = 65536 -# If you do decide to increase the object_chunk_size, then consider lowering -# this value to one. Up to "put_queue_length" object_chunk_size'd buffers can -# be queued to the object server for processing. Given one proxy server worker -# can handle up to 1,024 connections, by default, it will consume 10 * 65,536 -# * 1,024 bytes of memory in the worse case (default values). Be sure the -# amount of memory available on the system can accommodate increased values -# for object_chunk_size. -put_queue_depth = 10 - -[filter:tempauth] -use = egg:swift#tempauth -# Here you need to add users explicitly. See the OpenStack Swift Deployment -# Guide for more information. The user and user64 directives take the -# following form: -# user__ = [group] [group] [...] [storage_url] -# user64__ = [group] [group] [...] [storage_url] -# Where you use user64 for accounts and/or usernames that include underscores. -# -# NOTE (and WARNING): The account name must match the device name specified -# when generating the account, container, and object build rings. -# -# E.g. -# user_ufo0_admin = abc123 .admin - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/ufo/etc/swift.conf-gluster b/ufo/etc/swift.conf-gluster deleted file mode 100644 index 25c3ca1..0000000 --- a/ufo/etc/swift.conf-gluster +++ /dev/null @@ -1,91 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. It is STRONGLY recommended to leave this value at -# the default (5 * 2**30 + 2). - -# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting -# web service handle such a size? I think with UFO, we need to keep with the -# default size from Swift and encourage users to research what size their web -# services infrastructure can handle. - -max_file_size = 18446744073709551616 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/ufo/gluster-swift-ufo.spec b/ufo/gluster-swift-ufo.spec deleted file mode 100644 index 8c0167c..0000000 --- a/ufo/gluster-swift-ufo.spec +++ /dev/null @@ -1,79 +0,0 @@ -############################################################################################################ -# Command to build rpms.# -# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz # -############################################################################################################ -# Setting up the environment. # -# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES # -# * Copy the contents of gluster directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} # -# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec # -# For more information refer # -# http://fedoraproject.org/wiki/How_to_create_an_RPM_package # -############################################################################################################ - -%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} -%endif - -%define _confdir %{_sysconfdir}/swift -%define _ufo_version 1.1 -%define _ufo_release 4 - -Summary : GlusterFS Unified File and Object Storage. -Name : gluster-swift-ufo -Version : %{_ufo_version} -Release : %{_ufo_release} -Group : Application/File -Vendor : Red Hat Inc. -Source0 : %{name}-%{version}-%{release}.tar.gz -Packager : gluster-users@gluster.org -License : Apache -BuildArch: noarch -Requires : memcached -Requires : openssl -Requires : python -Requires : openstack-swift >= 1.4.8 -Requires : openstack-swift-account >= 1.4.8 -Requires : openstack-swift-container >= 1.4.8 -Requires : openstack-swift-object >= 1.4.8 -Requires : openstack-swift-proxy >= 1.4.8 -Obsoletes: gluster-swift -Obsoletes: gluster-swift-plugin - -%description -Gluster Unified File and Object Storage unifies NAS and object storage -technology. This provides a system for data storage that enables users to access -the same data as an object and as a file, simplifying management and controlling -storage costs. - -%prep -%setup -q - -%build -%{__python} setup.py build - -%install -rm -rf %{buildroot} - -%{__python} setup.py install -O1 --skip-build --root %{buildroot} - -mkdir -p %{buildroot}/%{_confdir}/ -cp -r etc/* %{buildroot}/%{_confdir}/ - -mkdir -p %{buildroot}/%{_bindir}/ -cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/ - -%clean -rm -rf %{buildroot} - -%files -%defattr(-,root,root) -%{python_sitelib}/gluster -%{python_sitelib}/gluster_swift_ufo-%{version}-*.egg-info -%{_bindir}/gluster-swift-gen-builders -%dir %{_confdir} -%config %{_confdir}/account-server/1.conf-gluster -%config %{_confdir}/container-server/1.conf-gluster -%config %{_confdir}/object-server/1.conf-gluster -%config %{_confdir}/swift.conf-gluster -%config %{_confdir}/proxy-server.conf-gluster -%config %{_confdir}/fs.conf-gluster diff --git a/ufo/gluster/__init__.py b/ufo/gluster/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/__init__.py b/ufo/gluster/swift/__init__.py deleted file mode 100644 index ef350fd..0000000 --- a/ufo/gluster/swift/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" Gluster Swift UFO """ - -class Version(object): - def __init__(self, canonical_version, final): - self.canonical_version = canonical_version - self.final = final - - @property - def pretty_version(self): - if self.final: - return self.canonical_version - else: - return '%s-dev' % (self.canonical_version,) - - -_version = Version('1.1', False) -__version__ = _version.pretty_version -__canonical_version__ = _version.canonical_version diff --git a/ufo/gluster/swift/account/__init__.py b/ufo/gluster/swift/account/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/account/server.py b/ufo/gluster/swift/account/server.py deleted file mode 100644 index aeaabc9..0000000 --- a/ufo/gluster/swift/account/server.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Account Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.account import server -from gluster.swift.common.DiskDir import DiskAccount - - -class AccountController(server.AccountController): - def _get_account_broker(self, drive, part, account): - """ - Overriden to provide the GlusterFS specific broker that talks to - Gluster for the information related to servicing a given request - instead of talking to a database. - - :param drive: drive that holds the container - :param part: partition the container is in - :param account: account name - :returns: DiskDir object - """ - return DiskAccount(self.root, drive, account, self.logger) - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI account server apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return AccountController(conf) diff --git a/ufo/gluster/swift/common/DiskDir.py b/ufo/gluster/swift/common/DiskDir.py deleted file mode 100644 index 18d08cc..0000000 --- a/ufo/gluster/swift/common/DiskDir.py +++ /dev/null @@ -1,496 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os, errno - -from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \ - mkdirs, validate_account, validate_container, is_marker, \ - get_container_details, get_account_details, get_container_metadata, \ - create_container_metadata, create_account_metadata, DEFAULT_GID, \ - DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ - write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ - X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ - X_CONTAINER_COUNT, CONTAINER, os_path -from gluster.swift.common import Glusterfs - -from swift.common.constraints import CONTAINER_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, TRUE_VALUES - - -DATADIR = 'containers' - -# Create a dummy db_file in /etc/swift -_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') -if _unittests_enabled in TRUE_VALUES: - _tmp_dir = '/tmp/gluster_unit_tests' - try: - os.mkdir(_tmp_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - _db_file = os.path.join(_tmp_dir, 'db_file.db') -else: - _db_file = '/etc/swift/db_file.db' -if not os.path.exists(_db_file): - file(_db_file, 'w+') - - -def _read_metadata(dd): - """ Filter read metadata so that it always returns a tuple that includes - some kind of timestamp. With 1.4.8 of the Swift integration the - timestamps were not stored. Here we fabricate timestamps for volumes - where the existing data has no timestamp (that is, stored data is not - a tuple), allowing us a measure of backward compatibility. - - FIXME: At this time it does not appear that the timestamps on each - metadata are used for much, so this should not hurt anything. - """ - metadata_i = read_metadata(dd) - metadata = {} - timestamp = 0 - for key, value in metadata_i.iteritems(): - if not isinstance(value, tuple): - value = (value, timestamp) - metadata[key] = value - return metadata - - -class DiskCommon(object): - def is_deleted(self): - return not os_path.exists(self.datadir) - - def filter_prefix(self, objects, prefix): - """ - Accept sorted list. - """ - found = 0 - filtered_objs = [] - for object_name in objects: - if object_name.startswith(prefix): - filtered_objs.append(object_name) - found = 1 - else: - if found: - break - return filtered_objs - - def filter_delimiter(self, objects, delimiter, prefix): - """ - Accept sorted list. - Objects should start with prefix. - """ - filtered_objs=[] - for object_name in objects: - tmp_obj = object_name.replace(prefix, '', 1) - sufix = tmp_obj.split(delimiter, 1) - new_obj = prefix + sufix[0] - if new_obj and new_obj not in filtered_objs: - filtered_objs.append(new_obj) - - return filtered_objs - - def filter_marker(self, objects, marker): - """ - TODO: We can traverse in reverse order to optimize. - Accept sorted list. - """ - filtered_objs=[] - found = 0 - if objects[-1] < marker: - return filtered_objs - for object_name in objects: - if object_name > marker: - filtered_objs.append(object_name) - - return filtered_objs - - def filter_end_marker(self, objects, end_marker): - """ - Accept sorted list. - """ - filtered_objs=[] - for object_name in objects: - if object_name < end_marker: - filtered_objs.append(object_name) - else: - break - - return filtered_objs - - def filter_limit(self, objects, limit): - filtered_objs=[] - for i in range(0, limit): - filtered_objs.append(objects[i]) - - return filtered_objs - - -class DiskDir(DiskCommon): - """ - Manage object files on disk. - - :param path: path to devices on the node - :param drive: gluster volume drive name - :param account: account name for the object - :param container: container name for the object - :param logger: account or container server logging object - :param uid: user ID container object should assume - :param gid: group ID container object should assume - """ - - def __init__(self, path, drive, account, container, logger, - uid=DEFAULT_UID, gid=DEFAULT_GID): - self.root = path - if container: - self.container = container - else: - self.container = None - if self.container: - self.datadir = os.path.join(path, drive, self.container) - else: - self.datadir = os.path.join(path, drive) - self.account = account - assert logger is not None - self.logger = logger - self.metadata = {} - self.container_info = None - self.object_info = None - self.uid = int(uid) - self.gid = int(gid) - self.db_file = _db_file - self.dir_exists = os_path.exists(self.datadir) - if self.dir_exists: - try: - self.metadata = _read_metadata(self.datadir) - except EOFError: - create_container_metadata(self.datadir) - else: - return - if self.container: - if not self.metadata: - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_container(self.metadata): - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not self.metadata: - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_account(self.metadata): - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - - def empty(self): - return dir_empty(self.datadir) - - def delete(self): - if self.empty(): - #For delete account. - if os_path.ismount(self.datadir): - clean_metadata(self.datadir) - else: - rmdirs(self.datadir) - self.dir_exists = False - - def put_metadata(self, metadata): - """ - Write metadata to directory/container. - """ - write_metadata(self.datadir, metadata) - self.metadata = metadata - - def put(self, metadata): - """ - Create and write metatdata to directory/container. - :param metadata: Metadata to write. - """ - if not self.dir_exists: - mkdirs(self.datadir) - - os.chown(self.datadir, self.uid, self.gid) - write_metadata(self.datadir, metadata) - self.metadata = metadata - self.dir_exists = True - - def put_obj(self, content_length, timestamp): - ocnt = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) - self.metadata[X_PUT_TIMESTAMP] = timestamp - bused = self.metadata[X_BYTES_USED][0] - self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) - #TODO: define update_metadata instad of writing whole metadata again. - self.put_metadata(self.metadata) - - def delete_obj(self, content_length): - ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) - bused, timestamp = self.metadata[X_BYTES_USED] - self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) - self.put_metadata(self.metadata) - - def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) - self.metadata[X_BYTES_USED] = (0, put_timestamp) - ccnt = self.metadata[X_CONTAINER_COUNT][0] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) - self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) - self.put_metadata(self.metadata) - - def delete_container(self, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, 0) - self.metadata[X_BYTES_USED] = (0, 0) - ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) - self.put_metadata(self.metadata) - - def unlink(self): - """ - Remove directory/container if empty. - """ - if dir_empty(self.datadir): - rmdirs(self.datadir) - - def list_objects_iter(self, limit, marker, end_marker, - prefix, delimiter, path): - """ - Returns tuple of name, created_at, size, content_type, etag. - """ - if path: - prefix = path = path.rstrip('/') + '/' - delimiter = '/' - if delimiter and not prefix: - prefix = '' - - self.update_object_count() - - objects, object_count, bytes_used = self.object_info - - if objects and prefix: - objects = self.filter_prefix(objects, prefix) - - if objects and delimiter: - objects = self.filter_delimiter(objects, delimiter, prefix) - - if objects and marker: - objects = self.filter_marker(objects, marker) - - if objects and end_marker: - objects = self.filter_end_marker(objects, end_marker) - - if objects and limit: - if len(objects) > limit: - objects = self.filter_limit(objects, limit) - - container_list = [] - if objects: - for obj in objects: - list_item = [] - list_item.append(obj) - obj_path = os.path.join(self.datadir, obj) - metadata = read_metadata(obj_path) - if not metadata or not validate_object(metadata): - metadata = create_object_metadata(obj_path) - if metadata: - list_item.append(metadata[X_TIMESTAMP]) - list_item.append(int(metadata[X_CONTENT_LENGTH])) - list_item.append(metadata[X_CONTENT_TYPE]) - list_item.append(metadata[X_ETAG]) - container_list.append(list_item) - - return container_list - - def update_object_count(self): - if not self.object_info: - self.object_info = get_container_details(self.datadir) - - objects, object_count, bytes_used = self.object_info - - if X_OBJECTS_COUNT not in self.metadata \ - or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ - or X_BYTES_USED not in self.metadata \ - or int(self.metadata[X_BYTES_USED][0]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = (object_count, 0) - self.metadata[X_BYTES_USED] = (bytes_used, 0) - write_metadata(self.datadir, self.metadata) - - def update_container_count(self): - if not self.container_info: - self.container_info = get_account_details(self.datadir) - - containers, container_count = self.container_info - - if X_CONTAINER_COUNT not in self.metadata \ - or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: - self.metadata[X_CONTAINER_COUNT] = (container_count, 0) - write_metadata(self.datadir, self.metadata) - - def get_info(self, include_metadata=False): - """ - Get global data for the container. - :returns: dict with keys: account, container, object_count, bytes_used, - hash, id, created_at, put_timestamp, delete_timestamp, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, and reported_bytes_used. - If include_metadata is set, metadata is included as a key - pointing to a dict of tuples of the metadata - """ - # TODO: delete_timestamp, reported_put_timestamp - # reported_delete_timestamp, reported_object_count, - # reported_bytes_used, created_at - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the object counts in case they changed behind our back. - self.update_object_count() - - data = {'account' : self.account, 'container' : self.container, - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], - 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], - 'delete_timestamp' : '1', - 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', - 'reported_object_count' : '1', 'reported_bytes_used' : '1'} - if include_metadata: - data['metadata'] = self.metadata - return data - - def put_object(self, name, timestamp, size, content_type, - etag, deleted=0): - # TODO: Implement the specifics of this func. - pass - - def initialize(self, timestamp): - pass - - def update_put_timestamp(self, timestamp): - """ - Create the container if it doesn't exist and update the timestamp - """ - if not os_path.exists(self.datadir): - self.put(self.metadata) - - def delete_object(self, name, timestamp): - # TODO: Implement the delete object - pass - - def delete_db(self, timestamp): - """ - Delete the container - """ - self.unlink() - - def update_metadata(self, metadata): - assert self.metadata, "Valid container/account metadata should have been created by now" - if metadata: - new_metadata = self.metadata.copy() - new_metadata.update(metadata) - if new_metadata != self.metadata: - write_metadata(self.datadir, new_metadata) - self.metadata = new_metadata - - -class DiskAccount(DiskDir): - def __init__(self, root, drive, account, logger): - super(DiskAccount, self).__init__(root, drive, account, None, logger) - assert self.dir_exists - - def list_containers_iter(self, limit, marker, end_marker, - prefix, delimiter): - """ - Return tuple of name, object_count, bytes_used, 0(is_subdir). - Used by account server. - """ - if delimiter and not prefix: - prefix = '' - - self.update_container_count() - - containers, container_count = self.container_info - - if containers: - containers.sort() - - if containers and prefix: - containers = self.filter_prefix(containers, prefix) - - if containers and delimiter: - containers = self.filter_delimiter(containers, delimiter, prefix) - - if containers and marker: - containers = self.filter_marker(containers, marker) - - if containers and end_marker: - containers = self.filter_end_marker(containers, end_marker) - - if containers and limit: - if len(containers) > limit: - containers = self.filter_limit(containers, limit) - - account_list = [] - if containers: - for cont in containers: - list_item = [] - metadata = None - list_item.append(cont) - cont_path = os.path.join(self.datadir, cont) - metadata = _read_metadata(cont_path) - if not metadata or not validate_container(metadata): - metadata = create_container_metadata(cont_path) - - if metadata: - list_item.append(metadata[X_OBJECTS_COUNT][0]) - list_item.append(metadata[X_BYTES_USED][0]) - list_item.append(0) - account_list.append(list_item) - - return account_list - - def get_info(self, include_metadata=False): - """ - Get global data for the account. - :returns: dict with keys: account, created_at, put_timestamp, - delete_timestamp, container_count, object_count, - bytes_used, hash, id - """ - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the container counts in case they changed behind our back. - self.update_container_count() - - data = {'account' : self.account, 'created_at' : '1', - 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], - 'hash' : '', 'id' : ''} - - if include_metadata: - data['metadata'] = self.metadata - return data - - def get_container_timestamp(self, container): - cont_path = os.path.join(self.datadir, container) - metadata = read_metadata(cont_path) - - return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None diff --git a/ufo/gluster/swift/common/DiskFile.py b/ufo/gluster/swift/common/DiskFile.py deleted file mode 100644 index 900bd49..0000000 --- a/ufo/gluster/swift/common/DiskFile.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import errno -import random -from hashlib import md5 -from contextlib import contextmanager -from swift.common.utils import normalize_timestamp, renamer -from swift.common.exceptions import DiskFileNotExist -from gluster.swift.common.exceptions import AlreadyExistsAsDir -from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \ - create_object_metadata, do_open, do_close, do_unlink, do_chown, \ - do_listdir, read_metadata, write_metadata, os_path, do_fsync -from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \ - X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \ - X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \ - FILE_TYPE, DEFAULT_UID, DEFAULT_GID - -import logging -from swift.obj.server import DiskFile - - -DEFAULT_DISK_CHUNK_SIZE = 65536 -# keep these lower-case -DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) - - -def _adjust_metadata(metadata): - # Fix up the metadata to ensure it has a proper value for the - # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE - # metadata values. - content_type = metadata['Content-Type'] - if not content_type: - # FIXME: How can this be that our caller supplied us with metadata - # that has a content type that evaluates to False? - # - # FIXME: If the file exists, we would already know it is a - # directory. So why are we assuming it is a file object? - metadata['Content-Type'] = FILE_TYPE - x_object_type = FILE - else: - x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE - metadata[X_TYPE] = OBJECT - metadata[X_OBJECT_TYPE] = x_object_type - return metadata - - -class Gluster_DiskFile(DiskFile): - """ - Manage object files on disk. - - :param path: path to devices on the node/mount path for UFO. - :param device: device name/account_name for UFO. - :param partition: partition on the device the object lives in - :param account: account name for the object - :param container: container name for the object - :param obj: object name for the object - :param logger: logger object for writing out log file messages - :param keep_data_fp: if True, don't close the fp, otherwise close it - :param disk_chunk_Size: size of chunks on file reads - :param uid: user ID disk object should assume (file or directory) - :param gid: group ID disk object should assume (file or directory) - """ - - def __init__(self, path, device, partition, account, container, obj, - logger, keep_data_fp=False, - disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, - uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): - self.disk_chunk_size = disk_chunk_size - self.iter_hook = iter_hook - # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/, - # etc. - obj = obj.strip(os.path.sep) - if os.path.sep in obj: - self._obj_path, self._obj = os.path.split(obj) - else: - self._obj_path = '' - self._obj = obj - - if self._obj_path: - self.name = os.path.join(container, self._obj_path) - else: - self.name = container - # Absolute path for object directory. - self.datadir = os.path.join(path, device, self.name) - self.device_path = os.path.join(path, device) - self._container_path = os.path.join(path, device, container) - self._is_dir = False - self.tmppath = None - self.logger = logger - self.metadata = {} - self.meta_file = None - self.fp = None - self.iter_etag = None - self.started_at_0 = False - self.read_to_eof = False - self.quarantined_dir = None - self.keep_cache = False - self.uid = int(uid) - self.gid = int(gid) - - # Don't store a value for data_file until we know it exists. - self.data_file = None - data_file = os.path.join(self.datadir, self._obj) - if not os_path.exists(data_file): - return - - self.data_file = os.path.join(data_file) - self.metadata = read_metadata(data_file) - if not self.metadata: - create_object_metadata(data_file) - self.metadata = read_metadata(data_file) - - if not validate_object(self.metadata): - create_object_metadata(data_file) - self.metadata = read_metadata(data_file) - - self.filter_metadata() - - if os_path.isdir(data_file): - self._is_dir = True - else: - if keep_data_fp: - # The caller has an assumption that the "fp" field of this - # object is an file object if keep_data_fp is set. However, - # this implementation of the DiskFile object does not need to - # open the file for internal operations. So if the caller - # requests it, we'll just open the file for them. - self.fp = do_open(data_file, 'rb') - - def close(self, verify_file=True): - """ - Close the file. Will handle quarantining file if necessary. - - :param verify_file: Defaults to True. If false, will not check - file to see if it needs quarantining. - """ - #Marker directory - if self._is_dir: - return - if self.fp: - do_close(self.fp) - self.fp = None - - def is_deleted(self): - """ - Check if the file is deleted. - - :returns: True if the file doesn't exist or has been flagged as - deleted. - """ - return not self.data_file - - def _create_dir_object(self, dir_path): - #TODO: if object already exists??? - if os_path.exists(dir_path) and not os_path.isdir(dir_path): - self.logger.error("Deleting file %s", dir_path) - do_unlink(dir_path) - #If dir aleady exist just override metadata. - mkdirs(dir_path) - do_chown(dir_path, self.uid, self.gid) - create_object_metadata(dir_path) - - def put_metadata(self, metadata, tombstone=False): - """ - Short hand for putting metadata to .meta and .ts files. - - :param metadata: dictionary of metadata to be written - :param tombstone: whether or not we are writing a tombstone - """ - if tombstone: - # We don't write tombstone files. So do nothing. - return - assert self.data_file is not None, "put_metadata: no file to put metadata into" - metadata = _adjust_metadata(metadata) - write_metadata(self.data_file, metadata) - self.metadata = metadata - self.filter_metadata() - - def put(self, fd, metadata, extension='.data'): - """ - Finalize writing the file on disk, and renames it from the temp file to - the real location. This should be called after the data has been - written to the temp file. - - :param fd: file descriptor of the temp file - :param metadata: dictionary of metadata to be written - :param extension: extension to be used when making the file - """ - # Our caller will use '.data' here; we just ignore it since we map the - # URL directly to the file system. - extension = '' - - metadata = _adjust_metadata(metadata) - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - if not self.data_file: - self.data_file = os.path.join(self.datadir, self._obj) - self._create_dir_object(self.data_file) - self.put_metadata(metadata) - return - - # Check if directory already exists. - if self._is_dir: - # FIXME: How can we have a directory and it not be marked as a - # MARKER_DIR (see above)? - msg = 'File object exists as a directory: %s' % self.data_file - raise AlreadyExistsAsDir(msg) - - timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) - write_metadata(self.tmppath, metadata) - if X_CONTENT_LENGTH in metadata: - self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) - do_fsync(fd) - if self._obj_path: - dir_objs = self._obj_path.split('/') - assert len(dir_objs) >= 1 - tmp_path = self._container_path - for dir_name in dir_objs: - tmp_path = os.path.join(tmp_path, dir_name) - self._create_dir_object(tmp_path) - - newpath = os.path.join(self.datadir, self._obj) - renamer(self.tmppath, newpath) - do_chown(newpath, self.uid, self.gid) - self.metadata = metadata - self.data_file = newpath - self.filter_metadata() - return - - def unlinkold(self, timestamp): - """ - Remove any older versions of the object file. Any file that has an - older timestamp than timestamp will be deleted. - - :param timestamp: timestamp to compare with each file - """ - if not self.metadata or self.metadata['X-Timestamp'] >= timestamp: - return - - assert self.data_file, \ - "Have metadata, %r, but no data_file" % self.metadata - - if self._is_dir: - # Marker directory object - if not rmdirs(self.data_file): - logging.error('Unable to delete dir object: %s', self.data_file) - return - else: - # File object - do_unlink(self.data_file) - - self.metadata = {} - self.data_file = None - - def get_data_file_size(self): - """ - Returns the os_path.getsize for the file. Raises an exception if this - file does not match the Content-Length stored in the metadata. Or if - self.data_file does not exist. - - :returns: file size as an int - :raises DiskFileError: on file size mismatch. - :raises DiskFileNotExist: on file not existing (including deleted) - """ - #Marker directory. - if self._is_dir: - return 0 - try: - file_size = 0 - if self.data_file: - file_size = os_path.getsize(self.data_file) - if X_CONTENT_LENGTH in self.metadata: - metadata_size = int(self.metadata[X_CONTENT_LENGTH]) - if file_size != metadata_size: - self.metadata[X_CONTENT_LENGTH] = file_size - write_metadata(self.data_file, self.metadata) - - return file_size - except OSError as err: - if err.errno != errno.ENOENT: - raise - raise DiskFileNotExist('Data File does not exist.') - - def filter_metadata(self): - if X_TYPE in self.metadata: - self.metadata.pop(X_TYPE) - if X_OBJECT_TYPE in self.metadata: - self.metadata.pop(X_OBJECT_TYPE) - - @contextmanager - def mkstemp(self): - """Contextmanager to make a temporary file.""" - - # Creating intermidiate directories and corresponding metadata. - # For optimization, check if the subdirectory already exists, - # if exists, then it means that it also has its metadata. - # Not checking for container, since the container should already - # exist for the call to come here. - if not os_path.exists(self.datadir): - path = self._container_path - subdir_list = self._obj_path.split(os.path.sep) - for i in range(len(subdir_list)): - path = os.path.join(path, subdir_list[i]); - if not os_path.exists(path): - self._create_dir_object(path) - - tmpfile = '.' + self._obj + '.' + md5(self._obj + \ - str(random.random())).hexdigest() - - self.tmppath = os.path.join(self.datadir, tmpfile) - fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL) - try: - yield fd - finally: - try: - do_close(fd) - except OSError: - pass - tmppath, self.tmppath = self.tmppath, None - try: - do_unlink(tmppath) - except OSError as err: - if err.errno != errno.ENOENT: - raise diff --git a/ufo/gluster/swift/common/Glusterfs.py b/ufo/gluster/swift/common/Glusterfs.py deleted file mode 100644 index a0c8126..0000000 --- a/ufo/gluster/swift/common/Glusterfs.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os, sys, fcntl, time, errno -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import TRUE_VALUES, search_tree -from gluster.swift.common.fs_utils import mkdirs - -# -# Read the fs.conf file once at startup (module load) -# -_fs_conf = ConfigParser() -MOUNT_IP = 'localhost' -OBJECT_ONLY = False -RUN_DIR='/var/run/swift' -SWIFT_DIR = '/etc/swift' -_do_getsize = False -if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): - try: - MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') - except (NoSectionError, NoOptionError): - pass - try: - OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass - try: - RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift') - except (NoSectionError, NoOptionError): - pass - - try: - _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \ - "no") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass - -NAME = 'glusterfs' - - -def _busy_wait(full_mount_path): - # Iterate for definite number of time over a given - # interval for successful mount - for i in range(0, 5): - if os.path.ismount(os.path.join(full_mount_path)): - return True - time.sleep(2) - logging.error('Busy wait for mount timed out for mount %s', full_mount_path) - return False - -def mount(root, drive): - # FIXME: Possible thundering herd problem here - - el = _get_export_list() - for export in el: - if drive == export: - break - else: - logging.error('No export found in %r matching drive, %s', el, drive) - return False - - # NOTE: root is typically the default value of /mnt/gluster-object - full_mount_path = os.path.join(root, drive) - if not os.path.isdir(full_mount_path): - mkdirs(full_mount_path) - - lck_file = os.path.join(RUN_DIR, '%s.lock' %drive); - - if not os.path.exists(RUN_DIR): - mkdirs(RUN_DIR) - - fd = os.open(lck_file, os.O_CREAT|os.O_RDWR) - with os.fdopen(fd, 'r+b') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) - except IOError as ex: - if ex.errno in (errno.EACCES, errno.EAGAIN): - # This means that some other process is mounting the - # filesystem, so wait for the mount process to complete - return _busy_wait(full_mount_path) - else: - raise ex - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ - full_mount_path) - if os.system(mnt_cmd) or not _busy_wait(full_mount_path): - logging.error('Mount failed %s: %s', NAME, mnt_cmd) - return False - return True - -def unmount(full_mount_path): - # FIXME: Possible thundering herd problem here - - umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path - if os.system(umnt_cmd): - logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) - -def _get_export_list(): - cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP - - export_list = [] - - if os.system(cmnd + ' >> /dev/null'): - logging.error('Getting volume info failed for %s', NAME) - else: - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) - - return export_list - -def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"): - """Read the object-server's configuration file and return - the device value""" - - mnt_dir = '' - conf_files = search_tree(conf_dir, conf_file, '.conf') - if not conf_files: - raise Exception("Config file not found") - - _conf = ConfigParser() - if _conf.read(conf_files[0]): - try: - mnt_dir = _conf.get('DEFAULT', 'devices', '') - except (NoSectionError, NoOptionError): - raise - return os.path.join(mnt_dir, vol_name) diff --git a/ufo/gluster/swift/common/__init__.py b/ufo/gluster/swift/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/common/constraints.py b/ufo/gluster/swift/common/constraints.py deleted file mode 100644 index 11f626b..0000000 --- a/ufo/gluster/swift/common/constraints.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - from webob.exc import HTTPBadRequest -except ImportError: - from swift.common.swob import HTTPBadRequest -import swift.common.constraints -import swift.common.ring as _ring -from gluster.swift.common import Glusterfs, ring - -if hasattr(swift.common.constraints, 'constraints_conf_int'): - MAX_OBJECT_NAME_COMPONENT_LENGTH = \ - swift.common.constraints.constraints_conf_int( - 'max_object_name_component_length', 255) -else: - MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 - -def validate_obj_name_component(obj): - if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: - return 'too long (%d)' % len(obj) - if obj == '.' or obj == '..': - return 'cannot be . or ..' - return '' - -# Save the original check object creation -__check_object_creation = swift.common.constraints.check_object_creation - -# Define our new one which invokes the original -def gluster_check_object_creation(req, object_name): - """ - Check to ensure that everything is alright about an object to be created. - Monkey patches swift.common.constraints.check_object_creation, invoking - the original, and then adding an additional check for individual object - name components. - - :param req: HTTP request object - :param object_name: name of object to be created - :raises HTTPRequestEntityTooLarge: the object is too large - :raises HTTPLengthRequered: missing content-length header and not - a chunked request - :raises HTTPBadRequest: missing or bad content-type header, or - bad metadata - """ - ret = __check_object_creation(req, object_name) - - if ret is None: - for obj in object_name.split('/'): - reason = validate_obj_name_component(obj) - if reason: - bdy = 'Invalid object name "%s", component "%s" %s' \ - % (object_name, obj, reason) - ret = HTTPBadRequest(body=bdy, - request=req, - content_type='text/plain') - - return ret - -# Replace the original check object creation with ours -swift.common.constraints.check_object_creation = gluster_check_object_creation - -# Save the original check mount -__check_mount = swift.common.constraints.check_mount - -# Define our new one which invokes the original -def gluster_check_mount(root, drive): - # FIXME: Potential performance optimization here to not call the original - # check mount which makes two stat calls. We could do what they do with - # just one. - if __check_mount(root, drive): - return True - - return Glusterfs.mount(root, drive) - -# Replace the original check mount with ours -swift.common.constraints.check_mount = gluster_check_mount - -# Save the original Ring class -__Ring = _ring.Ring - -# Replace the original Ring class -_ring.Ring = ring.Ring diff --git a/ufo/gluster/swift/common/exceptions.py b/ufo/gluster/swift/common/exceptions.py deleted file mode 100644 index d9357db..0000000 --- a/ufo/gluster/swift/common/exceptions.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class GlusterfsException(Exception): - pass - -class FileOrDirNotFoundError(GlusterfsException): - pass - -class NotDirectoryError(GlusterfsException): - pass - -class AlreadyExistsAsDir(GlusterfsException): - pass - diff --git a/ufo/gluster/swift/common/fs_utils.py b/ufo/gluster/swift/common/fs_utils.py deleted file mode 100644 index 0613a26..0000000 --- a/ufo/gluster/swift/common/fs_utils.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno -import os.path as os_path -from eventlet import tpool -from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ - NotDirectoryError - -def do_walk(*args, **kwargs): - return os.walk(*args, **kwargs) - -def do_write(fd, msg): - try: - cnt = os.write(fd, msg) - except OSError as err: - logging.exception("Write failed, err: %s", str(err)) - raise - return cnt - -def do_mkdir(path): - try: - os.mkdir(path) - except OSError as err: - if err.errno != errno.EEXIST: - logging.exception("Mkdir failed on %s err: %s", path, err.strerror) - raise - return True - -def do_makedirs(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno != errno.EEXIST: - logging.exception("Makedirs failed on %s err: %s", path, err.strerror) - raise - return True - -def do_listdir(path): - try: - buf = os.listdir(path) - except OSError as err: - logging.exception("Listdir failed on %s err: %s", path, err.strerror) - raise - return buf - -def do_chown(path, uid, gid): - try: - os.chown(path, uid, gid) - except OSError as err: - logging.exception("Chown failed on %s err: %s", path, err.strerror) - raise - return True - -def do_stat(path): - try: - #Check for fd. - if isinstance(path, int): - buf = os.fstat(path) - else: - buf = os.stat(path) - except OSError as err: - logging.exception("Stat failed on %s err: %s", path, err.strerror) - raise - return buf - -def do_open(path, mode): - if isinstance(mode, int): - try: - fd = os.open(path, mode) - except OSError as err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - else: - try: - fd = open(path, mode) - except IOError as err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - return fd - -def do_close(fd): - #fd could be file or int type. - try: - if isinstance(fd, int): - os.close(fd) - else: - fd.close() - except OSError as err: - logging.exception("Close failed on %s err: %s", fd, err.strerror) - raise - return True - -def do_unlink(path, log = True): - try: - os.unlink(path) - except OSError as err: - if err.errno != errno.ENOENT: - if log: - logging.exception("Unlink failed on %s err: %s", path, err.strerror) - raise - return True - -def do_rmdir(path): - try: - os.rmdir(path) - except OSError as err: - if err.errno != errno.ENOENT: - logging.exception("Rmdir failed on %s err: %s", path, err.strerror) - raise - res = False - else: - res = True - return res - -def do_rename(old_path, new_path): - try: - os.rename(old_path, new_path) - except OSError as err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - err.strerror) - raise - return True - -def mkdirs(path): - """ - Ensures the path is a directory or makes it if not. Errors if the path - exists but is a file or on permissions failure. - - :param path: path to create - """ - if not os.path.isdir(path): - do_makedirs(path) - -def dir_empty(path): - """ - Return true if directory/container is empty. - :param path: Directory path. - :returns: True/False. - """ - if os.path.isdir(path): - files = do_listdir(path) - return not files - elif not os.path.exists(path): - raise FileOrDirNotFoundError() - raise NotDirectoryError() - -def rmdirs(path): - if not os.path.isdir(path): - return False - try: - os.rmdir(path) - except OSError as err: - if err.errno != errno.ENOENT: - logging.error("rmdirs failed on %s, err: %s", path, err.strerror) - return False - return True - -def do_fsync(fd): - try: - tpool.execute(os.fsync, fd) - except OSError as err: - logging.exception("fsync failed with err: %s", err.strerror) - raise - return True diff --git a/ufo/gluster/swift/common/middleware/__init__.py b/ufo/gluster/swift/common/middleware/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/common/middleware/gluster.py b/ufo/gluster/swift/common/middleware/gluster.py deleted file mode 100644 index ab63c51..0000000 --- a/ufo/gluster/swift/common/middleware/gluster.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Noop Middleware that simply allows us to monkey patch the constraints -import gluster.swift.common.constraints - -class Gluster(object): - """ - Noop middleware for use with the proxy server to get paste.deploy to load - this middleware such that the plugin constraints monkey patch the common - constraints ahead of their use. - """ - def __init__(self, app, conf): - self.app = app - self.conf = conf - - def __call__(self, env, start_response): - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def gluster_filter(app): - return Gluster(app, conf) - return gluster_filter diff --git a/ufo/gluster/swift/common/ring.py b/ufo/gluster/swift/common/ring.py deleted file mode 100644 index 06aab8d..0000000 --- a/ufo/gluster/swift/common/ring.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ConfigParser import ConfigParser -from swift.common.ring import ring -from swift.common.utils import search_tree -from gluster.swift.common.Glusterfs import SWIFT_DIR - -reseller_prefix = "AUTH_" -conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf') -if conf_files: - conf_file = conf_files[0] - -_conf = ConfigParser() -if conf_files and _conf.read(conf_file): - if _conf.defaults().get("reseller_prefix", None): - reseller_prefix = _conf.defaults().get("reseller_prefix") - else: - for key, value in _conf._sections.items(): - if value.get("reseller_prefix", None): - reseller_prefix = value["reseller_prefix"] - break - -if not reseller_prefix.endswith('_'): - reseller_prefix = reseller_prefix + '_' - -class Ring(ring.Ring): - def _get_part_nodes(self, part): - seen_ids = set() - nodes = [dev for dev in self._devs \ - if dev['device'] == self.acc_name \ - and not (dev['id'] in seen_ids \ - or seen_ids.add(dev['id']))] - if not nodes: - nodes = [self.false_node] - return nodes - - def get_part_nodes(self, part): - """ - Get the nodes that are responsible for the partition. If one - node is responsible for more than one replica of the same - partition, it will only appear in the output once. - - :param part: partition to get nodes for - :returns: list of node dicts - - See :func:`get_nodes` for a description of the node dicts. - """ - return self._get_part_nodes(part) - - def get_nodes(self, account, container=None, obj=None): - """ - Get the partition and nodes for an account/container/object. - If a node is responsible for more than one replica, it will - only appear in the output once. - :param account: account name - :param container: container name - :param obj: object name - :returns: a tuple of (partition, list of node dicts) - - Each node dict will have at least the following keys: - ====== =============================================================== - id unique integer identifier amongst devices - weight a float of the relative weight of this device as compared to - others; this indicates how many partitions the builder will try - to assign to this device - zone integer indicating which zone the device is in; a given - partition will not be assigned to multiple devices within the - same zone - ip the ip address of the device - port the tcp port of the device - device the device's name on disk (sdb1, for example) - meta general use 'extra' field; for example: the online date, the - hardware description - ====== =============================================================== - """ - self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \ - 'meta': '', 'device': 'volume_not_in_ring', \ - 'port': 6012} - if account.startswith(reseller_prefix): - self.acc_name = account.replace(reseller_prefix, '', 1) - else: - self.acc_name = account - - part = 0 - return part, self._get_part_nodes(part) - - - def get_more_nodes(self, part): - """ - Generator to get extra nodes for a partition for hinted handoff. - - :param part: partition to get handoff nodes for - :returns: generator of node dicts - - See :func:`get_nodes` for a description of the node dicts. - Should never be called in the swift UFO environment, so yield nothing - """ - yield self.false_node diff --git a/ufo/gluster/swift/common/utils.py b/ufo/gluster/swift/common/utils.py deleted file mode 100644 index f2cd8de..0000000 --- a/ufo/gluster/swift/common/utils.py +++ /dev/null @@ -1,496 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno -import xattr -import random -from hashlib import md5 -from eventlet import sleep -import cPickle as pickle -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from gluster.swift.common.fs_utils import * -from gluster.swift.common import Glusterfs - -X_CONTENT_TYPE = 'Content-Type' -X_CONTENT_LENGTH = 'Content-Length' -X_TIMESTAMP = 'X-Timestamp' -X_PUT_TIMESTAMP = 'X-PUT-Timestamp' -X_TYPE = 'X-Type' -X_ETAG = 'ETag' -X_OBJECTS_COUNT = 'X-Object-Count' -X_BYTES_USED = 'X-Bytes-Used' -X_CONTAINER_COUNT = 'X-Container-Count' -X_OBJECT_TYPE = 'X-Object-Type' -DIR_TYPE = 'application/directory' -ACCOUNT = 'Account' -METADATA_KEY = 'user.swift.metadata' -MAX_XATTR_SIZE = 65536 -CONTAINER = 'container' -DIR = 'dir' -MARKER_DIR = 'marker_dir' -TEMP_DIR = 'tmp' -ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR -FILE = 'file' -FILE_TYPE = 'application/octet-stream' -OBJECT = 'Object' -OBJECT_TYPE = 'application/octet-stream' -DEFAULT_UID = -1 -DEFAULT_GID = -1 -PICKLE_PROTOCOL = 2 -CHUNK_SIZE = 65536 -MEMCACHE_KEY_PREFIX = 'gluster.swift.' -MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' -MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' - -def read_metadata(path): - """ - Helper function to read the pickled metadata from a File/Directory. - - :param path: File/Directory to read metadata from. - - :returns: dictionary of metadata - """ - metadata = None - metadata_s = '' - key = 0 - while metadata is None: - try: - metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - if key > 0: - # No errors reading the xattr keys, but since we have not - # been able to find enough chunks to get a successful - # unpickle operation, we consider the metadata lost, and - # drop the existing data so that the internal state can be - # recreated. - clean_metadata(path) - # We either could not find any metadata key, or we could find - # some keys, but were not successful in performing the - # unpickling (missing keys perhaps)? Either way, just report - # to the caller we have no metadata. - metadata = {} - else: - logging.exception("xattr.getxattr failed on %s key %s err: %s", - path, key, str(err)) - # Note that we don't touch the keys on errors fetching the - # data since it could be a transient state. - raise - else: - try: - # If this key provides all or the remaining part of the pickle - # data, we don't need to keep searching for more keys. This - # means if we only need to store data in N xattr key/value - # pair, we only need to invoke xattr get N times. With large - # keys sizes we are shooting for N = 1. - metadata = pickle.loads(metadata_s) - assert isinstance(metadata, dict) - except EOFError, pickle.UnpicklingError: - # We still are not able recognize this existing data collected - # as a pickled object. Make sure we loop around to try to get - # more from another xattr key. - metadata = None - key += 1 - return metadata - -def write_metadata(path, metadata): - """ - Helper function to write pickled metadata for a File/Directory. - - :param path: File/Directory path to write the metadata - :param metadata: dictionary to metadata write - """ - assert isinstance(metadata, dict) - metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) - key = 0 - while metastr: - try: - xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) - except IOError as err: - logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err)) - raise - metastr = metastr[MAX_XATTR_SIZE:] - key += 1 - -def clean_metadata(path): - key = 0 - while True: - try: - xattr.removexattr(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - break - raise - key += 1 - -def check_user_xattr(path): - if not os_path.exists(path): - return False - try: - xattr.setxattr(path, 'user.test.key1', 'value1') - except IOError as err: - logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) - raise - try: - xattr.removexattr(path, 'user.test.key1') - except IOError as err: - logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) - #Remove xattr may fail in case of concurrent remove. - return True - -def validate_container(metadata): - if not metadata: - logging.warn('validate_container: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys(): - #logging.warn('validate_container: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == CONTAINER: - return True - - logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) - return False - -def validate_account(metadata): - if not metadata: - logging.warn('validate_account: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys() or \ - X_CONTAINER_COUNT not in metadata.keys(): - #logging.warn('validate_account: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == ACCOUNT: - return True - - logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) - return False - -def validate_object(metadata): - if not metadata: - logging.warn('validate_object: No metadata') - return False - - if X_TIMESTAMP not in metadata.keys() or \ - X_CONTENT_TYPE not in metadata.keys() or \ - X_ETAG not in metadata.keys() or \ - X_CONTENT_LENGTH not in metadata.keys() or \ - X_TYPE not in metadata.keys() or \ - X_OBJECT_TYPE not in metadata.keys(): - #logging.warn('validate_object: Metadata missing entries: %s' % metadata) - return False - - if metadata[X_TYPE] == OBJECT: - return True - - logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) - return False - -def is_marker(metadata): - if not metadata: - logging.warn('is_marker: No metadata') - return False - - if X_OBJECT_TYPE not in metadata.keys(): - logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) - return False - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - return True - else: - return False - -def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, - bytes_used=0, obj_list=[]): - # strip the prefix off, also stripping the leading and trailing slashes - obj_path = path.replace(cont_path, '').strip(os.path.sep) - - for obj_name in src_list: - if obj_path: - obj_list.append(os.path.join(obj_path, obj_name)) - else: - obj_list.append(obj_name) - - object_count += 1 - - if Glusterfs._do_getsize and reg_file: - bytes_used += os_path.getsize(os.path.join(path, obj_name)) - sleep() - - return object_count, bytes_used - -def update_list(path, cont_path, dirs=[], files=[], object_count=0, - bytes_used=0, obj_list=[]): - if files: - object_count, bytes_used = _update_list(path, cont_path, files, True, - object_count, bytes_used, - obj_list) - if dirs: - object_count, bytes_used = _update_list(path, cont_path, dirs, False, - object_count, bytes_used, - obj_list) - return object_count, bytes_used - - -class ContainerDetails(object): - def __init__(self, bytes_used, object_count, obj_list, dir_list): - self.bytes_used = bytes_used - self.object_count = object_count - self.obj_list = obj_list - self.dir_list = dir_list - - -def _get_container_details_from_fs(cont_path): - """ - get container details by traversing the filesystem - """ - bytes_used = 0 - object_count = 0 - obj_list = [] - dir_list = [] - - if os_path.isdir(cont_path): - for (path, dirs, files) in do_walk(cont_path): - object_count, bytes_used = update_list(path, cont_path, dirs, files, - object_count, bytes_used, - obj_list) - - dir_list.append((path, do_stat(path).st_mtime)) - sleep() - - return ContainerDetails(bytes_used, object_count, obj_list, dir_list) - -def get_container_details(cont_path, memcache=None): - """ - Return object_list, object_count and bytes_used. - """ - mkey = '' - if memcache: - mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path - cd = memcache.get(mkey) - if cd: - if not cd.dir_list: - cd = None - else: - for (path, mtime) in cd.dir_list: - if mtime != do_stat(path).st_mtime: - cd = None - else: - cd = None - if not cd: - cd = _get_container_details_from_fs(cont_path) - if memcache: - memcache.set(mkey, cd) - return cd.obj_list, cd.object_count, cd.bytes_used - - -class AccountDetails(object): - """ A simple class to store the three pieces of information associated - with an account: - - 1. The last known modification time - 2. The count of containers in the following list - 3. The list of containers - """ - def __init__(self, mtime, container_count, container_list): - self.mtime = mtime - self.container_count = container_count - self.container_list = container_list - - -def _get_account_details_from_fs(acc_path, acc_stats): - container_list = [] - container_count = 0 - - if not acc_stats: - acc_stats = do_stat(acc_path) - is_dir = (acc_stats.st_mode & 0040000) != 0 - if is_dir: - for name in do_listdir(acc_path): - if name.lower() == TEMP_DIR \ - or name.lower() == ASYNCDIR \ - or not os_path.isdir(os.path.join(acc_path, name)): - continue - container_count += 1 - container_list.append(name) - - return AccountDetails(acc_stats.st_mtime, container_count, container_list) - -def get_account_details(acc_path, memcache=None): - """ - Return container_list and container_count. - """ - acc_stats = None - mkey = '' - if memcache: - mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path - ad = memcache.get(mkey) - if ad: - # FIXME: Do we really need to stat the file? If we are object - # only, then we can track the other Swift HTTP APIs that would - # modify the account and invalidate the cached entry there. If we - # are not object only, are we even called on this path? - acc_stats = do_stat(acc_path) - if ad.mtime != acc_stats.st_mtime: - ad = None - else: - ad = None - if not ad: - ad = _get_account_details_from_fs(acc_path, acc_stats) - if memcache: - memcache.set(mkey, ad) - return ad.container_list, ad.container_count - -def _get_etag(path): - etag = md5() - with open(path, 'rb') as fp: - while True: - chunk = fp.read(CHUNK_SIZE) - if chunk: - etag.update(chunk) - else: - break - return etag.hexdigest() - -def get_object_metadata(obj_path): - """ - Return metadata of object. - """ - try: - stats = do_stat(obj_path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - metadata = {} - else: - is_dir = (stats.st_mode & 0040000) != 0 - metadata = { - X_TYPE: OBJECT, - X_TIMESTAMP: normalize_timestamp(stats.st_ctime), - X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, - X_OBJECT_TYPE: DIR if is_dir else FILE, - X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, - X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), - } - return metadata - -def _add_timestamp(metadata_i): - # At this point we have a simple key/value dictionary, turn it into - # key/(value,timestamp) pairs. - timestamp = 0 - metadata = {} - for key, value_i in metadata_i.iteritems(): - if not isinstance(value_i, tuple): - metadata[key] = (value_i, timestamp) - else: - metadata[key] = value_i - return metadata - -def get_container_metadata(cont_path, memcache=None): - objects = [] - object_count = 0 - bytes_used = 0 - objects, object_count, bytes_used = get_container_details(cont_path, memcache) - metadata = {X_TYPE: CONTAINER, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)), - X_OBJECTS_COUNT: object_count, - X_BYTES_USED: bytes_used} - return _add_timestamp(metadata) - -def get_account_metadata(acc_path, memcache=None): - containers = [] - container_count = 0 - containers, container_count = get_account_details(acc_path, memcache) - metadata = {X_TYPE: ACCOUNT, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)), - X_OBJECTS_COUNT: 0, - X_BYTES_USED: 0, - X_CONTAINER_COUNT: container_count} - return _add_timestamp(metadata) - -def restore_metadata(path, metadata): - meta_orig = read_metadata(path) - if meta_orig: - meta_new = meta_orig.copy() - meta_new.update(metadata) - else: - meta_new = metadata - if meta_orig != meta_new: - write_metadata(path, meta_new) - return meta_new - -def create_object_metadata(obj_path): - metadata = get_object_metadata(obj_path) - return restore_metadata(obj_path, metadata) - -def create_container_metadata(cont_path, memcache=None): - metadata = get_container_metadata(cont_path, memcache) - return restore_metadata(cont_path, metadata) - -def create_account_metadata(acc_path, memcache=None): - metadata = get_account_metadata(acc_path, memcache) - return restore_metadata(acc_path, metadata) - -def write_pickle(obj, dest, tmp=None, pickle_protocol=0): - """ - Ensure that a pickle file gets written to disk. The file is first written - to a tmp file location in the destination directory path, ensured it is - synced to disk, then moved to its final destination name. - - This version takes advantage of Gluster's dot-prefix-dot-suffix naming - where the a file named ".thefile.name.9a7aasv" is hashed to the same - Gluster node as "thefile.name". This ensures the renaming of a temp file - once written does not move it to another Gluster node. - - :param obj: python object to be pickled - :param dest: path of final destination file - :param tmp: path to tmp to use, defaults to None (ignored) - :param pickle_protocol: protocol to pickle the obj with, defaults to 0 - """ - dirname = os.path.dirname(dest) - basename = os.path.basename(dest) - tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest() - tmppath = os.path.join(dirname, tmpname) - with open(tmppath, 'wb') as fo: - pickle.dump(obj, fo, pickle_protocol) - # TODO: This flush() method call turns into a flush() system call - # We'll need to wrap this as well, but we would do this by writing - #a context manager for our own open() method which returns an object - # in fo which makes the gluster API call. - fo.flush() - do_fsync(fo) - do_rename(tmppath, dest) - -# Over-ride Swift's utils.write_pickle with ours -import swift.common.utils -swift.common.utils.write_pickle = write_pickle diff --git a/ufo/gluster/swift/container/__init__.py b/ufo/gluster/swift/container/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/container/server.py b/ufo/gluster/swift/container/server.py deleted file mode 100644 index c5792aa..0000000 --- a/ufo/gluster/swift/container/server.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Container Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.container import server -from gluster.swift.common.DiskDir import DiskDir - - -class ContainerController(server.ContainerController): - def _get_container_broker(self, drive, part, account, container): - """ - Overriden to provide the GlusterFS specific broker that talks to - Gluster for the information related to servicing a given request - instead of talking to a database. - - :param drive: drive that holds the container - :param part: partition the container is in - :param account: account name - :param container: container name - :returns: DiskDir object - """ - return DiskDir(self.root, drive, account, container, self.logger) - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI container server apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return ContainerController(conf) diff --git a/ufo/gluster/swift/obj/__init__.py b/ufo/gluster/swift/obj/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/obj/server.py b/ufo/gluster/swift/obj/server.py deleted file mode 100644 index 1c2b6cb..0000000 --- a/ufo/gluster/swift/obj/server.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Object Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints -import gluster.swift.common.utils - -from swift.obj import server -from gluster.swift.common.DiskFile import Gluster_DiskFile - -# Monkey patch the object server module to use Gluster's DiskFile definition -server.DiskFile = Gluster_DiskFile - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI object server apps""" - conf = global_conf.copy() - conf.update(local_conf) - return server.ObjectController(conf) diff --git a/ufo/gluster/swift/proxy/__init__.py b/ufo/gluster/swift/proxy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/gluster/swift/proxy/server.py b/ufo/gluster/swift/proxy/server.py deleted file mode 100644 index 792a97d..0000000 --- a/ufo/gluster/swift/proxy/server.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.proxy import server - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI proxy apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return server.Application(conf) diff --git a/ufo/setup.py b/ufo/setup.py deleted file mode 100644 index a483102..0000000 --- a/ufo/setup.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup, find_packages - -from gluster.swift import __canonical_version__ as version - - -name = 'gluster_swift_ufo' - - -setup( - name=name, - version=version, - description='Gluster Swift/UFO', - license='Apache License (2.0)', - author='Red Hat, Inc.', - author_email='gluster-users@gluster.org', - url='https://gluster.org/', - packages=find_packages(exclude=['test', 'bin']), - test_suite='nose.collector', - classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.6', - 'Environment :: No Input/Output (Daemon)', - ], - install_requires=[], # removed for better compat - scripts=[ - 'bin/gluster-swift-gen-builders', - ], - entry_points={ - 'paste.app_factory': [ - 'proxy=gluster.swift.proxy.server:app_factory', - 'object=gluster.swift.obj.server:app_factory', - 'container=gluster.swift.container.server:app_factory', - 'account=gluster.swift.account.server:app_factory', - ], - 'paste.filter_factory': [ - 'gluster=gluster.swift.common.middleware.gluster:filter_factory', - ], - }, - ) diff --git a/ufo/test/__init__.py b/ufo/test/__init__.py deleted file mode 100644 index 50b24ed..0000000 --- a/ufo/test/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -# See http://code.google.com/p/python-nose/issues/detail?id=373 -# The code below enables nosetests to work with i18n _() blocks - -import __builtin__ -import sys -import os -from ConfigParser import MissingSectionHeaderError -from StringIO import StringIO - -from swift.common.utils import readconf - -setattr(__builtin__, '_', lambda x: x) - - -# Work around what seems to be a Python bug. -# c.f. https://bugs.launchpad.net/swift/+bug/820185. -import logging -logging.raiseExceptions = False - - -def get_config(section_name=None, defaults=None): - """ - Attempt to get a test config dictionary. - - :param section_name: the section to read (all sections if not defined) - :param defaults: an optional dictionary namespace of defaults - """ - config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', - '/etc/swift/test.conf') - config = {} - if defaults is not None: - config.update(defaults) - - try: - config = readconf(config_file, section_name) - except SystemExit: - if not os.path.exists(config_file): - print >>sys.stderr, \ - 'Unable to read test config %s - file not found' \ - % config_file - elif not os.access(config_file, os.R_OK): - print >>sys.stderr, \ - 'Unable to read test config %s - permission denied' \ - % config_file - else: - print >>sys.stderr, \ - 'Unable to read test config %s - section %s not found' \ - % (config_file, section_name) - return config diff --git a/ufo/test/unit/__init__.py b/ufo/test/unit/__init__.py deleted file mode 100644 index cb24764..0000000 --- a/ufo/test/unit/__init__.py +++ /dev/null @@ -1,95 +0,0 @@ -""" Gluster Swift Unit Tests """ - -import logging -from collections import defaultdict -from test import get_config -from swift.common.utils import TRUE_VALUES - - -class NullLoggingHandler(logging.Handler): - - def emit(self, record): - pass - - -class FakeLogger(object): - # a thread safe logger - - def __init__(self, *args, **kwargs): - self._clear() - self.level = logging.NOTSET - if 'facility' in kwargs: - self.facility = kwargs['facility'] - - def _clear(self): - self.log_dict = defaultdict(list) - - def _store_in(store_name): - def stub_fn(self, *args, **kwargs): - self.log_dict[store_name].append((args, kwargs)) - return stub_fn - - error = _store_in('error') - info = _store_in('info') - warning = _store_in('warning') - debug = _store_in('debug') - - def exception(self, *args, **kwargs): - self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) - - # mock out the StatsD logging methods: - increment = _store_in('increment') - decrement = _store_in('decrement') - timing = _store_in('timing') - timing_since = _store_in('timing_since') - update_stats = _store_in('update_stats') - set_statsd_prefix = _store_in('set_statsd_prefix') - - def setFormatter(self, obj): - self.formatter = obj - - def close(self): - self._clear() - - def set_name(self, name): - # don't touch _handlers - self._name = name - - def acquire(self): - pass - - def release(self): - pass - - def createLock(self): - pass - - def emit(self, record): - pass - - def handle(self, record): - pass - - def flush(self): - pass - - def handleError(self, record): - pass - - -original_syslog_handler = logging.handlers.SysLogHandler - - -def fake_syslog_handler(): - for attr in dir(original_syslog_handler): - if attr.startswith('LOG'): - setattr(FakeLogger, attr, - copy.copy(getattr(logging.handlers.SysLogHandler, attr))) - FakeLogger.priority_map = \ - copy.deepcopy(logging.handlers.SysLogHandler.priority_map) - - logging.handlers.SysLogHandler = FakeLogger - - -if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES: - fake_syslog_handler() diff --git a/ufo/test/unit/common/__init__.py b/ufo/test/unit/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ufo/test/unit/common/data/README.rings b/ufo/test/unit/common/data/README.rings deleted file mode 100644 index 6457501..0000000 --- a/ufo/test/unit/common/data/README.rings +++ /dev/null @@ -1,3 +0,0 @@ -The unit tests expect certain ring data built using the following command: - - ../../../../bin/gluster-swift-gen-builders test iops \ No newline at end of file diff --git a/ufo/test/unit/common/data/account.builder b/ufo/test/unit/common/data/account.builder deleted file mode 100644 index 090ba4b..0000000 Binary files a/ufo/test/unit/common/data/account.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/account.ring.gz b/ufo/test/unit/common/data/account.ring.gz deleted file mode 100644 index 6d4c854..0000000 Binary files a/ufo/test/unit/common/data/account.ring.gz and /dev/null differ diff --git a/ufo/test/unit/common/data/account_tree.tar.bz2 b/ufo/test/unit/common/data/account_tree.tar.bz2 deleted file mode 100644 index cb23e4d..0000000 Binary files a/ufo/test/unit/common/data/account_tree.tar.bz2 and /dev/null differ diff --git a/ufo/test/unit/common/data/backups/1365124498.account.builder b/ufo/test/unit/common/data/backups/1365124498.account.builder deleted file mode 100644 index 090ba4b..0000000 Binary files a/ufo/test/unit/common/data/backups/1365124498.account.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/backups/1365124498.container.builder b/ufo/test/unit/common/data/backups/1365124498.container.builder deleted file mode 100644 index 733d27d..0000000 Binary files a/ufo/test/unit/common/data/backups/1365124498.container.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/backups/1365124498.object.builder b/ufo/test/unit/common/data/backups/1365124498.object.builder deleted file mode 100644 index ff877ec..0000000 Binary files a/ufo/test/unit/common/data/backups/1365124498.object.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/backups/1365124499.object.builder b/ufo/test/unit/common/data/backups/1365124499.object.builder deleted file mode 100644 index 8b8cd6c..0000000 Binary files a/ufo/test/unit/common/data/backups/1365124499.object.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/container.builder b/ufo/test/unit/common/data/container.builder deleted file mode 100644 index 733d27d..0000000 Binary files a/ufo/test/unit/common/data/container.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/container.ring.gz b/ufo/test/unit/common/data/container.ring.gz deleted file mode 100644 index 592b84b..0000000 Binary files a/ufo/test/unit/common/data/container.ring.gz and /dev/null differ diff --git a/ufo/test/unit/common/data/container_tree.tar.bz2 b/ufo/test/unit/common/data/container_tree.tar.bz2 deleted file mode 100644 index b4a1492..0000000 Binary files a/ufo/test/unit/common/data/container_tree.tar.bz2 and /dev/null differ diff --git a/ufo/test/unit/common/data/object.builder b/ufo/test/unit/common/data/object.builder deleted file mode 100644 index 8b8cd6c..0000000 Binary files a/ufo/test/unit/common/data/object.builder and /dev/null differ diff --git a/ufo/test/unit/common/data/object.ring.gz b/ufo/test/unit/common/data/object.ring.gz deleted file mode 100644 index d2f7192..0000000 Binary files a/ufo/test/unit/common/data/object.ring.gz and /dev/null differ diff --git a/ufo/test/unit/common/test_Glusterfs.py b/ufo/test/unit/common/test_Glusterfs.py deleted file mode 100644 index 7de060a..0000000 --- a/ufo/test/unit/common/test_Glusterfs.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import os, fcntl, errno, shutil -from tempfile import mkdtemp -import gluster.swift.common.Glusterfs as gfs - -def mock_os_path_ismount(path): - return True - -def mock_get_export_list(): - return ['test', 'test2'] - -def mock_os_system(cmd): - return False - -def mock_fcntl_lockf(f, *a, **kw): - raise IOError(errno.EAGAIN) - -def _init(): - global _RUN_DIR, _OS_SYSTEM, _FCNTL_LOCKF - global _OS_PATH_ISMOUNT, __GET_EXPORT_LIST - - _RUN_DIR = gfs.RUN_DIR - _OS_SYSTEM = os.system - _FCNTL_LOCKF = fcntl.lockf - _OS_PATH_ISMOUNT = os.path.ismount - __GET_EXPORT_LIST = gfs._get_export_list - -def _init_mock_variables(tmpdir): - os.system = mock_os_system - os.path.ismount = mock_os_path_ismount - gfs.RUN_DIR = os.path.join(tmpdir, 'var/run/swift') - gfs._get_export_list = mock_get_export_list - -def _reset_mock_variables(): - gfs.RUN_DIR = _RUN_DIR - gfs._get_export_list = __GET_EXPORT_LIST - - os.system = _OS_SYSTEM - fcntl.lockf = _FCNTL_LOCKF - os.path.ismount = _OS_PATH_ISMOUNT - -class TestGlusterfs(unittest.TestCase): - """ Tests for common.GlusterFS """ - - def setUp(self): - _init() - - def test_mount(self): - try: - tmpdir = mkdtemp() - root = os.path.join(tmpdir, 'mnt/gluster-object') - drive = 'test' - - _init_mock_variables(tmpdir) - assert gfs.mount(root, drive) - finally: - _reset_mock_variables() - shutil.rmtree(tmpdir) - - def test_mount_egain(self): - try: - tmpdir = mkdtemp() - root = os.path.join(tmpdir, 'mnt/gluster-object') - drive = 'test' - - _init_mock_variables(tmpdir) - assert gfs.mount(root, drive) - fcntl.lockf = mock_fcntl_lockf - assert gfs.mount(root, drive) - finally: - _reset_mock_variables() - shutil.rmtree(tmpdir) - - def test_mount_get_export_list_err(self): - gfs._get_export_list = mock_get_export_list - assert not gfs.mount(None, 'drive') - _reset_mock_variables() - - def tearDown(self): - _reset_mock_variables() diff --git a/ufo/test/unit/common/test_diskfile.py b/ufo/test/unit/common/test_diskfile.py deleted file mode 100644 index 85d539a..0000000 --- a/ufo/test/unit/common/test_diskfile.py +++ /dev/null @@ -1,932 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for gluster.swift.common.DiskFile """ - -import os -import stat -import errno -import unittest -import tempfile -import shutil -from hashlib import md5 -from swift.common.utils import normalize_timestamp -from swift.common.exceptions import DiskFileNotExist -import gluster.swift.common.DiskFile -import gluster.swift.common.utils -from gluster.swift.common.DiskFile import Gluster_DiskFile, \ - AlreadyExistsAsDir -from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \ - X_OBJECT_TYPE -from test_utils import _initxattr, _destroyxattr -from test.unit import FakeLogger - - -_metadata = {} - -def _mock_read_metadata(filename): - if filename in _metadata: - md = _metadata[filename] - else: - md = {} - return md - -def _mock_write_metadata(filename, metadata): - _metadata[filename] = metadata - -def _mock_clear_metadata(): - _metadata = {} - - -class MockException(Exception): - pass - - -def _mock_rmdirs(p): - raise MockException("gluster.swift.common.DiskFile.rmdirs() called") - -def _mock_do_listdir(p): - raise MockException("gluster.swift.common.DiskFile.do_listdir() called") - -def _mock_do_unlink(f): - ose = OSError() - ose.errno = errno.ENOENT - raise ose - - -def _mock_do_unlink_eacces_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -def _mock_getsize_eaccess_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -def _mock_do_rmdir_eacces_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -class MockRenamerCalled(Exception): - pass - - -def _mock_renamer(a, b): - raise MockRenamerCalled() - - -class TestDiskFile(unittest.TestCase): - """ Tests for gluster.swift.common.DiskFile """ - - def setUp(self): - self.lg = FakeLogger() - _initxattr() - _mock_clear_metadata() - self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata - self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata - gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata - gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata - self._saved_ut_wm = gluster.swift.common.utils.write_metadata - self._saved_ut_rm = gluster.swift.common.utils.read_metadata - gluster.swift.common.utils.write_metadata = _mock_write_metadata - gluster.swift.common.utils.read_metadata = _mock_read_metadata - - def tearDown(self): - self.lg = None - _destroyxattr() - gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm - gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm - gluster.swift.common.utils.write_metadata = self._saved_ut_wm - gluster.swift.common.utils.read_metadata = self._saved_ut_rm - - def test_constructor_no_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == "/tmp/foo/vol0/bar" - assert gdf.device_path == "/tmp/foo/vol0" - assert gdf._container_path == "/tmp/foo/vol0/bar" - assert gdf.disk_chunk_size == 65536 - assert gdf.iter_hook == None - assert gdf.logger == self.lg - assert gdf.uid == DEFAULT_UID - assert gdf.gid == DEFAULT_GID - assert gdf.metadata == {} - assert gdf.meta_file == None - assert gdf.data_file == None - assert gdf.fp == None - assert gdf.iter_etag == None - assert not gdf.started_at_0 - assert not gdf.read_to_eof - assert gdf.quarantined_dir == None - assert not gdf.keep_cache - assert not gdf._is_dir - - def test_constructor_leadtrail_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "b/a" - assert gdf.name == "bar/b/a" - assert gdf.datadir == "/tmp/foo/vol0/bar/b/a" - assert gdf.device_path == "/tmp/foo/vol0" - - def test_constructor_no_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - stats = os.stat(the_file) - ts = normalize_timestamp(stats.st_ctime) - etag = md5() - etag.update("1234") - etag = etag.hexdigest() - exp_md = { - 'Content-Length': 4, - 'ETag': etag, - 'X-Timestamp': ts, - 'Content-Type': 'application/octet-stream'} - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'file', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_invalid_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - inv_md = { - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = inv_md - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata != inv_md - finally: - shutil.rmtree(td) - - def test_constructor_isdir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'dir', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_dir] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_keep_data_fp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg, keep_data_fp=True) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is not None - finally: - shutil.rmtree(td) - - def test_constructor_chunk_size(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, disk_chunk_size=8192) - assert gdf.disk_chunk_size == 8192 - - def test_constructor_iter_hook(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, iter_hook='hook') - assert gdf.iter_hook == 'hook' - - def test_close(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - # Should be a no-op, as by default is_dir is False, but fp is None - gdf.close() - - gdf._is_dir = True - gdf.fp = "123" - # Should still be a no-op as is_dir is True (marker directory) - gdf.close() - assert gdf.fp == "123" - - gdf._is_dir = False - saved_dc = gluster.swift.common.DiskFile.do_close - self.called = False - def our_do_close(fp): - self.called = True - gluster.swift.common.DiskFile.do_close = our_do_close - try: - gdf.close() - assert self.called - assert gdf.fp is None - finally: - gluster.swift.common.DiskFile.do_close = saved_dc - - def test_is_deleted(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.is_deleted() - gdf.data_file = "/tmp/foo/bar" - assert not gdf.is_deleted() - - def test_create_dir_object(self): - td = tempfile.mkdtemp() - the_dir = os.path.join(td, "vol0", "bar", "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - gdf._create_dir_object(the_dir) - assert os.path.isdir(the_dir) - assert the_dir in _metadata - finally: - shutil.rmtree(td) - - def test_create_dir_object_exists(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_path) - with open(the_dir, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - def _mock_do_chown(p, u, g): - assert u == DEFAULT_UID - assert g == DEFAULT_GID - dc = gluster.swift.common.DiskFile.do_chown - gluster.swift.common.DiskFile.do_chown = _mock_do_chown - try: - gdf._create_dir_object(the_dir) - finally: - gluster.swift.common.DiskFile.do_chown = dc - assert os.path.isdir(the_dir) - assert the_dir in _metadata - finally: - shutil.rmtree(td) - - def test_put_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "z") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - md = { 'Content-Type': 'application/octet-stream', 'a': 'b' } - gdf.put_metadata(md.copy()) - assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md) - assert _metadata[the_dir] == md - finally: - shutil.rmtree(td) - - def test_put_w_tombstone(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - - gdf.put_metadata({'x': '1'}, tombstone=True) - assert gdf.metadata == {} - - def test_put_w_meta_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_file_no_content_type(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir_create(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - assert gdf.metadata == {} - newmd = { - 'Content-Length': 0, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/directory'} - gdf.put(None, newmd, extension='.dir') - assert gdf.data_file == the_dir - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - origmd = gdf.metadata.copy() - origfmd = _metadata[the_dir] - newmd = gdf.metadata.copy() - # FIXME: This is a hack to get to the code-path; it is not clear - # how this can happen normally. - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - try: - gdf.put(None, newmd, extension='.data') - except AlreadyExistsAsDir: - pass - else: - self.fail("Expected to encounter 'already-exists-as-dir' exception") - assert gdf.metadata == origmd - assert _metadata[the_dir] == origfmd - finally: - shutil.rmtree(td) - - def test_put(self): - td = tempfile.mkdtemp() - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == os.path.join(td, "vol0", "bar") - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - def test_put_obj_path(self): - the_obj_path = os.path.join("b", "a") - the_file = os.path.join(the_obj_path, "z") - td = tempfile.mkdtemp() - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - the_file, self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == the_obj_path - assert gdf.name == os.path.join("bar", "b", "a") - assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a") - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - def test_unlinkold_no_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir - try: - gdf.unlinkold(None) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir - - def test_unlinkold_same_timestamp(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.metadata['X-Timestamp'] = 1 - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir - try: - gdf.unlinkold(1) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir - - def test_unlinkold_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_not_found(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - # Handle the case the file is not in the directory listing. - os.unlink(the_file) - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_unlink_error(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - - stats = os.stat(the_path) - os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) - - # Handle the case do_unlink() raises an OSError - __os_unlink = os.unlink - os.unlink = _mock_do_unlink_eacces_err - try: - gdf.unlinkold(normalize_timestamp(later)) - except OSError as e: - assert e.errno == errno.EACCES - else: - self.fail("Excepted an OSError when unlinking file") - finally: - os.unlink = __os_unlink - os.chmod(the_path, stats.st_mode) - - assert os.path.isdir(gdf.datadir) - assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_is_dir_failure(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - stats = os.stat(gdf.datadir) - os.chmod(gdf.datadir, 0) - __os_rmdir = os.rmdir - os.rmdir = _mock_do_rmdir_eacces_err - try: - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - finally: - os.chmod(gdf.datadir, stats.st_mode) - os.rmdir = __os_rmdir - assert os.path.isdir(gdf.datadir) - assert os.path.isdir(gdf.data_file) - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.metadata['Content-Length'] - gdf.metadata['Content-Length'] = 3 - assert 4 == gdf.get_data_file_size() - assert 4 == gdf.metadata['Content-Length'] - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dne(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - - def test_get_data_file_size_dne_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - gdf.data_file = gdf.data_file + ".dne" - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - finally: - shutil.rmtree(td) - - def test_get_data_file_size_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - stats = os.stat(the_path) - os.chmod(the_path, 0) - __os_path_getsize = os.path.getsize - os.path.getsize = _mock_getsize_eaccess_err - try: - s = gdf.get_data_file_size() - except OSError as err: - assert err.errno == errno.EACCES - else: - self.fail("Expected OSError exception") - finally: - os.path.getsize = __os_path_getsize - os.chmod(the_path, stats.st_mode) - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert 0 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_filter_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.filter_metadata() - assert gdf.metadata == {} - - gdf.metadata[X_TYPE] = 'a' - gdf.metadata[X_OBJECT_TYPE] = 'b' - gdf.metadata['foobar'] = 'c' - gdf.filter_metadata() - assert X_TYPE not in gdf.metadata - assert X_OBJECT_TYPE not in gdf.metadata - assert 'foobar' in gdf.metadata - - def test_mkstemp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_close(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - # At the end of previous with block a close on fd is called. - # Calling os.close on the same fd will raise an OSError - # exception and we must catch it. - try: - os.close(fd) - except OSError as err: - pass - else: - self.fail("Exception expected") - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_unlink(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - os.unlink(saved_tmppath) - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) diff --git a/ufo/test/unit/common/test_fs_utils.py b/ufo/test/unit/common/test_fs_utils.py deleted file mode 100644 index 186e07d..0000000 --- a/ufo/test/unit/common/test_fs_utils.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import random -import unittest -from tempfile import mkdtemp, mkstemp -from gluster.swift.common import fs_utils as fs -from gluster.swift.common.exceptions import NotDirectoryError, \ - FileOrDirNotFoundError - -class TestUtils(unittest.TestCase): - """ Tests for common.utils """ - - def test_do_walk(self): - try: - # create directory structure - tmpparent = mkdtemp() - tmpdirs = [] - tmpfiles = [] - for i in range(5): - tmpdirs.append(mkdtemp(dir=tmpparent).rsplit(os.path.sep, 1)[1]) - tmpfiles.append(mkstemp(dir=tmpparent)[1].rsplit(os.path.sep, \ - 1)[1]) - - for path, dirnames, filenames in fs.do_walk(tmpparent): - assert path == tmpparent - assert dirnames.sort() == tmpdirs.sort() - assert filenames.sort() == tmpfiles.sort() - break - finally: - shutil.rmtree(tmpparent) - - def test_do_open(self): - try: - fd, tmpfile = mkstemp() - f = fs.do_open(tmpfile, 'r') - try: - f.write('test') - except IOError as err: - pass - else: - self.fail("IOError expected") - finally: - f.close() - os.close(fd) - os.remove(tmpfile) - - def test_do_open_err(self): - try: - fs.do_open(os.path.join('/tmp', str(random.random())), 'r') - except IOError: - pass - else: - self.fail("IOError expected") - - def test_do_write(self): - try: - fd, tmpfile = mkstemp() - cnt = fs.do_write(fd, "test") - assert cnt == len("test") - finally: - os.close(fd) - os.remove(tmpfile) - - def test_do_write_err(self): - try: - fd, tmpfile = mkstemp() - fd1 = os.open(tmpfile, os.O_RDONLY) - fs.do_write(fd1, "test") - except OSError: - pass - else: - self.fail("OSError expected") - finally: - os.close(fd) - os.close(fd1) - - def test_do_mkdir(self): - try: - path = os.path.join('/tmp', str(random.random())) - fs.do_mkdir(path) - assert os.path.exists(path) - assert fs.do_mkdir(path) - finally: - os.rmdir(path) - - def test_do_mkdir_err(self): - try: - path = os.path.join('/tmp', str(random.random()), str(random.random())) - fs.do_mkdir(path) - except OSError: - pass - else: - self.fail("OSError expected") - - - def test_do_makedirs(self): - try: - subdir = os.path.join('/tmp', str(random.random())) - path = os.path.join(subdir, str(random.random())) - fs.do_makedirs(path) - assert os.path.exists(path) - assert fs.do_makedirs(path) - finally: - shutil.rmtree(subdir) - - def test_do_listdir(self): - try: - tmpdir = mkdtemp() - subdir = [] - for i in range(5): - subdir.append(mkdtemp(dir=tmpdir).rsplit(os.path.sep, 1)[1]) - - assert subdir.sort() == fs.do_listdir(tmpdir).sort() - finally: - shutil.rmtree(tmpdir) - - def test_do_listdir_err(self): - try: - path = os.path.join('/tmp', str(random.random())) - fs.do_listdir(path) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_do_stat(self): - try: - tmpdir = mkdtemp() - fd, tmpfile = mkstemp(dir=tmpdir) - buf1 = os.stat(tmpfile) - buf2 = fs.do_stat(fd) - buf3 = fs.do_stat(tmpfile) - - assert buf1 == buf2 - assert buf1 == buf3 - finally: - os.close(fd) - os.remove(tmpfile) - os.rmdir(tmpdir) - - def test_do_stat_err(self): - try: - fs.do_stat(os.path.join('/tmp', str(random.random()))) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_do_close(self): - try: - fd, tmpfile = mkstemp() - fs.do_close(fd); - try: - os.write(fd, "test") - except OSError: - pass - else: - self.fail("OSError expected") - fp = open(tmpfile) - fs.do_close(fp) - finally: - os.remove(tmpfile) - - def test_do_unlink(self): - try: - fd, tmpfile = mkstemp() - fs.do_unlink(tmpfile) - assert not os.path.exists(tmpfile) - assert fs.do_unlink(os.path.join('/tmp', str(random.random()))) - finally: - os.close(fd) - - def test_do_unlink_err(self): - try: - tmpdir = mkdtemp() - fs.do_unlink(tmpdir) - except OSError: - pass - else: - self.fail('OSError expected') - finally: - os.rmdir(tmpdir) - - def test_do_rmdir(self): - tmpdir = mkdtemp() - fs.do_rmdir(tmpdir) - assert not os.path.exists(tmpdir) - assert not fs.do_rmdir(os.path.join('/tmp', str(random.random()))) - - def test_do_rmdir_err(self): - try: - fd, tmpfile = mkstemp() - fs.do_rmdir(tmpfile) - except OSError: - pass - else: - self.fail('OSError expected') - finally: - os.close(fd) - os.remove(tmpfile) - - def test_do_rename(self): - try: - srcpath = mkdtemp() - destpath = os.path.join('/tmp', str(random.random())) - fs.do_rename(srcpath, destpath) - assert not os.path.exists(srcpath) - assert os.path.exists(destpath) - finally: - os.rmdir(destpath) - - def test_do_rename_err(self): - try: - srcpath = os.path.join('/tmp', str(random.random())) - destpath = os.path.join('/tmp', str(random.random())) - fs.do_rename(srcpath, destpath) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_dir_empty(self): - try: - tmpdir = mkdtemp() - subdir = mkdtemp(dir=tmpdir) - assert not fs.dir_empty(tmpdir) - assert fs.dir_empty(subdir) - finally: - shutil.rmtree(tmpdir) - - def test_dir_empty_err(self): - try: - try: - assert fs.dir_empty(os.path.join('/tmp', str(random.random()))) - except FileOrDirNotFoundError: - pass - else: - self.fail("FileOrDirNotFoundError exception expected") - - fd, tmpfile = mkstemp() - try: - fs.dir_empty(tmpfile) - except NotDirectoryError: - pass - else: - self.fail("NotDirectoryError exception expected") - finally: - os.close(fd) - os.unlink(tmpfile) - - def test_rmdirs(self): - try: - tmpdir = mkdtemp() - subdir = mkdtemp(dir=tmpdir) - fd, tmpfile = mkstemp(dir=tmpdir) - assert not fs.rmdirs(tmpfile) - assert not fs.rmdirs(tmpdir) - assert fs.rmdirs(subdir) - assert not os.path.exists(subdir) - finally: - os.close(fd) - shutil.rmtree(tmpdir) diff --git a/ufo/test/unit/common/test_ring.py b/ufo/test/unit/common/test_ring.py deleted file mode 100644 index 8b7509c..0000000 --- a/ufo/test/unit/common/test_ring.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest -import gluster.swift.common.constraints -import swift.common.utils -from gluster.swift.common.ring import Ring - - -class TestRing(unittest.TestCase): - """ Tests for common.utils """ - - def setUp(self): - swift.common.utils.HASH_PATH_SUFFIX = 'endcap' - swiftdir = os.path.join(os.getcwd(), "common", "data") - self.ring = Ring(swiftdir, ring_name='object') - - def test_first_device(self): - part, node = self.ring.get_nodes('test') - assert node[0]['device'] == 'test' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'test' - for node in self.ring.get_more_nodes(0): - assert node['device'] == 'volume_not_in_ring' - - def test_invalid_device(self): - part, node = self.ring.get_nodes('test2') - assert node[0]['device'] == 'volume_not_in_ring' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'volume_not_in_ring' - - def test_second_device(self): - part, node = self.ring.get_nodes('iops') - assert node[0]['device'] == 'iops' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'iops' - for node in self.ring.get_more_nodes(0): - assert node['device'] == 'volume_not_in_ring' - - def test_second_device_with_reseller_prefix(self): - part, node = self.ring.get_nodes('AUTH_iops') - assert node[0]['device'] == 'iops' diff --git a/ufo/test/unit/common/test_utils.py b/ufo/test/unit/common/test_utils.py deleted file mode 100644 index c645509..0000000 --- a/ufo/test/unit/common/test_utils.py +++ /dev/null @@ -1,1020 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for common.utils """ - -import os -import unittest -import errno -import xattr -import cPickle as pickle -import tempfile -import hashlib -import tarfile -import shutil -from collections import defaultdict -from swift.common.utils import normalize_timestamp -from gluster.swift.common import utils, Glusterfs - -# -# Somewhat hacky way of emulating the operation of xattr calls. They are made -# against a dictionary that stores the xattr key/value pairs. -# -_xattrs = {} -_xattr_op_cnt = defaultdict(int) -_xattr_set_err = {} -_xattr_get_err = {} -_xattr_rem_err = {} - -def _xkey(path, key): - return "%s:%s" % (path, key) - -def _setxattr(path, key, value, *args, **kwargs): - _xattr_op_cnt['set'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_set_err: - e = IOError() - e.errno = _xattr_set_err[xkey] - raise e - global _xattrs - _xattrs[xkey] = value - -def _getxattr(path, key, *args, **kwargs): - _xattr_op_cnt['get'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_get_err: - e = IOError() - e.errno = _xattr_get_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - ret_val = _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - return ret_val - -def _removexattr(path, key, *args, **kwargs): - _xattr_op_cnt['remove'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_rem_err: - e = IOError() - e.errno = _xattr_rem_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - del _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - -def _initxattr(): - global _xattrs - _xattrs = {} - global _xattr_op_cnt - _xattr_op_cnt = defaultdict(int) - global _xattr_set_err, _xattr_get_err, _xattr_rem_err - _xattr_set_err = {} - _xattr_get_err = {} - _xattr_rem_err = {} - - # Save the current methods - global _xattr_set; _xattr_set = xattr.setxattr - global _xattr_get; _xattr_get = xattr.getxattr - global _xattr_remove; _xattr_remove = xattr.removexattr - - # Monkey patch the calls we use with our internal unit test versions - xattr.setxattr = _setxattr - xattr.getxattr = _getxattr - xattr.removexattr = _removexattr - -def _destroyxattr(): - # Restore the current methods just in case - global _xattr_set; xattr.setxattr = _xattr_set - global _xattr_get; xattr.getxattr = _xattr_get - global _xattr_remove; xattr.removexattr = _xattr_remove - # Destroy the stored values and - global _xattrs; _xattrs = None - - -class SimMemcache(object): - def __init__(self): - self._d = {} - - def get(self, key): - return self._d.get(key, None) - - def set(self, key, value): - self._d[key] = value - - -class TestUtils(unittest.TestCase): - """ Tests for common.utils """ - - def setUp(self): - _initxattr() - - def tearDown(self): - _destroyxattr() - - def test_write_metadata(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - utils.write_metadata(path, orig_d) - xkey = _xkey(path, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert orig_d == pickle.loads(_xattrs[xkey]) - assert _xattr_op_cnt['set'] == 1 - - def test_write_metadata_err(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattr_set_err[xkey] = errno.EOPNOTSUPP - try: - utils.write_metadata(path, orig_d) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert len(_xattrs.keys()) == 0 - assert _xattr_op_cnt['set'] == 1 - else: - self.fail("Expected an IOError exception on write") - - def test_write_metadata_multiple(self): - # At 64 KB an xattr key/value pair, this should generate three keys. - path = "/tmp/foo/w" - orig_d = { 'bar' : 'x' * 150000 } - utils.write_metadata(path, orig_d) - assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys()) - payload = '' - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - assert xkey in _xattrs - assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE - payload += _xattrs[xkey] - assert orig_d == pickle.loads(payload) - assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt - - def test_clean_metadata(self): - path = "/tmp/foo/c" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - utils.clean_metadata(path) - assert _xattr_op_cnt['remove'] == 4, "%r" % _xattr_op_cnt - - def test_clean_metadata_err(self): - path = "/tmp/foo/c" - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps({ 'a': 'y' }, utils.PICKLE_PROTOCOL) - _xattr_rem_err[xkey] = errno.EOPNOTSUPP - try: - utils.clean_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert _xattr_op_cnt['remove'] == 1, "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on remove") - - def test_read_metadata(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_notfound(self): - path = "/tmp/foo/r" - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_err(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - _xattr_get_err[xkey] = errno.EOPNOTSUPP - try: - res_d = utils.read_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on get") - - def test_read_metadata_multiple(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - - def test_read_metadata_multiple_one_missing(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,2): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert len(expected_p) <= utils.MAX_XATTR_SIZE - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) - - def test_restore_metadata_none(self): - # No initial metadata - path = "/tmp/foo/i" - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata(self): - # Initial metadata - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'a': 'z', 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata_nochange(self): - # Initial metadata but no changes - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, {}) - expected_d = { 'a': 'z' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt - - def test_add_timestamp_empty(self): - orig = {} - res = utils._add_timestamp(orig) - assert res == {} - - def test_add_timestamp_none(self): - orig = { 'a': 1, 'b': 2, 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_mixed(self): - orig = { 'a': 1, 'b': (2, 1), 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_all(self): - orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == orig - - def test_get_etag_empty(self): - tf = tempfile.NamedTemporaryFile() - hd = utils._get_etag(tf.name) - assert hd == hashlib.md5().hexdigest() - - def test_get_etag(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123' * utils.CHUNK_SIZE) - tf.file.flush() - hd = utils._get_etag(tf.name) - tf.file.seek(0) - md5 = hashlib.md5() - while True: - chunk = tf.file.read(utils.CHUNK_SIZE) - if not chunk: - break - md5.update(chunk) - assert hd == md5.hexdigest() - - def test_get_object_metadata_dne(self): - md = utils.get_object_metadata("/tmp/doesNotEx1st") - assert md == {} - - def test_get_object_metadata_err(self): - tf = tempfile.NamedTemporaryFile() - try: - md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) - except OSError as e: - assert e.errno != errno.ENOENT - else: - self.fail("Expected exception") - - obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, - utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) - - def test_get_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123'); tf.file.flush() - md = utils.get_object_metadata(tf.name) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_get_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - md = utils.get_object_metadata(td) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_create_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('4567'); tf.file.flush() - r_md = utils.create_object_metadata(tf.name) - - xkey = _xkey(tf.name, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_create_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_object_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_get_container_metadata(self): - def _mock_get_container_details(path, memcache=None): - o_list = [ 'a', 'b', 'c' ] - o_count = 3 - b_used = 47 - return o_list, o_count, b_used - orig_gcd = utils.get_container_details - utils.get_container_details = _mock_get_container_details - td = tempfile.mkdtemp() - try: - exp_md = { - utils.X_TYPE: (utils.CONTAINER, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (3, 0), - utils.X_BYTES_USED: (47, 0), - } - md = utils.get_container_metadata(td) - assert md == exp_md - finally: - utils.get_container_details = orig_gcd - os.rmdir(td) - - def test_get_account_metadata(self): - def _mock_get_account_details(path, memcache=None): - c_list = [ '123', 'abc' ] - c_count = 2 - return c_list, c_count - orig_gad = utils.get_account_details - utils.get_account_details = _mock_get_account_details - td = tempfile.mkdtemp() - try: - exp_md = { - utils.X_TYPE: (utils.ACCOUNT, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (0, 0), - utils.X_BYTES_USED: (0, 0), - utils.X_CONTAINER_COUNT: (2, 0), - } - md = utils.get_account_metadata(td) - assert md == exp_md - finally: - utils.get_account_details = orig_gad - os.rmdir(td) - - cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, - utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] - - def test_create_container_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_container_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.cont_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.CONTAINER, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - finally: - os.rmdir(td) - - acct_keys = [val for val in cont_keys] - acct_keys.append(utils.X_CONTAINER_COUNT) - - def test_create_account_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_account_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.acct_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - assert md[utils.X_CONTAINER_COUNT] == (0, 0) - finally: - os.rmdir(td) - - def test_container_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path): - bu = 5 - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - try: - retval = utils.get_container_details(the_path) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - - def test_container_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - # If it did not properly use memcache, the default mocked version - # of get details from fs would return 5 bytes used instead of the - # 6 we specified above. - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_key(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + "u", cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_dir_list(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we miss due to mtimes not matching - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(200) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - orig_gcdff = utils._get_account_details_from_fs - utils._get_account_details_from_fs = mock_get_account_details_from_fs - try: - retval = utils.get_account_details(the_path) - ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (ad.container_list, ad.container_count) - finally: - utils._get_account_details_from_fs = orig_gcdff - - def test_account_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - assert retval == (ad.container_list, ad.container_count) - wrong_ad = mock_get_account_details_from_fs(the_path, None) - assert wrong_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path + 'u', ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - ad.mtime = 200 - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_get_account_details_from_fs(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - ad = utils._get_account_details_from_fs(td, None) - assert ad.mtime == os.path.getmtime(td) - assert ad.container_count == 3 - assert set(ad.container_list) == set(['c1', 'c2', 'c3']) - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_container_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - cd = utils._get_container_details_from_fs(tf.name) - assert cd.bytes_used == 0 - assert cd.object_count == 0 - assert cd.obj_list == [] - assert cd.dir_list == [] - - def test_get_container_details_from_fs(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - cd = utils._get_container_details_from_fs(td) - assert cd.bytes_used == 0, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) - assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', - 'dir1/file1', 'dir1/file2' - ]), repr(cd.obj_list) - - full_dir1 = os.path.join(td, 'dir1') - full_dir2 = os.path.join(td, 'dir2') - full_dir3 = os.path.join(td, 'dir3') - exp_dir_dict = { td: os.path.getmtime(td), - full_dir1: os.path.getmtime(full_dir1), - full_dir2: os.path.getmtime(full_dir2), - full_dir3: os.path.getmtime(full_dir3), - } - for d,m in cd.dir_list: - assert d in exp_dir_dict - assert exp_dir_dict[d] == m - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - - def test_get_container_details_from_fs_do_getsize_true(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - __do_getsize = Glusterfs._do_getsize - Glusterfs._do_getsize = True - - cd = utils._get_container_details_from_fs(td) - assert cd.bytes_used == 30, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) - assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', - 'dir1/file1', 'dir1/file2' - ]), repr(cd.obj_list) - - full_dir1 = os.path.join(td, 'dir1') - full_dir2 = os.path.join(td, 'dir2') - full_dir3 = os.path.join(td, 'dir3') - exp_dir_dict = { td: os.path.getmtime(td), - full_dir1: os.path.getmtime(full_dir1), - full_dir2: os.path.getmtime(full_dir2), - full_dir3: os.path.getmtime(full_dir3), - } - for d,m in cd.dir_list: - assert d in exp_dir_dict - assert exp_dir_dict[d] == m - finally: - Glusterfs._do_getsize = __do_getsize - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_account_details_from_fs_notadir_w_stats(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, os.stat(tf.name)) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] - - def test_get_account_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, None) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] - - def test_write_pickle(self): - td = tempfile.mkdtemp() - try: - fpp = os.path.join(td, 'pp') - utils.write_pickle('pickled peppers', fpp) - with open(fpp, "rb") as f: - contents = f.read() - s = pickle.loads(contents) - assert s == 'pickled peppers', repr(s) - finally: - shutil.rmtree(td) - - def test_write_pickle_ignore_tmp(self): - tf = tempfile.NamedTemporaryFile() - td = tempfile.mkdtemp() - try: - fpp = os.path.join(td, 'pp') - # Also test an explicity pickle protocol - utils.write_pickle('pickled peppers', fpp, tmp=tf.name, pickle_protocol=2) - with open(fpp, "rb") as f: - contents = f.read() - s = pickle.loads(contents) - assert s == 'pickled peppers', repr(s) - with open(tf.name, "rb") as f: - contents = f.read() - assert contents == '' - finally: - shutil.rmtree(td) - - def test_check_user_xattr_bad_path(self): - assert False == utils.check_user_xattr("/tmp/foo/bar/check/user/xattr") - - def test_check_user_xattr_bad_set(self): - td = tempfile.mkdtemp() - xkey = _xkey(td, 'user.test.key1') - _xattr_set_err[xkey] = errno.EOPNOTSUPP - try: - assert False == utils.check_user_xattr(td) - except IOError: - pass - else: - self.fail("Expected IOError") - finally: - shutil.rmtree(td) - - def test_check_user_xattr_bad_remove(self): - td = tempfile.mkdtemp() - xkey = _xkey(td, 'user.test.key1') - _xattr_rem_err[xkey] = errno.EOPNOTSUPP - try: - utils.check_user_xattr(td) - except IOError: - self.fail("Unexpected IOError") - finally: - shutil.rmtree(td) - - def test_check_user_xattr(self): - td = tempfile.mkdtemp() - try: - assert utils.check_user_xattr(td) - finally: - shutil.rmtree(td) - - def test_validate_container_empty(self): - ret = utils.validate_container({}) - assert ret == False - - def test_validate_container_missing_keys(self): - ret = utils.validate_container({ 'foo': 'bar' }) - assert ret == False - - def test_validate_container_bad_type(self): - md = { utils.X_TYPE: ('bad', 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0) } - ret = utils.validate_container(md) - assert ret == False - - def test_validate_container_good_type(self): - md = { utils.X_TYPE: (utils.CONTAINER, 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0) } - ret = utils.validate_container(md) - assert ret - - def test_validate_account_empty(self): - ret = utils.validate_account({}) - assert ret == False - - def test_validate_account_missing_keys(self): - ret = utils.validate_account({ 'foo': 'bar' }) - assert ret == False - - def test_validate_account_bad_type(self): - md = { utils.X_TYPE: ('bad', 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0), - utils.X_CONTAINER_COUNT: ('na', 0) } - ret = utils.validate_account(md) - assert ret == False - - def test_validate_account_good_type(self): - md = { utils.X_TYPE: (utils.ACCOUNT, 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0), - utils.X_CONTAINER_COUNT: ('na', 0) } - ret = utils.validate_account(md) - assert ret - - def test_validate_object_empty(self): - ret = utils.validate_object({}) - assert ret == False - - def test_validate_object_missing_keys(self): - ret = utils.validate_object({ 'foo': 'bar' }) - assert ret == False - - def test_validate_object_bad_type(self): - md = { utils.X_TIMESTAMP: 'na', - utils.X_CONTENT_TYPE: 'na', - utils.X_ETAG: 'bad', - utils.X_CONTENT_LENGTH: 'na', - utils.X_TYPE: 'bad', - utils.X_OBJECT_TYPE: 'na' } - ret = utils.validate_object(md) - assert ret == False - - def test_validate_object_good_type(self): - md = { utils.X_TIMESTAMP: 'na', - utils.X_CONTENT_TYPE: 'na', - utils.X_ETAG: 'bad', - utils.X_CONTENT_LENGTH: 'na', - utils.X_TYPE: utils.OBJECT, - utils.X_OBJECT_TYPE: 'na' } - ret = utils.validate_object(md) - assert ret - - def test_is_marker_empty(self): - assert False == utils.is_marker(None) - - def test_is_marker_missing(self): - assert False == utils.is_marker( { 'foo': 'bar' } ) - - def test_is_marker_not_marker(self): - md = { utils.X_OBJECT_TYPE: utils.DIR } - assert False == utils.is_marker(md) - - def test_is_marker(self): - md = { utils.X_OBJECT_TYPE: utils.MARKER_DIR } - assert utils.is_marker(md) diff --git a/ufo/tools/test-requires b/ufo/tools/test-requires deleted file mode 100644 index ef8bf14..0000000 --- a/ufo/tools/test-requires +++ /dev/null @@ -1,6 +0,0 @@ -coverage -nose -nosexcover -openstack.nose_plugin -nosehtmloutput -mock>=0.8.0 diff --git a/ufo/tox.ini b/ufo/tox.ini deleted file mode 100644 index 792bb41..0000000 --- a/ufo/tox.ini +++ /dev/null @@ -1,25 +0,0 @@ -[tox] -envlist = py26,py27 - -[testenv] -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_OPENSTACK=1 - NOSE_OPENSTACK_COLOR=1 - NOSE_OPENSTACK_RED=0.05 - NOSE_OPENSTACK_YELLOW=0.025 - NOSE_OPENSTACK_SHOW_ELAPSED=1 - NOSE_OPENSTACK_STDOUT=1 -deps = - https://launchpad.net/swift/grizzly/1.8.0/+download/swift-1.8.0.tar.gz - -r{toxinidir}/tools/test-requires -changedir = {toxinidir}/test/unit -commands = nosetests -v --exe --with-coverage --cover-package gluster --cover-erase {posargs} - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv:cover] -setenv = NOSE_WITH_COVERAGE=1 - -[testenv:venv] -commands = {posargs} diff --git a/ufo/unittests.sh b/ufo/unittests.sh deleted file mode 100755 index a02be77..0000000 --- a/ufo/unittests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd $(dirname $0)/test/unit -nosetests --exe --with-coverage --cover-package gluster --cover-erase $@ -saved_status=$? -rm -f .coverage -exit $saved_status diff --git a/unittests.sh b/unittests.sh new file mode 100755 index 0000000..a02be77 --- /dev/null +++ b/unittests.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd $(dirname $0)/test/unit +nosetests --exe --with-coverage --cover-package gluster --cover-erase $@ +saved_status=$? +rm -f .coverage +exit $saved_status -- cgit