Final forward port of PDQ performance patches

Change-Id: I4ef131b3cc7648d4571a4d854029efb1aff8b901
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/5305
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>

6 files changed, 710 insertions, 212 deletions

diff --git a/test/unit/common/test_Glusterfs.py b/test/unit/common/test_Glusterfs.py
index f36f601..bcb233c 100644
--- a/test/unit/common/test_Glusterfs.py
+++ b/test/unit/common/test_Glusterfs.py
@@ -50,7 +50,12 @@ def _init():
 def _init_mock_variables(tmpdir):
     os.system            = mock_os_system
     os.path.ismount      = mock_os_path_ismount
-    gfs.RUN_DIR          = os.path.join(tmpdir, 'var/run/swift')
+    try:
+        os.makedirs(os.path.join(tmpdir, "var", "run"))
+    except OSError as err:
+        if err.errno != errno.EEXIST:
+            raise
+    gfs.RUN_DIR          = os.path.join(tmpdir, 'var', 'run', 'swift')
     gfs._get_export_list = mock_get_export_list
 
 def _reset_mock_variables():
@@ -108,8 +113,16 @@ class TestGlusterfs(unittest.TestCase):
             shutil.rmtree(tmpdir)
 
     def test_mount_get_export_list_err(self):
-        gfs._get_export_list = mock_get_export_list
-        assert not gfs.mount(None, 'drive')
+        try:
+            tmpdir = mkdtemp()
+            root   = os.path.join(tmpdir, 'mnt/gluster-object')
+            drive  = 'test3'
+
+            _init_mock_variables(tmpdir)
+            gfs._get_export_list = mock_get_export_list
+            assert not gfs.mount(root, drive)
+        finally:
+            shutil.rmtree(tmpdir)
 
     def tearDown(self):
         _reset_mock_variables()
diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py
index e2769ab..7139094 100644
--- a/test/unit/common/test_constraints.py
+++ b/test/unit/common/test_constraints.py
@@ -73,6 +73,7 @@ class TestConstraints(unittest.TestCase):
         self.assertTrue(cnt.validate_obj_name_component('tests'*(max_obj_len/5+1)))
         self.assertTrue(cnt.validate_obj_name_component('.'))
         self.assertTrue(cnt.validate_obj_name_component('..'))
+        self.assertTrue(cnt.validate_obj_name_component(''))
 
     def test_gluster_check_object_creation(self):
         with patch('gluster.swift.common.constraints.__check_object_creation',
@@ -83,14 +84,3 @@ class TestConstraints(unittest.TestCase):
         with patch('gluster.swift.common.constraints.__check_object_creation',
                    mock_check_object_creation):
             self.assertTrue(cnt.gluster_check_object_creation(None, 'dir/.'))
-
-    def test_gluster_check_mount(self):
-        with patch('gluster.swift.common.constraints.__check_mount',
-                   mock_check_mount):
-            self.assertTrue(cnt.gluster_check_mount('/tmp/drive', 'vol0'))
-
-        with patch('gluster.swift.common.constraints.__check_mount',
-                   mock_check_mount_err):
-            with patch('gluster.swift.common.Glusterfs.mount',
-                       mock_glusterfs_mount):
-                self.assertTrue(cnt.gluster_check_mount('/tmp/drive', 'vol0'))
diff --git a/test/unit/common/test_diskfile.py b/test/unit/common/test_diskfile.py
index 857ba9d..6367888 100644
--- a/test/unit/common/test_diskfile.py
+++ b/test/unit/common/test_diskfile.py
@@ -21,15 +21,15 @@ import errno
 import unittest
 import tempfile
 import shutil
+from mock import patch
 from hashlib import md5
 from swift.common.utils import normalize_timestamp
 from swift.common.exceptions import DiskFileNotExist, DiskFileError
 import gluster.swift.common.DiskFile
 import gluster.swift.common.utils
-from gluster.swift.common.DiskFile import Gluster_DiskFile, \
-    AlreadyExistsAsDir
+from gluster.swift.common.DiskFile import Gluster_DiskFile
 from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \
-    X_OBJECT_TYPE
+    X_OBJECT_TYPE, DIR_OBJECT
 from test_utils import _initxattr, _destroyxattr
 from test.unit import FakeLogger
 
@@ -60,21 +60,6 @@ def _mock_rmobjdir(p):
 def _mock_do_fsync(fd):
     return
 
-def _mock_os_unlink_eacces_err(f):
-    ose = OSError()
-    ose.errno = errno.EACCES
-    raise ose
-
-def _mock_getsize_eaccess_err(f):
-    ose = OSError()
-    ose.errno = errno.EACCES
-    raise ose
-
-def _mock_do_rmdir_eacces_err(f):
-    ose = OSError()
-    ose.errno = errno.EACCES
-    raise ose
-
 class MockRenamerCalled(Exception):
     pass
 
@@ -293,7 +278,7 @@ class TestDiskFile(unittest.TestCase):
         gdf._is_dir = True
         gdf.fp = "123"
         # Should still be a no-op as is_dir is True (marker directory)
-        gdf.close()
+        self.assertRaises(AssertionError, gdf.close)
         assert gdf.fp == "123"
 
         gdf._is_dir = False
@@ -317,17 +302,39 @@ class TestDiskFile(unittest.TestCase):
         gdf.data_file = "/tmp/foo/bar"
         assert not gdf.is_deleted()
 
-    def test_create_dir_object(self):
+    def test_create_dir_object_no_md(self):
         td = tempfile.mkdtemp()
-        the_dir = os.path.join(td, "vol0", "bar", "dir")
+        the_cont = os.path.join(td, "vol0", "bar")
+        the_dir = "dir"
         try:
+            os.makedirs(the_cont)
             gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
-                                   "dir/z", self.lg)
+                                   os.path.join(the_dir, "z"), self.lg)
             # Not created, dir object path is different, just checking
             assert gdf._obj == "z"
             gdf._create_dir_object(the_dir)
-            assert os.path.isdir(the_dir)
-            assert the_dir in _metadata
+            full_dir_path = os.path.join(the_cont, the_dir)
+            assert os.path.isdir(full_dir_path)
+            assert full_dir_path not in _metadata
+        finally:
+            shutil.rmtree(td)
+
+    def test_create_dir_object_with_md(self):
+        td = tempfile.mkdtemp()
+        the_cont = os.path.join(td, "vol0", "bar")
+        the_dir = "dir"
+        try:
+            os.makedirs(the_cont)
+            gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
+                                   os.path.join(the_dir, "z"), self.lg)
+            # Not created, dir object path is different, just checking
+            assert gdf._obj == "z"
+            dir_md = {'Content-Type': 'application/directory',
+                      X_OBJECT_TYPE: DIR_OBJECT}
+            gdf._create_dir_object(the_dir, dir_md)
+            full_dir_path = os.path.join(the_cont, the_dir)
+            assert os.path.isdir(full_dir_path)
+            assert full_dir_path in _metadata
         finally:
             shutil.rmtree(td)
 
@@ -357,6 +364,32 @@ class TestDiskFile(unittest.TestCase):
         finally:
             shutil.rmtree(td)
 
+    def test_create_dir_object_do_stat_failure(self):
+        td = tempfile.mkdtemp()
+        the_path = os.path.join(td, "vol0", "bar")
+        the_dir = os.path.join(the_path, "dir")
+        try:
+            os.makedirs(the_path)
+            with open(the_dir, "wb") as fd:
+                fd.write("1234")
+            gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
+                                   "dir/z", self.lg)
+            # Not created, dir object path is different, just checking
+            assert gdf._obj == "z"
+            def _mock_do_chown(p, u, g):
+                assert u == DEFAULT_UID
+                assert g == DEFAULT_GID
+            dc = gluster.swift.common.DiskFile.do_chown
+            gluster.swift.common.DiskFile.do_chown = _mock_do_chown
+            self.assertRaises(DiskFileError,
+                    gdf._create_dir_object,
+                    the_dir)
+            gluster.swift.common.DiskFile.do_chown = dc
+            self.assertFalse(os.path.isdir(the_dir))
+            self.assertFalse(the_dir in _metadata)
+        finally:
+            shutil.rmtree(td)
+
     def test_put_metadata(self):
         td = tempfile.mkdtemp()
         the_path = os.path.join(td, "vol0", "bar")
@@ -452,21 +485,24 @@ class TestDiskFile(unittest.TestCase):
 
     def test_put_w_marker_dir_create(self):
         td = tempfile.mkdtemp()
-        the_path = os.path.join(td, "vol0", "bar")
-        the_dir = os.path.join(the_path, "dir")
+        the_cont = os.path.join(td, "vol0", "bar")
+        the_dir = os.path.join(the_cont, "dir")
         try:
+            os.makedirs(the_cont)
             gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
                                    "dir", self.lg)
             assert gdf.metadata == {}
             newmd = {
-                'Content-Length': 0,
                 'ETag': 'etag',
                 'X-Timestamp': 'ts',
                 'Content-Type': 'application/directory'}
             gdf.put(None, newmd, extension='.dir')
             assert gdf.data_file == the_dir
-            assert gdf.metadata == newmd
-            assert _metadata[the_dir] == newmd
+            for key,val in newmd.items():
+                assert gdf.metadata[key] == val
+                assert _metadata[the_dir][key] == val
+            assert gdf.metadata[X_OBJECT_TYPE] == DIR_OBJECT
+            assert _metadata[the_dir][X_OBJECT_TYPE] == DIR_OBJECT
         finally:
             shutil.rmtree(td)
 
@@ -487,10 +523,11 @@ class TestDiskFile(unittest.TestCase):
             newmd['X-Object-Meta-test'] = '1234'
             try:
                 gdf.put(None, newmd, extension='.data')
-            except AlreadyExistsAsDir:
+            except DiskFileError:
                 pass
             else:
-                self.fail("Expected to encounter 'already-exists-as-dir' exception")
+                self.fail("Expected to encounter"
+                          " 'already-exists-as-dir' exception")
             assert gdf.metadata == origmd
             assert _metadata[the_dir] == origfmd
         finally:
@@ -498,13 +535,15 @@ class TestDiskFile(unittest.TestCase):
 
     def test_put(self):
         td = tempfile.mkdtemp()
+        the_cont = os.path.join(td, "vol0", "bar")
         try:
+            os.makedirs(the_cont)
             gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
                                    "z", self.lg)
             assert gdf._obj == "z"
             assert gdf._obj_path == ""
             assert gdf.name == "bar"
-            assert gdf.datadir == os.path.join(td, "vol0", "bar")
+            assert gdf.datadir == the_cont
             assert gdf.data_file is None
 
             body = '1234\n'
@@ -656,20 +695,22 @@ class TestDiskFile(unittest.TestCase):
 
             later = float(gdf.metadata['X-Timestamp']) + 1
 
-            stats = os.stat(the_path)
-            os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
+            def _mock_os_unlink_eacces_err(f):
+                raise OSError(errno.EACCES, os.strerror(errno.EACCES))
 
-            # Handle the case os_unlink() raises an OSError
-            __os_unlink = os.unlink
-            os.unlink = _mock_os_unlink_eacces_err
+            stats = os.stat(the_path)
             try:
-                gdf.unlinkold(normalize_timestamp(later))
-            except OSError as e:
-                assert e.errno == errno.EACCES
-            else:
-                self.fail("Excepted an OSError when unlinking file")
+                os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
+
+                # Handle the case os_unlink() raises an OSError
+                with patch("os.unlink", _mock_os_unlink_eacces_err):
+                    try:
+                        gdf.unlinkold(normalize_timestamp(later))
+                    except OSError as e:
+                        assert e.errno == errno.EACCES
+                    else:
+                        self.fail("Excepted an OSError when unlinking file")
             finally:
-                os.unlink = __os_unlink
                 os.chmod(the_path, stats.st_mode)
 
             assert os.path.isdir(gdf.datadir)
@@ -695,32 +736,6 @@ class TestDiskFile(unittest.TestCase):
         finally:
             shutil.rmtree(td)
 
-    def test_unlinkold_is_dir_failure(self):
-        td = tempfile.mkdtemp()
-        the_path = os.path.join(td, "vol0", "bar")
-        the_dir = os.path.join(the_path, "d")
-        try:
-            os.makedirs(the_dir)
-            gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
-                                   "d", self.lg, keep_data_fp=True)
-            assert gdf.data_file == the_dir
-            assert gdf._is_dir
-
-            stats = os.stat(gdf.datadir)
-            os.chmod(gdf.datadir, 0)
-            __os_rmdir = os.rmdir
-            os.rmdir = _mock_do_rmdir_eacces_err
-            try:
-                later = float(gdf.metadata['X-Timestamp']) + 1
-                gdf.unlinkold(normalize_timestamp(later))
-            finally:
-                os.chmod(gdf.datadir, stats.st_mode)
-                os.rmdir = __os_rmdir
-            assert os.path.isdir(gdf.datadir)
-            self.assertTrue(gdf.data_file is None)
-        finally:
-            shutil.rmtree(td)
-
     def test_get_data_file_size(self):
         td = tempfile.mkdtemp()
         the_path = os.path.join(td, "vol0", "bar")
@@ -806,17 +821,20 @@ class TestDiskFile(unittest.TestCase):
             assert gdf.data_file == the_file
             assert not gdf._is_dir
             stats = os.stat(the_path)
-            os.chmod(the_path, 0)
-            __os_path_getsize = os.path.getsize
-            os.path.getsize = _mock_getsize_eaccess_err
             try:
-                s = gdf.get_data_file_size()
-            except OSError as err:
-                assert err.errno == errno.EACCES
-            else:
-                self.fail("Expected OSError exception")
+                os.chmod(the_path, 0)
+
+                def _mock_getsize_eaccess_err(f):
+                    raise OSError(errno.EACCES, os.strerror(errno.EACCES))
+
+                with patch("os.path.getsize", _mock_getsize_eaccess_err):
+                    try:
+                        s = gdf.get_data_file_size()
+                    except OSError as err:
+                        assert err.errno == errno.EACCES
+                    else:
+                        self.fail("Expected OSError exception")
             finally:
-                os.path.getsize = __os_path_getsize
                 os.chmod(the_path, stats.st_mode)
         finally:
             shutil.rmtree(td)
diff --git a/test/unit/common/test_fs_utils.py b/test/unit/common/test_fs_utils.py
index 910199e..1828092 100644
--- a/test/unit/common/test_fs_utils.py
+++ b/test/unit/common/test_fs_utils.py
@@ -24,7 +24,8 @@ from mock import patch
 from tempfile import mkdtemp, mkstemp
 from gluster.swift.common import fs_utils as fs
 from gluster.swift.common.exceptions import NotDirectoryError, \
-    FileOrDirNotFoundError
+    FileOrDirNotFoundError, GlusterFileSystemOSError, \
+    GlusterFileSystemIOError
 
 def mock_os_fsync(fd):
     return True
@@ -54,9 +55,135 @@ class TestFsUtils(unittest.TestCase):
         finally:
             shutil.rmtree(tmpparent)
 
+    def test_do_ismount_path_does_not_exist(self):
+        tmpdir = mkdtemp()
+        try:
+            assert False == fs.do_ismount(os.path.join(tmpdir, 'bar'))
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_path_not_mount(self):
+        tmpdir = mkdtemp()
+        try:
+            assert False == fs.do_ismount(tmpdir)
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_path_error(self):
+
+        def _mock_os_lstat(path):
+            raise OSError(13, "foo")
+
+        tmpdir = mkdtemp()
+        try:
+            with patch("os.lstat", _mock_os_lstat):
+                try:
+                    fs.do_ismount(tmpdir)
+                except GlusterFileSystemOSError as err:
+                    pass
+                else:
+                    self.fail("Expected GlusterFileSystemOSError")
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_path_is_symlink(self):
+        tmpdir = mkdtemp()
+        try:
+            link = os.path.join(tmpdir, "tmp")
+            os.symlink("/tmp", link)
+            assert False == fs.do_ismount(link)
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_path_is_root(self):
+        assert True == fs.do_ismount('/')
+
+    def test_do_ismount_parent_path_error(self):
+
+        _os_lstat = os.lstat
+
+        def _mock_os_lstat(path):
+            if path.endswith(".."):
+                raise OSError(13, "foo")
+            else:
+                return _os_lstat(path)
+
+        tmpdir = mkdtemp()
+        try:
+            with patch("os.lstat", _mock_os_lstat):
+                try:
+                    fs.do_ismount(tmpdir)
+                except GlusterFileSystemOSError as err:
+                    pass
+                else:
+                    self.fail("Expected GlusterFileSystemOSError")
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_successes_dev(self):
+
+        _os_lstat = os.lstat
+
+        class MockStat(object):
+            def __init__(self, mode, dev, ino):
+                self.st_mode = mode
+                self.st_dev = dev
+                self.st_ino = ino
+
+        def _mock_os_lstat(path):
+            if path.endswith(".."):
+                parent = _os_lstat(path)
+                return MockStat(parent.st_mode, parent.st_dev + 1,
+                                parent.st_ino)
+            else:
+                return _os_lstat(path)
+
+        tmpdir = mkdtemp()
+        try:
+            with patch("os.lstat", _mock_os_lstat):
+                try:
+                    fs.do_ismount(tmpdir)
+                except GlusterFileSystemOSError as err:
+                    self.fail("Unexpected exception")
+                else:
+                    pass
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_do_ismount_successes_ino(self):
+
+        _os_lstat = os.lstat
+
+        class MockStat(object):
+            def __init__(self, mode, dev, ino):
+                self.st_mode = mode
+                self.st_dev = dev
+                self.st_ino = ino
+
+        def _mock_os_lstat(path):
+            if path.endswith(".."):
+                return _os_lstat(path)
+            else:
+                parent_path = os.path.join(path, "..")
+                child = _os_lstat(path)
+                parent = _os_lstat(parent_path)
+                return MockStat(child.st_mode, parent.st_ino,
+                                child.st_dev)
+
+        tmpdir = mkdtemp()
+        try:
+            with patch("os.lstat", _mock_os_lstat):
+                try:
+                    fs.do_ismount(tmpdir)
+                except GlusterFileSystemOSError as err:
+                    self.fail("Unexpected exception")
+                else:
+                    pass
+        finally:
+            shutil.rmtree(tmpdir)
 
     def test_do_open(self):
-        fd, tmpfile = mkstemp()
+        _fd, tmpfile = mkstemp()
         try:
             f = fs.do_open(tmpfile, 'r')
             try:
@@ -67,27 +194,36 @@ class TestFsUtils(unittest.TestCase):
                 self.fail("IOError expected")
             finally:
                 f.close()
+
+            fd = fs.do_open(tmpfile, os.O_RDONLY)
+            try:
+                os.write(fd, 'test')
+            except OSError as err:
+                pass
+            else:
+                self.fail("OSError expected")
+            finally:
+                os.close(fd)
         finally:
-            os.close(fd)
+            os.close(_fd)
             os.remove(tmpfile)
 
-
     def test_do_open_err(self):
         try:
             fs.do_open(os.path.join('/tmp', str(random.random())), 'r')
-        except IOError:
+        except GlusterFileSystemIOError:
             pass
         else:
-            self.fail("IOError expected")
+            self.fail("GlusterFileSystemIOError expected")
 
     def test_do_open_err_int_mode(self):
         try:
             fs.do_open(os.path.join('/tmp', str(random.random())),
                        os.O_RDONLY)
-        except OSError:
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail("OSError expected")
+            self.fail("GlusterFileSystemOSError expected")
 
     def test_do_write(self):
         fd, tmpfile = mkstemp()
@@ -104,13 +240,13 @@ class TestFsUtils(unittest.TestCase):
             fd1 = os.open(tmpfile, os.O_RDONLY)
             try:
                 fs.do_write(fd1, "test")
-            except OSError:
+            except GlusterFileSystemOSError:
                 pass
             else:
-                self.fail("OSError expected")
+                self.fail("GlusterFileSystemOSError expected")
             finally:
                 os.close(fd1)
-        except OSError as ose:
+        except GlusterFileSystemOSError as ose:
             self.fail("Open failed with %s" %ose.strerror)
         finally:
             os.close(fd)
@@ -126,6 +262,72 @@ class TestFsUtils(unittest.TestCase):
         finally:
             shutil.rmtree(subdir)
 
+    def test_mkdirs_already_dir(self):
+        tmpdir = mkdtemp()
+        try:
+            fs.mkdirs(tmpdir)
+        except (GlusterFileSystemOSError, OSError):
+            self.fail("Unexpected exception")
+        else:
+            pass
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_mkdirs(self):
+        tmpdir = mkdtemp()
+        try:
+            fs.mkdirs(os.path.join(tmpdir, "a", "b", "c"))
+        except OSError:
+            self.fail("Unexpected exception")
+        else:
+            pass
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_mkdirs_existing_file(self):
+        tmpdir = mkdtemp()
+        fd, tmpfile = mkstemp(dir=tmpdir)
+        try:
+            fs.mkdirs(tmpfile)
+        except OSError:
+            pass
+        else:
+            self.fail("Expected GlusterFileSystemOSError exception")
+        finally:
+            os.close(fd)
+            shutil.rmtree(tmpdir)
+
+    def test_mkdirs_existing_file_on_path(self):
+        tmpdir = mkdtemp()
+        fd, tmpfile = mkstemp(dir=tmpdir)
+        try:
+            fs.mkdirs(os.path.join(tmpfile, 'b'))
+        except OSError:
+            pass
+        else:
+            self.fail("Expected GlusterFileSystemOSError exception")
+        finally:
+            os.close(fd)
+            shutil.rmtree(tmpdir)
+
+    def test_do_mkdir(self):
+        try:
+            path = os.path.join('/tmp', str(random.random()))
+            fs.do_mkdir(path)
+            assert os.path.exists(path)
+            assert fs.do_mkdir(path) is None
+        finally:
+            os.rmdir(path)
+
+    def test_do_mkdir_err(self):
+        try:
+            path = os.path.join('/tmp', str(random.random()), str(random.random()))
+            fs.do_mkdir(path)
+        except GlusterFileSystemOSError:
+            pass
+        else:
+            self.fail("GlusterFileSystemOSError expected")
+
     def test_do_listdir(self):
         tmpdir = mkdtemp()
         try:
@@ -141,38 +343,106 @@ class TestFsUtils(unittest.TestCase):
         try:
             path = os.path.join('/tmp', str(random.random()))
             fs.do_listdir(path)
-        except OSError:
+        except GlusterFileSystemOSError:
+            pass
+        else:
+            self.fail("GlusterFileSystemOSError expected")
+
+    def test_do_fstat(self):
+        tmpdir = mkdtemp()
+        try:
+            fd, tmpfile = mkstemp(dir=tmpdir)
+            buf1 = os.stat(tmpfile)
+            buf2 = fs.do_fstat(fd)
+
+            assert buf1 == buf2
+        finally:
+            os.close(fd)
+            os.remove(tmpfile)
+            os.rmdir(tmpdir)
+
+    def test_do_fstat_err(self):
+        try:
+            fs.do_fstat(1000)
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail("OSError expected")
+            self.fail("Expected GlusterFileSystemOSError")
+
 
     def test_do_stat(self):
         tmpdir = mkdtemp()
         try:
             fd, tmpfile = mkstemp(dir=tmpdir)
             buf1 = os.stat(tmpfile)
-            buf2 = fs.do_stat(fd)
-            buf3 = fs.do_stat(tmpfile)
+            buf2 = fs.do_stat(tmpfile)
 
             assert buf1 == buf2
-            assert buf1 == buf3
         finally:
             os.close(fd)
             os.remove(tmpfile)
             os.rmdir(tmpdir)
 
+    def test_do_stat_enoent(self):
+        res = fs.do_stat(os.path.join('/tmp', str(random.random())))
+        assert res is None
+
     def test_do_stat_err(self):
+
+        def mock_os_stat_eacces(path):
+            raise OSError(errno.EACCES, os.strerror(errno.EACCES))
+
         try:
-            fs.do_stat(os.path.join('/tmp', str(random.random())))
-        except OSError:
+            with patch('os.stat', mock_os_stat_eacces):
+                fs.do_stat('/tmp')
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail("OSError expected")
+            self.fail("GlusterFileSystemOSError expected")
+
+    def test_do_stat_eio_once(self):
+        count = [0]
+        _os_stat = os.stat
+
+        def mock_os_stat_eio(path):
+            count[0] += 1
+            if count[0] <= 1:
+                raise OSError(errno.EIO, os.strerror(errno.EIO))
+            return _os_stat(path)
+
+        with patch('os.stat', mock_os_stat_eio):
+            fs.do_stat('/tmp') is not None
+
+    def test_do_stat_eio_twice(self):
+        count = [0]
+        _os_stat = os.stat
+
+        def mock_os_stat_eio(path):
+            count[0] += 1
+            if count[0] <= 2:
+                raise OSError(errno.EIO, os.strerror(errno.EIO))
+            return _os_stat(path)
+
+        with patch('os.stat', mock_os_stat_eio):
+            fs.do_stat('/tmp') is not None
+
+    def test_do_stat_eio_ten(self):
+
+        def mock_os_stat_eio(path):
+            raise OSError(errno.EIO, os.strerror(errno.EIO))
+
+        try:
+            with patch('os.stat', mock_os_stat_eio):
+                fs.do_stat('/tmp')
+        except GlusterFileSystemOSError:
+            pass
+        else:
+            self.fail("GlusterFileSystemOSError expected")
 
     def test_do_close(self):
         fd, tmpfile = mkstemp()
         try:
-            fs.do_close(fd);
+            fs.do_close(fd)
             try:
                 os.write(fd, "test")
             except OSError:
@@ -184,26 +454,43 @@ class TestFsUtils(unittest.TestCase):
         finally:
             os.remove(tmpfile)
 
-    def test_do_close_err(self):
+    def test_do_close_err_fd(self):
         fd, tmpfile = mkstemp()
         try:
-            fs.do_close(fd);
+            fs.do_close(fd)
 
             try:
-                fs.do_close(fd);
-            except OSError:
+                fs.do_close(fd)
+            except GlusterFileSystemOSError:
                 pass
             else:
-                self.fail("OSError expected")
+                self.fail("GlusterFileSystemOSError expected")
+        finally:
+            os.remove(tmpfile)
+
+    def test_do_close_err_fp(self):
+        fd, tmpfile = mkstemp()
+        os.close(fd)
+        fp = open(tmpfile, 'w')
+        try:
+            fd = fp.fileno()
+            os.close(fd)
+            try:
+                fs.do_close(fp)
+            except GlusterFileSystemIOError:
+                pass
+            else:
+                self.fail("GlusterFileSystemIOError expected")
         finally:
             os.remove(tmpfile)
 
     def test_do_unlink(self):
         fd, tmpfile = mkstemp()
         try:
-            fs.do_unlink(tmpfile)
+            assert fs.do_unlink(tmpfile) is None
             assert not os.path.exists(tmpfile)
-            assert fs.do_unlink(os.path.join('/tmp', str(random.random())))
+            res = fs.do_unlink(os.path.join('/tmp', str(random.random())))
+            assert res is None
         finally:
             os.close(fd)
 
@@ -211,10 +498,10 @@ class TestFsUtils(unittest.TestCase):
         tmpdir = mkdtemp()
         try:
             fs.do_unlink(tmpdir)
-        except OSError:
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail('OSError expected')
+            self.fail('GlusterFileSystemOSError expected')
         finally:
             os.rmdir(tmpdir)
 
@@ -233,10 +520,10 @@ class TestFsUtils(unittest.TestCase):
             srcpath = os.path.join('/tmp', str(random.random()))
             destpath = os.path.join('/tmp', str(random.random()))
             fs.do_rename(srcpath, destpath)
-        except OSError:
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail("OSError expected")
+            self.fail("GlusterFileSystemOSError expected")
 
     def test_dir_empty(self):
         tmpdir = mkdtemp()
@@ -248,15 +535,28 @@ class TestFsUtils(unittest.TestCase):
             shutil.rmtree(tmpdir)
 
     def test_dir_empty_err(self):
-        try:
+        def _mock_os_listdir(path):
+            raise OSError(13, "foo")
+
+        with patch("os.listdir", _mock_os_listdir):
             try:
-                assert fs.dir_empty(os.path.join('/tmp', str(random.random())))
-            except FileOrDirNotFoundError:
+                fs.dir_empty("/tmp")
+            except GlusterFileSystemOSError:
                 pass
             else:
-                self.fail("FileOrDirNotFoundError exception expected")
+                self.fail("GlusterFileSystemOSError exception expected")
+
+    def test_dir_empty_notfound(self):
+        try:
+            assert fs.dir_empty(os.path.join('/tmp', str(random.random())))
+        except FileOrDirNotFoundError:
+            pass
+        else:
+            self.fail("FileOrDirNotFoundError exception expected")
 
-            fd, tmpfile = mkstemp()
+    def test_dir_empty_notdir(self):
+        fd, tmpfile = mkstemp()
+        try:
             try:
                 fs.dir_empty(tmpfile)
             except NotDirectoryError:
@@ -267,14 +567,26 @@ class TestFsUtils(unittest.TestCase):
             os.close(fd)
             os.unlink(tmpfile)
 
-    def test_rmdirs(self):
+    def test_do_rmdir(self):
         tmpdir = mkdtemp()
         try:
             subdir = mkdtemp(dir=tmpdir)
             fd, tmpfile = mkstemp(dir=tmpdir)
-            assert not fs.rmdirs(tmpfile)
-            assert not fs.rmdirs(tmpdir)
-            assert fs.rmdirs(subdir)
+            try:
+                fs.do_rmdir(tmpfile)
+            except GlusterFileSystemOSError:
+                pass
+            else:
+                self.fail("Expected GlusterFileSystemOSError")
+            assert os.path.exists(subdir)
+            try:
+                fs.do_rmdir(tmpdir)
+            except GlusterFileSystemOSError:
+                pass
+            else:
+                self.fail("Expected GlusterFileSystemOSError")
+            assert os.path.exists(subdir)
+            fs.do_rmdir(subdir)
             assert not os.path.exists(subdir)
         finally:
             os.close(fd)
@@ -290,11 +602,12 @@ class TestFsUtils(unittest.TestCase):
             else:
                 try:
                     fs.do_chown(subdir, 20000, 20000)
-                except OSError as ex:
+                except GlusterFileSystemOSError as ex:
                     if ex.errno != errno.EPERM:
-                        self.fail("Expected OSError")
+                        self.fail(
+                            "Expected GlusterFileSystemOSError(errno=EPERM)")
                 else:
-                        self.fail("Expected OSError")
+                    self.fail("Expected GlusterFileSystemOSError")
         finally:
             shutil.rmtree(tmpdir)
 
@@ -308,11 +621,12 @@ class TestFsUtils(unittest.TestCase):
             else:
                 try:
                     fs.do_chown(tmpfile, 20000, 20000)
-                except OSError as ex:
+                except GlusterFileSystemOSError as ex:
                     if ex.errno != errno.EPERM:
-                        self.fail("Expected OSError")
+                        self.fail(
+                            "Expected GlusterFileSystemOSError(errno=EPERM")
                 else:
-                        self.fail("Expected OSError")
+                    self.fail("Expected GlusterFileSystemOSError")
         finally:
             os.close(fd)
             shutil.rmtree(tmpdir)
@@ -321,10 +635,10 @@ class TestFsUtils(unittest.TestCase):
         try:
             fs.do_chown(os.path.join('/tmp', str(random.random())),
                         20000, 20000)
-        except OSError:
+        except GlusterFileSystemOSError:
             pass
         else:
-            self.fail("Expected OSError")
+            self.fail("Expected GlusterFileSystemOSError")
 
     def test_fchown(self):
         tmpdir = mkdtemp()
@@ -336,11 +650,12 @@ class TestFsUtils(unittest.TestCase):
             else:
                 try:
                     fs.do_fchown(fd, 20000, 20000)
-                except OSError as ex:
+                except GlusterFileSystemOSError as ex:
                     if ex.errno != errno.EPERM:
-                        self.fail("Expected OSError")
+                        self.fail(
+                            "Expected GlusterFileSystemOSError(errno=EPERM)")
                 else:
-                        self.fail("Expected OSError")
+                    self.fail("Expected GlusterFileSystemOSError")
         finally:
             os.close(fd)
             shutil.rmtree(tmpdir)
@@ -356,11 +671,12 @@ class TestFsUtils(unittest.TestCase):
             else:
                 try:
                     fs.do_fchown(fd_rd, 20000, 20000)
-                except OSError as ex:
+                except GlusterFileSystemOSError as ex:
                     if ex.errno != errno.EPERM:
-                        self.fail("Expected OSError")
+                        self.fail(
+                            "Expected GlusterFileSystemOSError(errno=EPERM)")
                 else:
-                    self.fail("Expected OSError")
+                    self.fail("Expected GlusterFileSystemOSError")
         finally:
             os.close(fd_rd)
             os.close(fd)
@@ -374,8 +690,8 @@ class TestFsUtils(unittest.TestCase):
                 os.write(fd, 'test')
                 with patch('eventlet.tpool.execute', mock_tpool_execute):
                     with patch('os.fsync', mock_os_fsync):
-                        assert fs.do_fsync(fd)
-            except OSError as ose:
+                        assert fs.do_fsync(fd) is None
+            except GlusterFileSystemOSError as ose:
                 self.fail('Opening a temporary file failed with %s' %ose.strerror)
             else:
                 os.close(fd)
@@ -390,13 +706,13 @@ class TestFsUtils(unittest.TestCase):
             os.write(fd, 'test')
             with patch('eventlet.tpool.execute', mock_tpool_execute):
                 with patch('os.fsync', mock_os_fsync):
-                    assert fs.do_fsync(fd)
+                    assert fs.do_fsync(fd) is None
                 os.close(fd)
                 try:
                     fs.do_fsync(fd)
-                except OSError:
+                except GlusterFileSystemOSError:
                     pass
                 else:
-                    self.fail("Expected OSError")
+                    self.fail("Expected GlusterFileSystemOSError")
         finally:
             shutil.rmtree(tmpdir)
diff --git a/test/unit/common/test_ring.py b/test/unit/common/test_ring.py
index 4cbb28c..ca9fc5c 100644
--- a/test/unit/common/test_ring.py
+++ b/test/unit/common/test_ring.py
@@ -21,7 +21,7 @@ from gluster.swift.common.ring import Ring
 
 
 class TestRing(unittest.TestCase):
-    """ Tests for common.utils """
+    """ Tests for common.ring """
 
     def setUp(self):
         swift.common.utils.HASH_PATH_SUFFIX = 'endcap'
@@ -66,4 +66,3 @@ class TestRing(unittest.TestCase):
     def test_invalid_partition(self):
         nodes = self.ring.get_part_nodes(0)
         self.assertEqual(nodes[0]['device'], 'volume_not_in_ring')
-
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 20984b1..6622f45 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -25,8 +25,10 @@ import hashlib
 import tarfile
 import shutil
 from collections import defaultdict
+from mock import patch
 from swift.common.utils import normalize_timestamp
 from gluster.swift.common import utils, Glusterfs
+from gluster.swift.common.exceptions import GlusterFileSystemOSError
 
 #
 # Somewhat hacky way of emulating the operation of xattr calls. They are made
@@ -128,6 +130,10 @@ class SimMemcache(object):
         self._d[key] = value
 
 
+def _mock_os_fsync(fd):
+    return
+
+
 class TestUtils(unittest.TestCase):
     """ Tests for common.utils """
 
@@ -338,8 +344,9 @@ class TestUtils(unittest.TestCase):
     def test_get_object_metadata_err(self):
         tf = tempfile.NamedTemporaryFile()
         try:
-            md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st"))
-        except OSError as e:
+            md = utils.get_object_metadata(
+                os.path.join(tf.name, "doesNotEx1st"))
+        except GlusterFileSystemOSError as e:
             assert e.errno != errno.ENOENT
         else:
             self.fail("Expected exception")
@@ -520,7 +527,7 @@ class TestUtils(unittest.TestCase):
         finally:
             os.rmdir(td)
 
-    def test_get_account_details_from_fs(self):
+    def test_get_account_details(self):
         orig_cwd = os.getcwd()
         td = tempfile.mkdtemp()
         try:
@@ -528,23 +535,28 @@ class TestUtils(unittest.TestCase):
             os.chdir(td)
             tf.extractall()
 
-            ad = utils._get_account_details_from_fs(td)
-            assert ad.mtime == os.path.getmtime(td)
-            assert ad.container_count == 3
-            assert set(ad.container_list) == set(['c1', 'c2', 'c3'])
+            container_list, container_count = utils.get_account_details(td)
+            assert container_count == 3
+            assert set(container_list) == set(['c1', 'c2', 'c3'])
         finally:
             os.chdir(orig_cwd)
             shutil.rmtree(td)
 
-    def test_get_container_details_from_fs_notadir(self):
+    def test_get_account_details_notadir(self):
+        tf = tempfile.NamedTemporaryFile()
+        container_list, container_count = utils.get_account_details(tf.name)
+        assert container_count == 0
+        assert container_list == []
+
+    def test_get_container_details_notadir(self):
         tf = tempfile.NamedTemporaryFile()
-        cd = utils._get_container_details_from_fs(tf.name)
-        assert cd.bytes_used == 0
-        assert cd.object_count == 0
-        assert cd.obj_list == []
-        assert cd.dir_list == []
+        obj_list, object_count, bytes_used = \
+            utils.get_container_details(tf.name)
+        assert bytes_used == 0
+        assert object_count == 0
+        assert obj_list == []
 
-    def test_get_container_details_from_fs(self):
+    def test_get_container_details(self):
         orig_cwd = os.getcwd()
         td = tempfile.mkdtemp()
         try:
@@ -552,13 +564,14 @@ class TestUtils(unittest.TestCase):
             os.chdir(td)
             tf.extractall()
 
-            cd = utils._get_container_details_from_fs(td)
-            assert cd.bytes_used == 0, repr(cd.bytes_used)
+            obj_list, object_count, bytes_used = \
+                utils.get_container_details(td)
+            assert bytes_used == 0, repr(bytes_used)
             # Should not include the directories
-            assert cd.object_count == 5, repr(cd.object_count)
-            assert set(cd.obj_list) == set(['file1', 'file3', 'file2',
-                                   'dir1/file1', 'dir1/file2'
-                                   ]), repr(cd.obj_list)
+            assert object_count == 5, repr(object_count)
+            assert set(obj_list) == set(['file1', 'file3', 'file2',
+                                         'dir1/file1', 'dir1/file2'
+                                         ]), repr(obj_list)
 
             full_dir1 = os.path.join(td, 'dir1')
             full_dir2 = os.path.join(td, 'dir2')
@@ -568,14 +581,11 @@ class TestUtils(unittest.TestCase):
                              full_dir2: os.path.getmtime(full_dir2),
                              full_dir3: os.path.getmtime(full_dir3),
                              }
-            for d,m in cd.dir_list:
-                assert d in exp_dir_dict
-                assert exp_dir_dict[d] == m
         finally:
             os.chdir(orig_cwd)
             shutil.rmtree(td)
 
-    def test_get_container_details_from_fs_ufo(self):
+    def test_get_container_details_ufo(self):
         orig_cwd = os.getcwd()
         __obj_only = Glusterfs.OBJECT_ONLY
         td = tempfile.mkdtemp()
@@ -586,13 +596,14 @@ class TestUtils(unittest.TestCase):
 
             Glusterfs.OBJECT_ONLY = False
 
-            cd = utils._get_container_details_from_fs(td)
-            assert cd.bytes_used == 0, repr(cd.bytes_used)
-            assert cd.object_count == 8, repr(cd.object_count)
-            assert set(cd.obj_list) == set(['file1', 'file3', 'file2',
-                                   'dir3', 'dir1', 'dir2',
-                                   'dir1/file1', 'dir1/file2'
-                                   ]), repr(cd.obj_list)
+            obj_list, object_count, bytes_used = \
+                utils.get_container_details(td)
+            assert bytes_used == 0, repr(bytes_used)
+            assert object_count == 8, repr(object_count)
+            assert set(obj_list) == set(['file1', 'file3', 'file2',
+                                         'dir3', 'dir1', 'dir2',
+                                         'dir1/file1', 'dir1/file2'
+                                         ]), repr(obj_list)
 
             full_dir1 = os.path.join(td, 'dir1')
             full_dir2 = os.path.join(td, 'dir2')
@@ -602,9 +613,6 @@ class TestUtils(unittest.TestCase):
                              full_dir2: os.path.getmtime(full_dir2),
                              full_dir3: os.path.getmtime(full_dir3),
                              }
-            for d,m in cd.dir_list:
-                assert d in exp_dir_dict
-                assert exp_dir_dict[d] == m
         finally:
             os.chdir(orig_cwd)
             shutil.rmtree(td)
@@ -621,12 +629,13 @@ class TestUtils(unittest.TestCase):
 
             Glusterfs._do_getsize = True
 
-            cd = utils._get_container_details_from_fs(td)
-            assert cd.bytes_used == 30, repr(cd.bytes_used)
-            assert cd.object_count == 5, repr(cd.object_count)
-            assert set(cd.obj_list) == set(['file1', 'file3', 'file2',
-                                   'dir1/file1', 'dir1/file2'
-                                   ]), repr(cd.obj_list)
+            obj_list, object_count, bytes_used = \
+                utils.get_container_details(td)
+            assert bytes_used == 30, repr(bytes_used)
+            assert object_count == 5, repr(object_count)
+            assert set(obj_list) == set(['file1', 'file3', 'file2',
+                                         'dir1/file1', 'dir1/file2'
+                                         ]), repr(obj_list)
 
             full_dir1 = os.path.join(td, 'dir1')
             full_dir2 = os.path.join(td, 'dir2')
@@ -636,33 +645,18 @@ class TestUtils(unittest.TestCase):
                              full_dir2: os.path.getmtime(full_dir2),
                              full_dir3: os.path.getmtime(full_dir3),
                              }
-            for d,m in cd.dir_list:
-                assert d in exp_dir_dict
-                assert exp_dir_dict[d] == m
         finally:
             Glusterfs._do_getsize = __do_getsize
             os.chdir(orig_cwd)
             shutil.rmtree(td)
 
-    def test_get_account_details_from_fs_notadir_w_stats(self):
-        tf = tempfile.NamedTemporaryFile()
-        ad = utils._get_account_details_from_fs(tf.name)
-        assert ad.mtime == os.path.getmtime(tf.name)
-        assert ad.container_count == 0
-        assert ad.container_list == []
-
-    def test_get_account_details_from_fs_notadir(self):
-        tf = tempfile.NamedTemporaryFile()
-        ad = utils._get_account_details_from_fs(tf.name)
-        assert ad.mtime == os.path.getmtime(tf.name)
-        assert ad.container_count == 0
-        assert ad.container_list == []
-
     def test_write_pickle(self):
         td = tempfile.mkdtemp()
         try:
             fpp = os.path.join(td, 'pp')
-            utils.write_pickle('pickled peppers', fpp)
+            # FIXME: Remove this patch when coverage.py can handle eventlet
+            with patch("os.fsync", _mock_os_fsync):
+                utils.write_pickle('pickled peppers', fpp)
             with open(fpp, "rb") as f:
                 contents = f.read()
             s = pickle.loads(contents)
@@ -676,7 +670,10 @@ class TestUtils(unittest.TestCase):
         try:
             fpp = os.path.join(td, 'pp')
             # Also test an explicity pickle protocol
-            utils.write_pickle('pickled peppers', fpp, tmp=tf.name, pickle_protocol=2)
+            # FIXME: Remove this patch when coverage.py can handle eventlet
+            with patch("os.fsync", _mock_os_fsync):
+                utils.write_pickle('pickled peppers', fpp, tmp=tf.name,
+                                   pickle_protocol=2)
             with open(fpp, "rb") as f:
                 contents = f.read()
             s = pickle.loads(contents)
@@ -855,3 +852,168 @@ class TestUtilsDirObjects(unittest.TestCase):
         self.assertFalse(utils.rmobjdir(self.rootdir))
         self._clear_dir_object(self.dirs[0])
         self.assertTrue(utils.rmobjdir(self.rootdir))
+
+    def test_rmobjdir_metadata_errors(self):
+
+        def _mock_rm(path):
+            print "_mock_rm-metadata_errors(%s)" % path
+            if path.endswith("dir3"):
+                raise OSError(13, "foo")
+            return {}
+
+        _orig_rm = utils.read_metadata
+        utils.read_metadata = _mock_rm
+        try:
+            try:
+                utils.rmobjdir(self.rootdir)
+            except OSError:
+                pass
+            else:
+                self.fail("Expected OSError")
+        finally:
+            utils.read_metadata = _orig_rm
+
+    def test_rmobjdir_metadata_enoent(self):
+
+        def _mock_rm(path):
+            print "_mock_rm-metadata_enoent(%s)" % path
+            shutil.rmtree(path)
+            raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+
+        # Remove the files
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        _orig_rm = utils.read_metadata
+        utils.read_metadata = _mock_rm
+        try:
+            try:
+                self.assertTrue(utils.rmobjdir(self.rootdir))
+            except OSError:
+                self.fail("Unexpected OSError")
+            else:
+                pass
+        finally:
+            utils.read_metadata = _orig_rm
+
+    def test_rmobjdir_rmdir_enoent(self):
+
+        seen = [0]
+        _orig_rm = utils.do_rmdir
+
+        def _mock_rm(path):
+            print "_mock_rm-rmdir_enoent(%s)" % path
+            if path == self.rootdir and not seen[0]:
+                seen[0] = 1
+                raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY))
+            else:
+                shutil.rmtree(path)
+                raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+
+        # Remove the files
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        utils.do_rmdir = _mock_rm
+        try:
+            try:
+                self.assertTrue(utils.rmobjdir(self.rootdir))
+            except OSError:
+                self.fail("Unexpected OSError")
+            else:
+                pass
+        finally:
+            utils.do_rmdir = _orig_rm
+
+    def test_rmobjdir_rmdir_error(self):
+
+        seen = [0]
+        _orig_rm = utils.do_rmdir
+
+        def _mock_rm(path):
+            print "_mock_rm-rmdir_enoent(%s)" % path
+            if path == self.rootdir and not seen[0]:
+                seen[0] = 1
+                raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY))
+            else:
+                raise OSError(errno.EACCES, os.strerror(errno.EACCES))
+
+        # Remove the files
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        utils.do_rmdir = _mock_rm
+        try:
+            try:
+                utils.rmobjdir(self.rootdir)
+            except OSError:
+                pass
+            else:
+                self.fail("Expected OSError")
+        finally:
+            utils.do_rmdir = _orig_rm
+
+    def test_rmobjdir_files_left_in_top_dir(self):
+
+        seen = [0]
+        _orig_rm = utils.do_rmdir
+
+        def _mock_rm(path):
+            print "_mock_rm-files_left_in_top_dir(%s)" % path
+            if path == self.rootdir:
+                if not seen[0]:
+                    seen[0] = 1
+                    raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY))
+                else:
+                    return _orig_rm(path)
+            else:
+                shutil.rmtree(path)
+                raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+
+        # Remove the files, leaving the ones at the root
+        for f in self.files:
+            if f.startswith('dir'):
+                os.unlink(os.path.join(self.rootdir, f))
+
+        utils.do_rmdir = _mock_rm
+        try:
+            try:
+                self.assertFalse(utils.rmobjdir(self.rootdir))
+            except OSError:
+                self.fail("Unexpected OSError")
+            else:
+                pass
+        finally:
+            utils.do_rmdir = _orig_rm
+
+    def test_rmobjdir_error_final_rmdir(self):
+
+        seen = [0]
+        _orig_rm = utils.do_rmdir
+
+        def _mock_rm(path):
+            print "_mock_rm-files_left_in_top_dir(%s)" % path
+            if path == self.rootdir:
+                if not seen[0]:
+                    seen[0] = 1
+                    raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY))
+                else:
+                    raise OSError(errno.EACCES, os.strerror(errno.EACCES))
+            else:
+                shutil.rmtree(path)
+                raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+
+        # Remove the files, leaving the ones at the root
+        for f in self.files:
+            os.unlink(os.path.join(self.rootdir, f))
+
+        utils.do_rmdir = _mock_rm
+        try:
+            try:
+                utils.rmobjdir(self.rootdir)
+            except OSError:
+                pass
+            else:
+                self.fail("Expected OSError")
+        finally:
+            utils.do_rmdir = _orig_rm