Fix infinite loop for temp file renames on ENOENT

For whatever reason, it appears that GlusterFS, or perhaps FUSE can
continuously return ENOENT on a rename system call even when we have
double checked that there is no reason to do so. That is a bug for
that sub system. However, our response to that bug can result in an
infinite loop, which is bad.

This code reduces that to 10 attempts.

In addition, we restructed the open retry loop to match, providing
module constants for the upper bounds of both retry loops.

BUG: XXXXXX (https://bugzilla.redhat.com/show_bug.cgi?id=XXXXXX)

Change-Id: Ia2d6dd427daba3ea0461863c5ffe3aef27c88f9b
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/5670
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>

1 files changed, 52 insertions, 4 deletions

diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py
index 4686a19..819abe7 100644
--- a/test/unit/obj/test_diskfile.py
+++ b/test/unit/obj/test_diskfile.py
@@ -25,14 +25,17 @@ import mock
 from mock import patch
 from hashlib import md5
 
-import gluster.swift.common.utils
-import gluster.swift.obj.diskfile
 from swift.common.utils import normalize_timestamp
-from gluster.swift.obj.diskfile import DiskFile
 from swift.common.exceptions import DiskFileNotExist, DiskFileError, \
     DiskFileNoSpace
+
+from gluster.swift.common.exceptions import GlusterFileSystemOSError
+import gluster.swift.common.utils
+import gluster.swift.obj.diskfile
+from gluster.swift.obj.diskfile import DiskFile
 from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \
     X_OBJECT_TYPE, DIR_OBJECT
+
 from test.unit.common.test_utils import _initxattr, _destroyxattr
 from test.unit import FakeLogger
 
@@ -565,7 +568,6 @@ class TestDiskFile(unittest.TestCase):
         finally:
             shutil.rmtree(td)
 
-
     def test_put_ENOSPC(self):
         td = tempfile.mkdtemp()
         the_cont = os.path.join(td, "vol0", "bar")
@@ -589,6 +591,7 @@ class TestDiskFile(unittest.TestCase):
                 'ETag': etag,
                 'Content-Length': '5',
                 }
+
             def mock_open(*args, **kwargs):
                 raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))
 
@@ -605,6 +608,51 @@ class TestDiskFile(unittest.TestCase):
         finally:
             shutil.rmtree(td)
 
+    def test_put_rename_ENOENT(self):
+        td = tempfile.mkdtemp()
+        the_cont = os.path.join(td, "vol0", "bar")
+        try:
+            os.makedirs(the_cont)
+            gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg)
+            assert gdf._obj == "z"
+            assert gdf._obj_path == ""
+            assert gdf.name == "bar"
+            assert gdf.datadir == the_cont
+            assert gdf.data_file is None
+
+            body = '1234\n'
+            etag = md5()
+            etag.update(body)
+            etag = etag.hexdigest()
+            metadata = {
+                'X-Timestamp': '1234',
+                'Content-Type': 'file',
+                'ETag': etag,
+                'Content-Length': '5',
+                }
+
+            def mock_sleep(*args, **kwargs):
+                # Return without sleep, no need to dely unit tests
+                return
+
+            def mock_rename(*args, **kwargs):
+                raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+
+            with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
+                with mock.patch("os.rename", mock_rename):
+                    try:
+                        with gdf.writer() as dw:
+                            assert dw.tmppath is not None
+                            tmppath = dw.tmppath
+                            dw.write(body)
+                            dw.put(metadata)
+                    except GlusterFileSystemOSError:
+                        pass
+                    else:
+                        self.fail("Expected exception DiskFileError")
+        finally:
+            shutil.rmtree(td)
+
     def test_put_obj_path(self):
         the_obj_path = os.path.join("b", "a")
         the_file = os.path.join(the_obj_path, "z")