[Testfix] Increase timeouts and fix I/O errors

Problem:
--------

Problem 1:
In the latest runs the following testcases fail
with wait timeout mostly on rebalance with an exception
on test_stack_overflow which fails on layout:
1.functional.dht.test_stack_overflow.TestStackOverflow_cplex_dispersed_glusterfs.test_stack_overflow
2.functional.dht.test_rebalance_dir_file_from_multiple_clients.RebalanceValidation_cplex_dispersed_glusterfs.test_expanding_volume_when_io_in_progress
3.functional.dht.test_restart_glusterd_after_rebalance.RebalanceValidation_cplex_dispersed_glusterfs.test_restart_glusterd_after_rebalance
4.functional.dht.test_stop_glusterd_while_rebalance_in_progress.RebalanceValidation_cplex_dispersed_glusterfs.test_stop_glusterd_while_rebalance_in_progress
5.functional.dht.test_rebalance_with_hidden_files.RebalanceValidation_cplex_dispersed_glusterfs.test_rebalance_with_hidden_files

This is mostly observed on disprese volumes which
is expected as in most cases disprese volumes take
more time than pure replicated or distributed volumes
due to it's design.

Problem 2:
Another issue which was observed was that
test_rebalance_with_hidden_files failing on I/O with
distributed volume type with the below stack trace:

Traceback (most recent call last):
  File "/usr/share/glustolibs/io/scripts/file_dir_ops.py", line 1246, in <module>
    rc = args.func(args)
  File "/usr/share/glustolibs/io/scripts/file_dir_ops.py", line 374, in create_files
    base_file_name, file_types)
  File "/usr/share/glustolibs/io/scripts/file_dir_ops.py", line 341, in _create_files
    ret = pool.map(lambda file_tuple: _create_file(*file_tuple), files)
  File "/usr/lib64/python2.7/multiprocessing/pool.py", line 250, in map
    return self.map_async(func, iterable, chunksize).get()
  File "/usr/lib64/python2.7/multiprocessing/pool.py", line 554, in get
    raise self._value
IOError: [Errno 17] File exists: '/mnt/testvol_distributed_glusterfs/.1.txt'

Solution:
--------

Problem 1
Increasing or adding timeout so that wait timeouts are not observed.

Problem 2
Adding counter logic to fix the I/O failure.

Change-Id: I917137abdeb2e3844ee666258235f6ccc854ee9f
Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>

5 files changed, 11 insertions, 6 deletions

diff --git a/tests/functional/dht/test_rebalance_dir_file_from_multiple_clients.py b/tests/functional/dht/test_rebalance_dir_file_from_multiple_clients.py
index dd80479bb..add72aec5 100644
--- a/tests/functional/dht/test_rebalance_dir_file_from_multiple_clients.py
+++ b/tests/functional/dht/test_rebalance_dir_file_from_multiple_clients.py
@@ -201,7 +201,7 @@ class RebalanceValidation(GlusterBaseClass):
         # Wait for rebalance to complete
         g.log.info("Waiting for rebalance to complete")
         ret = wait_for_rebalance_to_complete(self.mnode, self.volname,
-                                             timeout=600)
+                                             timeout=1800)
         self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
                               "%s", self.volname))
         g.log.info("Rebalance status on volume %s: Complete",
diff --git a/tests/functional/dht/test_rebalance_with_hidden_files.py b/tests/functional/dht/test_rebalance_with_hidden_files.py
index b0cb9b340..40fe7b976 100644
--- a/tests/functional/dht/test_rebalance_with_hidden_files.py
+++ b/tests/functional/dht/test_rebalance_with_hidden_files.py
@@ -91,16 +91,19 @@ class RebalanceValidation(GlusterBaseClass):
         # Start IO on mounts
         g.log.info("Starting IO on all mounts...")
         self.all_mounts_procs = []
+        counter = 1
         for mount_obj in self.mounts:
             g.log.info("Starting IO on %s:%s", mount_obj.client_system,
                        mount_obj.mountpoint)
             cmd = ("/usr/bin/env python %s create_files "
-                   "--base-file-name . -f 99 %s" % (
+                   "--base-file-name .file%d -f 99 %s" % (
                        self.script_upload_path,
+                       counter,
                        mount_obj.mountpoint))
             proc = g.run_async(mount_obj.client_system, cmd,
                                user=mount_obj.user)
             self.all_mounts_procs.append(proc)
+            counter += 100
 
         # validate IO
         self.assertTrue(
@@ -173,7 +176,8 @@ class RebalanceValidation(GlusterBaseClass):
 
         # Wait for rebalance to complete
         g.log.info("Waiting for rebalance to complete")
-        ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
+        ret = wait_for_rebalance_to_complete(self.mnode, self.volname,
+                                             timeout=1800)
         self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
                               "%s", self.volname))
         g.log.info("Rebalance is successfully complete on the volume %s",
diff --git a/tests/functional/dht/test_restart_glusterd_after_rebalance.py b/tests/functional/dht/test_restart_glusterd_after_rebalance.py
index b4436968d..408e309a1 100644
--- a/tests/functional/dht/test_restart_glusterd_after_rebalance.py
+++ b/tests/functional/dht/test_restart_glusterd_after_rebalance.py
@@ -131,7 +131,8 @@ class RebalanceValidation(GlusterBaseClass):
 
         # Wait for rebalance to complete
         g.log.info("Waiting for rebalance to complete")
-        ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
+        ret = wait_for_rebalance_to_complete(self.mnode, self.volname,
+                                             timeout=1800)
         self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
                               "%s", self.volname))
         g.log.info("Rebalance is successfully complete on the volume %s",
diff --git a/tests/functional/dht/test_stack_overflow.py b/tests/functional/dht/test_stack_overflow.py
index b165bdf65..c371add63 100644
--- a/tests/functional/dht/test_stack_overflow.py
+++ b/tests/functional/dht/test_stack_overflow.py
@@ -110,7 +110,7 @@ class TestStackOverflow(GlusterBaseClass):
 
         # Wait for fix-layout to complete
         ret = wait_for_fix_layout_to_complete(self.mnode, self.volname,
-                                              timeout=300)
+                                              timeout=3000)
         self.assertTrue(ret, ("Fix-layout failed on volume %s",
                               self.volname))
         g.log.info("Fix-layout is successful on "
diff --git a/tests/functional/dht/test_stop_glusterd_while_rebalance_in_progress.py b/tests/functional/dht/test_stop_glusterd_while_rebalance_in_progress.py
index 83a0ee991..350cb4e1a 100644
--- a/tests/functional/dht/test_stop_glusterd_while_rebalance_in_progress.py
+++ b/tests/functional/dht/test_stop_glusterd_while_rebalance_in_progress.py
@@ -167,7 +167,7 @@ class RebalanceValidation(GlusterBaseClass):
         # Wait for rebalance to complete
         g.log.info("Waiting for rebalance to complete")
         ret = wait_for_rebalance_to_complete(self.mnode, self.volname,
-                                             timeout=600)
+                                             timeout=1800)
         self.assertTrue(ret, ("Rebalance is either timed out or failed"
                               "%s", self.volname))
         g.log.info("Volume %s: Rebalance completed successfully",