changelog: fix br-state-check.t failure for brick_mux

Problem: Sometime br-state-check.t crash while runnning
         for brick multiplex and command in test case is
         taking 2 minutes for detach a brick

Solution: Update code in changelog xlator specific to wait
          on all connection before cleanup rpc threads and
          cleanup rpc object only in non brick mux scenario

BUG: 1577672
Change-Id: I16e257c1e127744a815000b87bd8b7b8d9c51e1b
fixes: bz#1577672
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

2 files changed, 39 insertions, 1 deletions

diff --git a/xlators/features/changelog/src/changelog-rpc-common.c b/xlators/features/changelog/src/changelog-rpc-common.c
index 1745f218b08..056519232cf 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.c
+++ b/xlators/features/changelog/src/changelog-rpc-common.c
@@ -280,7 +280,12 @@ changelog_rpc_server_destroy (xlator_t *this, rpcsvc_t *rpc, char *sockfile,
                 rpc->rxpool = NULL;
         }
 
-        GF_FREE (rpc);
+        /* TODO Avoid freeing rpc object in case of brick multiplex
+           after freeing rpc object svc->rpclock corrupted and it takes
+           more time to detach a brick
+        */
+        if (!this->cleanup_starting)
+                GF_FREE (rpc);
 }
 
 rpcsvc_t *
diff --git a/xlators/features/changelog/src/changelog-rpc.c b/xlators/features/changelog/src/changelog-rpc.c
index 852c0694f9a..793e7a8aff4 100644
--- a/xlators/features/changelog/src/changelog-rpc.c
+++ b/xlators/features/changelog/src/changelog-rpc.c
@@ -157,6 +157,9 @@ void
 changelog_destroy_rpc_listner (xlator_t *this, changelog_priv_t *priv)
 {
         char sockfile[UNIX_PATH_MAX] = {0,};
+        changelog_clnt_t     *c_clnt  = &priv->connections;
+        changelog_rpc_clnt_t *crpc = NULL;
+        int                   nofconn = 0;
 
         /* sockfile path could have been saved to avoid this */
         CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
@@ -165,6 +168,36 @@ changelog_destroy_rpc_listner (xlator_t *this, changelog_priv_t *priv)
                                       priv->rpc, sockfile,
                                       changelog_rpcsvc_notify,
                                       changelog_programs);
+
+        /* TODO Below approach is not perfect to wait for cleanup
+           all active connections without this code brick process
+           can be crash in case of brick multiplexing if any in-progress
+           request process on rpc by changelog xlator after
+           cleanup resources
+        */
+
+        if (c_clnt) {
+                do {
+                        nofconn = 0;
+                        LOCK (&c_clnt->active_lock);
+                                list_for_each_entry (crpc, &c_clnt->active, list) {
+                                          nofconn++;
+                                }
+                        UNLOCK (&c_clnt->active_lock);
+                        LOCK (&c_clnt->wait_lock);
+                                list_for_each_entry (crpc, &c_clnt->waitq, list) {
+                                          nofconn++;
+                                }
+                        UNLOCK (&c_clnt->wait_lock);
+                        pthread_mutex_lock (&c_clnt->pending_lock);
+                                list_for_each_entry (crpc, &c_clnt->pending, list) {
+                                          nofconn++;
+                                }
+                        pthread_mutex_unlock (&c_clnt->pending_lock);
+
+                } while (nofconn);  /* Wait for all connection cleanup */
+        }
+
         (void) changelog_cleanup_rpc_threads (this, priv);
 }