core: heketi-cli is throwing error "target is busy"

Problem: When rpc-transport-disconnect happens, server_connection_cleanup_flush_cbk() 
         is supposed to call rpc_transport_unref() after open-files on 
         that transport are flushed per transport.But open-fd-count is 
         maintained in bound_xl->fd_count, which can be incremented/decremented 
         cumulatively in server_connection_cleanup() by all transport 
         disconnect paths. So instead of rpc_transport_unref() happening 
         per transport, it ends up doing it only once after all the files 
         on all the transports for the brick are flushed leading to 
         rpc-leaks.

Solution: To avoid races maintain fd_cnt at client instead of maintaining
          on brick

Credits: Pranith Kumar Karampuri
Change-Id: I6e8ea37a61f82d9aefb227c5b3ab57a7a36850e6
fixes: bz#1668190
Signed-off-by: Mohit Agrawal <moagrawal@redhat.com>

5 files changed, 5 insertions, 6 deletions

diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c
index 0f98a66a123..e875c8b6b69 100644
--- a/libglusterfs/src/client_t.c
+++ b/libglusterfs/src/client_t.c
@@ -224,6 +224,7 @@ gf_client_get(xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid,
 
         GF_ATOMIC_INIT(client->bind, 1);
         GF_ATOMIC_INIT(client->count, 1);
+        GF_ATOMIC_INIT(client->fd_cnt, 0);
 
         client->auth.flavour = cred->flavour;
         if (cred->flavour != AUTH_NONE) {
diff --git a/libglusterfs/src/glusterfs/client_t.h b/libglusterfs/src/glusterfs/client_t.h
index 29f50daf03b..8ef3665a9c2 100644
--- a/libglusterfs/src/glusterfs/client_t.h
+++ b/libglusterfs/src/glusterfs/client_t.h
@@ -47,6 +47,8 @@ typedef struct _client {
     inode_t *subdir_inode;
     uuid_t subdir_gfid;
     int32_t opversion;
+    /* Variable to save fd_count for detach brick */
+    gf_atomic_t fd_cnt;
 } client_t;
 
 #define GF_CLIENTCTX_INITIAL_SIZE 8
diff --git a/libglusterfs/src/glusterfs/xlator.h b/libglusterfs/src/glusterfs/xlator.h
index 0c39f4bd3cc..7002657d0bc 100644
--- a/libglusterfs/src/glusterfs/xlator.h
+++ b/libglusterfs/src/glusterfs/xlator.h
@@ -856,9 +856,6 @@ struct _xlator {
     /* Flag to understand how this xlator is categorized */
     gf_category_t category;
 
-    /* Variable to save fd_count for detach brick */
-    gf_atomic_t fd_cnt;
-
     /* Variable to save xprt associated for detach brick */
     gf_atomic_t xprtrefcnt;
 
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index bd568660863..dc1e88770fb 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -614,7 +614,6 @@ xlator_init(xlator_t *xl)
 
     xl->instance_name = NULL;
     GF_ATOMIC_INIT(xl->xprtrefcnt, 0);
-    GF_ATOMIC_INIT(xl->fd_cnt, 0);
     if (!xl->init) {
         gf_msg(xl->name, GF_LOG_WARNING, 0, LG_MSG_INIT_FAILED,
                "No init() found");
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 8ad2d8492ed..1a34239d8b9 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -257,7 +257,7 @@ server_connection_cleanup_flush_cbk(call_frame_t *frame, void *cookie,
         victim = client->bound_xl;
 
     if (victim) {
-        fd_cnt = GF_ATOMIC_DEC(victim->fd_cnt);
+        fd_cnt = GF_ATOMIC_DEC(client->fd_cnt);
         if (!fd_cnt && conf && detach) {
             pthread_mutex_lock(&conf->mutex);
             {
@@ -399,7 +399,7 @@ server_connection_cleanup(xlator_t *this, client_t *client, int32_t flags,
             if (fd_cnt) {
                 if (fd_exist)
                     (*fd_exist) = _gf_true;
-                GF_ATOMIC_ADD(bound_xl->fd_cnt, fd_cnt);
+                GF_ATOMIC_ADD(client->fd_cnt, fd_cnt);
             }
         }