From f13023862edc868d4da87609412341aecd041ed8 Mon Sep 17 00:00:00 2001
From: Zhang Huan <zhanghuan@open-fs.com>
Date: Wed, 29 Nov 2017 17:36:55 +0800
Subject: protocol/client: reduce lock contention

Current use of a per-client mutex to protect fdctx introduces lock
contentions when there are dozens of file operations active.

Use finer grain spinlock to reduce contention, and put retrieving
fdctx out of lock.

Change-Id: Iea3e2eb481e76a5d73a582ba81529180c5b88248
BUG: 1519598
Signed-off-by: Zhang Huan <zhanghuan@open-fs.com>
---
 xlators/protocol/client/src/client-handshake.c | 31 +++++++++++++-------------
 1 file changed, 16 insertions(+), 15 deletions(-)

(limited to 'xlators/protocol/client/src/client-handshake.c')

diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 6429dc2e8ae..68ea20ffed7 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -164,12 +164,12 @@ clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx,
         GF_VALIDATE_OR_GOTO (this->name, conf, out);
         GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
 
-        pthread_mutex_lock (&conf->lock);
+        pthread_spin_lock (&conf->fd_lock);
         {
                 fdctx->remote_fd     = -1;
                 fdctx->lk_heal_state = GF_LK_HEAL_DONE;
         }
-        pthread_mutex_unlock (&conf->lock);
+        pthread_spin_unlock (&conf->fd_lock);
 
         ret = 0;
 out:
@@ -488,11 +488,11 @@ client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
 
         if (!clnt_fd_lk_local_error_status (this, local) &&
             clnt_fd_lk_local_unref (this, local) == 0) {
-                pthread_mutex_lock (&conf->lock);
+                pthread_spin_lock (&conf->fd_lock);
                 {
                         fdctx->lk_heal_state = GF_LK_HEAL_DONE;
                 }
-                pthread_mutex_unlock (&conf->lock);
+                pthread_spin_unlock (&conf->fd_lock);
 
                 fdctx->reopen_done (fdctx, fdctx->remote_fd, this);
         }
@@ -634,7 +634,7 @@ client_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
 
         conf = this->private;
 
-        pthread_mutex_lock (&conf->lock);
+        pthread_spin_lock (&conf->fd_lock);
         {
                 fdctx->remote_fd = rfd;
                 fdctx->reopen_attempts = 0;
@@ -644,7 +644,7 @@ client_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
                 else
                         destroy = _gf_true;
         }
-        pthread_mutex_unlock (&conf->lock);
+        pthread_spin_unlock (&conf->fd_lock);
 
         if (destroy)
                 client_fdctx_destroy (this, fdctx);
@@ -726,7 +726,7 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        pthread_mutex_lock (&conf->lock);
+        pthread_spin_lock (&conf->fd_lock);
         {
                 if (!fdctx->released) {
                         if (conf->lk_heal &&
@@ -737,7 +737,7 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         }
                 }
         }
-        pthread_mutex_unlock (&conf->lock);
+        pthread_spin_unlock (&conf->fd_lock);
 
         ret = 0;
 
@@ -975,11 +975,12 @@ client_attempt_reopen (fd_t *fd, xlator_t *this)
                 goto out;
 
         conf = this->private;
-        pthread_mutex_lock (&conf->lock);
+
+        fdctx = this_fd_get_ctx (fd, this);
+        if (!fdctx)
+                goto out;
+        pthread_spin_lock (&conf->fd_lock);
         {
-                fdctx = this_fd_get_ctx (fd, this);
-                if (!fdctx)
-                        goto unlock;
                 if (__is_fd_reopen_in_progress (fdctx))
                         goto unlock;
                 if (fdctx->remote_fd != -1)
@@ -994,7 +995,7 @@ client_attempt_reopen (fd_t *fd, xlator_t *this)
                 }
         }
 unlock:
-        pthread_mutex_unlock (&conf->lock);
+        pthread_spin_unlock (&conf->fd_lock);
         if (reopen)
                 protocol_client_reopen (fdctx, this);
 out:
@@ -1017,7 +1018,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
         conf = this->private;
         INIT_LIST_HEAD (&reopen_head);
 
-        pthread_mutex_lock (&conf->lock);
+        pthread_spin_lock (&conf->fd_lock);
         {
                 list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
                                           sfd_pos) {
@@ -1030,7 +1031,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                         count++;
                 }
         }
-        pthread_mutex_unlock (&conf->lock);
+        pthread_spin_unlock (&conf->fd_lock);
 
         /* Delay notifying CHILD_UP to parents
            until all locks are recovered */
-- 
cgit