glusterfs: Use gcc builtin ATOMIC operator to increase/decreate refcount.

Problem: In glusterfs code base we call mutex_lock/unlock to take
         reference/dereference for a object.Sometime it could be
         reason for lock contention also.

Solution: There is no need to use mutex to increase/decrease ref
          counter, instead of using mutex use gcc builtin ATOMIC
          operation.

Test:   I have not observed yet how much performance gain after apply
        this patch specific to glusterfs but i have tested same
        with below small program(mutex and atomic both) and
        get good difference.

static int numOuterLoops;
static void *
threadFunc(void *arg)
{
    int j;

    for (j = 0; j < numOuterLoops; j++) {
            __atomic_add_fetch (&glob, 1,__ATOMIC_ACQ_REL);
    }
    return NULL;
}

int
main(int argc, char *argv[])
{
    int opt, s, j;
    int numThreads;
    pthread_t *thread;
    int verbose;
    int64_t n = 0;

    if (argc < 2 ) {
     printf(" Please provide 2 args Num of threads && Outer Loop\n");
     exit (-1);
    }
    numThreads = atoi(argv[1]);
    numOuterLoops = atoi (argv[2]);

    if (1) {
        printf("\tthreads: %d; outer loops: %d;\n",
                numThreads, numOuterLoops);
    }

    thread = calloc(numThreads, sizeof(pthread_t));
    if (thread == NULL) {
        printf ("calloc error so exit\n");
        exit (-1);
    }

    __atomic_store (&glob, &n, __ATOMIC_RELEASE);
    for (j = 0; j < numThreads; j++) {
        s = pthread_create(&thread[j], NULL, threadFunc, NULL);
        if (s != 0) {
            printf ("pthread_create failed so exit\n");
            exit (-1);
        }
    }

    for (j = 0; j < numThreads; j++) {
        s = pthread_join(thread[j], NULL);
        if (s != 0) {
            printf ("pthread_join failed so exit\n");
            exit (-1);
        }
    }
    printf("glob value is %ld\n",__atomic_load_n (&glob,__ATOMIC_RELAXED));

    exit(0);
}

   time ./thr_count 800 800000
   threads: 800; outer loops: 800000;
   glob value is 640000000

real	1m10.288s
user	0m57.269s
sys	3m31.565s

time ./thr_count_atomic 800 800000
     threads: 800; outer loops: 800000;
glob value is 640000000

real	0m20.313s
user	1m20.558s
sys	0m0.028

Change-Id: Ie5030a52ea264875e002e108dd4b207b15ab7cc7
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

4 files changed, 9 insertions, 22 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index cb59408dcf6..06aed0a80ae 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1740,11 +1740,8 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                rpc->refcount++;
-        }
-        pthread_mutex_unlock (&rpc->lock);
+
+        GF_ATOMIC_INC (rpc->refcount);
         return rpc;
 }
 
@@ -1806,11 +1803,9 @@ rpc_clnt_unref (struct rpc_clnt *rpc)
 
         if (!rpc)
                 return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                count = --rpc->refcount;
-        }
-        pthread_mutex_unlock (&rpc->lock);
+
+        count = GF_ATOMIC_DEC (rpc->refcount);
+
         if (!count) {
                 rpc_clnt_trigger_destroy (rpc);
                 return NULL;
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 428ffb80650..6503ca5a573 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -187,7 +187,7 @@ typedef struct rpc_clnt {
         struct mem_pool       *saved_frames_pool;
 
         glusterfs_ctx_t       *ctx;
-        int                   refcount;
+        gf_atomic_t           refcount;
         int                   auth_null;
         char                  disabled;
         xlator_t             *owner;
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index 40b5917f9b8..40c41ab7016 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -493,11 +493,7 @@ rpc_transport_ref (rpc_transport_t *this)
 
 	GF_VALIDATE_OR_GOTO("rpc_transport", this, fail);
 
-	pthread_mutex_lock (&this->lock);
-	{
-		this->refcount ++;
-	}
-	pthread_mutex_unlock (&this->lock);
+        GF_ATOMIC_INC (this->refcount);
 
 	return_this = this;
 fail:
@@ -513,11 +509,7 @@ rpc_transport_unref (rpc_transport_t *this)
 
 	GF_VALIDATE_OR_GOTO("rpc_transport", this, fail);
 
-	pthread_mutex_lock (&this->lock);
-	{
-                refcount = --this->refcount;
-	}
-	pthread_mutex_unlock (&this->lock);
+        refcount = GF_ATOMIC_DEC (this->refcount);
 
 	if (refcount == 0) {
                 if (this->mydata)
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 33f474e42c6..332efb45c25 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -184,7 +184,7 @@ struct rpc_transport {
         void                      *xl;       /* Used for THIS */
         void                      *mydata;
         pthread_mutex_t            lock;
-        int32_t                    refcount;
+        gf_atomic_t                refcount;
 
         int32_t                    outstanding_rpc_count;