glusterfs: Use gcc builtin ATOMIC operator to increase/decreate refcount.

Problem: In glusterfs code base we call mutex_lock/unlock to take
         reference/dereference for a object.Sometime it could be
         reason for lock contention also.

Solution: There is no need to use mutex to increase/decrease ref
          counter, instead of using mutex use gcc builtin ATOMIC
          operation.

Test:   I have not observed yet how much performance gain after apply
        this patch specific to glusterfs but i have tested same
        with below small program(mutex and atomic both) and
        get good difference.

static int numOuterLoops;
static void *
threadFunc(void *arg)
{
    int j;

    for (j = 0; j < numOuterLoops; j++) {
            __atomic_add_fetch (&glob, 1,__ATOMIC_ACQ_REL);
    }
    return NULL;
}

int
main(int argc, char *argv[])
{
    int opt, s, j;
    int numThreads;
    pthread_t *thread;
    int verbose;
    int64_t n = 0;

    if (argc < 2 ) {
     printf(" Please provide 2 args Num of threads && Outer Loop\n");
     exit (-1);
    }
    numThreads = atoi(argv[1]);
    numOuterLoops = atoi (argv[2]);

    if (1) {
        printf("\tthreads: %d; outer loops: %d;\n",
                numThreads, numOuterLoops);
    }

    thread = calloc(numThreads, sizeof(pthread_t));
    if (thread == NULL) {
        printf ("calloc error so exit\n");
        exit (-1);
    }

    __atomic_store (&glob, &n, __ATOMIC_RELEASE);
    for (j = 0; j < numThreads; j++) {
        s = pthread_create(&thread[j], NULL, threadFunc, NULL);
        if (s != 0) {
            printf ("pthread_create failed so exit\n");
            exit (-1);
        }
    }

    for (j = 0; j < numThreads; j++) {
        s = pthread_join(thread[j], NULL);
        if (s != 0) {
            printf ("pthread_join failed so exit\n");
            exit (-1);
        }
    }
    printf("glob value is %ld\n",__atomic_load_n (&glob,__ATOMIC_RELAXED));

    exit(0);
}

   time ./thr_count 800 800000
   threads: 800; outer loops: 800000;
   glob value is 640000000

real	1m10.288s
user	0m57.269s
sys	3m31.565s

time ./thr_count_atomic 800 800000
     threads: 800; outer loops: 800000;
glob value is 640000000

real	0m20.313s
user	1m20.558s
sys	0m0.028

Change-Id: Ie5030a52ea264875e002e108dd4b207b15ab7cc7
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

1 files changed, 7 insertions, 45 deletions

diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 76584fc9cde..d4f0d89c338 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -126,7 +126,7 @@ __iobuf_arena_destroy_iobufs (struct iobuf_arena *iobuf_arena)
 
         iobuf = iobuf_arena->iobufs;
         for (i = 0; i < iobuf_cnt; i++) {
-                GF_ASSERT (iobuf->ref == 0);
+                GF_ASSERT (GF_ATOMIC_GET(iobuf->ref) == 0);
 
                 LOCK_DESTROY (&iobuf->lock);
                 list_del_init (&iobuf->list);
@@ -529,23 +529,6 @@ out:
 
 
 struct iobuf *
-__iobuf_ref (struct iobuf *iobuf)
-{
-        iobuf->ref++;
-
-        return iobuf;
-}
-
-
-struct iobuf *
-__iobuf_unref (struct iobuf *iobuf)
-{
-        iobuf->ref--;
-
-        return iobuf;
-}
-
-struct iobuf *
 __iobuf_get (struct iobuf_arena *iobuf_arena, size_t page_size)
 {
         struct iobuf      *iobuf        = NULL;
@@ -619,7 +602,7 @@ iobuf_get_from_stdalloc (struct iobuf_pool *iobuf_pool, size_t page_size)
         LOCK_INIT (&iobuf->lock);
 
         /* Hold a ref because you are allocating and using it */
-        iobuf->ref = 1;
+        GF_ATOMIC_INIT (iobuf->ref, 1);
 
         ret = 0;
 out:
@@ -835,12 +818,7 @@ iobuf_unref (struct iobuf *iobuf)
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf, out);
 
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_unref (iobuf);
-                ref = iobuf->ref;
-        }
-        UNLOCK (&iobuf->lock);
+        ref = GF_ATOMIC_DEC (iobuf->ref);
 
         if (!ref)
                 iobuf_put (iobuf);
@@ -854,12 +832,7 @@ struct iobuf *
 iobuf_ref (struct iobuf *iobuf)
 {
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf, out);
-
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_ref (iobuf);
-        }
-        UNLOCK (&iobuf->lock);
+        GF_ATOMIC_INC (iobuf->ref);
 
 out:
         return iobuf;
@@ -888,8 +861,7 @@ iobref_new ()
 
         LOCK_INIT (&iobref->lock);
 
-        iobref->ref++;
-
+        GF_ATOMIC_INIT (iobref->ref, 1);
         return iobref;
 }
 
@@ -898,12 +870,7 @@ struct iobref *
 iobref_ref (struct iobref *iobref)
 {
         GF_VALIDATE_OR_GOTO ("iobuf", iobref, out);
-
-        LOCK (&iobref->lock);
-        {
-                iobref->ref++;
-        }
-        UNLOCK (&iobref->lock);
+        GF_ATOMIC_INC (iobref->ref);
 
 out:
         return iobref;
@@ -940,12 +907,7 @@ iobref_unref (struct iobref *iobref)
         int ref = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobref, out);
-
-        LOCK (&iobref->lock);
-        {
-                ref = (--iobref->ref);
-        }
-        UNLOCK (&iobref->lock);
+        ref = GF_ATOMIC_DEC (iobref->ref);
 
         if (!ref)
                 iobref_destroy (iobref);