libglusterfs: provide standardized atomic operations

The current macros ATOMIC_INCREMENT() and ATOMIC_DECREMENT() expect a
lock as first argument. There are at least two issues with this
approach:

  1. this lock is unused on architectures that have atomic operations
  2. some structures use a single lock for multiple variables

By defining a gf_atomic_t type, the unused lock can be removed, saving a
few bytes on modern architectures.

Because the gf_atomic_t type locates the lock for the variable (in case
of older architectures), each variable is protected the same on all
architectures. This makes the behaviour across all architectures more
equal (per variable locking, by a gf_lock_t or compiler optimization).

BUG: 1437037
Change-Id: Ic164892b06ea676e6a9566f8a98b7faf0efe76d6
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: https://review.gluster.org/16963
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

9 files changed, 143 insertions, 58 deletions

diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index e9e690ee4bd..2311c53645a 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -53,7 +53,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \
 	syncop-utils.h parse-utils.h libglusterfs-messages.h tw.h \
 	lvm-defaults.h quota-common-utils.h rot-buffs.h \
 	compat-uuid.h upcall-utils.h throttle-tbf.h events.h\
-	compound-fop-utils.h
+	compound-fop-utils.h atomic.h
 
 libglusterfs_ladir = $(includedir)/glusterfs
 
diff --git a/libglusterfs/src/atomic.h b/libglusterfs/src/atomic.h
new file mode 100644
index 00000000000..71fcb1ee972
--- /dev/null
+++ b/libglusterfs/src/atomic.h
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _ATOMIC_H
+#define _ATOMIC_H
+
+#include <inttypes.h>
+
+#if defined(HAVE_ATOMIC_BUILTINS) || defined(HAVE_SYNC_BUILTINS)
+/* optimized implementation, macros only */
+
+typedef struct gf_atomic_t {
+        int64_t    cnt;
+} gf_atomic_t;
+
+#if defined(HAVE_ATOMIC_BUILTINS)
+
+/* all macros have a 'gf_atomic_t' as 1st argument */
+#define GF_ATOMIC_INIT(op, n)     __atomic_store (&(op.cnt), __ATOMIC_RELEASE)
+#define GF_ATOMIC_GET(op)         __atomic_load (&(op.cnt), __ATOMIC_ACQUIRE)
+#define GF_ATOMIC_INC(op)         __atomic_add_and_fetch (&(op.cnt), 1, \
+                                                          __ATOMIC_ACQ_REL)
+#define GF_ATOMIC_DEC(op)         __atomic_sub_and_fetch (&(op.cnt), 1, \
+                                                          __ATOMIC_ACQ_REL)
+#define GF_ATOMIC_ADD(op, n)      __atomic_add_and_fetch (&(op.cnt), n, \
+                                                          __ATOMIC_ACQ_REL)
+#define GF_ATOMIC_SUB(op, n)      __atomic_sub_and_fetch (&(op.cnt), n, \
+                                                          __ATOMIC_ACQ_REL)
+
+#else /* !HAVE_ATOMIC_BUILTINS, but HAVE_SYNC_BUILTINS */
+
+/* all macros have a 'gf_atomic_t' as 1st argument */
+#define GF_ATOMIC_INIT(op, n)     ({ op.cnt = n; __sync_synchronize (); })
+#define GF_ATOMIC_GET(op)         __sync_add_and_fetch (&(op.cnt), 0)
+#define GF_ATOMIC_INC(op)         __sync_add_and_fetch (&(op.cnt), 1)
+#define GF_ATOMIC_DEC(op)         __sync_sub_and_fetch (&(op.cnt), 1)
+#define GF_ATOMIC_ADD(op, n)      __sync_add_and_fetch (&(op.cnt), n)
+#define GF_ATOMIC_SUB(op, n)      __sync_sub_and_fetch (&(op.cnt), n)
+
+#endif /* HAVE_ATOMIC_BUILTINS || HAVE_SYNC_BUILTINS */
+
+#else /* no HAVE_(ATOMIC|SYNC)_BUILTINS */
+/* fallback implementation, using small inline functions to improve type
+ * checking while compiling */
+
+#include "locking.h"
+
+typedef struct gf_atomic_t {
+        int64_t    cnt;
+        gf_lock_t  lk;
+} gf_atomic_t;
+
+
+static inline void
+gf_atomic_init (gf_atomic_t *op, int64_t cnt)
+{
+        LOCK_INIT (&op->lk);
+        op->cnt = cnt;
+}
+
+
+static inline uint64_t
+gf_atomic_get (gf_atomic_t *op)
+{
+        uint64_t ret;
+
+        LOCK (&op->lk);
+        {
+               ret = op->cnt;
+        }
+        UNLOCK (&op->lk);
+
+        return ret;
+}
+
+
+static inline int64_t
+gf_atomic_add (gf_atomic_t *op, int64_t n)
+{
+        uint64_t ret;
+
+        LOCK (&op->lk);
+        {
+                op->cnt += n;
+                ret = op->cnt;
+        }
+        UNLOCK (&op->lk);
+
+        return ret;
+}
+
+
+#define GF_ATOMIC_INIT(op, cnt)   gf_atomic_init (&op, cnt)
+#define GF_ATOMIC_GET(op)         gf_atomic_get (&op)
+#define GF_ATOMIC_INC(op)         gf_atomic_add (&op, 1)
+#define GF_ATOMIC_DEC(op)         gf_atomic_add (&op, -1)
+#define GF_ATOMIC_ADD(op, n)      gf_atomic_add (&op, n)
+#define GF_ATOMIC_SUB(op, n)      gf_atomic_add (&op, -n)
+
+#endif /* HAVE_ATOMIC_SYNC_OPS */
+
+#endif /* _ATOMIC_H */
diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c
index c20c4089ec3..1adfef5c7e3 100644
--- a/libglusterfs/src/client_t.c
+++ b/libglusterfs/src/client_t.c
@@ -192,8 +192,7 @@ gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid)
                                         memcmp (cred->authdata,
                                                 client->auth.data,
                                                 client->auth.len) == 0))) {
-                                INCREMENT_ATOMIC (client->ref.lock,
-                                                  client->ref.bind);
+                                GF_ATOMIC_INC (client->bind);
                                 goto unlock;
                         }
                 }
@@ -207,7 +206,6 @@ gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid)
                 client->this = this;
 
                 LOCK_INIT (&client->scratch_ctx.lock);
-                LOCK_INIT (&client->ref.lock);
 
                 client->client_uid = gf_strdup (client_uid);
                 if (client->client_uid == NULL) {
@@ -229,8 +227,8 @@ gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid)
                         goto unlock;
                 }
 
-                /* no need to do these atomically here */
-                client->ref.bind = client->ref.count = 1;
+                GF_ATOMIC_INIT (client->bind, 1);
+                GF_ATOMIC_INIT (client->count, 1);
 
                 client->auth.flavour = cred->flavour;
                 if (cred->flavour != AUTH_NONE) {
@@ -277,9 +275,10 @@ unlock:
 
         if (client)
                 gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
-                                  "%s: bind_ref: %d, ref: %d",
-                                  client->client_uid, client->ref.bind,
-                                  client->ref.count);
+                                  "%s: bind_ref: %"GF_PRI_ATOMIC", ref: "
+                                  "%"GF_PRI_ATOMIC, client->client_uid,
+                                  GF_ATOMIC_GET (client->bind),
+                                  GF_ATOMIC_GET (client->count));
         return client;
 }
 
@@ -295,14 +294,15 @@ gf_client_put (client_t *client, gf_boolean_t *detached)
         if (detached)
                 *detached = _gf_false;
 
-        bind_ref = DECREMENT_ATOMIC (client->ref.lock, client->ref.bind);
+        bind_ref = GF_ATOMIC_DEC (client->bind);
         if (bind_ref == 0)
                 unref = _gf_true;
 
         gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF, "%s: "
-                          "bind_ref: %d, ref: %d, unref: %d",
-                          client->client_uid, client->ref.bind,
-                          client->ref.count, unref);
+                          "bind_ref: %"GF_PRI_ATOMIC", ref: %"GF_PRI_ATOMIC", "
+                          "unref: %d", client->client_uid,
+                          GF_ATOMIC_GET (client->bind),
+                          GF_ATOMIC_GET (client->count), unref);
         if (unref) {
                 if (detached)
                         *detached = _gf_true;
@@ -322,10 +322,10 @@ gf_client_ref (client_t *client)
                 return NULL;
         }
 
-        INCREMENT_ATOMIC (client->ref.lock, client->ref.count);
+        GF_ATOMIC_INC (client->count);
         gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT, "%s: "
-                          "ref-count %d", client->client_uid,
-                          client->ref.count);
+                          "ref-count %"GF_PRI_ATOMIC, client->client_uid,
+                          GF_ATOMIC_GET (client->count));
         return client;
 }
 
@@ -360,7 +360,6 @@ client_destroy (client_t *client)
         clienttable = client->this->ctx->clienttable;
 
         LOCK_DESTROY (&client->scratch_ctx.lock);
-        LOCK_DESTROY (&client->ref.lock);
 
         LOCK (&clienttable->lock);
         {
@@ -419,7 +418,7 @@ gf_client_disconnect (client_t *client)
 void
 gf_client_unref (client_t *client)
 {
-        int refcount;
+        uint64_t refcount;
 
         if (!client) {
                 gf_msg_callingfn ("client_t", GF_LOG_ERROR, EINVAL,
@@ -427,10 +426,10 @@ gf_client_unref (client_t *client)
                 return;
         }
 
-        refcount = DECREMENT_ATOMIC (client->ref.lock, client->ref.count);
+        refcount = GF_ATOMIC_DEC (client->count);
         gf_msg_callingfn ("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT, "%s: "
-                          "ref-count %d", client->client_uid,
-                          (int)client->ref.count);
+                          "ref-count %"GF_PRI_ATOMIC, client->client_uid,
+                          refcount);
         if (refcount == 0) {
                 gf_msg (THIS->name, GF_LOG_INFO, 0, LG_MSG_DISCONNECT_CLIENT,
                         "Shutting down connection %s", client->client_uid);
@@ -586,7 +585,8 @@ client_dump (client_t *client, char *prefix)
                 return;
 
         memset(key, 0, sizeof key);
-        gf_proc_dump_write("refcount", "%d", client->ref.count);
+        gf_proc_dump_write("refcount", GF_PRI_ATOMIC,
+                           GF_ATOMIC_GET (client->count));
 }
 
 
@@ -780,7 +780,8 @@ gf_client_dump_fdtables (xlator_t *this)
 
                         gf_proc_dump_build_key (key, "conn", "%d.ref",
                                                         count);
-                        gf_proc_dump_write (key, "%d", client->ref.count);
+                        gf_proc_dump_write (key, GF_PRI_ATOMIC,
+                                            GF_ATOMIC_GET (client->count));
                         if (client->bound_xl) {
                                 gf_proc_dump_build_key (key, "conn",
                                                         "%d.bound_xl", count);
diff --git a/libglusterfs/src/client_t.h b/libglusterfs/src/client_t.h
index 29ea7f29ce8..31f1bd048ed 100644
--- a/libglusterfs/src/client_t.h
+++ b/libglusterfs/src/client_t.h
@@ -13,6 +13,7 @@
 
 #include "glusterfs.h"
 #include "locking.h"  /* for gf_lock_t, not included by glusterfs.h */
+#include "atomic.h" /* for gf_atomic_t */
 
 struct client_ctx {
         void     *ctx_key;
@@ -26,11 +27,8 @@ typedef struct _client_t {
                 unsigned short       count;
                 struct client_ctx   *ctx;
         }            scratch_ctx;
-        struct {
-                gf_lock_t            lock;
-                volatile int         bind;
-                volatile int         count;
-        }            ref;
+        gf_atomic_t                  bind;
+        gf_atomic_t                  count;
         xlator_t    *bound_xl;
         xlator_t    *this;
         int          tbl_index;
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 1c8547265d1..24ce0683f7a 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -90,23 +90,6 @@
 #define THIS (*__glusterfs_this_location())
 #define DECLARE_OLD_THIS        xlator_t *old_THIS = THIS
 
-/*
- * a more comprehensive feature test is shown at
- * http://lists.iptel.org/pipermail/semsdev/2010-October/005075.html
- * this is sufficient for RHEL5 i386 builds
- */
-#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) && !defined(__i386__)
-# define INCREMENT_ATOMIC(lk, op) __sync_add_and_fetch(&op, 1)
-# define DECREMENT_ATOMIC(lk, op) __sync_sub_and_fetch(&op, 1)
-#else
-/* These are only here for old gcc, e.g. on RHEL5 i386.
- * We're not ever going to use this in an if stmt,
- * but let's be pedantically correct for style points */
-# define INCREMENT_ATOMIC(lk, op) do { LOCK (&lk); ++op; UNLOCK (&lk); } while (0)
-/* this is a gcc 'statement expression', it works with llvm/clang too */
-# define DECREMENT_ATOMIC(lk, op) ({ LOCK (&lk); --op; UNLOCK (&lk); op; })
-#endif
-
 xlator_t **__glusterfs_this_location (void);
 xlator_t *glusterfs_this_get (void);
 int glusterfs_this_set (xlator_t *);
diff --git a/libglusterfs/src/logging.h b/libglusterfs/src/logging.h
index a6e318dc3fa..78d57888f35 100644
--- a/libglusterfs/src/logging.h
+++ b/libglusterfs/src/logging.h
@@ -37,6 +37,7 @@
 #endif
 #define GF_PRI_BLKSIZE     PRId32
 #define GF_PRI_SIZET       "zu"
+#define GF_PRI_ATOMIC      PRIu64
 
 #ifdef GF_DARWIN_HOST_OS
 #define GF_PRI_TIME "ld"
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index af839099ff8..4b600f4681a 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -80,7 +80,7 @@ gf_mem_set_acct_info (xlator_t *xl, char **alloc_ptr, size_t size,
         }
         UNLOCK(&xl->mem_acct->rec[type].lock);
 
-        INCREMENT_ATOMIC (xl->mem_acct->lock, xl->mem_acct->refcnt);
+        GF_ATOMIC_INC (xl->mem_acct->refcnt);
 
         header = (struct mem_header *) ptr;
         header->type = type;
@@ -326,7 +326,7 @@ __gf_free (void *free_ptr)
         }
         UNLOCK (&mem_acct->rec[header->type].lock);
 
-        if (DECREMENT_ATOMIC (mem_acct->lock, mem_acct->refcnt) == 0) {
+        if (GF_ATOMIC_DEC (mem_acct->refcnt) == 0) {
                 FREE (mem_acct);
         }
 
diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h
index 0dc186341b2..1b27119cf6c 100644
--- a/libglusterfs/src/mem-pool.h
+++ b/libglusterfs/src/mem-pool.h
@@ -13,6 +13,7 @@
 
 #include "list.h"
 #include "locking.h"
+#include "atomic.h"
 #include "logging.h"
 #include "mem-types.h"
 #include <stdlib.h>
@@ -48,14 +49,7 @@ struct mem_acct_rec {
 
 struct mem_acct {
         uint32_t            num_types;
-        /*
-         * The lock is only used on ancient platforms (e.g. RHEL5) to keep
-         * refcnt increment/decrement atomic.  We could even make its existence
-         * conditional on the right set of version/feature checks, but it's so
-         * lightweight that it's not worth the obfuscation.
-         */
-        gf_lock_t           lock;
-        unsigned int        refcnt;
+        gf_atomic_t         refcnt;
         struct mem_acct_rec rec[0];
 };
 
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index 0d09b3fbc82..408012e7846 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -596,8 +596,7 @@ xlator_mem_acct_init (xlator_t *xl, int num_types)
         memset (xl->mem_acct, 0, sizeof(struct mem_acct));
 
         xl->mem_acct->num_types = num_types;
-        LOCK_INIT (&xl->mem_acct->lock);
-        xl->mem_acct->refcnt = 1;
+        GF_ATOMIC_INIT (xl->mem_acct->refcnt, 1);
 
         for (i = 0; i < num_types; i++) {
                 memset (&xl->mem_acct->rec[i], 0, sizeof(struct mem_acct_rec));
@@ -654,7 +653,7 @@ xlator_memrec_free (xlator_t *xl)
                 for (i = 0; i < mem_acct->num_types; i++) {
                         LOCK_DESTROY (&(mem_acct->rec[i].lock));
                 }
-                if (DECREMENT_ATOMIC (mem_acct->lock, mem_acct->refcnt) == 0) {
+                if (GF_ATOMIC_DEC (mem_acct->refcnt) == 0) {
                         FREE (mem_acct);
                         xl->mem_acct = NULL;
                 }