core: run many bricks within one glusterfsd process

This patch adds support for multiple brick translator stacks running in
a single brick server process.  This reduces our per-brick memory usage
by approximately 3x, and our appetite for TCP ports even more.  It also
creates potential to avoid process/thread thrashing, and to improve QoS
by scheduling more carefully across the bricks, but realizing that
potential will require further work.

Multiplexing is controlled by the "cluster.brick-multiplex" global
option.  By default it's off, and bricks are started in separate
processes as before.  If multiplexing is enabled, then *compatible*
bricks (mostly those with the same transport options) will be started in
the same process.

Backport of:
> Change-Id: I45059454e51d6f4cbb29a4953359c09a408695cb
> BUG: 1385758
> Reviewed-on: https://review.gluster.org/14763

Change-Id: I4bce9080f6c93d50171823298fdf920258317ee8
BUG: 1418091
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: https://review.gluster.org/16496
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>

1 files changed, 21 insertions, 30 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 00b84e076c3..bc6cddea7f7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -315,7 +315,7 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
 
         sleep (5);
 
-        ret = glusterd_rebalance_rpc_create (volinfo, _gf_false);
+        ret = glusterd_rebalance_rpc_create (volinfo);
 
         //FIXME: this cbk is passed as NULL in all occurrences. May be
         //we never needed it.
@@ -363,8 +363,7 @@ out:
 }
 
 int
-glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
-                               gf_boolean_t reconnect)
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo)
 {
         dict_t                  *options = NULL;
         char                     sockfile[PATH_MAX] = {0,};
@@ -383,35 +382,27 @@ glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
         if (!defrag)
                 goto out;
 
-        //rpc obj for rebalance process already in place.
-        if (glusterd_defrag_rpc_get (defrag)) {
-                ret = 0;
-                glusterd_defrag_rpc_put (defrag);
-                goto out;
-        }
         GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
-        /* If reconnecting check if defrag sockfile exists in the new location
+        /* Check if defrag sockfile exists in the new location
          * in /var/run/ , if it does not try the old location
          */
-        if (reconnect) {
-                ret = sys_stat (sockfile, &buf);
-                /* TODO: Remove this once we don't need backward compatibility
-                 * with the older path
-                 */
-                if (ret && (errno == ENOENT)) {
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                GD_MSG_FILE_OP_FAILED, "Rebalance sockfile "
-                                "%s does not exist. Trying old path.",
-                                sockfile);
-                        GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
-                                                           priv);
-                        ret =sys_stat (sockfile, &buf);
-                        if (ret && (ENOENT == errno)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_REBAL_NO_SOCK_FILE, "Rebalance "
-                                        "sockfile %s does not exist", sockfile);
-                                goto out;
-                        }
+        ret = sys_stat (sockfile, &buf);
+        /* TODO: Remove this once we don't need backward compatibility
+         * with the older path
+         */
+        if (ret && (errno == ENOENT)) {
+                gf_msg (this->name, GF_LOG_WARNING, errno,
+                        GD_MSG_FILE_OP_FAILED, "Rebalance sockfile "
+                        "%s does not exist. Trying old path.",
+                        sockfile);
+                GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
+                                                   priv);
+                ret =sys_stat (sockfile, &buf);
+                if (ret && (ENOENT == errno)) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_REBAL_NO_SOCK_FILE, "Rebalance "
+                                "sockfile %s does not exist", sockfile);
+                        goto out;
                 }
         }
 
@@ -429,7 +420,7 @@ glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
 
         glusterd_volinfo_ref (volinfo);
         ret = glusterd_rpc_create (&defrag->rpc, options,
-                                   glusterd_defrag_notify, volinfo);
+                                   glusterd_defrag_notify, volinfo, _gf_true);
         if (ret) {
                 gf_msg (THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
                         "Glusterd RPC creation failed");