mgmt/glusterd: add option to specify a different base-port

This is (arguably) a hack to work around a bug in libvirt which is not
well behaved wrt to using TCP ports in the unreserved space between
49152-65535. (See RFC 6335)

Normally glusterd starts and binds to the first available port in range,
usually 49152. libvirt's live migration also tries to use ports in this
range, but has no fallback to use (an)other port(s) when the one it wants
is already in use.

libvirt cannot fix this in time for their impending release. This is
submitted to gerrit to provide some minimal visibility upstream to justify
hacking this change (as a temporary patch) into the glusterfs-3.4.1 RPMs
for Fedora 18-21 until libvirt can fix their implementation.

Change-Id: Ie77b00ac60730d1e48907dd0b38ddae92f3ac345
BUG: 987555
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
Reviewed-on: http://review.gluster.org/6147
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Tested-by: Niels de Vos <ndevos@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavan Pichai <rpichai@redhat.com>

5 files changed, 19 insertions, 10 deletions

diff --git a/doc/glusterd.vol b/doc/glusterd.vol
index de17d8fd8f9..9bac52ab7fe 100644
--- a/doc/glusterd.vol
+++ b/doc/glusterd.vol
@@ -5,4 +5,5 @@ volume management
     option transport.socket.keepalive-time 10
     option transport.socket.keepalive-interval 2
     option transport.socket.read-fail-log off
+#   option base-port 49152
 end-volume
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index aab6744a418..7dec27cadc9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -52,8 +52,8 @@ pmap_port_isfree (int port)
 }
 
 
-struct pmap_registry *
-pmap_registry_new (void)
+static struct pmap_registry *
+pmap_registry_new (xlator_t *this)
 {
         struct pmap_registry *pmap = NULL;
         int                   i = 0;
@@ -69,8 +69,8 @@ pmap_registry_new (void)
                         pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
         }
 
-        pmap->base_port = GF_IANA_PRIV_PORTS_START;
-        pmap->last_alloc = GF_IANA_PRIV_PORTS_START;
+        pmap->base_port = ((glusterd_conf_t *)(this->private))->base_port;
+        pmap->last_alloc = ((glusterd_conf_t *)(this->private))->base_port;
 
         return pmap;
 }
@@ -86,7 +86,7 @@ pmap_registry_get (xlator_t *this)
 
         pmap = priv->pmap;
         if (!pmap) {
-                pmap = pmap_registry_new ();
+                pmap = pmap_registry_new (this);
                 if (!pmap)
                         return NULL;
                 priv->pmap = pmap;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index ae0c4e83be9..1790c5aec24 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1484,7 +1484,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                     strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
                                 gf_string2int (value, &brickinfo->port);
 
-                                if (brickinfo->port < GF_IANA_PRIV_PORTS_START){
+                                if (brickinfo->port < priv->base_port){
                                         /* This is required to adhere to the
                                            IANA standards */
                                         brickinfo->port = 0;
@@ -1500,8 +1500,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                     strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
                                 gf_string2int (value, &brickinfo->rdma_port);
 
-                                if (brickinfo->rdma_port <
-                                    GF_IANA_PRIV_PORTS_START){
+                                if (brickinfo->rdma_port < priv->base_port) {
                                         /* This is required to adhere to the
                                            IANA standards */
                                         brickinfo->rdma_port = 0;
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 785e67af66d..62c4a578555 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -916,7 +916,6 @@ init (xlator_t *this)
         int                first_time        = 0;
         char              *mountbroker_root  = NULL;
         int                i                 = 0;
-
 #ifdef DEBUG
         char              *valgrind_str      = NULL;
 #endif
@@ -1101,6 +1100,12 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
+        conf->base_port = GF_IANA_PRIV_PORTS_START;
+        if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "base-port override: %d", conf->base_port);
+        }
+
         /* Set option to run bricks on valgrind if enabled in glusterd.vol */
 #ifdef DEBUG
         conf->valgrind = _gf_false;
@@ -1116,7 +1121,6 @@ init (xlator_t *this)
                 }
         }
 #endif
-
         this->private = conf;
         (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
 
@@ -1309,5 +1313,9 @@ struct volume_options options[] = {
           .description = "Sets the quorum percentage for the trusted "
           "storage pool."
         },
+        { .key = {"base-port"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Sets the base port for portmap query"
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 00331258e97..10545749612 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -148,6 +148,7 @@ typedef struct {
         dict_t             *opts;
         synclock_t      big_lock;
         gf_boolean_t    restart_done;
+        uint32_t        base_port;
 } glusterd_conf_t;