From b69a4f1e31726260a07f883cefac28ed29cc6a12 Mon Sep 17 00:00:00 2001 From: "Anand V. Avati" Date: Thu, 12 Aug 2010 05:05:01 +0000 Subject: portmap functionality in glusterd Signed-off-by: Anand V. Avati Signed-off-by: Anand V. Avati BUG: 1314 (portmapper functionality) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1314 --- xlators/mgmt/glusterd/src/Makefile.am | 4 +- xlators/mgmt/glusterd/src/glusterd-pmap.c | 336 +++++++++++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-pmap.h | 55 +++++ xlators/mgmt/glusterd/src/glusterd-utils.c | 12 +- xlators/mgmt/glusterd/src/glusterd.c | 8 + xlators/mgmt/glusterd/src/glusterd.h | 3 + 6 files changed, 415 insertions(+), 3 deletions(-) create mode 100644 xlators/mgmt/glusterd/src/glusterd-pmap.c create mode 100644 xlators/mgmt/glusterd/src/glusterd-pmap.h (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index adc7ee30bf9..f3abd96ef9e 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -2,7 +2,9 @@ xlator_LTLIBRARIES = glusterd.la xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt glusterd_la_LDFLAGS = -module -avoidversion glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c glusterd-op-sm.c \ - glusterd-utils.c glusterd3_1-mops.c glusterd-store.c glusterd-handshake.c + glusterd-utils.c glusterd3_1-mops.c glusterd-store.c glusterd-handshake.c \ + glusterd-pmap.c + glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\ $(top_builddir)/rpc/xdr/src/libgfxdr.la\ $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c new file mode 100644 index 00000000000..76ad7298764 --- /dev/null +++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c @@ -0,0 +1,336 @@ +/* + Copyright (c) 2010 Gluster, Inc. + This file is part of GlusterFS. + + GlusterFS is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + GlusterFS is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see + . +*/ + + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "xlator.h" +#include "glusterfs.h" +#include "compat-errno.h" + +#include "glusterd.h" +#include "glusterd-utils.h" + +#include "portmap.h" +#include "protocol-common.h" +#include "rpcsvc.h" + +#include +#include +#include + + +int +pmap_port_isfree (int port) +{ + struct sockaddr_in sin; + int sock = -1; + int ret = 0; + + memset (&sin, 0, sizeof (sin)); + sin.sin_family = PF_INET; + sin.sin_port = hton16 (port); + + sock = socket (PF_INET, SOCK_STREAM, 0); + if (sock == -1) + return -1; + + ret = bind (sock, (struct sockaddr *)&sin, sizeof (sin)); + close (sock); + + return ret == 0 ? 1 : 0; +} + + +struct pmap_registry * +pmap_registry_new (void) +{ + struct pmap_registry *pmap = NULL; + int i = 0; + + pmap = CALLOC (sizeof (*pmap), 1); + if (!pmap) + return NULL; + + for (i = 0; i < 65536; i++) { + if (!pmap_port_isfree (i)) + pmap->ports[i].used = 1; + } + + pmap->base_port = 6969; + pmap->last_alloc = 6969; + + return pmap; +} + + +struct pmap_registry * +pmap_registry_get (xlator_t *this) +{ + glusterd_conf_t *priv = NULL; + struct pmap_registry *pmap = NULL; + + priv = this->private; + + pmap = priv->pmap; + if (!pmap) { + pmap = pmap_registry_new (); + if (!pmap) + return NULL; + priv->pmap = pmap; + } + + return pmap; +} + + +int +pmap_registry_search (xlator_t *this, const char *brickname) +{ + glusterd_conf_t *priv = NULL; + struct pmap_registry *pmap = NULL; + int p = 0; + int port = 0; + + priv = this->private; + + pmap = priv->pmap; + + for (p = pmap->base_port; p < 65535; p++) { + if (strcmp (pmap->ports[p].brickname, brickname) == 0) { + port = p; + break; + } + } + + return port; +} + + +int +pmap_registry_alloc (xlator_t *this) +{ + glusterd_conf_t *priv = NULL; + struct pmap_registry *pmap = NULL; + int p = 0; + int port = 0; + + priv = this->private; + + pmap = priv->pmap; + + for (p = pmap->base_port; p < 65535; p++) { + if (pmap->ports[p].used) + continue; + + if (pmap_port_isfree (p)) { + pmap->ports[p].used = 1; + port = p; + break; + } + } + + return port; +} + + + +int +pmap_registry_bind (xlator_t *this, int port, const char *brickname) +{ + glusterd_conf_t *priv = NULL; + struct pmap_registry *pmap = NULL; + int p = 0; + + + priv = this->private; + + pmap = priv->pmap; + + pmap->ports[p].used = 1; + if (pmap->ports[p].brickname) + free (pmap->ports[p].brickname); + pmap->ports[p].brickname = strdup (brickname); + + return 0; +} + + +typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data); + + +static int +xdr_to_glusterfs_req (rpcsvc_request_t *req, void *arg, gfs_serialize_t sfunc) +{ + int ret = -1; + + if (!req) + return -1; + + ret = sfunc (req->msg[0], arg); + + if (ret > 0) + ret = 0; + + return ret; +} + + +int +gluster_pmap_portbybrick (rpcsvc_request_t *req) +{ + pmap_port_by_brick_req args = {0,}; + pmap_port_by_brick_rsp rsp = {0,}; + char *brick = NULL; + int port = 0; + + + if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_port_by_brick_req)) { + req->rpc_err = GARBAGE_ARGS; + goto fail; + } + + brick = args.brick; + + port = pmap_registry_search (THIS, brick); + + rsp.op_ret = 0; + rsp.port = port; + +fail: + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (gd_serialize_t)xdr_from_pmap_port_by_brick_rsp); + + return 0; +} + + +int +gluster_pmap_brickbyport (rpcsvc_request_t *req) +{ + pmap_brick_by_port_req args = {0,}; + pmap_brick_by_port_rsp rsp = {0,}; + + + if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_brick_by_port_req)) { + req->rpc_err = GARBAGE_ARGS; + goto fail; + } + + +fail: + + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (gd_serialize_t)xdr_from_pmap_brick_by_port_rsp); + + return 0; +} + + + +int +gluster_pmap_signup (rpcsvc_request_t *req) +{ + pmap_signup_req args = {0,}; + pmap_signup_rsp rsp = {0,}; + + + if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_signup_req)) { + req->rpc_err = GARBAGE_ARGS; + goto fail; + } + + +fail: + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (gd_serialize_t)xdr_from_pmap_signup_rsp); + + return 0; +} + + +int +gluster_pmap_signin (rpcsvc_request_t *req) +{ + pmap_signin_req args = {0,}; + pmap_signin_rsp rsp = {0,}; + + + if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_signin_req)) { + req->rpc_err = GARBAGE_ARGS; + goto fail; + } + + +fail: + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (gd_serialize_t)xdr_from_pmap_signin_rsp); + + return 0; +} + + + +int +gluster_pmap_signout (rpcsvc_request_t *req) +{ + pmap_signout_req args = {0,}; + pmap_signout_rsp rsp = {0,}; + + + if (xdr_to_glusterfs_req (req, &args, xdr_to_pmap_signout_req)) { + //failed to decode msg; + req->rpc_err = GARBAGE_ARGS; + goto fail; + } + + +fail: + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (gd_serialize_t)xdr_from_pmap_signout_rsp); + + return 0; +} + + +rpcsvc_actor_t gluster_pmap_actors[] = { + [GF_PMAP_NULL] = {"NULL", GF_HNDSK_NULL, NULL, NULL, NULL }, + [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, + gluster_pmap_portbybrick, NULL, NULL }, + [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, + gluster_pmap_brickbyport, NULL, NULL }, + [GF_PMAP_SIGNIN] = {"SIGNIN", GF_PMAP_SIGNIN, + gluster_pmap_signin, NULL, NULL }, + [GF_PMAP_SIGNOUT] = {"SIGNOUT", GF_PMAP_SIGNOUT, + gluster_pmap_signout, NULL, NULL }, + [GF_PMAP_SIGNUP] = {"SIGNUP", GF_PMAP_SIGNUP, + gluster_pmap_signup, NULL, NULL }, +}; + + +struct rpcsvc_program gluster_pmap_prog = { + .progname = "Gluster Portmap", + .prognum = GLUSTER_PMAP_PROGRAM, + .progver = GLUSTER_PMAP_VERSION, + .actors = gluster_pmap_actors, + .numactors = GF_PMAP_MAXVALUE, +}; diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h new file mode 100644 index 00000000000..be028157efa --- /dev/null +++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h @@ -0,0 +1,55 @@ +/* + Copyright (c) 2010 Gluster, Inc. + This file is part of GlusterFS. + + GlusterFS is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + GlusterFS is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see + . +*/ + +#ifndef _GLUSTERD_PMAP_H_ +#define _GLUSTERD_PMAP_H_ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include +#include "uuid.h" + +#include "glusterfs.h" +#include "xlator.h" +#include "logging.h" +#include "call-stub.h" +#include "fd.h" +#include "byte-order.h" +#include "glusterd.h" +#include "rpcsvc.h" + + +struct pmap_port_status { + int used; + char *brickname; +}; + +struct pmap_registry { + int base_port; + int last_alloc; + struct pmap_port_status ports[65536]; +}; + +int pmap_registry_alloc (xlator_t *this); +int pmap_registry_bind (xlator_t *this, int port, const char *brickname); + +#endif diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index c17582341c0..c135e75264f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -716,7 +716,7 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo) int32_t glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, - glusterd_brickinfo_t *brickinfo, + glusterd_brickinfo_t *brickinfo, int32_t count) { int32_t ret = -1; @@ -727,6 +727,7 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, char path[PATH_MAX] = {0,}; char cmd_str[8192] = {0,}; char rundir[PATH_MAX] = {0,}; + int port = 0; GF_ASSERT (volinfo); GF_ASSERT (brickinfo); @@ -746,12 +747,19 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, goto out; } + port = pmap_registry_alloc (THIS); + GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname, count); snprintf (volfile, PATH_MAX, "%s/%s-%s-%d.vol", path, brickinfo->hostname, volinfo->volname, count); - snprintf (cmd_str, 8192, "glusterfs -f %s -p %s", volfile, pidfile); + + snprintf (cmd_str, 8192, + "glusterfs --xlator-option server-tcp.listen-port=%d -f %s -p %s", + port, volfile, pidfile); ret = system (cmd_str); + if (ret == 0) + pmap_registry_bind (THIS, port, brickinfo->path); out: return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 7d888d944fb..d2b271a1b5f 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -49,6 +49,7 @@ static uuid_t glusterd_uuid; extern struct rpcsvc_program glusterd1_mop_prog; extern struct rpcsvc_program gluster_handshake_prog; extern struct rpc_clnt_program glusterd3_1_mgmt_prog; +extern struct rpcsvc_program gluster_pmap_prog; extern glusterd_op_info_t opinfo; @@ -267,6 +268,13 @@ init (xlator_t *this) goto out; } + ret = rpcsvc_program_register (rpc, gluster_pmap_prog); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "rpcsvc_program_register returned %d", ret); + goto out; + } + gluster_handshake_prog.options = this->options; ret = rpcsvc_program_register (rpc, gluster_handshake_prog); if (ret) diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 2f0937a326b..600381243de 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -42,6 +42,8 @@ #include "rpcsvc.h" #include "glusterd-sm.h" #include "glusterd1-xdr.h" +#include "glusterd-pmap.h" + #define GLUSTERD_MAX_VOLUME_NAME 1000 @@ -64,6 +66,7 @@ typedef struct { char workdir[PATH_MAX]; rpcsvc_t *rpc; rpc_clnt_prog_t *mgmt; + struct pmap_registry *pmap; struct list_head volumes; struct list_head hostnames; glusterd_store_handle_t *handle; -- cgit