From d4eb4ef28073fc8f5430802d9de9344e551711bb Mon Sep 17 00:00:00 2001 From: Shreyas Siravara Date: Tue, 8 Aug 2017 12:09:25 +0530 Subject: gfproxy: Introduce new server-side daemon called GFProxy Summmary: Adds a new server-side daemon called gfproxyd & a new FUSE client called gfproxy-client Updates: #242 BUG: 1428063 Change-Id: I83210098d3a381922bc64fed1110ae1b76e6519f Tested-by: Shreyas Siravara Reviewed-by: Kevin Vigor Signed-off-by: Shreyas Siravara Signed-off-by: Poornima G --- xlators/mgmt/glusterd/src/glusterd-handshake.c | 44 ++++ xlators/mgmt/glusterd/src/glusterd-utils.c | 39 ++++ xlators/mgmt/glusterd/src/glusterd-utils.h | 8 + xlators/mgmt/glusterd/src/glusterd-volgen.c | 277 ++++++++++++++++++++++-- xlators/mgmt/glusterd/src/glusterd-volgen.h | 3 +- xlators/mgmt/glusterd/src/glusterd-volume-set.c | 5 + xlators/mgmt/glusterd/src/glusterd.h | 5 + 7 files changed, 360 insertions(+), 21 deletions(-) (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index c7e419c5691..8c397325260 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -264,6 +264,50 @@ build_volfile_path (char *volume_id, char *path, } + volid_ptr = strstr (volume_id, "gfproxy-client/"); + if (volid_ptr) { + volid_ptr = strchr (volid_ptr, '/'); + if (!volid_ptr) { + ret = -1; + goto out; + } + volid_ptr++; + + ret = glusterd_volinfo_find (volid_ptr, &volinfo); + if (ret == -1) { + gf_log (this->name, GF_LOG_ERROR, + "Couldn't find volinfo"); + goto out; + } + + glusterd_get_gfproxy_client_volfile (volinfo, path, path_len); + + ret = 0; + goto out; + } + + volid_ptr = strstr (volume_id, "gfproxy/"); + if (volid_ptr) { + volid_ptr = strchr (volid_ptr, '/'); + if (!volid_ptr) { + ret = -1; + goto out; + } + volid_ptr++; + + ret = glusterd_volinfo_find (volid_ptr, &volinfo); + if (ret == -1) { + gf_log (this->name, GF_LOG_ERROR, + "Couldn't find volinfo"); + goto out; + } + + glusterd_get_gfproxyd_volfile (volinfo, path, path_len); + + ret = 0; + goto out; + } + volid_ptr = strstr (volume_id, "/snaps/"); if (volid_ptr) { ret = get_snap_volname_and_volinfo (volid_ptr, &volname, diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index fd70516a289..623a3841f89 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -12515,6 +12515,45 @@ out: return ret; } +void +glusterd_get_gfproxy_client_volfile (glusterd_volinfo_t *volinfo, + char *path, int path_len) +{ + char workdir[PATH_MAX] = {0, }; + glusterd_conf_t *priv = THIS->private; + + GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv); + + switch (volinfo->transport_type) { + case GF_TRANSPORT_TCP: + snprintf (path, path_len, + "%s/trusted-%s.tcp-gfproxy-fuse.vol", + workdir, volinfo->volname); + break; + + case GF_TRANSPORT_RDMA: + snprintf (path, path_len, + "%s/trusted-%s.rdma-gfproxy-fuse.vol", + workdir, volinfo->volname); + break; + default: + break; + } +} + +void +glusterd_get_gfproxyd_volfile (glusterd_volinfo_t *volinfo, + char *path, int path_len) +{ + char workdir[PATH_MAX] = {0, }; + glusterd_conf_t *priv = THIS->private; + + GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv); + + snprintf (path, path_len, "%s/%s.gfproxyd.vol", workdir, + volinfo->volname); +} + void glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo, char *path, int path_len) diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 65ea7565d3a..7e94c94ffc0 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -738,6 +738,14 @@ void glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo, char *path, int path_len); +void +glusterd_get_gfproxy_client_volfile (glusterd_volinfo_t *volinfo, + char *path, int path_len); + +void +glusterd_get_gfproxyd_volfile (glusterd_volinfo_t *volinfo, + char *path, int path_len); + int32_t glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo, glusterd_brickinfo_t *dup_brickinfo); diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index f3d4d08bf05..93b6e95d5f9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -58,6 +58,20 @@ extern struct volopt_map_entry glusterd_volopt_map[]; } \ } while (0 /* CONSTCOND */) +/** + * Needed for GFProxy + */ +#define GF_PROXY_DAEMON_PORT 40000 +#define GF_PROXY_DAEMON_PORT_STR "40000" + +static int +volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *set_dict, void *param); + +static int +build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *mod_dict); + /********************************************* * * xlator generation / graph manipulation API @@ -1466,6 +1480,75 @@ server_spec_extended_option_handler (volgen_graph_t *graph, static void get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo); +static int +gfproxy_server_graph_builder (volgen_graph_t *graph, + glusterd_volinfo_t *volinfo, + dict_t *set_dict, void *param) +{ + xlator_t *xl = NULL; + /*char *value = NULL;*/ + char transt[16] = {0, }; + char key[1024] = {0, }; + /*char port_str[7] = {0, };*/ + int ret = 0; + char *username = NULL; + char *password = NULL; + /*int rclusters = 0;*/ + + /* We are a trusted client */ + ret = dict_set_uint32 (set_dict, "trusted-client", GF_CLIENT_TRUSTED); + if (ret != 0) + goto out; + + ret = dict_set_str (set_dict, "gfproxy-server", "on"); + if (ret != 0) + goto out; + + /* Build the client section of the graph first */ + build_client_graph (graph, volinfo, set_dict); + + /* Clear this setting so that future users of set_dict do not end up + * thinking they are a gfproxy server */ + dict_del (set_dict, "gfproxy-server"); + dict_del (set_dict, "trusted-client"); + + /* Then add the server to it */ + get_vol_transport_type (volinfo, transt); + xl = volgen_graph_add (graph, "protocol/server", volinfo->volname); + if (!xl) + goto out; + + ret = xlator_set_option (xl, "listen-port", GF_PROXY_DAEMON_PORT_STR); + if (ret != 0) + goto out; + + ret = xlator_set_option (xl, "transport-type", transt); + if (ret != 0) + goto out; + + /* Set username and password */ + username = glusterd_auth_get_username (volinfo); + password = glusterd_auth_get_password (volinfo); + if (username) { + snprintf (key, sizeof (key), "auth.login.%s-server.allow", + volinfo->volname); + ret = xlator_set_option (xl, key, username); + if (ret) + return -1; + } + + if (password) { + snprintf (key, sizeof (key), "auth.login.%s.password", + username); + ret = xlator_set_option (xl, key, password); + if (ret != 0) + goto out; + } + +out: + return ret; +} + static int brick_graph_add_posix (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, dict_t *set_dict, glusterd_brickinfo_t *brickinfo) @@ -2718,6 +2801,42 @@ out: return -1; } +static int +gfproxy_server_perfxl_option_handler (volgen_graph_t *graph, + struct volopt_map_entry *vme, + void *param) +{ + + GF_ASSERT (param); + + /* write-behind is the *not* allowed for gfproxy-servers */ + if (strstr (vme->key, "write-behind")) { + return 0; + } + + perfxl_option_handler (graph, vme, param); + + return 0; +} + +static int +gfproxy_client_perfxl_option_handler (volgen_graph_t *graph, + struct volopt_map_entry *vme, + void *param) +{ + + GF_ASSERT (param); + + /* write-behind is the only allowed "perf" for gfproxy-clients */ + if (!strstr (vme->key, "write-behind")) + return 0; + + perfxl_option_handler (graph, vme, param); + + return 0; +} + + static int nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme, void *param) @@ -2946,8 +3065,10 @@ _free_xlator_opt_key (char *key) } static xlator_t * -volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, - char *hostname, char *subvol, char *xl_id, +volgen_graph_build_client (volgen_graph_t *graph, + glusterd_volinfo_t *volinfo, + char *hostname, char *port, + char *subvol, char *xl_id, char *transt, dict_t *set_dict) { xlator_t *xl = NULL; @@ -2978,6 +3099,12 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, goto err; } + if (port) { + ret = xlator_set_option (xl, "remote-port", port); + if (ret) + goto err; + } + ret = xlator_set_option (xl, "remote-subvolume", subvol); if (ret) goto err; @@ -3001,7 +3128,8 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, ret = dict_get_uint32 (set_dict, "trusted-client", &client_type); - if (!ret && client_type == GF_CLIENT_TRUSTED) { + if (!ret && (client_type == GF_CLIENT_TRUSTED + || client_type == GF_CLIENT_TRUSTED_PROXY)) { str = NULL; str = glusterd_auth_get_username (volinfo); if (str) { @@ -3085,7 +3213,9 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, i = 0; cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) { xl = volgen_graph_build_client (graph, volinfo, - brick->hostname, brick->path, + brick->hostname, + NULL, + brick->path, brick->brick_id, transt, set_dict); if (!xl) { @@ -3317,8 +3447,9 @@ volgen_graph_build_snapview_client (volgen_graph_t *graph, get_transport_type (volinfo, set_dict, transt, _gf_false); - prot_clnt = volgen_graph_build_client (graph, volinfo, NULL, subvol, - xl_id, transt, set_dict); + prot_clnt = volgen_graph_build_client (graph, volinfo, + NULL, NULL, subvol, + xl_id, transt, set_dict); if (!prot_clnt) { ret = -1; goto out; @@ -3932,15 +4063,36 @@ static int client_graph_set_perf_options(volgen_graph_t *graph, char *volname = NULL; int ret = 0; + /* + * Logic to make sure gfproxy-client gets custom performance translators + */ + ret = dict_get_str_boolean (set_dict, "gfproxy-client", 0); + if (ret == 1) { + return volgen_graph_set_options_generic ( + graph, set_dict, volinfo, + &gfproxy_client_perfxl_option_handler); + } + + /* + * Logic to make sure gfproxy-server gets custom performance translators + */ + ret = dict_get_str_boolean (set_dict, "gfproxy-server", 0); + if (ret == 1) { + return volgen_graph_set_options_generic ( + graph, set_dict, volinfo, + &gfproxy_server_perfxl_option_handler); + } + /* * Logic to make sure NFS doesn't have performance translators by * default for a volume */ - volname = volinfo->volname; ret = client_graph_set_rda_options (graph, volinfo, set_dict); if (ret < 0) return ret; + volname = volinfo->volname; + tmp_data = dict_get (set_dict, "nfs-volume-file"); if (!tmp_data) return volgen_graph_set_options_generic(graph, set_dict, @@ -4172,25 +4324,48 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, gf_boolean_t ob = _gf_false; int uss_enabled = -1; xlator_t *this = THIS; + char *subvol = NULL; + size_t subvol_namelen = 0; GF_ASSERT (this); GF_ASSERT (conf); - volname = volinfo->volname; - ret = volgen_graph_build_clients (graph, volinfo, set_dict, - param); - if (ret) + ret = dict_get_str_boolean (set_dict, "gfproxy-client", 0); + if (ret == -1) goto out; - if (volinfo->type == GF_CLUSTER_TYPE_TIER) - ret = volume_volgen_graph_build_clusters_tier - (graph, volinfo, _gf_false); - else - ret = volume_volgen_graph_build_clusters - (graph, volinfo, _gf_false); + volname = volinfo->volname; + if (ret == 0) { + ret = volgen_graph_build_clients (graph, volinfo, set_dict, + param); + if (ret) + goto out; - if (ret == -1) - goto out; + if (volinfo->type == GF_CLUSTER_TYPE_TIER) + ret = volume_volgen_graph_build_clusters_tier + (graph, volinfo, _gf_false); + else + ret = volume_volgen_graph_build_clusters + (graph, volinfo, _gf_false); + + if (ret == -1) + goto out; + } else { + ret = dict_get_str (set_dict, + "config.gfproxyd-remote-host", &tmp); + if (ret == -1) + goto out; + + subvol_namelen = strlen (volinfo->volname) + + strlen ("-server") + 1; + subvol = alloca (subvol_namelen); + snprintf (subvol, subvol_namelen, + "%s-server", volinfo->volname); + + volgen_graph_build_client (graph, volinfo, tmp, + GF_PROXY_DAEMON_PORT_STR, subvol, + "gfproxy", "tcp", set_dict); + } ret = dict_get_str_boolean (set_dict, "features.shard", _gf_false); if (ret == -1) @@ -5325,6 +5500,21 @@ get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo, brickinfo->hostname, brick); } +static void +get_gfproxyd_filepath (char *filename, glusterd_volinfo_t *volinfo) +{ + char path[PATH_MAX] = {0, }; + glusterd_conf_t *priv = NULL; + + priv = THIS->private; + + GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv); + + snprintf (filename, PATH_MAX, + "%s/%s.gfproxyd.vol", path, + volinfo->volname); +} + gf_boolean_t glusterd_is_valid_volfpath (char *volname, char *brick) { @@ -5369,6 +5559,32 @@ out: return ret; } +static int +glusterd_generate_gfproxyd_volfile (glusterd_volinfo_t *volinfo) +{ + volgen_graph_t graph = {0, }; + char filename[PATH_MAX] = {0, }; + int ret = -1; + + GF_ASSERT (volinfo); + + get_gfproxyd_filepath (filename, volinfo); + + struct glusterd_gfproxyd_info info = { + .port = GF_PROXY_DAEMON_PORT, + }; + + ret = build_graph_generic (&graph, volinfo, + NULL, &info, + &gfproxy_server_graph_builder); + if (ret == 0) + ret = volgen_write_volfile (&graph, filename); + + volgen_graph_free (&graph); + + return ret; +} + static int glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *brickinfo, @@ -5661,7 +5877,8 @@ glusterd_generate_client_per_brick_volfile (glusterd_volinfo_t *volinfo) cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) { xl = volgen_graph_build_client (&graph, volinfo, - brick->hostname, brick->path, + brick->hostname, + NULL, brick->path, brick->brick_id, "tcp", dict); if (!xl) { @@ -5792,6 +6009,11 @@ generate_client_volfiles (glusterd_volinfo_t *volinfo, ret = glusterd_get_trusted_client_filepath (filepath, volinfo, type); + } else if (client_type == GF_CLIENT_TRUSTED_PROXY) { + glusterd_get_gfproxy_client_volfile (volinfo, + filepath, + PATH_MAX); + ret = dict_set_str (dict, "gfproxy-client", "on"); } else { ret = glusterd_get_client_filepath (filepath, volinfo, @@ -6037,6 +6259,7 @@ build_bitd_volume_graph (volgen_graph_t *graph, xl = volgen_graph_build_client (&cgraph, volinfo, brickinfo->hostname, + NULL, brickinfo->path, brickinfo->brick_id, transt, set_dict); @@ -6198,6 +6421,7 @@ build_scrub_volume_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, xl = volgen_graph_build_client (&cgraph, volinfo, brickinfo->hostname, + NULL, brickinfo->path, brickinfo->brick_id, transt, set_dict); @@ -6329,12 +6553,25 @@ glusterd_create_volfiles (glusterd_volinfo_t *volinfo) goto out; } + ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED_PROXY); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Could not generate gfproxy client volfiles"); + goto out; + } + ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER); if (ret) gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL, "Could not generate client volfiles"); + + ret = glusterd_generate_gfproxyd_volfile (volinfo); + if (ret) + gf_log (this->name, GF_LOG_ERROR, + "Could not generate gfproxy volfiles"); + out: return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h index aa4359c2589..2a8a6e6aed9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.h +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h @@ -57,7 +57,8 @@ typedef enum { GF_CLIENT_TRUSTED, - GF_CLIENT_OTHER + GF_CLIENT_OTHER, + GF_CLIENT_TRUSTED_PROXY, } glusterd_client_type_t; /* It indicates the type of volfile that the graph is built for */ diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index c4ad3152725..d3f31d0c343 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -2820,6 +2820,11 @@ struct volopt_map_entry glusterd_volopt_map[] = { .option = "!config", .op_version = 2 }, + { .key = "config.gfproxyd-remote-host", + .voltype = "configuration", + .option = "gfproxyd-remote-host", + .op_version = 2 + }, { .key = GLUSTERD_QUORUM_TYPE_KEY, .voltype = "mgmt/glusterd", .value = "off", diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 8beec6c29cf..281a6c198f1 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -254,6 +254,11 @@ struct glusterd_brick_proc { typedef struct glusterd_brick_proc glusterd_brick_proc_t; +struct glusterd_gfproxyd_info { + short port; + char *logfile; +}; + struct gf_defrag_brickinfo_ { char *name; int files; -- cgit