diff options
Diffstat (limited to 'xlators/mgmt/glusterd/src/glusterd-volgen.c')
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.c | 351 |
1 files changed, 340 insertions, 11 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index ae095bf7c..f42d596ba 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -103,7 +103,6 @@ xlator_instantiate_va (const char *type, const char *format, va_list arg) return NULL; } -#ifdef __not_used_as_of_now_ static xlator_t * xlator_instantiate (const char *type, const char *format, ...) { @@ -116,7 +115,6 @@ xlator_instantiate (const char *type, const char *format, ...) return xl; } -#endif static int volgen_xlator_link (xlator_t *pxl, xlator_t *cxl) @@ -1445,6 +1443,312 @@ server_spec_extended_option_handler (volgen_graph_t *graph, static void get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo); +xlator_t * +add_one_peer (volgen_graph_t *graph, glusterd_brickinfo_t *peer, + char *volname, uint16_t index) +{ + xlator_t *kid; + + kid = volgen_graph_add_nolink (graph, "protocol/client", + "%s-client-%u", volname, + index++); + if (!kid) { + return NULL; + } + + /* TBD: figure out where to get the proper transport list */ + if (xlator_set_option(kid,"transport-type","socket")) { + return NULL; + } + if (xlator_set_option(kid,"remote-host",peer->hostname)) { + return NULL; + } + if (xlator_set_option(kid,"remote-subvolume",peer->path)) { + return NULL; + } + /* TBD: deal with RDMA, SSL */ + + return kid; +} + +void +assign_groups (glusterd_volinfo_t *volinfo) +{ + glusterd_brickinfo_t *brickinfo = NULL; + uint16_t group_num = 0; + int in_group = 0; + uuid_t tmp_uuid; + + list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + if (in_group == 0) { + uuid_generate(tmp_uuid); + } + brickinfo->group = group_num; + uuid_copy(brickinfo->nsr_uuid,tmp_uuid); + if (++in_group >= volinfo->replica_count) { + in_group = 0; + ++group_num; + } + } +} + +int +add_nsr_stuff (volgen_graph_t *graph, char *volname, + glusterd_brickinfo_t *brickinfo, glusterd_volinfo_t *volinfo, + char *changelog_basepath) +{ + xlator_t *me; + xlator_t *kid; + glusterd_brickinfo_t *peer; + uint16_t index = 0; + //uint32_t i=0; + char *leader_opt; + uint32_t replica_group_size = 1; + char dst[NSR_MAX_PATH_SIZE]; + char local_path[NSR_MAX_PATH_SIZE]; + char local_name[NSR_MAX_PATH_SIZE]; + char hosts[NSR_MAX_PATH_SIZE * NSR_MAX_REPLICA_GROUP_SIZE]; + char remote_names[NSR_MAX_REPLICA_GROUP_SIZE * NSR_MAX_PATH_SIZE]; + char filepath[PATH_MAX] = {0,}; + char lp[PATH_MAX] = {0,}; + xlator_t *xl = NULL; + char s[256]; + char transt[16] = {0,}; + char auth[256]; + char c_d[NSR_MAX_PATH_SIZE]; + char *username = NULL, *password = NULL; + gf_boolean_t enable_recon = _gf_false; + static uint32_t nsr_port = 27000; + + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr.recon") > 0) { + enable_recon = _gf_true; + } + + volgen_graph_t ng = {0,}; + char path[PATH_MAX] = {0,}; + char *ptr = NULL, *this = NULL, *that = NULL; + glusterd_conf_t *priv = NULL; + + + priv = THIS->private; + remote_names[0] = '\0'; + that = gf_strdup (brickinfo->hostname); + this = gf_strdup (brickinfo->path); + ptr = strchr (this, '/'); + while (ptr) { + *ptr = '-'; + ptr = strchr (this, '/'); + } + GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv); + snprintf (dst, PATH_MAX, + "%s/%s/%s:%s", + path, + GLUSTERD_BRICK_INFO_DIR, + that, + this); + + /* Create the NSR xlator, but defer linkage for now. */ + me = xlator_instantiate ("cluster/nsr", "%s-nsr", volname); + if (!me || volgen_xlator_link(me,first_of(graph))) { + return -1; + } + + strcpy(local_name, brickinfo->hostname); + strcpy(local_path, brickinfo->hostname); + strcat(local_name, ":"); + strcat(local_name, brickinfo->path); + strcpy(hosts, brickinfo->hostname); + + peer = list_prev (brickinfo, &volinfo->bricks, + glusterd_brickinfo_t, brick_list); + /* Check leader status while we have this pointer in hand. */ + leader_opt = (!peer || (peer->group != brickinfo->group)) ? "yes" + : "no"; + if (xlator_set_option(me,"vol-name",volname)) + return -1; + if (xlator_set_option(me,"my-name",local_name)) + return -1; + if (xlator_set_option(me,"leader",leader_opt)) + return -1; + if (xlator_set_option(me,"subvol-uuid", + uuid_utoa(brickinfo->nsr_uuid))) { + return -1; + } + +#define FILL_REMOTE_NAMES { \ + strcat(remote_names, \ + peer->hostname); \ + strcat(remote_names, \ + ":"); \ + strcat(remote_names, \ + peer->path); \ + strcat(remote_names, \ + ","); \ + strcat(hosts, ","); \ + strcat(hosts, \ + peer->hostname); \ + replica_group_size++; \ +} + + /* Now get on with the show. */ + while (peer) { + if (peer->group != brickinfo->group) { + break; + } + gf_log ("glusterd", GF_LOG_INFO, + "%s:%s needs client for %s:%s", + brickinfo->hostname, brickinfo->path, + peer->hostname, peer->path); + kid = add_one_peer (graph, peer, volname, index++); + if (!kid || volgen_xlator_link(me,kid)) { + return -1; + } + FILL_REMOTE_NAMES; + peer = list_prev (peer, &volinfo->bricks, + glusterd_brickinfo_t, brick_list); + } + + peer = list_next (brickinfo, &volinfo->bricks, + glusterd_brickinfo_t, brick_list); + while (peer) { + if (peer->group != brickinfo->group) { + break; + } + gf_log ("glusterd", GF_LOG_INFO, + "%s:%s needs client for %s:%s", + brickinfo->hostname, brickinfo->path, + peer->hostname, peer->path); + kid = add_one_peer (graph, peer, volname, index++); + if (!kid || volgen_xlator_link(me,kid)) { + return -1; + } + FILL_REMOTE_NAMES; + peer = list_next (peer, &volinfo->bricks, + glusterd_brickinfo_t, brick_list); + } + + // to remove the final "," + if (strlen(remote_names)) { + remote_names[strlen(remote_names) - 1] = '\0'; + } + if (xlator_set_option(me,"etcd-servers",hosts)) + return -1; + + // Finish linkage to client file + glusterfs_graph_set_first(&graph->graph,me); + + if (enable_recon == _gf_false) + return 0; + + /* Now fill in the various files required for reconciliation */ + snprintf (filepath, PATH_MAX, + "%s-nsr-recon.vol", + dst); + gf_log ("glusterd", GF_LOG_INFO, + "writing nsr recon volfile in %s\n", + filepath); +#if 0 + strcpy(lp, local_name); +#else + strcpy(lp, brickinfo->path); +#endif + strcat(lp,"/recon"); + bzero(&ng, sizeof(ng)); + xl = volgen_graph_add_as (&ng, "cluster/nsr_recon",lp); + if (!xl) + return -1; + sprintf(s,"%d",replica_group_size); + if (xlator_set_option(xl, "replica-group-size", s) == -1) + return -1; + if (xlator_set_option(xl, "local-member", local_name) == -1) + return -1; + if (xlator_set_option(xl, "replica-group-members", remote_names) == -1) + return -1; + if (xlator_set_option(xl,"vol-name",volname)) + return -1; + if (xlator_set_option(xl,"changelog-dir",changelog_basepath)) + return -1; + if (xlator_set_option(xl,"base-dir",brickinfo->path)) + return -1; + + xl = volgen_graph_add (&ng, "protocol/server", lp); + if (!xl) + return -1; + get_vol_transport_type (volinfo, transt); + if(xlator_set_option (xl, "transport-type", transt) == -1) + return -1; + sprintf(s,"%d",nsr_port); + if(xlator_set_option (xl, "transport.socket.listen-port", s) == -1) + return -1; + strcpy(auth, "auth.addr."); + strcat(auth, lp); + strcat(auth, ".allow"); + if(xlator_set_option (xl, auth, "*") == -1) + return -1; + if(xlator_set_option (xl, "rpc-auth.auth-null", "off") == -1) + return -1; + if(xlator_set_option (xl, "rpc-auth.auth-unix", "off") == -1) + return -1; + if(xlator_set_option (xl, "rpc-auth.auth-glusterfs", "off") == -1) + return -1; + if(volgen_write_volfile(&ng, filepath) == -1) + return -1; + + bzero(&ng, sizeof(ng)); + kid = volgen_graph_add_nolink (&ng, "protocol/client", + "%s-client-%u", lp, 0); + if (!kid) + return -1; + if (xlator_set_option(kid,"remote-host",brickinfo->hostname)) + return -1; +#if 0 + strcpy(lp, brickinfo->path); + strcat(lp,"/recon"); +#endif + if (xlator_set_option(kid,"remote-subvolume",lp)) + return -1; + if(xlator_set_option (kid, "transport-type", transt) == -1) + return -1; + sprintf(s,"%d",nsr_port++); + if(xlator_set_option (kid, "remote-port", s) == -1) + return -1; + snprintf (c_d, PATH_MAX, + "%s/%s/con:%s:%s", + path, + GLUSTERD_BRICK_INFO_DIR, + that, this); + if (volgen_write_volfile(&ng, c_d)) + return -1; + + bzero(&ng, sizeof(ng)); + kid = volgen_graph_add_nolink (&ng, "protocol/client", + "%s-client-%u", lp, 0); + if (!kid) + return -1; + if (xlator_set_option(kid,"remote-host",brickinfo->hostname)) + return -1; + if (xlator_set_option(kid,"remote-subvolume",brickinfo->path)) + return -1; + if(xlator_set_option (kid, "transport-type", transt) == -1) + return -1; + username = glusterd_auth_get_username (volinfo); + password = glusterd_auth_get_password (volinfo); + if(xlator_set_option (kid, "username", username) == -1) + return -1; + if(xlator_set_option (kid, "password", password) == -1) + return -1; + snprintf (c_d, PATH_MAX, + "%s/%s/data:%s:%s", + path, + GLUSTERD_BRICK_INFO_DIR, that, + this); + if (volgen_write_volfile(&ng, c_d)) + return -1; + + return 0; + +} + static int server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, dict_t *set_dict, void *param) @@ -1561,10 +1865,17 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, if (ret) return -1; + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + ret = xlator_set_option (xl, "encoding", "ascii"); + if (ret) + return -1; + } + ret = check_and_add_debug_xl (graph, set_dict, volname, "changelog"); if (ret) return -1; + xl = volgen_graph_add (graph, "features/access-control", volname); if (!xl) return -1; @@ -1643,9 +1954,19 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, return -1; } - xl = volgen_graph_add (graph, "features/index", volname); - if (!xl) - return -1; + /* TBD: conditionalize on NSR being enabled */ + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, + changelog_basepath); + if (ret) { + return -1; + } + } + else { + xl = volgen_graph_add (graph, "features/index", volname); + if (!xl) + return -1; + } snprintf (index_basepath, sizeof (index_basepath), "%s/%s", path, ".glusterfs/indices"); @@ -2470,8 +2791,8 @@ volume_volgen_graph_build_clusters (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, gf_boolean_t is_quotad) { - char *replicate_args[] = {"cluster/replicate", - "%s-replicate-%d"}; + char *replicate_type = "cluster/replicate"; + char *replicate_fmt = "%s-replicate-%d"; char *stripe_args[] = {"cluster/stripe", "%s-stripe-%d"}; int rclusters = 0; @@ -2485,12 +2806,16 @@ volume_volgen_graph_build_clusters (volgen_graph_t *graph, if (volinfo->dist_leaf_count == 1) goto build_distribute; + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + replicate_type = "cluster/nsrc"; + } + /* All other cases, it will have one or the other cluster type */ switch (volinfo->type) { case GF_CLUSTER_TYPE_REPLICATE: clusters = volgen_graph_build_clusters (graph, volinfo, - replicate_args[0], - replicate_args[1], + replicate_type, + replicate_fmt, volinfo->brick_count, volinfo->replica_count); if (clusters < 0) @@ -2510,8 +2835,8 @@ volume_volgen_graph_build_clusters (volgen_graph_t *graph, if (volinfo->replica_count == 0) goto out; clusters = volgen_graph_build_clusters (graph, volinfo, - replicate_args[0], - replicate_args[1], + replicate_type, + replicate_fmt, volinfo->brick_count, volinfo->replica_count); if (clusters < 0) @@ -3539,6 +3864,10 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo) } } + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + assign_groups(volinfo); + } + list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { gf_log ("", GF_LOG_DEBUG, "Found a brick - %s:%s", brickinfo->hostname, |