diff options
Diffstat (limited to 'xlators/experimental/jbr-client')
| -rw-r--r-- | xlators/experimental/jbr-client/Makefile.am | 3 | ||||
| -rw-r--r-- | xlators/experimental/jbr-client/src/Makefile.am | 32 | ||||
| -rw-r--r-- | xlators/experimental/jbr-client/src/fop-template.c | 113 | ||||
| -rwxr-xr-x | xlators/experimental/jbr-client/src/gen-fops.py | 57 | ||||
| -rw-r--r-- | xlators/experimental/jbr-client/src/jbr-messages.h | 105 | ||||
| -rw-r--r-- | xlators/experimental/jbr-client/src/jbrc.c | 320 | ||||
| -rw-r--r-- | xlators/experimental/jbr-client/src/jbrc.h | 27 |
7 files changed, 657 insertions, 0 deletions
diff --git a/xlators/experimental/jbr-client/Makefile.am b/xlators/experimental/jbr-client/Makefile.am new file mode 100644 index 00000000000..a985f42a877 --- /dev/null +++ b/xlators/experimental/jbr-client/Makefile.am @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = diff --git a/xlators/experimental/jbr-client/src/Makefile.am b/xlators/experimental/jbr-client/src/Makefile.am new file mode 100644 index 00000000000..58f399f0607 --- /dev/null +++ b/xlators/experimental/jbr-client/src/Makefile.am @@ -0,0 +1,32 @@ +xlator_LTLIBRARIES = jbrc.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental + +nodist_jbrc_la_SOURCES = jbrc-cg.c +CLEANFILES = $(nodist_jbrc_la_SOURCES) + +jbrc_la_LDFLAGS = -module -avoid-version +jbrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la + +noinst_HEADERS = \ + $(top_srcdir)/xlators/lib/src/libxlator.h \ + $(top_srcdir)/glusterfsd/src/glusterfsd.h \ + jbrc.h jbr-messages.h + +AM_CPPFLAGS = $(GF_CPPFLAGS) \ + -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \ + -I$(top_srcdir)/rpc/rpc-lib/src + +AM_CFLAGS = -Wall $(GF_CFLAGS) + +JBRC_PREFIX = $(top_srcdir)/xlators/experimental/jbr-client/src +JBRC_GEN_FOPS = $(JBRC_PREFIX)/gen-fops.py +JBRC_TEMPLATES = $(JBRC_PREFIX)/fop-template.c +JBRC_WRAPPER = $(JBRC_PREFIX)/jbrc.c +noinst_PYTHON = $(JBRC_GEN_FOPS) +EXTRA_DIST = $(JBRC_TEMPLATES) $(JBRC_WRAPPER) + +jbrc-cg.c: $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER) + $(PYTHON) $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER) > $@ + +uninstall-local: + rm -f $(DESTDIR)$(xlatordir)/jbr.so diff --git a/xlators/experimental/jbr-client/src/fop-template.c b/xlators/experimental/jbr-client/src/fop-template.c new file mode 100644 index 00000000000..7719f511f01 --- /dev/null +++ b/xlators/experimental/jbr-client/src/fop-template.c @@ -0,0 +1,113 @@ +/* template-name fop */ +int32_t +jbrc_@NAME@ (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + jbrc_local_t *local = NULL; + xlator_t *target_xl = ACTIVE_CHILD(this); + + local = mem_get(this->local_pool); + if (!local) { + goto err; + } + + local->stub = fop_@NAME@_stub (frame, jbrc_@NAME@_continue, + @SHORT_ARGS@); + if (!local->stub) { + goto err; + } + local->curr_xl = target_xl; + local->scars = 0; + + frame->local = local; + STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, target_xl, + target_xl, target_xl->fops->@NAME@, + @SHORT_ARGS@); + return 0; + +err: + if (local) { + mem_put(local); + } + STACK_UNWIND_STRICT (@NAME@, frame, -1, ENOMEM, + @ERROR_ARGS@); + return 0; +} + +/* template-name cbk */ +int32_t +jbrc_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, + @LONG_ARGS@) +{ + jbrc_local_t *local = frame->local; + xlator_t *last_xl = cookie; + xlator_t *next_xl; + jbrc_private_t *priv = this->private; + struct timespec spec; + + if (op_ret != (-1)) { + if (local->scars) { + gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_RETRY_MSG, + HILITE("retried %p OK"), frame->local); + } + priv->active = last_xl; + goto unwind; + } + if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) { + goto unwind; + } + + /* TBD: get leader ID from xdata? */ + next_xl = next_xlator(this, last_xl); + /* + * We can't just give up after we've tried all bricks, because it's + * quite likely that a new leader election just hasn't finished yet. + * We also shouldn't retry endlessly, and especially not at a high + * rate, but that's good enough while we work on other things. + * + * TBD: implement slow/finite retry via a worker thread + */ + if (!next_xl || (local->scars >= SCAR_LIMIT)) { + gf_msg (this->name, GF_LOG_DEBUG, 0, J_MSG_RETRY_MSG, + HILITE("ran out of retries for %p"), frame->local); + goto unwind; + } + + local->curr_xl = next_xl; + local->scars += 1; + spec.tv_sec = 1; + spec.tv_nsec = 0; + /* + * WARNING + * + * Just calling gf_timer_call_after like this leaves open the + * possibility that writes will get reordered, if a first write is + * rescheduled and then a second comes along to find an updated + * priv->active before the first actually executes. We might need to + * implement a stricter (and more complicated) queuing mechanism to + * ensure absolute consistency in this case. + */ + if (gf_timer_call_after(this->ctx, spec, jbrc_retry_cb, local)) { + return 0; + } + +unwind: + call_stub_destroy(local->stub); + STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno, + @SHORT_ARGS@); + return 0; +} + +/* template-name cont-func */ +int32_t +jbrc_@NAME@_continue (call_frame_t *frame, xlator_t *this, + @LONG_ARGS@) +{ + jbrc_local_t *local = frame->local; + + STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, local->curr_xl, + local->curr_xl, local->curr_xl->fops->@NAME@, + @SHORT_ARGS@); + return 0; +} diff --git a/xlators/experimental/jbr-client/src/gen-fops.py b/xlators/experimental/jbr-client/src/gen-fops.py new file mode 100755 index 00000000000..4d9451f7177 --- /dev/null +++ b/xlators/experimental/jbr-client/src/gen-fops.py @@ -0,0 +1,57 @@ +#!/usr/bin/python + +import os +import re +import string +import sys + +curdir = os.path.dirname(sys.argv[0]) +gendir = os.path.join(curdir,'../../../../libglusterfs/src') +sys.path.append(gendir) +from generator import ops, fop_subs, cbk_subs, generate + +# We really want the callback argument list, even when we're generating fop +# code, so we propagate here. +# TBD: this should probably be right in generate.py +for k, v in cbk_subs.iteritems(): + fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@'] + +# Stolen from old codegen.py +def load_templates (path): + templates = {} + tmpl_re = re.compile("/\* template-name (.*) \*/") + templates = {} + t_name = None + for line in open(path,"r").readlines(): + if not line: + break + m = tmpl_re.match(line) + if m: + if t_name: + templates[t_name] = string.join(t_contents,'') + t_name = m.group(1).strip() + t_contents = [] + elif t_name: + t_contents.append(line) + if t_name: + templates[t_name] = string.join(t_contents,'') + return templates + +# Stolen from gen_fdl.py +def gen_client (templates): + for name, value in ops.iteritems(): + if name == 'getspec': + # It's not real if it doesn't have a stub function. + continue + print generate(templates['cbk'],name,cbk_subs) + print generate(templates['cont-func'],name,fop_subs) + print generate(templates['fop'],name,fop_subs) + +tmpl = load_templates(sys.argv[1]) +for l in open(sys.argv[2],'r').readlines(): + if l.find('#pragma generate') != -1: + print "/* BEGIN GENERATED CODE - DO NOT MODIFY */" + gen_client(tmpl) + print "/* END GENERATED CODE */" + else: + print l[:-1] diff --git a/xlators/experimental/jbr-client/src/jbr-messages.h b/xlators/experimental/jbr-client/src/jbr-messages.h new file mode 100644 index 00000000000..61fa725d56a --- /dev/null +++ b/xlators/experimental/jbr-client/src/jbr-messages.h @@ -0,0 +1,105 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _JBR_MESSAGES_H_ +#define _JBR_MESSAGES_H_ + +#include "glfs-message-id.h" + +/* NOTE: Rules for message additions + * 1) Each instance of a message is _better_ left with a unique message ID, even + * if the message format is the same. Reasoning is that, if the message + * format needs to change in one instance, the other instances are not + * impacted or the new change does not change the ID of the instance being + * modified. + * 2) Addition of a message, + * - Should increment the GLFS_NUM_MESSAGES + * - Append to the list of messages defined, towards the end + * - Retain macro naming as glfs_msg_X (for redability across developers) + * NOTE: Rules for message format modifications + * 3) Check acorss the code if the message ID macro in question is reused + * anywhere. If reused then then the modifications should ensure correctness + * everywhere, or needs a new message ID as (1) above was not adhered to. If + * not used anywhere, proceed with the required modification. + * NOTE: Rules for message deletion + * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used + * anywhere, then can be deleted, but will leave a hole by design, as + * addition rules specify modification to the end of the list and not filling + * holes. + */ + +#define JBR_COMP_BASE GLFS_MSGID_COMP_JBR +#define GLFS_NUM_MESSAGES 1 +#define GLFS_MSGID_END (JBR_COMP_BASE + GLFS_NUM_MESSAGES + 1) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_INIT_FAIL (JBR_COMP_BASE + 1) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_RETRY_MSG (JBR_COMP_BASE + 2) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_MEM_ERR (JBR_COMP_BASE + 3) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_DICT_FLR (JBR_COMP_BASE + 4) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_GENERIC (JBR_COMP_BASE + 5) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_INVALID (JBR_COMP_BASE + 6) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_NO_DATA (JBR_COMP_BASE + 7) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_SYS_CALL_FAILURE (JBR_COMP_BASE + 8) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + */ +#define J_MSG_QUORUM_NOT_MET (JBR_COMP_BASE + 9) + +#endif /* _JBR_MESSAGES_H_ */ diff --git a/xlators/experimental/jbr-client/src/jbrc.c b/xlators/experimental/jbr-client/src/jbrc.c new file mode 100644 index 00000000000..9bb9346c5c0 --- /dev/null +++ b/xlators/experimental/jbr-client/src/jbrc.c @@ -0,0 +1,320 @@ +/* + Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "call-stub.h" +#include "defaults.h" +#include "timer.h" +#include "xlator.h" +#include "jbr-messages.h" +#include "jbrc.h" +#include "statedump.h" + +#define SCAR_LIMIT 20 +#define HILITE(x) ("[1;33m"x"[0m") + +/* + * The fops are actually generated by gen-fops.py; the rest was mostly copied + * from defaults.c (commit cd253754 on 27 August 2013). + */ + +enum gf_dht_mem_types_ { + gf_mt_jbrc_private_t = gf_common_mt_end + 1, + gf_mt_jbrc_end +}; + +char *JBRC_XATTR = "user.jbr.active"; + +static inline +xlator_t * +ACTIVE_CHILD (xlator_t *parent) +{ + jbrc_private_t *priv = parent->private; + + return priv ? priv->active : FIRST_CHILD(parent); +} + +xlator_t * +next_xlator (xlator_t *this, xlator_t *prev) +{ + xlator_list_t *trav; + + for (trav = this->children; trav; trav = trav->next) { + if (trav->xlator == prev) { + return trav->next ? trav->next->xlator + : this->children->xlator; + } + } + + return NULL; +} + +void +jbrc_retry_cb (void *cb_arg) +{ + jbrc_local_t *local = cb_arg; + + gf_msg (__func__, GF_LOG_INFO, 0, J_MSG_RETRY_MSG, + HILITE("retrying %p"), local); + call_resume_wind(local->stub); +} + +#pragma generate + +int32_t +jbrc_forget (xlator_t *this, inode_t *inode) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL, + "xlator does not implement forget_cbk"); + return 0; +} + + +int32_t +jbrc_releasedir (xlator_t *this, fd_t *fd) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL, + "xlator does not implement releasedir_cbk"); + return 0; +} + +int32_t +jbrc_release (xlator_t *this, fd_t *fd) +{ + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL, + "xlator does not implement release_cbk"); + return 0; +} + +struct xlator_fops fops = { + .lookup = jbrc_lookup, + .stat = jbrc_stat, + .fstat = jbrc_fstat, + .truncate = jbrc_truncate, + .ftruncate = jbrc_ftruncate, + .access = jbrc_access, + .readlink = jbrc_readlink, + .mknod = jbrc_mknod, + .mkdir = jbrc_mkdir, + .unlink = jbrc_unlink, + .rmdir = jbrc_rmdir, + .symlink = jbrc_symlink, + .rename = jbrc_rename, + .link = jbrc_link, + .create = jbrc_create, + .open = jbrc_open, + .readv = jbrc_readv, + .writev = jbrc_writev, + .flush = jbrc_flush, + .fsync = jbrc_fsync, + .opendir = jbrc_opendir, + .readdir = jbrc_readdir, + .readdirp = jbrc_readdirp, + .fsyncdir = jbrc_fsyncdir, + .statfs = jbrc_statfs, + .setxattr = jbrc_setxattr, + .getxattr = jbrc_getxattr, + .fsetxattr = jbrc_fsetxattr, + .fgetxattr = jbrc_fgetxattr, + .removexattr = jbrc_removexattr, + .fremovexattr = jbrc_fremovexattr, + .lk = jbrc_lk, + .inodelk = jbrc_inodelk, + .finodelk = jbrc_finodelk, + .entrylk = jbrc_entrylk, + .fentrylk = jbrc_fentrylk, + .rchecksum = jbrc_rchecksum, + .xattrop = jbrc_xattrop, + .fxattrop = jbrc_fxattrop, + .setattr = jbrc_setattr, + .fsetattr = jbrc_fsetattr, + .fallocate = jbrc_fallocate, + .discard = jbrc_discard, +}; + +struct xlator_cbks cbks = { +}; + + +int32_t +mem_acct_init (xlator_t *this) +{ + int ret = -1; + + GF_VALIDATE_OR_GOTO ("jbrc", this, out); + + ret = xlator_mem_acct_init (this, gf_mt_jbrc_end + 1); + + if (ret != 0) { + gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR, + "Memory accounting init failed"); + return ret; + } +out: + return ret; +} + + +int32_t +jbrc_init (xlator_t *this) +{ + jbrc_private_t *priv = NULL; + xlator_list_t *trav = NULL; + + this->local_pool = mem_pool_new (jbrc_local_t, 128); + if (!this->local_pool) { + gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR, + "failed to create jbrc_local_t pool"); + goto err; + } + + priv = GF_CALLOC (1, sizeof (*priv), gf_mt_jbrc_private_t); + if (!priv) { + goto err; + } + + for (trav = this->children; trav; trav = trav->next) { + ++(priv->n_children); + } + + priv->active = FIRST_CHILD(this); + this->private = priv; + return 0; + +err: + if (priv) { + GF_FREE(priv); + } + return -1; +} + +void +jbrc_fini (xlator_t *this) +{ + GF_FREE(this->private); +} + +int +jbrc_get_child_index (xlator_t *this, xlator_t *kid) +{ + xlator_list_t *trav; + int retval = -1; + + for (trav = this->children; trav; trav = trav->next) { + ++retval; + if (trav->xlator == kid) { + return retval; + } + } + + return -1; +} + +uint8_t +jbrc_count_up_kids (jbrc_private_t *priv) +{ + uint8_t retval = 0; + uint8_t i; + + for (i = 0; i < priv->n_children; ++i) { + if (priv->kid_state & (1 << i)) { + ++retval; + } + } + + return retval; +} + +int32_t +jbrc_notify (xlator_t *this, int32_t event, void *data, ...) +{ + int32_t ret = 0; + int32_t index = 0; + jbrc_private_t *priv = NULL; + + GF_VALIDATE_OR_GOTO (THIS->name, this, out); + priv = this->private; + GF_VALIDATE_OR_GOTO (this->name, priv, out); + + switch (event) { + case GF_EVENT_CHILD_UP: + index = jbrc_get_child_index(this, data); + if (index >= 0) { + priv->kid_state |= (1 << index); + priv->up_children = jbrc_count_up_kids(priv); + gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC, + "got CHILD_UP for %s, now %u kids", + ((xlator_t *)data)->name, + priv->up_children); + } + ret = default_notify (this, event, data); + break; + case GF_EVENT_CHILD_DOWN: + index = jbrc_get_child_index(this, data); + if (index >= 0) { + priv->kid_state &= ~(1 << index); + priv->up_children = jbrc_count_up_kids(priv); + gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC, + "got CHILD_DOWN for %s, now %u kids", + ((xlator_t *)data)->name, + priv->up_children); + } + break; + default: + ret = default_notify (this, event, data); + } + +out: + return ret; +} + +int +jbrc_priv_dump (xlator_t *this) +{ + jbrc_private_t *priv = NULL; + char key_prefix[GF_DUMP_MAX_BUF_LEN]; + xlator_list_t *trav = NULL; + int32_t i = -1; + + GF_VALIDATE_OR_GOTO (THIS->name, this, out); + priv = this->private; + GF_VALIDATE_OR_GOTO (this->name, priv, out); + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", + this->type, this->name); + gf_proc_dump_add_section(key_prefix); + + gf_proc_dump_write("up_children", "%u", priv->up_children); + + for (trav = this->children, i = 0; trav; trav = trav->next, i++) { + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "child_%d", i); + gf_proc_dump_write(key_prefix, "%s", trav->xlator->name); + } + +out: + return 0; +} + +struct xlator_dumpops dumpops = { + .priv = jbrc_priv_dump, +}; + +class_methods_t class_methods = { + .init = jbrc_init, + .fini = jbrc_fini, + .notify = jbrc_notify, +}; + +struct volume_options options[] = { + { .key = {NULL} }, +}; diff --git a/xlators/experimental/jbr-client/src/jbrc.h b/xlators/experimental/jbr-client/src/jbrc.h new file mode 100644 index 00000000000..c83259ca1bd --- /dev/null +++ b/xlators/experimental/jbr-client/src/jbrc.h @@ -0,0 +1,27 @@ +/* + Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _JBRC_H_ +#define _JBRC_H_ + +typedef struct { + xlator_t *active; + uint8_t up_children; + uint8_t n_children; + uint32_t kid_state; +} jbrc_private_t; + +typedef struct { + call_stub_t *stub; + xlator_t *curr_xl; + uint16_t scars; +} jbrc_local_t; + +#endif /* _JBRC_H_ */ |