metadisp: new translator for data and metadata separation

Summary:

feature/metadisp is an xlator for performing "metadata dispersal" across
multiple children. it does this by flattening the complex
POSIX paths into /$GFID style paths, then forwarding the
metadata operations to its first child and forwarding the
data operations to its second child.

The purpose of this xlator is to allow separation of data and metadata,
in cases where metadata might be stored in another format (embedded kv?),
on another disk (ssd), on another host (dht2).

Change-Id: I392c8bd0c867a3237d144aea327323f700a2728d
Updates: #816
Signed-Off-By: Sheena Artrip <sheenobu@fb.com>
Tested-By: Amar Tumballi <amar@kadalu.io>

16 files changed, 1152 insertions, 0 deletions

diff --git a/xlators/features/metadisp/Makefile.am b/xlators/features/metadisp/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/metadisp/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/metadisp/src/Makefile.am b/xlators/features/metadisp/src/Makefile.am
new file mode 100644
index 00000000000..1520ad8c424
--- /dev/null
+++ b/xlators/features/metadisp/src/Makefile.am
@@ -0,0 +1,38 @@
+noinst_PYTHON = gen-fops.py
+
+EXTRA_DIST = fops-tmpl.c
+
+xlator_LTLIBRARIES = metadisp.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+nodist_metadisp_la_SOURCES = fops.c
+
+BUILT_SOURCES = fops.c
+
+metadisp_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+metadisp_la_SOURCES = metadisp.c \
+		metadisp-unlink.c \
+		metadisp-stat.c \
+		metadisp-lookup.c \
+		metadisp-readdir.c \
+		metadisp-create.c \
+		metadisp-open.c \
+		metadisp-fsync.c \
+		metadisp-setattr.c \
+		backend.c
+
+metadisp_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = metadisp.h metadisp-fops.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+fops.c: fops-tmpl.c $(top_srcdir)/libglusterfs/src/generator.py gen-fops.py
+	PYTHONPATH=$(top_srcdir)/libglusterfs/src \
+	$(PYTHON) $(srcdir)/gen-fops.py $(srcdir)/fops-tmpl.c > $@
+
+CLEANFILES = $(nodist_metadisp_la_SOURCES)
diff --git a/xlators/features/metadisp/src/backend.c b/xlators/features/metadisp/src/backend.c
new file mode 100644
index 00000000000..ee2c25bfaa7
--- /dev/null
+++ b/xlators/features/metadisp/src/backend.c
@@ -0,0 +1,45 @@
+#define GFID_STR_LEN 37
+
+#include "metadisp.h"
+
+/*
+ * backend.c
+ *
+ * functions responsible for converting user-facing paths to backend-style
+ * "/$GFID" paths.
+ */
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc)
+{
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char gfid_buf[GFID_STR_LEN + 1] = {
+        0,
+    };
+    char *path = NULL;
+
+    GF_VALIDATE_OR_GOTO("metadisp", src_loc, out);
+    GF_VALIDATE_OR_GOTO("metadisp", dst_loc, out);
+
+    loc_copy(dst_loc, src_loc);
+    memcpy(dst_loc->pargfid, root, sizeof(root));
+    GF_FREE((char *)dst_loc->path);  // we are overwriting path so nuke
+                                     // whatever loc_copy gave us
+
+    uuid_utoa_r(gfid, gfid_buf);
+
+    path = GF_CALLOC(GFID_STR_LEN + 1, sizeof(char),
+                     gf_common_mt_char);  // freed via loc_wipe
+
+    path[0] = '/';
+    strncpy(path + 1, gfid_buf, GFID_STR_LEN);
+    path[GFID_STR_LEN] = 0;
+    dst_loc->path = path;
+    if (src_loc->name)
+        dst_loc->name = strrchr(dst_loc->path, '/');
+    if (dst_loc->name)
+        dst_loc->name++;
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/fops-tmpl.c b/xlators/features/metadisp/src/fops-tmpl.c
new file mode 100644
index 00000000000..4385b7dd5b7
--- /dev/null
+++ b/xlators/features/metadisp/src/fops-tmpl.c
@@ -0,0 +1,10 @@
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/xlator.h>
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+#pragma generate
diff --git a/xlators/features/metadisp/src/gen-fops.py b/xlators/features/metadisp/src/gen-fops.py
new file mode 100644
index 00000000000..8b5e120fdec
--- /dev/null
+++ b/xlators/features/metadisp/src/gen-fops.py
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+
+import sys
+from generator import fop_subs, generate
+
+FN_METADATA_CHILD_GENERIC = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                 @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ metadata");
+  STACK_WIND (frame, default_@NAME@_cbk,
+              METADATA_CHILD(this), METADATA_CHILD(this)->fops->@NAME@,
+              @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_GENERIC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ generic");
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          DATA_CHILD(this), DATA_CHILD(this)->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATAFD_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ datafd");
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATALOC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ dataloc");
+  loc_t backend_loc = {
+      0,
+  };
+  if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+      goto unwind;
+  }
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+
+unwind:
+  STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+  return 0;
+}
+"""
+
+FOPS_LINE_TEMPLATE = "\t.@NAME@ = metadisp_@NAME@,"
+
+skipped = [
+    "readdir",
+    "readdirp",
+    "lookup",
+    "fsync",
+    "stat",
+    "open",
+    "create",
+    "unlink",
+    "setattr",
+    # TODO: implement "inodelk",
+]
+
+
+def gen_fops():
+    done = skipped
+
+    #
+    # these are fops that wind to the DATA_CHILD
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "writev",
+        "readv",
+        "ftruncate",
+        "zerofill",
+        "discard",
+        "seek",
+        "fstat",
+    ]:
+        done = done + [name]
+        print(generate(FN_DATAFD_TEMPLATE, name, fop_subs))
+
+    for name in ["truncate"]:
+        done = done + [name]
+        print(generate(FN_DATALOC_TEMPLATE, name, fop_subs))
+
+    # these are fops that operate solely on dentries, folders,
+    # or extended attributes. Therefore, they must always
+    # wind to METADATA_CHILD and should never perform
+    # any path rewriting
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "mkdir",
+        "symlink",
+        "link",
+        "rename",
+        "mknod",
+        "opendir",
+        # "readdir,  # special-cased
+        # "readdirp, # special-cased
+        "fsyncdir",
+        # "setattr", # special-cased
+        "readlink",
+        "fentrylk",
+        "access",
+        # TODO: these wind to both,
+        # data for backend-attributes and metadata for the rest
+        "xattrop",
+        "setxattr",
+        "getxattr",
+        "removexattr",
+        "fgetxattr",
+        "fsetxattr",
+        "fremovexattr",
+    ]:
+
+        done = done + [name]
+        print(generate(FN_METADATA_CHILD_GENERIC, name, fop_subs))
+
+    print("struct xlator_fops fops = {")
+    for name in done:
+        print(generate(FOPS_LINE_TEMPLATE, name, fop_subs))
+
+    print("};")
+
+
+for l in open(sys.argv[1], "r").readlines():
+    if l.find("#pragma generate") != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_fops()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/metadisp/src/metadisp-create.c b/xlators/features/metadisp/src/metadisp-create.c
new file mode 100644
index 00000000000..f8c9798dd59
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-create.c
@@ -0,0 +1,101 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Create, like stat, is a two-step process. We send a create
+ * to the METADATA_CHILD, then send another create to the DATA_CHILD.
+ *
+ * We do the metadata child first to ensure that the ACLs are enforced.
+ */
+
+int32_t
+metadisp_create_dentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           inode_t *inode, struct iatt *buf,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                       dict_t *xdata)
+{
+    // create the backend data inode
+    STACK_WIND(frame, metadisp_create_dentry_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = cookie;
+    if (op_ret != 0) {
+        STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+    }
+
+    if (stub == NULL) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+
+    loc_t backend_loc = {
+        0,
+    };
+    call_stub_t *stub = NULL;
+    uuid_t *gfid_req = NULL;
+
+    RESOLVE_GFID_REQ(xdata, gfid_req, out);
+
+    if (build_backend_loc(*gfid_req, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    frame->local = loc;
+
+    stub = fop_create_stub(frame, metadisp_create_resume, &backend_loc, flags,
+                           mode, umask, fd, xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_create_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->create, loc, flags, mode,
+                      umask, fd, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/metadisp-fops.h b/xlators/features/metadisp/src/metadisp-fops.h
new file mode 100644
index 00000000000..56dd427cf34
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fops.h
@@ -0,0 +1,51 @@
+#ifndef GF_METADISP_FOPS_H_
+#define GF_METADISP_FOPS_H_
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/glusterfs.h>
+
+#include <sys/types.h>
+
+/* fops in here are defined in their own file. Every other fop is just defined
+ * inline of fops.c */
+
+int
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata);
+
+int
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *dict);
+
+int
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+int
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata);
+
+int
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata);
+
+int
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata);
+
+int
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+#endif
diff --git a/xlators/features/metadisp/src/metadisp-fsync.c b/xlators/features/metadisp/src/metadisp-fsync.c
new file mode 100644
index 00000000000..2e46fa84eac
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fsync.c
@@ -0,0 +1,54 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_fsync_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, default_fsync_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    stub = fop_fsync_stub(frame, metadisp_fsync_resume, fd, flags, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_fsync_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-lookup.c b/xlators/features/metadisp/src/metadisp-lookup.c
new file mode 100644
index 00000000000..27d90c9f746
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-lookup.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Lookup, like stat, is a two-step process for grabbing the metadata details
+ * as well as the data details.
+ */
+
+int32_t
+metadisp_backend_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xdata,
+                            struct iatt *postparent)
+{
+    METADISP_TRACE("backend_lookup_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
+}
+
+int32_t
+metadisp_backend_lookup_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *xdata)
+{
+    METADISP_TRACE("backend_lookup_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_lookup_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->lookup, &backend_loc, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+
+    // memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    METADISP_TRACE("lookup");
+    call_stub_t *stub = NULL;
+    stub = fop_lookup_stub(frame, metadisp_backend_lookup_resume, loc, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_lookup_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-open.c b/xlators/features/metadisp/src/metadisp-open.c
new file mode 100644
index 00000000000..64814afe636
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-open.c
@@ -0,0 +1,70 @@
+#include <glusterfs/call-stub.h>
+#include "metadisp.h"
+
+int32_t
+metadisp_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE("got open results %d %d", op_ret, op_errno);
+
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!stub) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     int32_t flags, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, NULL, DATA_CHILD(this),
+                      DATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    stub = fop_open_stub(frame, metadisp_open_resume, &backend_loc, flags, fd,
+                         xdata);
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(open, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-readdir.c b/xlators/features/metadisp/src/metadisp-readdir.c
new file mode 100644
index 00000000000..5f840b1e88f
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-readdir.c
@@ -0,0 +1,65 @@
+#include "metadisp.h"
+
+/**
+ * With a change to the posix xlator, readdir and readdirp are shockingly
+ * simple.
+ *
+ * The issue with separating the backend data of the files
+ * with the metadata is that readdirs must now read from multiple sources
+ * to coalesce the directory entries.
+ *
+ * The way we do this is to tell the METADATA_CHILD that when it's
+ * running readdirp, each file entry should have a stat wound to
+ * 'stat-source-of-truth'.
+ *
+ * see metadisp_stat for how it handles winds _from_posix.
+ */
+
+int32_t
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    /*
+     * Always use readdirp, even if the original was readdir. Why? Because NFS.
+     * There are multiple translations between Gluster, UNIX, and NFS stat
+     * structures in that path. One of them uses the type etc. from the stat
+     * structure, which is only filled in by readdirp. If we use readdir, the
+     * entries do actually go all the way back to the client and are visible in
+     * getdents, but then the readdir throws them away because of the
+     * uninitialized type.
+     */
+    GF_UNUSED int32_t ret;
+    if (!xdata) {
+        xdata = dict_new();
+    }
+
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
+}
+
+int32_t
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    GF_UNUSED int32_t ret;
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-setattr.c b/xlators/features/metadisp/src/metadisp-setattr.c
new file mode 100644
index 00000000000..6991cf644f3
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-setattr.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_backend_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *statpre, struct iatt *statpost,
+                             dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_backend_setattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                struct iatt *stbuf, int32_t valid,
+                                dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_setattr_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->setattr, &backend_loc, stbuf, valid,
+               xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(setattr, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(statpost->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    METADISP_TRACE("setattr");
+    call_stub_t *stub = NULL;
+    stub = fop_setattr_stub(frame, metadisp_backend_setattr_resume, loc, stbuf,
+                            valid, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_setattr_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                      xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-stat.c b/xlators/features/metadisp/src/metadisp-stat.c
new file mode 100644
index 00000000000..b06d0dbcddd
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-stat.c
@@ -0,0 +1,124 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The stat flow in METADISP is complicated because we must
+ * do ensure a few things:
+ *    1. stat, on the path within the metadata layer,
+ *       MUST get the backend FD of the data layer.
+ *        --- we wind to the metadata layer, then the data layer.
+ *
+ *    2. the metadata layer MUST be able to ask the data
+ *       layer for stat information.
+ *        --- this is 'syncop-internal-from-posix'
+ *
+ *    3. when the metadata exists BUT the data is missing,
+ *       we MUST mark the backend file as bad and heal it.
+ */
+
+int32_t
+metadisp_stat_backend_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
+{
+    METADISP_TRACE("got backend stat results %d %d", op_ret, op_errno);
+    if (op_errno == ENOENT) {
+        STACK_UNWIND_STRICT(open, frame, -1, ENODATA, NULL, NULL);
+        return 0;
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
+{
+    METADISP_TRACE("winding stat to path %s", loc->path);
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("bad object, sending EUCLEAN");
+        STACK_UNWIND_STRICT(open, frame, -1, EUCLEAN, NULL, NULL);
+        return 0;
+    }
+
+    STACK_WIND(frame, metadisp_stat_backend_cbk, SECOND_CHILD(this),
+               SECOND_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    METADISP_TRACE("got stat results %d %d", op_ret, op_errno);
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // only use the stub for the files
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    loc_t backend_loc = {
+        0,
+    };
+    METADISP_FILTER_ROOT(stat, loc, xdata);
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    if (dict_get_int32(xdata, "syncop-internal-from-posix", &ret) == 0) {
+        // if we've just been sent a stat from posix, then we know
+        // that we must send down a stat for a file to the second child.
+        //
+        // that means we can skip the stat for the first child and just
+        // send to the data disk.
+        METADISP_TRACE("got syncop-internal-from-posix");
+        STACK_WIND(frame, default_stat_cbk, DATA_CHILD(this),
+                   DATA_CHILD(this)->fops->stat, &backend_loc, xdata);
+        return 0;
+    }
+
+    // we do not know if the request is for a file, folder, etc. wind
+    // to first child to find out.
+    stub = fop_stat_stub(frame, metadisp_stat_resume, &backend_loc, xdata);
+    METADISP_TRACE("winding stat to first child %s", loc->path);
+    STACK_WIND_COOKIE(frame, metadisp_stat_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(stat, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-unlink.c b/xlators/features/metadisp/src/metadisp-unlink.c
new file mode 100644
index 00000000000..1f6a8eb35ce
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-unlink.c
@@ -0,0 +1,160 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The unlink flow in metadisp is complicated because we must
+ * do ensure that UNLINK causes both the metadata objects
+ * to get removed and the data objects to get removed.
+ */
+
+int32_t
+metadisp_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflag, dict_t *xdata)
+{
+    METADISP_TRACE("winding backend unlink to path %s", loc->path);
+    STACK_WIND(frame, default_unlink_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE(". %d %d", op_ret, op_errno);
+
+    int ret = 0;
+    call_stub_t *stub = NULL;
+    int nlink = 0;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
+    if (ret != 0) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto unwind;
+    }
+    METADISP_TRACE("frontend hardlink count %d %d", ret, nlink);
+    if (nlink > 1) {
+        goto unwind;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+    call_stub_t *stub = NULL;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // fail fast on empty gfid so we don't loop forever
+    if (gf_uuid_is_null(buf->ia_gfid)) {
+        op_ret = -1;
+        op_errno = ENODATA;
+        goto unwind;
+    }
+
+    // fill gfid since the stub is incomplete
+    memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    memcpy(stub->args.loc.pargfid, postparent->ia_gfid, sizeof(uuid_t));
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("winding lookup for unlink to path %s", loc->path);
+
+        // loop back to ourselves after a lookup
+        stub = fop_unlink_stub(frame, metadisp_unlink, loc, xflag, xdata);
+        STACK_WIND_COOKIE(frame, metadisp_unlink_lookup_cbk, stub,
+                          METADATA_CHILD(this),
+                          METADATA_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+    }
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    //
+    // ensure we get the link count on the unlink response, so we can
+    // account for hardlinks before winding to the backend.
+    // NOTE:
+    //   multiple xlators use GF_REQUEST_LINK_COUNT_XDATA. confirmation
+    //   is needed to ensure that multiple requests will work in the same
+    //   xlator stack.
+    //
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+
+    METADISP_TRACE("winding frontend unlink to path %s", loc->path);
+    stub = fop_unlink_stub(frame, metadisp_unlink_resume, &backend_loc, xflag,
+                           xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_unlink_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(unlink, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp.c b/xlators/features/metadisp/src/metadisp.c
new file mode 100644
index 00000000000..3c8f150cebc
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.c
@@ -0,0 +1,46 @@
+#include <glusterfs/call-stub.h>
+
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+int32_t
+init(xlator_t *this)
+{
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with children. exiting");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+/* defined in fops.c */
+struct xlator_fops fops;
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .op_version = {1},
+    .identifier = "metadisp",
+    .category = GF_EXPERIMENTAL,
+};
diff --git a/xlators/features/metadisp/src/metadisp.h b/xlators/features/metadisp/src/metadisp.h
new file mode 100644
index 00000000000..c8fd7a13c04
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef GF_METADISP_H_
+#define GF_METADISP_H_
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+#define METADATA_CHILD(_this) FIRST_CHILD(_this)
+#define DATA_CHILD(_this) SECOND_CHILD(_this)
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc);
+
+#define METADISP_TRACE(_args...) gf_log("metadisp", GF_LOG_INFO, _args)
+
+#define METADISP_FILTER_ROOT(_op, _args...)                                    \
+    if (strcmp(loc->path, "/") == 0) {                                         \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define METADISP_FILTER_ROOT_BY_GFID(_op, _gfid, _args...)                     \
+    if (__is_root_gfid(_gfid)) {                                               \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define RESOLVE_GFID_REQ(_dict, _dest, _lbl)                                   \
+    VALIDATE_OR_GOTO(dict_get_ptr(_dict, "gfid-req", (void **)&_dest) == 0,    \
+                     _lbl)
+
+#endif /* __TEMPLATE_H__ */