Add GF_FOP_IPC for inter-translator communication.

Several features - e.g. encryption, erasure codes, or NSR - involve
multiple cooperating translators which sometimes need a "private" means
of communication amongst themselves.  Historically we've used virtual or
synthetic xattrs, but that's not very elegant and clutters up the
getxattr/setxattr path which must also handle real xattr requests.  This
new fop should address that.

The only argument is an int32_t "op" which should be recognized by the
target translator.  It is recommended that translators using these
feature follow some convention regarding the ops that they define, to
avoid conflicts.  Using a hash of the target translator's type string as
a base for a series of ops would probably be a good start.  Any other
information can be passed in both directions using xdata.

The default behavior for this fop, as with any other, is to pass through
to FIRST_CHILD.  That makes use of this fop "transparent" to other
translators that were written before it existed, but it also means that
it only really works with pass-through translators.  If a routing
translator (such as DHT) or a fan-out translator (such as AFR) is
involved, the IPC might not reach its intended destination unless those
translators are modified to forward IPC fops along all paths.

If an IPC gets all the way to storage/posix it is considered an error,
much like an uncaught exception.  We don't actually *do* anything in
that case, but we do flag it as an error in the log.

Change-Id: I7f37c9247ee35536f8136c7aea758e6fe04616c4
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>

3 files changed, 216 insertions, 2 deletions

diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 99a6f6d74..55d62dd39 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -2104,6 +2104,55 @@ out:
 }
 
 int
+client3_3_ipc_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+        call_frame_t    *frame         = NULL;
+        gfs3_zerofill_rsp rsp          = {0,};
+        int              ret           = 0;
+        xlator_t *this                 = NULL;
+        dict_t  *xdata                 = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gf_common_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (ipc, frame,
+                             rsp.op_ret, gf_error_to_errno (rsp.op_errno),
+                             xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
 client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
@@ -6093,6 +6142,45 @@ unwind:
         return 0;
 }
 
+int32_t
+client3_3_ipc (call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args        = NULL;
+        clnt_conf_t       *conf        = NULL;
+        gfs3_ipc_req       req        = {0,};
+        int                op_errno    = ESTALE;
+        int                ret         = 0;
+
+        GF_ASSERT (frame);
+
+        if (!this || !data)
+                goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        req.op = args->cmd;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_IPC, client3_3_ipc_cbk,
+                                    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_ipc_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
 /* Table Specific to FOPS */
 
 
@@ -6144,7 +6232,8 @@ rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
         [GF_FOP_RELEASE]     = { "RELEASE",     client3_3_release },
         [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_3_releasedir },
         [GF_FOP_GETSPEC]     = { "GETSPEC",     client3_getspec },
-        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
+        [GF_FOP_FREMOVEXATTR]= { "FREMOVEXATTR",client3_3_fremovexattr },
+        [GF_FOP_IPC]         = { "IPC",         client3_3_ipc },
 };
 
 /* Used From RPC-CLNT library to log proper name of procedure based on number */
@@ -6196,6 +6285,7 @@ char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
 	[GFS3_OP_FALLOCATE]   = "FALLOCATE",
 	[GFS3_OP_DISCARD]     = "DISCARD",
         [GFS3_OP_ZEROFILL]    = "ZEROFILL",
+        [GFS3_OP_IPC]         = "IPC",
 
 };
 
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index b0a71d3f9..72ff35a04 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2068,6 +2068,38 @@ out:
 
 
 int32_t
+client_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+        int          ret              = -1;
+        clnt_conf_t *conf             = NULL;
+        rpc_clnt_procedure_t *proc    = NULL;
+        clnt_args_t  args             = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.cmd = op;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_IPC];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_IPC]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL);
+
+        return 0;
+}
+
+
+int32_t
 client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
                 int32_t flags)
 {
@@ -2788,6 +2820,7 @@ struct xlator_fops fops = {
 	.discard     = client_discard,
         .zerofill    = client_zerofill,
         .getspec     = client_getspec,
+        .ipc         = client_ipc,
 };
 
 
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index 8bdadec6b..bf674271c 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -2040,6 +2040,42 @@ out:
 }
 
 
+int
+server_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp      rsp   = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        req = frame->local;
+        state  = CALL_STATE (frame);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": IPC%"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
 /* Resume function section */
 
 int
@@ -3088,7 +3124,6 @@ err:
 }
 
 
-
 /* Fop section */
 
 int
@@ -3450,6 +3485,61 @@ out:
 }
 
 int
+server3_3_ipc (rpcsvc_request_t *req)
+{
+        call_frame_t    *frame          = NULL;
+        gfs3_ipc_req     args           = {0,};
+        int              ret            = -1;
+        int              op_errno       = 0;
+        dict_t          *xdata          = NULL;
+        xlator_t        *bound_xl       = NULL;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_ipc_req);
+        if (ret < 0) {
+                /*failed to decode msg*/;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                /* something wrong, mostly insufficient memory*/
+                req->rpc_err = GARBAGE_ARGS; /* TODO */
+                goto out;
+        }
+        frame->root->op = GF_FOP_IPC;
+
+        bound_xl = frame->root->client->bound_xl;
+        if (!bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (bound_xl, xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len,
+                                      ret, op_errno, out);
+
+        ret = 0;
+        STACK_WIND (frame, server_ipc_cbk, bound_xl, bound_xl->fops->ipc,
+                    args.op, xdata);
+        free(xdata);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                req->rpc_err = GARBAGE_ARGS;
+
+        return ret;
+}
+
+int
 server3_3_readlink (rpcsvc_request_t *req)
 {
         server_state_t    *state                 = NULL;
@@ -6170,6 +6260,7 @@ rpcsvc_actor_t glusterfs3_3_fop_actors[] = {
         [GFS3_OP_FALLOCATE]    = {"FALLOCATE",    GFS3_OP_FALLOCATE,    server3_3_fallocate,    NULL, 0, DRC_NA},
         [GFS3_OP_DISCARD]      = {"DISCARD",      GFS3_OP_DISCARD,      server3_3_discard,      NULL, 0, DRC_NA},
         [GFS3_OP_ZEROFILL]    =  {"ZEROFILL",     GFS3_OP_ZEROFILL,     server3_3_zerofill,     NULL, 0, DRC_NA},
+        [GFS3_OP_IPC]         =  {"IPC",          GFS3_OP_IPC,          server3_3_ipc,          NULL, 0, DRC_NA},
 };