summaryrefslogtreecommitdiffstats
path: root/xlators
diff options
context:
space:
mode:
authorJeff Darcy <jdarcy@redhat.com>2015-03-10 20:14:47 -0400
committerVijay Bellur <vbellur@redhat.com>2015-03-17 07:02:15 -0700
commit0d2bed70faed3c63f25ed9269dc55562973ef9b7 (patch)
treefee1995e88b7ae19d1cb27cf437ab76583d111e6 /xlators
parent6b3704990257643da54100d8581856a7d2c72f86 (diff)
every/where: add GF_FOP_IPC for inter-translator communication
Several features - e.g. encryption, erasure codes, or NSR - involve multiple cooperating translators which sometimes need a "private" means of communication amongst themselves. Historically we've used virtual or synthetic xattrs, but that's not very elegant and clutters up the getxattr/setxattr path which must also handle real xattr requests. This new fop should address that. The only argument is an int32_t "op" which should be recognized by the target translator. It is recommended that translators using these feature follow some convention regarding the ops that they define, to avoid conflicts. Using a hash of the target translator's type string as a base for a series of ops would probably be a good start. Any other information can be passed in both directions using xdata. The default behavior for this fop, as with any other, is to pass through to FIRST_CHILD. That makes use of this fop "transparent" to other translators that were written before it existed, but it also means that it only really works with pass-through translators. If a routing translator (such as DHT) or a fan-out translator (such as AFR) is involved, the IPC might not reach its intended destination unless those translators are modified to forward IPC fops along all paths. If an IPC gets all the way to storage/posix it is considered an error, much like an uncaught exception. We don't actually *do* anything in that case, but we do log it send back an EOPNOTSUPP error. This makes the "unrecognized opcode" condition distinguishable from the "no IPC support" condition (which would yield an RPC error instead) so clients can probe for the presence of a handler for their own favorite opcode and either use that or use old-school xattrs depending on the result. BUG: 1158628 Signed-off-by: Venky Shankar <vshankar@redhat.com> Signed-off-by: Jeff Darcy <jdarcy@redhat.com> Change-Id: I84af1b17babe5b30ec03ecf027ae37d09b873968 Reviewed-on: http://review.gluster.org/8812 Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Diffstat (limited to 'xlators')
-rw-r--r--xlators/performance/io-threads/src/io-threads.c1
-rw-r--r--xlators/protocol/client/src/client-rpc-fops.c184
-rw-r--r--xlators/protocol/client/src/client.c33
-rw-r--r--xlators/protocol/server/src/server-rpc-fops.c94
-rw-r--r--xlators/storage/posix/src/posix.c15
5 files changed, 280 insertions, 47 deletions
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 9c6ae5a..148e55a 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -341,6 +341,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
case GF_FOP_RELEASEDIR:
case GF_FOP_GETSPEC:
break;
+ case GF_FOP_IPC:
default:
return -EINVAL;
}
diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 4305fb3..7ca91e9 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -2103,6 +2103,55 @@ out:
}
int
+client3_3_ipc_cbk (struct rpc_req *req, struct iovec *iov, int count,
+ void *myframe)
+{
+ call_frame_t *frame = NULL;
+ gfs3_ipc_rsp rsp = {0,};
+ int ret = 0;
+ xlator_t *this = NULL;
+ dict_t *xdata = NULL;
+
+ this = THIS;
+
+ frame = myframe;
+
+ if (-1 == req->rpc_status) {
+ rsp.op_ret = -1;
+ rsp.op_errno = ENOTCONN;
+ goto out;
+ }
+ ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_ipc_rsp);
+ if (ret < 0) {
+ gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+ rsp.op_ret = -1;
+ rsp.op_errno = EINVAL;
+ goto out;
+ }
+
+ GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+ (rsp.xdata.xdata_len), ret,
+ rsp.op_errno, out);
+
+out:
+ if (rsp.op_ret == -1) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "remote operation failed: %s",
+ strerror (gf_error_to_errno (rsp.op_errno)));
+ }
+ CLIENT_STACK_UNWIND (ipc, frame,
+ rsp.op_ret, gf_error_to_errno (rsp.op_errno),
+ xdata);
+
+ free (rsp.xdata.xdata_val);
+
+ if (xdata)
+ dict_unref (xdata);
+
+ return 0;
+}
+
+int
client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe)
{
@@ -6098,58 +6147,98 @@ unwind:
return 0;
}
+int32_t
+client3_3_ipc (call_frame_t *frame, xlator_t *this, void *data)
+{
+ clnt_args_t *args = NULL;
+ clnt_conf_t *conf = NULL;
+ gfs3_ipc_req req = {0,};
+ int op_errno = ESTALE;
+ int ret = 0;
+
+ GF_ASSERT (frame);
+
+ if (!this || !data)
+ goto unwind;
+
+ args = data;
+ conf = this->private;
+
+ req.op = args->cmd;
+
+ GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+ req.xdata.xdata_len, op_errno, unwind);
+
+ ret = client_submit_request(this, &req, frame, conf->fops,
+ GFS3_OP_IPC, client3_3_ipc_cbk,
+ NULL, NULL, 0, NULL, 0, NULL,
+ (xdrproc_t) xdr_gfs3_ipc_req);
+ if (ret)
+ gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+ GF_FREE (req.xdata.xdata_val);
+
+ return 0;
+unwind:
+ CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+ GF_FREE (req.xdata.xdata_val);
+
+ return 0;
+}
+
/* Table Specific to FOPS */
rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
- [GF_FOP_NULL] = { "NULL", NULL},
- [GF_FOP_STAT] = { "STAT", client3_3_stat },
- [GF_FOP_READLINK] = { "READLINK", client3_3_readlink },
- [GF_FOP_MKNOD] = { "MKNOD", client3_3_mknod },
- [GF_FOP_MKDIR] = { "MKDIR", client3_3_mkdir },
- [GF_FOP_UNLINK] = { "UNLINK", client3_3_unlink },
- [GF_FOP_RMDIR] = { "RMDIR", client3_3_rmdir },
- [GF_FOP_SYMLINK] = { "SYMLINK", client3_3_symlink },
- [GF_FOP_RENAME] = { "RENAME", client3_3_rename },
- [GF_FOP_LINK] = { "LINK", client3_3_link },
- [GF_FOP_TRUNCATE] = { "TRUNCATE", client3_3_truncate },
- [GF_FOP_OPEN] = { "OPEN", client3_3_open },
- [GF_FOP_READ] = { "READ", client3_3_readv },
- [GF_FOP_WRITE] = { "WRITE", client3_3_writev },
- [GF_FOP_STATFS] = { "STATFS", client3_3_statfs },
- [GF_FOP_FLUSH] = { "FLUSH", client3_3_flush },
- [GF_FOP_FSYNC] = { "FSYNC", client3_3_fsync },
- [GF_FOP_SETXATTR] = { "SETXATTR", client3_3_setxattr },
- [GF_FOP_GETXATTR] = { "GETXATTR", client3_3_getxattr },
- [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr },
- [GF_FOP_OPENDIR] = { "OPENDIR", client3_3_opendir },
- [GF_FOP_FSYNCDIR] = { "FSYNCDIR", client3_3_fsyncdir },
- [GF_FOP_ACCESS] = { "ACCESS", client3_3_access },
- [GF_FOP_CREATE] = { "CREATE", client3_3_create },
- [GF_FOP_FTRUNCATE] = { "FTRUNCATE", client3_3_ftruncate },
- [GF_FOP_FSTAT] = { "FSTAT", client3_3_fstat },
- [GF_FOP_LK] = { "LK", client3_3_lk },
- [GF_FOP_LOOKUP] = { "LOOKUP", client3_3_lookup },
- [GF_FOP_READDIR] = { "READDIR", client3_3_readdir },
- [GF_FOP_INODELK] = { "INODELK", client3_3_inodelk },
- [GF_FOP_FINODELK] = { "FINODELK", client3_3_finodelk },
- [GF_FOP_ENTRYLK] = { "ENTRYLK", client3_3_entrylk },
- [GF_FOP_FENTRYLK] = { "FENTRYLK", client3_3_fentrylk },
- [GF_FOP_XATTROP] = { "XATTROP", client3_3_xattrop },
- [GF_FOP_FXATTROP] = { "FXATTROP", client3_3_fxattrop },
- [GF_FOP_FGETXATTR] = { "FGETXATTR", client3_3_fgetxattr },
- [GF_FOP_FSETXATTR] = { "FSETXATTR", client3_3_fsetxattr },
- [GF_FOP_RCHECKSUM] = { "RCHECKSUM", client3_3_rchecksum },
- [GF_FOP_SETATTR] = { "SETATTR", client3_3_setattr },
- [GF_FOP_FSETATTR] = { "FSETATTR", client3_3_fsetattr },
- [GF_FOP_READDIRP] = { "READDIRP", client3_3_readdirp },
- [GF_FOP_FALLOCATE] = { "FALLOCATE", client3_3_fallocate },
- [GF_FOP_DISCARD] = { "DISCARD", client3_3_discard },
- [GF_FOP_ZEROFILL] = { "ZEROFILL", client3_3_zerofill},
- [GF_FOP_RELEASE] = { "RELEASE", client3_3_release },
- [GF_FOP_RELEASEDIR] = { "RELEASEDIR", client3_3_releasedir },
- [GF_FOP_GETSPEC] = { "GETSPEC", client3_getspec },
+ [GF_FOP_NULL] = { "NULL", NULL},
+ [GF_FOP_STAT] = { "STAT", client3_3_stat },
+ [GF_FOP_READLINK] = { "READLINK", client3_3_readlink },
+ [GF_FOP_MKNOD] = { "MKNOD", client3_3_mknod },
+ [GF_FOP_MKDIR] = { "MKDIR", client3_3_mkdir },
+ [GF_FOP_UNLINK] = { "UNLINK", client3_3_unlink },
+ [GF_FOP_RMDIR] = { "RMDIR", client3_3_rmdir },
+ [GF_FOP_SYMLINK] = { "SYMLINK", client3_3_symlink },
+ [GF_FOP_RENAME] = { "RENAME", client3_3_rename },
+ [GF_FOP_LINK] = { "LINK", client3_3_link },
+ [GF_FOP_TRUNCATE] = { "TRUNCATE", client3_3_truncate },
+ [GF_FOP_OPEN] = { "OPEN", client3_3_open },
+ [GF_FOP_READ] = { "READ", client3_3_readv },
+ [GF_FOP_WRITE] = { "WRITE", client3_3_writev },
+ [GF_FOP_STATFS] = { "STATFS", client3_3_statfs },
+ [GF_FOP_FLUSH] = { "FLUSH", client3_3_flush },
+ [GF_FOP_FSYNC] = { "FSYNC", client3_3_fsync },
+ [GF_FOP_SETXATTR] = { "SETXATTR", client3_3_setxattr },
+ [GF_FOP_GETXATTR] = { "GETXATTR", client3_3_getxattr },
+ [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr },
+ [GF_FOP_OPENDIR] = { "OPENDIR", client3_3_opendir },
+ [GF_FOP_FSYNCDIR] = { "FSYNCDIR", client3_3_fsyncdir },
+ [GF_FOP_ACCESS] = { "ACCESS", client3_3_access },
+ [GF_FOP_CREATE] = { "CREATE", client3_3_create },
+ [GF_FOP_FTRUNCATE] = { "FTRUNCATE", client3_3_ftruncate },
+ [GF_FOP_FSTAT] = { "FSTAT", client3_3_fstat },
+ [GF_FOP_LK] = { "LK", client3_3_lk },
+ [GF_FOP_LOOKUP] = { "LOOKUP", client3_3_lookup },
+ [GF_FOP_READDIR] = { "READDIR", client3_3_readdir },
+ [GF_FOP_INODELK] = { "INODELK", client3_3_inodelk },
+ [GF_FOP_FINODELK] = { "FINODELK", client3_3_finodelk },
+ [GF_FOP_ENTRYLK] = { "ENTRYLK", client3_3_entrylk },
+ [GF_FOP_FENTRYLK] = { "FENTRYLK", client3_3_fentrylk },
+ [GF_FOP_XATTROP] = { "XATTROP", client3_3_xattrop },
+ [GF_FOP_FXATTROP] = { "FXATTROP", client3_3_fxattrop },
+ [GF_FOP_FGETXATTR] = { "FGETXATTR", client3_3_fgetxattr },
+ [GF_FOP_FSETXATTR] = { "FSETXATTR", client3_3_fsetxattr },
+ [GF_FOP_RCHECKSUM] = { "RCHECKSUM", client3_3_rchecksum },
+ [GF_FOP_SETATTR] = { "SETATTR", client3_3_setattr },
+ [GF_FOP_FSETATTR] = { "FSETATTR", client3_3_fsetattr },
+ [GF_FOP_READDIRP] = { "READDIRP", client3_3_readdirp },
+ [GF_FOP_FALLOCATE] = { "FALLOCATE", client3_3_fallocate },
+ [GF_FOP_DISCARD] = { "DISCARD", client3_3_discard },
+ [GF_FOP_ZEROFILL] = { "ZEROFILL", client3_3_zerofill},
+ [GF_FOP_RELEASE] = { "RELEASE", client3_3_release },
+ [GF_FOP_RELEASEDIR] = { "RELEASEDIR", client3_3_releasedir },
+ [GF_FOP_GETSPEC] = { "GETSPEC", client3_getspec },
[GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
+ [GF_FOP_IPC] = { "IPC", client3_3_ipc },
};
/* Used From RPC-CLNT library to log proper name of procedure based on number */
@@ -6201,6 +6290,7 @@ char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
[GFS3_OP_FALLOCATE] = "FALLOCATE",
[GFS3_OP_DISCARD] = "DISCARD",
[GFS3_OP_ZEROFILL] = "ZEROFILL",
+ [GFS3_OP_IPC] = "IPC",
};
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index ccb5845..d9e7ccd 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2144,6 +2144,38 @@ out:
int32_t
+client_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+ int ret = -1;
+ clnt_conf_t *conf = NULL;
+ rpc_clnt_procedure_t *proc = NULL;
+ clnt_args_t args = {0,};
+
+ conf = this->private;
+ if (!conf || !conf->fops)
+ goto out;
+
+ args.cmd = op;
+ args.xdata = xdata;
+
+ proc = &conf->fops->proctable[GF_FOP_IPC];
+ if (!proc) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "rpc procedure not found for %s",
+ gf_fop_list[GF_FOP_IPC]);
+ goto out;
+ }
+ if (proc->fn)
+ ret = proc->fn (frame, this, &args);
+out:
+ if (ret)
+ STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL);
+
+ return 0;
+}
+
+
+int32_t
client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
int32_t flags)
{
@@ -2943,6 +2975,7 @@ struct xlator_fops fops = {
.discard = client_discard,
.zerofill = client_zerofill,
.getspec = client_getspec,
+ .ipc = client_ipc,
};
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index 902a8e5..051b977 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -2026,6 +2026,42 @@ out:
}
+int
+server_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+ gf_common_rsp rsp = {0,};
+ server_state_t *state = NULL;
+ rpcsvc_request_t *req = NULL;
+
+ req = frame->local;
+ state = CALL_STATE (frame);
+
+ GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+ rsp.xdata.xdata_len, op_errno, out);
+
+ if (op_ret) {
+ gf_log (this->name, GF_LOG_INFO,
+ "%"PRId64": IPC%"PRId64" (%s) ==> (%s)",
+ frame->root->unique, state->resolve.fd_no,
+ uuid_utoa (state->resolve.gfid),
+ strerror (op_errno));
+ goto out;
+ }
+
+out:
+ rsp.op_ret = op_ret;
+ rsp.op_errno = gf_errno_to_error (op_errno);
+
+ server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+ (xdrproc_t) xdr_gf_common_rsp);
+
+ GF_FREE (rsp.xdata.xdata_val);
+
+ return 0;
+}
+
+
/* Resume function section */
int
@@ -3436,6 +3472,63 @@ out:
}
int
+server3_3_ipc (rpcsvc_request_t *req)
+{
+ call_frame_t *frame = NULL;
+ gfs3_ipc_req args = {0,};
+ int ret = -1;
+ int op_errno = 0;
+ dict_t *xdata = NULL;
+ xlator_t *bound_xl = NULL;
+
+ if (!req)
+ return ret;
+
+ ret = xdr_to_generic (req->msg[0], &args,
+ (xdrproc_t)xdr_gfs3_ipc_req);
+ if (ret < 0) {
+ /*failed to decode msg*/;
+ req->rpc_err = GARBAGE_ARGS;
+ goto out;
+ }
+
+ frame = get_frame_from_request (req);
+ if (!frame) {
+ /* something wrong, mostly insufficient memory*/
+ req->rpc_err = GARBAGE_ARGS; /* TODO */
+ goto out;
+ }
+ frame->root->op = GF_FOP_IPC;
+
+ bound_xl = frame->root->client->bound_xl;
+ if (!bound_xl) {
+ /* auth failure, request on subvolume without setvolume */
+ req->rpc_err = GARBAGE_ARGS;
+ goto out;
+ }
+
+ GF_PROTOCOL_DICT_UNSERIALIZE (bound_xl, xdata,
+ args.xdata.xdata_val,
+ args.xdata.xdata_len,
+ ret, op_errno, out);
+
+ ret = 0;
+ STACK_WIND (frame, server_ipc_cbk, bound_xl, bound_xl->fops->ipc,
+ args.op, xdata);
+ if (xdata) {
+ dict_unref(xdata);
+ }
+
+out:
+ free (args.xdata.xdata_val);
+
+ if (op_errno)
+ req->rpc_err = GARBAGE_ARGS;
+
+ return ret;
+}
+
+int
server3_3_readlink (rpcsvc_request_t *req)
{
server_state_t *state = NULL;
@@ -6165,6 +6258,7 @@ rpcsvc_actor_t glusterfs3_3_fop_actors[GLUSTER_FOP_PROCCNT] = {
[GFS3_OP_FALLOCATE] = {"FALLOCATE", GFS3_OP_FALLOCATE, server3_3_fallocate, NULL, 0, DRC_NA},
[GFS3_OP_DISCARD] = {"DISCARD", GFS3_OP_DISCARD, server3_3_discard, NULL, 0, DRC_NA},
[GFS3_OP_ZEROFILL] = {"ZEROFILL", GFS3_OP_ZEROFILL, server3_3_zerofill, NULL, 0, DRC_NA},
+ [GFS3_OP_IPC] = {"IPC", GFS3_OP_IPC, server3_3_ipc, NULL, 0, DRC_NA},
};
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 47afed7..fc6ec99 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -885,6 +885,20 @@ err:
}
+static int32_t
+posix_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+ /*
+ * IPC is for inter-translator communication. If one gets here, it
+ * means somebody sent one that nobody else recognized, which is an
+ * error much like an uncaught exception.
+ */
+ gf_log (this->name, GF_LOG_ERROR, "GF_LOG_IPC(%d) not handled", op);
+ STACK_UNWIND_STRICT (ipc, frame, -1, -EOPNOTSUPP, NULL);
+ return 0;
+
+}
+
int32_t
posix_opendir (call_frame_t *frame, xlator_t *this,
loc_t *loc, fd_t *fd, dict_t *xdata)
@@ -6193,6 +6207,7 @@ struct xlator_fops fops = {
.fallocate = _posix_fallocate,
.discard = posix_discard,
.zerofill = posix_zerofill,
+ .ipc = posix_ipc,
};
struct xlator_cbks cbks = {