From 0d2bed70faed3c63f25ed9269dc55562973ef9b7 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Tue, 10 Mar 2015 20:14:47 -0400 Subject: every/where: add GF_FOP_IPC for inter-translator communication Several features - e.g. encryption, erasure codes, or NSR - involve multiple cooperating translators which sometimes need a "private" means of communication amongst themselves. Historically we've used virtual or synthetic xattrs, but that's not very elegant and clutters up the getxattr/setxattr path which must also handle real xattr requests. This new fop should address that. The only argument is an int32_t "op" which should be recognized by the target translator. It is recommended that translators using these feature follow some convention regarding the ops that they define, to avoid conflicts. Using a hash of the target translator's type string as a base for a series of ops would probably be a good start. Any other information can be passed in both directions using xdata. The default behavior for this fop, as with any other, is to pass through to FIRST_CHILD. That makes use of this fop "transparent" to other translators that were written before it existed, but it also means that it only really works with pass-through translators. If a routing translator (such as DHT) or a fan-out translator (such as AFR) is involved, the IPC might not reach its intended destination unless those translators are modified to forward IPC fops along all paths. If an IPC gets all the way to storage/posix it is considered an error, much like an uncaught exception. We don't actually *do* anything in that case, but we do log it send back an EOPNOTSUPP error. This makes the "unrecognized opcode" condition distinguishable from the "no IPC support" condition (which would yield an RPC error instead) so clients can probe for the presence of a handler for their own favorite opcode and either use that or use old-school xattrs depending on the result. BUG: 1158628 Signed-off-by: Venky Shankar Signed-off-by: Jeff Darcy Change-Id: I84af1b17babe5b30ec03ecf027ae37d09b873968 Reviewed-on: http://review.gluster.org/8812 Reviewed-by: Vijay Bellur --- api/src/gfapi.aliases | 2 + api/src/gfapi.map | 7 +- api/src/glfs.c | 24 ++++ api/src/glfs.h | 8 ++ libglusterfs/src/Makefile.am | 2 +- libglusterfs/src/call-stub.c | 55 ++++++- libglusterfs/src/call-stub.h | 10 ++ libglusterfs/src/defaults.c | 31 ++++ libglusterfs/src/defaults.h | 9 ++ libglusterfs/src/globals.c | 3 +- libglusterfs/src/glusterfs.h | 3 +- libglusterfs/src/syncop.c | 46 ++++++ libglusterfs/src/syncop.h | 4 + libglusterfs/src/xlator.c | 1 + libglusterfs/src/xlator.h | 10 ++ rpc/rpc-lib/src/protocol-common.h | 1 + rpc/xdr/src/glusterfs3-xdr.x | 12 ++ tests/features/ipc.t | 21 +++ tests/features/ipctest.py | 28 ++++ xlators/performance/io-threads/src/io-threads.c | 1 + xlators/protocol/client/src/client-rpc-fops.c | 184 ++++++++++++++++++------ xlators/protocol/client/src/client.c | 33 +++++ xlators/protocol/server/src/server-rpc-fops.c | 94 ++++++++++++ xlators/storage/posix/src/posix.c | 15 ++ 24 files changed, 552 insertions(+), 52 deletions(-) create mode 100755 tests/features/ipc.t create mode 100755 tests/features/ipctest.py diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases index 6c0a6413098..2ab7d443eb5 100644 --- a/api/src/gfapi.aliases +++ b/api/src/gfapi.aliases @@ -125,6 +125,8 @@ _pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1 _pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0 _pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0 +_pub_glfs_ipc _glfs_ipc$GFAPI_3.7.0 + _priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0 _priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0 _priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0 diff --git a/api/src/gfapi.map b/api/src/gfapi.map index a29f392dc53..39202e1883f 100644 --- a/api/src/gfapi.map +++ b/api/src/gfapi.map @@ -145,10 +145,15 @@ GFAPI_3.6.0 { glfs_h_access; } GFAPI_3.5.1; +GFAPI_3.7.0 { + global: + glfs_ipc; +} GFAPI_3.6.0; + GFAPI_PRIVATE_3.7.0 { global: glfs_free_from_ctx; glfs_new_from_ctx; glfs_resolve; -} GFAPI_3.6.0; +} GFAPI_3.7.0; diff --git a/api/src/glfs.c b/api/src/glfs.c index 421374d9731..f23481bbb4c 100644 --- a/api/src/glfs.c +++ b/api/src/glfs.c @@ -1067,3 +1067,27 @@ pub_glfs_get_volfile (struct glfs *fs, void *buf, size_t len) GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0); +int +pub_glfs_ipc (struct glfs *fs, int opcode) +{ + xlator_t *subvol = NULL; + int ret; + + __glfs_entry_fs (fs); + + subvol = glfs_active_subvol (fs); + if (!subvol) { + ret = -1; + errno = EIO; + goto out; + } + + ret = syncop_ipc (subvol, opcode, NULL, NULL); + DECODE_SYNCOP_ERR (ret); + +out: + glfs_subvol_done (fs, subvol); + return ret; +} + +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ipc, 3.7.0); diff --git a/api/src/glfs.h b/api/src/glfs.h index 3ef822ed3f1..9ee772741e6 100644 --- a/api/src/glfs.h +++ b/api/src/glfs.h @@ -760,6 +760,14 @@ int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock) __THROW glfs_fd_t *glfs_dup (glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0); +/* + * No xdata support for now. Nobody needs this call at all yet except for the + * test script, and that doesn't need xdata. Adding dict_t support and a new + * header-file requirement doesn't seem worth it until the need is greater. + */ +int glfs_ipc (glfs_fd_t *fd, int cmd) __THROW + GFAPI_PUBLIC(glfs_ipc, 3.7.0); + __END_DECLS #endif /* !_GLFS_H */ diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am index 3d1de02e894..02f4462e6b0 100644 --- a/libglusterfs/src/Makefile.am +++ b/libglusterfs/src/Makefile.am @@ -5,7 +5,7 @@ libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \ -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \ -I$(top_srcdir)/rpc/rpc-lib/src/ -I$(CONTRIBDIR)/rbtree \ -I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \ - -DSBIN_DIR=\"$(sbindir)\" + -DSBIN_DIR=\"$(sbindir)\" -lm libglusterfs_la_LIBADD = @LEXLIB@ $(ZLIB_LIBS) libglusterfs_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c index 7e94ee3c001..ee2e7c93337 100644 --- a/libglusterfs/src/call-stub.c +++ b/libglusterfs/src/call-stub.c @@ -2297,7 +2297,53 @@ out: } -static void +call_stub_t * +fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + call_stub_t *stub = NULL; + + GF_VALIDATE_OR_GOTO ("call-stub", frame, out); + + stub = stub_new (frame, 0, GF_FOP_IPC); + GF_VALIDATE_OR_GOTO ("call-stub", stub, out); + + stub->fn_cbk.ipc = fn; + + stub->args_cbk.op_ret = op_ret; + stub->args_cbk.op_errno = op_errno; + + if (xdata) + stub->args_cbk.xdata = dict_ref (xdata); +out: + return stub; +} + +call_stub_t * +fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn, + int32_t op, dict_t *xdata) +{ + call_stub_t *stub = NULL; + + GF_VALIDATE_OR_GOTO ("call-stub", frame, out); + GF_VALIDATE_OR_GOTO ("call-stub", fn, out); + + stub = stub_new (frame, 1, GF_FOP_IPC); + GF_VALIDATE_OR_GOTO ("call-stub", stub, out); + + stub->fn.ipc = fn; + + stub->args.cmd = op; + + if (xdata) + stub->args.xdata = dict_ref (xdata); +out: + return stub; + +} + + +void call_resume_wind (call_stub_t *stub) { GF_VALIDATE_OR_GOTO ("call-stub", stub, out); @@ -2529,6 +2575,10 @@ call_resume_wind (call_stub_t *stub) stub->args.fd, stub->args.offset, stub->args.size, stub->args.xdata); break; + case GF_FOP_IPC: + stub->fn.ipc (stub->frame, stub->frame->this, + stub->args.cmd, stub->args.xdata); + break; default: gf_log_callingfn ("call-stub", GF_LOG_ERROR, @@ -2736,6 +2786,9 @@ call_resume_unwind (call_stub_t *stub) STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat, &stub->args_cbk.poststat, stub->args_cbk.xdata); break; + case GF_FOP_IPC: + STUB_UNWIND (stub, ipc, stub->args_cbk.xdata); + break; default: gf_log_callingfn ("call-stub", GF_LOG_ERROR, diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h index eba1413a1e6..fe110775fc6 100644 --- a/libglusterfs/src/call-stub.h +++ b/libglusterfs/src/call-stub.h @@ -72,6 +72,7 @@ typedef struct { fop_fallocate_t fallocate; fop_discard_t discard; fop_zerofill_t zerofill; + fop_ipc_t ipc; } fn; union { @@ -119,6 +120,7 @@ typedef struct { fop_fallocate_cbk_t fallocate; fop_discard_cbk_t discard; fop_zerofill_cbk_t zerofill; + fop_ipc_cbk_t ipc; } fn_cbk; struct { @@ -761,6 +763,14 @@ fop_zerofill_cbk_stub(call_frame_t *frame, struct iatt *statpre, struct iatt *statpost, dict_t *xdata); +call_stub_t * +fop_ipc_stub (call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata); + +call_stub_t * +fop_ipc_cbk_stub (call_frame_t *frame, fop_ipc_cbk_t fn, + int32_t op_ret, int32_t op_errno, dict_t *xdata); + + void call_resume (call_stub_t *stub); void call_stub_destroy (call_stub_t *stub); void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno); diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c index a4f8f924b17..ac08a70cf32 100644 --- a/libglusterfs/src/defaults.c +++ b/libglusterfs/src/defaults.c @@ -1295,6 +1295,16 @@ default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this, return 0; } + +int32_t +default_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + STACK_UNWIND_STRICT (ipc, frame, op_ret, op_errno, xdata); + return 0; +} + + /* RESUME */ int32_t @@ -1726,6 +1736,17 @@ default_zerofill_resume(call_frame_t *frame, xlator_t *this, fd_t *fd, } +int32_t +default_ipc_resume (call_frame_t *frame, xlator_t *this, int32_t op, + dict_t *xdata) +{ + STACK_WIND (frame, default_ipc_cbk, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->ipc, + op, xdata); + return 0; +} + + /* FOPS */ int32_t @@ -2161,6 +2182,16 @@ default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, } +int32_t +default_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) +{ + STACK_WIND_TAIL (frame, + FIRST_CHILD(this), FIRST_CHILD(this)->fops->ipc, + op, xdata); + return 0; +} + + int32_t default_forget (xlator_t *this, inode_t *inode) { diff --git a/libglusterfs/src/defaults.h b/libglusterfs/src/defaults.h index 9bd5eb842ad..e29d62edfe1 100644 --- a/libglusterfs/src/defaults.h +++ b/libglusterfs/src/defaults.h @@ -263,6 +263,9 @@ int32_t default_zerofill(call_frame_t *frame, off_t offset, off_t len, dict_t *xdata); +int32_t default_ipc (call_frame_t *frame, xlator_t *this, int32_t op, + dict_t *xdata); + /* Resume */ int32_t default_getspec_resume (call_frame_t *frame, @@ -492,6 +495,9 @@ int32_t default_zerofill_resume(call_frame_t *frame, off_t offset, off_t len, dict_t *xdata); +int32_t default_ipc_resume (call_frame_t *frame, xlator_t *this, + int32_t op, dict_t *xdata); + /* _cbk_resume */ @@ -985,6 +991,9 @@ int32_t default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *pre, struct iatt *post, dict_t *xdata); +int32_t default_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata); + int32_t default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, char *spec_data); diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c index 57467ecde1d..bd1165ec50c 100644 --- a/libglusterfs/src/globals.c +++ b/libglusterfs/src/globals.c @@ -70,7 +70,8 @@ const char *gf_fop_list[GF_FOP_MAXVALUE] = { [GF_FOP_FREMOVEXATTR]= "FREMOVEXATTR", [GF_FOP_FALLOCATE] = "FALLOCATE", [GF_FOP_DISCARD] = "DISCARD", - [GF_FOP_ZEROFILL] = "ZEROFILL", + [GF_FOP_ZEROFILL] = "ZEROFILL", + [GF_FOP_IPC] = "IPC", }; /* THIS */ diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index d5a604d0341..8d7659b5015 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -238,7 +238,7 @@ typedef enum { GF_FOP_WRITE, GF_FOP_STATFS, GF_FOP_FLUSH, - GF_FOP_FSYNC, /* 15 */ + GF_FOP_FSYNC, /* 16 */ GF_FOP_SETXATTR, GF_FOP_GETXATTR, GF_FOP_REMOVEXATTR, @@ -271,6 +271,7 @@ typedef enum { GF_FOP_FALLOCATE, GF_FOP_DISCARD, GF_FOP_ZEROFILL, + GF_FOP_IPC, GF_FOP_MAXVALUE, } glusterfs_fop_t; diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c index 9df462321aa..e3321cf6ddb 100644 --- a/libglusterfs/src/syncop.c +++ b/libglusterfs/src/syncop.c @@ -2416,6 +2416,52 @@ syncop_zerofill(xlator_t *subvol, fd_t *fd, off_t offset, off_t len) } +int +syncop_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int op_ret, int op_errno, dict_t *xdata) +{ + struct syncargs *args = NULL; + + args = cookie; + + if (xdata) { + args->xdata = dict_ref(xdata); + } + + args->op_ret = op_ret; + args->op_errno = op_errno; + + __wake (args); + + return 0; +} + +int +syncop_ipc (xlator_t *subvol, int32_t op, dict_t *xdata_in, dict_t **xdata_out) +{ + struct syncargs args = {0, }; + + SYNCOP (subvol, (&args), syncop_ipc_cbk, subvol->fops->ipc, + op, xdata_in); + + if (args.xdata) { + if (xdata_out) { + /* + * We're passing this reference to the caller, along + * with the pointer itself. That means they're + * responsible for calling dict_unref at some point. + */ + *xdata_out = args.xdata; + } else { + dict_unref(args.xdata); + } + } + + if (args.op_ret < 0) + return -args.op_errno; + return args.op_ret; +} + int syncop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, struct gf_flock *flock, diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h index ec0d8a917e8..7f8ec7345b0 100644 --- a/libglusterfs/src/syncop.h +++ b/libglusterfs/src/syncop.h @@ -435,4 +435,8 @@ int syncop_lk (xlator_t *subvol, fd_t *fd, int cmd, struct gf_flock *flock); int syncop_inodelk (xlator_t *subvol, const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata_req, dict_t **xdata_rsp); + +int +syncop_ipc (xlator_t *subvol, int op, dict_t *xdata_in, dict_t **xdata_out); + #endif /* _SYNCOP_H */ diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c index b58247e52bc..49af7d2e0e6 100644 --- a/libglusterfs/src/xlator.c +++ b/libglusterfs/src/xlator.c @@ -82,6 +82,7 @@ fill_defaults (xlator_t *xl) SET_DEFAULT_FOP (fallocate); SET_DEFAULT_FOP (discard); SET_DEFAULT_FOP (zerofill); + SET_DEFAULT_FOP (ipc); SET_DEFAULT_FOP (getspec); diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h index 8e52bbb3010..e953ec04372 100644 --- a/libglusterfs/src/xlator.h +++ b/libglusterfs/src/xlator.h @@ -443,6 +443,10 @@ typedef int32_t (*fop_zerofill_cbk_t) (call_frame_t *frame, struct iatt *preop_stbuf, struct iatt *postop_stbuf, dict_t *xdata); +typedef int32_t (*fop_ipc_cbk_t) (call_frame_t *frame, void *cookie, + xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata); + typedef int32_t (*fop_lookup_t) (call_frame_t *frame, xlator_t *this, loc_t *loc, @@ -674,6 +678,7 @@ typedef int32_t (*fop_discard_t) (call_frame_t *frame, off_t offset, size_t len, dict_t *xdata); + typedef int32_t (*fop_zerofill_t) (call_frame_t *frame, xlator_t *this, fd_t *fd, @@ -681,6 +686,9 @@ typedef int32_t (*fop_zerofill_t) (call_frame_t *frame, off_t len, dict_t *xdata); +typedef int32_t (*fop_ipc_t) (call_frame_t *frame, xlator_t *this, int32_t op, + dict_t *xdata); + struct xlator_fops { fop_lookup_t lookup; fop_stat_t stat; @@ -727,6 +735,7 @@ struct xlator_fops { fop_fallocate_t fallocate; fop_discard_t discard; fop_zerofill_t zerofill; + fop_ipc_t ipc; /* these entries are used for a typechecking hack in STACK_WIND _only_ */ fop_lookup_cbk_t lookup_cbk; @@ -774,6 +783,7 @@ struct xlator_fops { fop_fallocate_cbk_t fallocate_cbk; fop_discard_cbk_t discard_cbk; fop_zerofill_cbk_t zerofill_cbk; + fop_ipc_cbk_t ipc_cbk; }; typedef int32_t (*cbk_forget_t) (xlator_t *this, diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h index 3d8cf11fa5f..ee59078de42 100644 --- a/rpc/rpc-lib/src/protocol-common.h +++ b/rpc/rpc-lib/src/protocol-common.h @@ -59,6 +59,7 @@ enum gf_fop_procnum { GFS3_OP_FALLOCATE, GFS3_OP_DISCARD, GFS3_OP_ZEROFILL, + GFS3_OP_IPC, GFS3_OP_MAXVALUE, } ; diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x index 81b0f201f89..0136aec2b08 100644 --- a/rpc/xdr/src/glusterfs3-xdr.x +++ b/rpc/xdr/src/glusterfs3-xdr.x @@ -640,6 +640,18 @@ struct gfs3_fstat_req { } ; +struct gfs3_ipc_req { + int op; + opaque xdata<>; +}; + +struct gfs3_ipc_rsp { + int op_ret; + int op_errno; + opaque xdata<>; +}; + + struct gf_setvolume_req { opaque dict<>; } ; diff --git a/tests/features/ipc.t b/tests/features/ipc.t new file mode 100755 index 00000000000..2aaca6620bf --- /dev/null +++ b/tests/features/ipc.t @@ -0,0 +1,21 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc + +cleanup; +mkdir -p $B0/1 +mkdir -p $M0 + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 $H0:$B0/1 +TEST $CLI volume start $V0 + +# This is a pretty lame test. Basically we just want to make sure that we +# get all the way through the translator stacks on client and server to get a +# simple error (95 = EOPNOTUPP) instead of a crash, RPC error, etc. +EXPECT 95 $PYTHON $(dirname $0)/ipctest.py $H0 $V0 + +cleanup; diff --git a/tests/features/ipctest.py b/tests/features/ipctest.py new file mode 100755 index 00000000000..0592bae3bbc --- /dev/null +++ b/tests/features/ipctest.py @@ -0,0 +1,28 @@ +#!/usr/bin/python + +import ctypes +import ctypes.util + +api = ctypes.CDLL(ctypes.util.find_library("gfapi")) +api.glfs_ipc.argtypes = [ ctypes.c_void_p, ctypes.c_int ] +api.glfs_ipc.restype = ctypes.c_int + +def do_ipc (host, volume): + fs = api.glfs_new(volume) + #api.glfs_set_logging(fs,"/dev/stderr",7) + api.glfs_set_volfile_server(fs,"tcp",host,24007) + + api.glfs_init(fs) + ret = api.glfs_ipc(fs,1470369258) + api.glfs_fini(fs) + + return ret + +if __name__ == "__main__": + import sys + + try: + res = apply(do_ipc,sys.argv[1:3]) + print res + except: + print "IPC failed (volume not started?)" diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c index 9c6ae5af12e..148e55ab297 100644 --- a/xlators/performance/io-threads/src/io-threads.c +++ b/xlators/performance/io-threads/src/io-threads.c @@ -341,6 +341,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub) case GF_FOP_RELEASEDIR: case GF_FOP_GETSPEC: break; + case GF_FOP_IPC: default: return -EINVAL; } diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c index 4305fb39627..7ca91e9880c 100644 --- a/xlators/protocol/client/src/client-rpc-fops.c +++ b/xlators/protocol/client/src/client-rpc-fops.c @@ -2102,6 +2102,55 @@ out: return 0; } +int +client3_3_ipc_cbk (struct rpc_req *req, struct iovec *iov, int count, + void *myframe) +{ + call_frame_t *frame = NULL; + gfs3_ipc_rsp rsp = {0,}; + int ret = 0; + xlator_t *this = NULL; + dict_t *xdata = NULL; + + this = THIS; + + frame = myframe; + + if (-1 == req->rpc_status) { + rsp.op_ret = -1; + rsp.op_errno = ENOTCONN; + goto out; + } + ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_ipc_rsp); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed"); + rsp.op_ret = -1; + rsp.op_errno = EINVAL; + goto out; + } + + GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val), + (rsp.xdata.xdata_len), ret, + rsp.op_errno, out); + +out: + if (rsp.op_ret == -1) { + gf_log (this->name, GF_LOG_WARNING, + "remote operation failed: %s", + strerror (gf_error_to_errno (rsp.op_errno))); + } + CLIENT_STACK_UNWIND (ipc, frame, + rsp.op_ret, gf_error_to_errno (rsp.op_errno), + xdata); + + free (rsp.xdata.xdata_val); + + if (xdata) + dict_unref (xdata); + + return 0; +} + int client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe) @@ -6098,58 +6147,98 @@ unwind: return 0; } +int32_t +client3_3_ipc (call_frame_t *frame, xlator_t *this, void *data) +{ + clnt_args_t *args = NULL; + clnt_conf_t *conf = NULL; + gfs3_ipc_req req = {0,}; + int op_errno = ESTALE; + int ret = 0; + + GF_ASSERT (frame); + + if (!this || !data) + goto unwind; + + args = data; + conf = this->private; + + req.op = args->cmd; + + GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val), + req.xdata.xdata_len, op_errno, unwind); + + ret = client_submit_request(this, &req, frame, conf->fops, + GFS3_OP_IPC, client3_3_ipc_cbk, + NULL, NULL, 0, NULL, 0, NULL, + (xdrproc_t) xdr_gfs3_ipc_req); + if (ret) + gf_log (this->name, GF_LOG_WARNING, "failed to send the fop"); + + GF_FREE (req.xdata.xdata_val); + + return 0; +unwind: + CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL); + GF_FREE (req.xdata.xdata_val); + + return 0; +} + /* Table Specific to FOPS */ rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = { - [GF_FOP_NULL] = { "NULL", NULL}, - [GF_FOP_STAT] = { "STAT", client3_3_stat }, - [GF_FOP_READLINK] = { "READLINK", client3_3_readlink }, - [GF_FOP_MKNOD] = { "MKNOD", client3_3_mknod }, - [GF_FOP_MKDIR] = { "MKDIR", client3_3_mkdir }, - [GF_FOP_UNLINK] = { "UNLINK", client3_3_unlink }, - [GF_FOP_RMDIR] = { "RMDIR", client3_3_rmdir }, - [GF_FOP_SYMLINK] = { "SYMLINK", client3_3_symlink }, - [GF_FOP_RENAME] = { "RENAME", client3_3_rename }, - [GF_FOP_LINK] = { "LINK", client3_3_link }, - [GF_FOP_TRUNCATE] = { "TRUNCATE", client3_3_truncate }, - [GF_FOP_OPEN] = { "OPEN", client3_3_open }, - [GF_FOP_READ] = { "READ", client3_3_readv }, - [GF_FOP_WRITE] = { "WRITE", client3_3_writev }, - [GF_FOP_STATFS] = { "STATFS", client3_3_statfs }, - [GF_FOP_FLUSH] = { "FLUSH", client3_3_flush }, - [GF_FOP_FSYNC] = { "FSYNC", client3_3_fsync }, - [GF_FOP_SETXATTR] = { "SETXATTR", client3_3_setxattr }, - [GF_FOP_GETXATTR] = { "GETXATTR", client3_3_getxattr }, - [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr }, - [GF_FOP_OPENDIR] = { "OPENDIR", client3_3_opendir }, - [GF_FOP_FSYNCDIR] = { "FSYNCDIR", client3_3_fsyncdir }, - [GF_FOP_ACCESS] = { "ACCESS", client3_3_access }, - [GF_FOP_CREATE] = { "CREATE", client3_3_create }, - [GF_FOP_FTRUNCATE] = { "FTRUNCATE", client3_3_ftruncate }, - [GF_FOP_FSTAT] = { "FSTAT", client3_3_fstat }, - [GF_FOP_LK] = { "LK", client3_3_lk }, - [GF_FOP_LOOKUP] = { "LOOKUP", client3_3_lookup }, - [GF_FOP_READDIR] = { "READDIR", client3_3_readdir }, - [GF_FOP_INODELK] = { "INODELK", client3_3_inodelk }, - [GF_FOP_FINODELK] = { "FINODELK", client3_3_finodelk }, - [GF_FOP_ENTRYLK] = { "ENTRYLK", client3_3_entrylk }, - [GF_FOP_FENTRYLK] = { "FENTRYLK", client3_3_fentrylk }, - [GF_FOP_XATTROP] = { "XATTROP", client3_3_xattrop }, - [GF_FOP_FXATTROP] = { "FXATTROP", client3_3_fxattrop }, - [GF_FOP_FGETXATTR] = { "FGETXATTR", client3_3_fgetxattr }, - [GF_FOP_FSETXATTR] = { "FSETXATTR", client3_3_fsetxattr }, - [GF_FOP_RCHECKSUM] = { "RCHECKSUM", client3_3_rchecksum }, - [GF_FOP_SETATTR] = { "SETATTR", client3_3_setattr }, - [GF_FOP_FSETATTR] = { "FSETATTR", client3_3_fsetattr }, - [GF_FOP_READDIRP] = { "READDIRP", client3_3_readdirp }, - [GF_FOP_FALLOCATE] = { "FALLOCATE", client3_3_fallocate }, - [GF_FOP_DISCARD] = { "DISCARD", client3_3_discard }, - [GF_FOP_ZEROFILL] = { "ZEROFILL", client3_3_zerofill}, - [GF_FOP_RELEASE] = { "RELEASE", client3_3_release }, - [GF_FOP_RELEASEDIR] = { "RELEASEDIR", client3_3_releasedir }, - [GF_FOP_GETSPEC] = { "GETSPEC", client3_getspec }, + [GF_FOP_NULL] = { "NULL", NULL}, + [GF_FOP_STAT] = { "STAT", client3_3_stat }, + [GF_FOP_READLINK] = { "READLINK", client3_3_readlink }, + [GF_FOP_MKNOD] = { "MKNOD", client3_3_mknod }, + [GF_FOP_MKDIR] = { "MKDIR", client3_3_mkdir }, + [GF_FOP_UNLINK] = { "UNLINK", client3_3_unlink }, + [GF_FOP_RMDIR] = { "RMDIR", client3_3_rmdir }, + [GF_FOP_SYMLINK] = { "SYMLINK", client3_3_symlink }, + [GF_FOP_RENAME] = { "RENAME", client3_3_rename }, + [GF_FOP_LINK] = { "LINK", client3_3_link }, + [GF_FOP_TRUNCATE] = { "TRUNCATE", client3_3_truncate }, + [GF_FOP_OPEN] = { "OPEN", client3_3_open }, + [GF_FOP_READ] = { "READ", client3_3_readv }, + [GF_FOP_WRITE] = { "WRITE", client3_3_writev }, + [GF_FOP_STATFS] = { "STATFS", client3_3_statfs }, + [GF_FOP_FLUSH] = { "FLUSH", client3_3_flush }, + [GF_FOP_FSYNC] = { "FSYNC", client3_3_fsync }, + [GF_FOP_SETXATTR] = { "SETXATTR", client3_3_setxattr }, + [GF_FOP_GETXATTR] = { "GETXATTR", client3_3_getxattr }, + [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr }, + [GF_FOP_OPENDIR] = { "OPENDIR", client3_3_opendir }, + [GF_FOP_FSYNCDIR] = { "FSYNCDIR", client3_3_fsyncdir }, + [GF_FOP_ACCESS] = { "ACCESS", client3_3_access }, + [GF_FOP_CREATE] = { "CREATE", client3_3_create }, + [GF_FOP_FTRUNCATE] = { "FTRUNCATE", client3_3_ftruncate }, + [GF_FOP_FSTAT] = { "FSTAT", client3_3_fstat }, + [GF_FOP_LK] = { "LK", client3_3_lk }, + [GF_FOP_LOOKUP] = { "LOOKUP", client3_3_lookup }, + [GF_FOP_READDIR] = { "READDIR", client3_3_readdir }, + [GF_FOP_INODELK] = { "INODELK", client3_3_inodelk }, + [GF_FOP_FINODELK] = { "FINODELK", client3_3_finodelk }, + [GF_FOP_ENTRYLK] = { "ENTRYLK", client3_3_entrylk }, + [GF_FOP_FENTRYLK] = { "FENTRYLK", client3_3_fentrylk }, + [GF_FOP_XATTROP] = { "XATTROP", client3_3_xattrop }, + [GF_FOP_FXATTROP] = { "FXATTROP", client3_3_fxattrop }, + [GF_FOP_FGETXATTR] = { "FGETXATTR", client3_3_fgetxattr }, + [GF_FOP_FSETXATTR] = { "FSETXATTR", client3_3_fsetxattr }, + [GF_FOP_RCHECKSUM] = { "RCHECKSUM", client3_3_rchecksum }, + [GF_FOP_SETATTR] = { "SETATTR", client3_3_setattr }, + [GF_FOP_FSETATTR] = { "FSETATTR", client3_3_fsetattr }, + [GF_FOP_READDIRP] = { "READDIRP", client3_3_readdirp }, + [GF_FOP_FALLOCATE] = { "FALLOCATE", client3_3_fallocate }, + [GF_FOP_DISCARD] = { "DISCARD", client3_3_discard }, + [GF_FOP_ZEROFILL] = { "ZEROFILL", client3_3_zerofill}, + [GF_FOP_RELEASE] = { "RELEASE", client3_3_release }, + [GF_FOP_RELEASEDIR] = { "RELEASEDIR", client3_3_releasedir }, + [GF_FOP_GETSPEC] = { "GETSPEC", client3_getspec }, [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr }, + [GF_FOP_IPC] = { "IPC", client3_3_ipc }, }; /* Used From RPC-CLNT library to log proper name of procedure based on number */ @@ -6201,6 +6290,7 @@ char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = { [GFS3_OP_FALLOCATE] = "FALLOCATE", [GFS3_OP_DISCARD] = "DISCARD", [GFS3_OP_ZEROFILL] = "ZEROFILL", + [GFS3_OP_IPC] = "IPC", }; diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index ccb584569ea..d9e7ccd0c4f 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -2143,6 +2143,38 @@ out: } +int32_t +client_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) +{ + int ret = -1; + clnt_conf_t *conf = NULL; + rpc_clnt_procedure_t *proc = NULL; + clnt_args_t args = {0,}; + + conf = this->private; + if (!conf || !conf->fops) + goto out; + + args.cmd = op; + args.xdata = xdata; + + proc = &conf->fops->proctable[GF_FOP_IPC]; + if (!proc) { + gf_log (this->name, GF_LOG_ERROR, + "rpc procedure not found for %s", + gf_fop_list[GF_FOP_IPC]); + goto out; + } + if (proc->fn) + ret = proc->fn (frame, this, &args); +out: + if (ret) + STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL); + + return 0; +} + + int32_t client_getspec (call_frame_t *frame, xlator_t *this, const char *key, int32_t flags) @@ -2943,6 +2975,7 @@ struct xlator_fops fops = { .discard = client_discard, .zerofill = client_zerofill, .getspec = client_getspec, + .ipc = client_ipc, }; diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c index 902a8e5f6b8..051b9771432 100644 --- a/xlators/protocol/server/src/server-rpc-fops.c +++ b/xlators/protocol/server/src/server-rpc-fops.c @@ -2026,6 +2026,42 @@ out: } +int +server_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + gf_common_rsp rsp = {0,}; + server_state_t *state = NULL; + rpcsvc_request_t *req = NULL; + + req = frame->local; + state = CALL_STATE (frame); + + GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val), + rsp.xdata.xdata_len, op_errno, out); + + if (op_ret) { + gf_log (this->name, GF_LOG_INFO, + "%"PRId64": IPC%"PRId64" (%s) ==> (%s)", + frame->root->unique, state->resolve.fd_no, + uuid_utoa (state->resolve.gfid), + strerror (op_errno)); + goto out; + } + +out: + rsp.op_ret = op_ret; + rsp.op_errno = gf_errno_to_error (op_errno); + + server_submit_reply(frame, req, &rsp, NULL, 0, NULL, + (xdrproc_t) xdr_gf_common_rsp); + + GF_FREE (rsp.xdata.xdata_val); + + return 0; +} + + /* Resume function section */ int @@ -3435,6 +3471,63 @@ out: return ret; } +int +server3_3_ipc (rpcsvc_request_t *req) +{ + call_frame_t *frame = NULL; + gfs3_ipc_req args = {0,}; + int ret = -1; + int op_errno = 0; + dict_t *xdata = NULL; + xlator_t *bound_xl = NULL; + + if (!req) + return ret; + + ret = xdr_to_generic (req->msg[0], &args, + (xdrproc_t)xdr_gfs3_ipc_req); + if (ret < 0) { + /*failed to decode msg*/; + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + frame = get_frame_from_request (req); + if (!frame) { + /* something wrong, mostly insufficient memory*/ + req->rpc_err = GARBAGE_ARGS; /* TODO */ + goto out; + } + frame->root->op = GF_FOP_IPC; + + bound_xl = frame->root->client->bound_xl; + if (!bound_xl) { + /* auth failure, request on subvolume without setvolume */ + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + GF_PROTOCOL_DICT_UNSERIALIZE (bound_xl, xdata, + args.xdata.xdata_val, + args.xdata.xdata_len, + ret, op_errno, out); + + ret = 0; + STACK_WIND (frame, server_ipc_cbk, bound_xl, bound_xl->fops->ipc, + args.op, xdata); + if (xdata) { + dict_unref(xdata); + } + +out: + free (args.xdata.xdata_val); + + if (op_errno) + req->rpc_err = GARBAGE_ARGS; + + return ret; +} + int server3_3_readlink (rpcsvc_request_t *req) { @@ -6165,6 +6258,7 @@ rpcsvc_actor_t glusterfs3_3_fop_actors[GLUSTER_FOP_PROCCNT] = { [GFS3_OP_FALLOCATE] = {"FALLOCATE", GFS3_OP_FALLOCATE, server3_3_fallocate, NULL, 0, DRC_NA}, [GFS3_OP_DISCARD] = {"DISCARD", GFS3_OP_DISCARD, server3_3_discard, NULL, 0, DRC_NA}, [GFS3_OP_ZEROFILL] = {"ZEROFILL", GFS3_OP_ZEROFILL, server3_3_zerofill, NULL, 0, DRC_NA}, + [GFS3_OP_IPC] = {"IPC", GFS3_OP_IPC, server3_3_ipc, NULL, 0, DRC_NA}, }; diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c index 47afed7fdad..fc6ec991c44 100644 --- a/xlators/storage/posix/src/posix.c +++ b/xlators/storage/posix/src/posix.c @@ -885,6 +885,20 @@ err: } +static int32_t +posix_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) +{ + /* + * IPC is for inter-translator communication. If one gets here, it + * means somebody sent one that nobody else recognized, which is an + * error much like an uncaught exception. + */ + gf_log (this->name, GF_LOG_ERROR, "GF_LOG_IPC(%d) not handled", op); + STACK_UNWIND_STRICT (ipc, frame, -1, -EOPNOTSUPP, NULL); + return 0; + +} + int32_t posix_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata) @@ -6193,6 +6207,7 @@ struct xlator_fops fops = { .fallocate = _posix_fallocate, .discard = posix_discard, .zerofill = posix_zerofill, + .ipc = posix_ipc, }; struct xlator_cbks cbks = { -- cgit