From db7d578da03a5d8bbc2169a45baea5e0e3ddc1e3 Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Fri, 1 Aug 2014 18:30:32 +0530 Subject: mount/fuse: Handle fd resolution failures Backport of http://review.gluster.org/8402 Problem: Even when the fd resolution failed, the fop is continuing on the new graph which may not have valid inode. This lead to NULL layout subvols in dht which lead to crash in fsync after graph migration. Fix: - Remove resolution error handling in FUSE_FOP as it was only added to handle fd migration failures. - check in fuse_resolve_done for fd resolution failures and fail the fop right away. - loc resolution failures are already handled in the corresponding fops. - Return errno from state->resolve.op_errno in resume functions. - Send error to fuse on frame allocation failures. - Removed unused variable state->resolved - Removed unused macro FUSE_FOP_COOKIE BUG: 1136835 Change-Id: I5074f7a9b177c54051ef37a4f73de7f8d1fcc5b7 Signed-off-by: Pranith Kumar K Reviewed-on: http://review.gluster.org/8595 Tested-by: Gluster Build System Reviewed-by: Niels de Vos --- tests/bugs/bug-1126048.c | 37 +++++++ tests/bugs/bug-1126048.t | 30 ++++++ xlators/mount/fuse/src/fuse-bridge.c | 79 +++++++++++---- xlators/mount/fuse/src/fuse-bridge.h | 178 +++++----------------------------- xlators/mount/fuse/src/fuse-helpers.c | 2 - xlators/mount/fuse/src/fuse-resolve.c | 10 +- 6 files changed, 149 insertions(+), 187 deletions(-) create mode 100644 tests/bugs/bug-1126048.c create mode 100755 tests/bugs/bug-1126048.t diff --git a/tests/bugs/bug-1126048.c b/tests/bugs/bug-1126048.c new file mode 100644 index 00000000000..2282bb2025e --- /dev/null +++ b/tests/bugs/bug-1126048.c @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include +#include +#include + +/* + * This function opens a file and to trigger migration failure, unlinks the + * file and performs graph switch (cmd passed in argv). If everything goes fine, + * fsync should fail without crashing the mount process. + */ +int +main (int argc, char **argv) +{ + int ret = 0; + int fd = 0; + char *cmd = argv[1]; + + printf ("cmd is: %s\n", cmd); + fd = open("a.txt", O_CREAT|O_RDWR); + if (fd < 0) + printf ("open failed: %s\n", strerror(errno)); + + ret = unlink("a.txt"); + if (ret < 0) + printf ("unlink failed: %s\n", strerror(errno)); + if (write (fd, "abc", 3) < 0) + printf ("Not able to print %s\n", strerror (errno)); + system(cmd); + sleep(1); //No way to confirm graph switch so sleep 1 + ret = fsync(fd); + if (ret < 0) + printf ("Not able to fsync %s\n", strerror (errno)); + return 0; +} diff --git a/tests/bugs/bug-1126048.t b/tests/bugs/bug-1126048.t new file mode 100755 index 00000000000..fe1120ba9e7 --- /dev/null +++ b/tests/bugs/bug-1126048.t @@ -0,0 +1,30 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc + +cleanup; + +function grep_for_ebadf { + $M0/bug-1126048 "gluster --mode=script volume add-brick $V0 $H0:$B0/brick2" | grep -i "Bad file descriptor" +} +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 $H0:$B0/brick1; +EXPECT 'Created' volinfo_field $V0 'Status'; + +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +TEST glusterfs -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=yes + +build_tester $(dirname $0)/bug-1126048.c + +TEST cp $(dirname $0)/bug-1126048 $M0 +cd $M0 +TEST grep_for_ebadf +TEST ls -l $M0 +cd - +TEST rm -f $(dirname $0)/bug-1126048 +cleanup; diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c index 26f806e0633..8ea02bc5f19 100644 --- a/xlators/mount/fuse/src/fuse-bridge.c +++ b/xlators/mount/fuse/src/fuse-bridge.c @@ -538,13 +538,33 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, return 0; } +void +fuse_fop_resume (fuse_state_t *state) +{ + fuse_resume_fn_t fn = NULL; + + /* + * Fail fd resolution failures right away. + */ + if (state->resolve.fd && state->resolve.op_ret < 0) { + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); + free_fuse_state (state); + return; + } + + fn = state->resume_fn; + fn (state); +} + void fuse_lookup_resume (fuse_state_t *state) { if (!state->loc.parent && !state->loc.inode) { gf_log ("fuse", GF_LOG_ERROR, "failed to resolve path %s", state->loc.path); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -774,7 +794,8 @@ fuse_getattr_resume (fuse_state_t *state) "%"PRIu64": GETATTR %"PRIu64" (%s) resolution failed", state->finh->unique, state->finh->nodeid, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1086,7 +1107,8 @@ fuse_setattr_resume (fuse_state_t *state) "%"PRIu64": SETATTR %"PRIu64" (%s) resolution failed", state->finh->unique, state->finh->nodeid, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1291,7 +1313,8 @@ fuse_access_resume (fuse_state_t *state) "%"PRIu64": ACCESS %"PRIu64" (%s) resolution failed", state->finh->unique, state->finh->nodeid, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1368,7 +1391,8 @@ fuse_readlink_resume (fuse_state_t *state) gf_log ("glusterfs-fuse", GF_LOG_ERROR, "READLINK %"PRIu64" (%s) resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1403,7 +1427,8 @@ fuse_mknod_resume (fuse_state_t *state) "MKNOD %"PRIu64"/%s (%s/%s) resolution failed", state->finh->nodeid, state->resolve.bname, uuid_utoa (state->resolve.gfid), state->resolve.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1473,7 +1498,8 @@ fuse_mkdir_resume (fuse_state_t *state) "MKDIR %"PRIu64" (%s/%s) resolution failed", state->finh->nodeid, uuid_utoa (state->resolve.gfid), state->resolve.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1535,7 +1561,8 @@ fuse_unlink_resume (fuse_state_t *state) "UNLINK %"PRIu64" (%s/%s) resolution failed", state->finh->nodeid, uuid_utoa (state->resolve.gfid), state->resolve.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1571,7 +1598,8 @@ fuse_rmdir_resume (fuse_state_t *state) "RMDIR %"PRIu64" (%s/%s) resolution failed", state->finh->nodeid, uuid_utoa (state->resolve.gfid), state->resolve.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1607,7 +1635,8 @@ fuse_symlink_resume (fuse_state_t *state) "SYMLINK %"PRIu64" (%s/%s) -> %s resolution failed", state->finh->nodeid, uuid_utoa (state->resolve.gfid), state->resolve.bname, state->name); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1723,7 +1752,8 @@ fuse_rename_resume (fuse_state_t *state) uuid_utoa_r (state->resolve2.gfid, loc2_uuid), state->resolve2.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1780,7 +1810,8 @@ fuse_link_resume (fuse_state_t *state) gf_log ("glusterfs-fuse", GF_LOG_WARNING, "fuse_loc_fill() failed %"PRIu64": LINK %s %s", state->finh->unique, state->loc2.path, state->loc.path); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -1931,7 +1962,8 @@ fuse_create_resume (fuse_state_t *state) "%"PRIu64" CREATE %s/%s resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid), state->resolve.bname); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -2037,7 +2069,8 @@ fuse_open_resume (fuse_state_t *state) "%"PRIu64": OPEN %s resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -2417,7 +2450,8 @@ fuse_opendir_resume (fuse_state_t *state) gf_log ("glusterfs-fuse", GF_LOG_WARNING, "%"PRIu64": OPENDIR (%s) resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -2948,7 +2982,8 @@ fuse_statfs_resume (fuse_state_t *state) "%"PRIu64": STATFS (%s) resolution fail", state->finh->unique, uuid_utoa (state->resolve.gfid)); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -2983,7 +3018,8 @@ fuse_setxattr_resume (fuse_state_t *state) "resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid), state->finh->nodeid, state->name); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -3280,7 +3316,8 @@ fuse_getxattr_resume (fuse_state_t *state) uuid_utoa (state->resolve.gfid), state->finh->nodeid, state->name); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -3425,7 +3462,8 @@ fuse_listxattr_resume (fuse_state_t *state) "resolution failed", state->finh->unique, uuid_utoa (state->resolve.gfid), state->finh->nodeid); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } @@ -3480,7 +3518,8 @@ fuse_removexattr_resume (fuse_state_t *state) state->finh->unique, uuid_utoa (state->resolve.gfid), state->finh->nodeid, state->name); - send_fuse_err (state->this, state->finh, ENOENT); + send_fuse_err (state->this, state->finh, + state->resolve.op_errno); free_fuse_state (state); return; } diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h index 34794b6ea45..1a85cd348b8 100644 --- a/xlators/mount/fuse/src/fuse-bridge.h +++ b/xlators/mount/fuse/src/fuse-bridge.h @@ -145,10 +145,17 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t; #define FUSE_FOP(state, ret, op_num, fop, args ...) \ do { \ - call_frame_t *frame = NULL; \ - xlator_t *xl = NULL; \ - int32_t op_ret = 0, op_errno = 0; \ - fuse_resolve_t *resolve = NULL; \ + xlator_t *xl = NULL; \ + call_frame_t *frame = NULL; \ + \ + xl = state->active_subvol; \ + if (!xl) { \ + gf_log_callingfn (state->this->name, GF_LOG_ERROR, \ + "No active subvolume"); \ + send_fuse_err (state->this, state->finh, ENOENT); \ + free_fuse_state (state); \ + break; \ + } \ \ frame = get_call_frame_for_req (state); \ if (!frame) { \ @@ -158,13 +165,7 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t; * better than trying to go on with a NULL \ * frame ... \ */ \ - gf_log_callingfn ("glusterfs-fuse", \ - GF_LOG_ERROR, \ - "FUSE message" \ - " unique %"PRIu64" opcode %d:" \ - " frame allocation failed", \ - state->finh->unique, \ - state->finh->opcode); \ + send_fuse_err (state->this, state->finh, ENOMEM); \ free_fuse_state (state); \ /* ideally, need to 'return', but let the */ \ /* calling function take care of it */ \ @@ -175,150 +176,15 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t; frame->root->op = op_num; \ frame->op = op_num; \ \ - if ( state->resolve_now ) { \ - resolve = state->resolve_now; \ - } else { \ - resolve = &(state->resolve); \ - } \ - \ - xl = state->active_subvol; \ - if (!xl) { \ - gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \ - "xl is NULL"); \ - op_errno = ENOENT; \ - op_ret = -1; \ - } else if (resolve->op_ret < 0) { \ - op_errno = resolve->op_errno; \ - op_ret = -1; \ - if (op_num == GF_FOP_LOOKUP) { \ - gf_log ("glusterfs-fuse", \ - (op_errno == ENOENT ? GF_LOG_TRACE \ - : GF_LOG_WARNING), \ - "%"PRIu64": %s() %s => -1 (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - resolve->resolve_loc.path, \ - strerror (op_errno)); \ - } else { \ - gf_log ("glusterfs-fuse", \ - GF_LOG_WARNING, \ - "%"PRIu64": %s() inode " \ - "migration of %s failed (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - resolve->resolve_loc.path, \ - strerror (op_errno)); \ - } \ - } else if (state->resolve2.op_ret < 0) { \ - op_errno = state->resolve2.op_errno; \ - op_ret = -1; \ - gf_log ("glusterfs-fuse", \ - GF_LOG_WARNING, \ - "%"PRIu64": %s() inode " \ - "migration of %s failed (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->resolve2.resolve_loc.path, \ - strerror (op_errno)); \ - } \ - \ - if (op_ret < 0) { \ - send_fuse_err (state->this, state->finh, op_errno); \ - free_fuse_state (state); \ - STACK_DESTROY (frame->root); \ - } else { \ - if (state->this->history) \ - gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \ - "(%s)", frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->loc.path, \ - (state->fd == NULL)? \ - uuid_utoa (state->loc.gfid): \ - uuid_utoa (state->fd->inode->gfid));\ - STACK_WIND (frame, ret, xl, xl->fops->fop, args); \ - } \ - \ - } while (0) - - -#define FUSE_FOP_COOKIE(state, xl, ret, cky, op_num, fop, args ...) \ - do { \ - call_frame_t *frame = NULL; \ - xlator_t *xl = NULL; \ - int32_t op_ret = 0, op_errno = 0; \ - \ - frame = get_call_frame_for_req (state); \ - if (!frame) { \ - gf_log ("glusterfs-fuse", \ - GF_LOG_ERROR, \ - "FUSE message" \ - " unique %"PRIu64" opcode %d:" \ - " frame allocation failed", \ - state->finh->unique, \ - state->finh->opcode); \ - free_fuse_state (state); \ - return 0; \ - } \ - \ - frame->root->state = state; \ - frame->root->op = op_num; \ - frame->op = op_num; \ - \ - xl = state->active_subvol; \ - if (!xl) { \ - gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \ - "xl is NULL"); \ - op_errno = ENOENT; \ - op_ret = -1; \ - } else if (state->resolve.op_ret < 0) { \ - op_errno = state->resolve.op_errno; \ - op_ret = -1; \ - if (op_num == GF_FOP_LOOKUP) { \ - gf_log ("glusterfs-fuse", \ - (op_errno == ENOENT ? GF_LOG_TRACE \ - : GF_LOG_WARNING), \ - "%"PRIu64": %s() %s => -1 (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->resolve.resolve_loc.path, \ - strerror (op_errno)); \ - } else { \ - gf_log ("glusterfs-fuse", \ - GF_LOG_WARNING, \ - "%"PRIu64": %s() inode " \ - "migration of %s failed (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->resolve.resolve_loc.path, \ - strerror (op_errno)); \ - } \ - } else if (state->resolve2.op_ret < 0) { \ - op_errno = state->resolve2.op_errno; \ - op_ret = -1; \ - gf_log ("glusterfs-fuse", \ - GF_LOG_WARNING, \ - "%"PRIu64": %s() inode " \ - "migration of %s failed (%s)", \ - frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->resolve2.resolve_loc.path, \ - strerror (op_errno)); \ - } \ - \ - if (op_ret < 0) { \ - send_fuse_err (state->this, state->finh, op_errno); \ - free_fuse_state (state); \ - STACK_DESTROY (frame->root); \ - } else { \ - if (xl->history) \ - gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \ - "(%s)", frame->root->unique, \ - gf_fop_list[frame->root->op], \ - state->loc.path, \ - uuid_utoa (state->loc.gfid)); \ - STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, \ - args); \ - } \ + if (state->this->history) \ + gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \ + "(%s)", frame->root->unique, \ + gf_fop_list[frame->root->op], \ + state->loc.path, \ + (state->fd == NULL)? \ + uuid_utoa (state->loc.gfid): \ + uuid_utoa (state->fd->inode->gfid)); \ + STACK_WIND (frame, ret, xl, xl->fops->fop, args); \ } while (0) #define GF_SELECT_LOG_LEVEL(_errno) \ @@ -446,7 +312,6 @@ typedef struct { inode_t *hint; u_char pargfid[16]; inode_t *parhint; - char *resolved; int op_ret; int op_errno; loc_t resolve_loc; @@ -533,5 +398,6 @@ int fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve, int fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve, fd_t *fd); int fuse_ignore_xattr_set (fuse_private_t *priv, char *key); +void fuse_fop_resume (fuse_state_t *state); int dump_history_fuse (circular_buffer_t *cb, void *data); #endif /* _GF_FUSE_BRIDGE_H_ */ diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c index 2774bdaa812..6584dcfb6fc 100644 --- a/xlators/mount/fuse/src/fuse-helpers.c +++ b/xlators/mount/fuse/src/fuse-helpers.c @@ -26,8 +26,6 @@ fuse_resolve_wipe (fuse_resolve_t *resolve) GF_FREE ((void *)resolve->bname); - GF_FREE ((void *)resolve->resolved); - if (resolve->fd) fd_unref (resolve->fd); diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c index 8565ce0e478..bf6d66e2f6a 100644 --- a/xlators/mount/fuse/src/fuse-resolve.c +++ b/xlators/mount/fuse/src/fuse-resolve.c @@ -652,20 +652,13 @@ fuse_resolve (fuse_state_t *state) return 0; } - static int fuse_resolve_done (fuse_state_t *state) { - fuse_resume_fn_t fn = NULL; - - fn = state->resume_fn; - - fn (state); - + fuse_fop_resume (state); return 0; } - /* * This function is called multiple times, once per resolving one location/fd. * state->resolve_now is used to decide which location/fd is to be resolved now @@ -710,7 +703,6 @@ fuse_resolve_continue (fuse_state_t *state) return 0; } - int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn) { -- cgit