From 7cc14db79d4579cfa50c30108fdf67ffa61cb94a Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Thu, 19 Apr 2012 14:10:44 +0530 Subject: glusterd,glusterfsd : Perform proper cleanup of connections and socket files Perform proper disconnections and cleanup in glusterd on stopping nfs-server and self-heal daemon processes to prevent glusterd from opening unneded unix domain socket connections. glusterfsd processes will cleanup the socket files properly in cleanup_and_exit() to prevent junk socket files. Also, fix rpcsvc_program_unregister() to preform the unregistering properly. Change-Id: I1c7302c1166cf43feba1c7a813c3dc10169dc53a BUG: 810089 Signed-off-by: Kaushal M Reviewed-on: http://review.gluster.com/3168 Tested-by: Gluster Build System Reviewed-by: Krishnan Parthasarathi --- glusterfsd/src/glusterfsd-mgmt.c | 70 +++++++++++++++++------------ glusterfsd/src/glusterfsd.c | 3 +- rpc/rpc-lib/src/rpcsvc.c | 30 ++++++++++--- xlators/mgmt/glusterd/src/glusterd-utils.c | 72 ++++++++++++++++++++++++++---- 4 files changed, 130 insertions(+), 45 deletions(-) diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index 814de1d9349..dac92d67fb4 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -191,38 +191,10 @@ glusterfs_terminate_response_send (rpcsvc_request_t *req, int op_ret) return ret; } -int -glusterfs_listener_stop (void) -{ - glusterfs_ctx_t *ctx = NULL; - cmd_args_t *cmd_args = NULL; - int ret = 0; - xlator_t *this = NULL; - - ctx = glusterfs_ctx_get (); - GF_ASSERT (ctx); - cmd_args = &ctx->cmd_args; - if (cmd_args->sock_file) { - ret = unlink (cmd_args->sock_file); - if (ret && (ENOENT == errno)) { - ret = 0; - } - } - - if (ret) { - this = THIS; - gf_log (this->name, GF_LOG_ERROR, "Failed to unlink linstener " - "socket %s, error: %s", cmd_args->sock_file, - strerror (errno)); - } - return ret; -} - int glusterfs_handle_terminate (rpcsvc_request_t *req) { - (void) glusterfs_listener_stop (); glusterfs_terminate_response_send (req, 0); cleanup_and_exit (SIGTERM); return 0; @@ -1791,6 +1763,48 @@ out: return ret; } +int +glusterfs_listener_stop (glusterfs_ctx_t *ctx) +{ + cmd_args_t *cmd_args = NULL; + rpcsvc_t *rpc = NULL; + rpcsvc_listener_t *listener = NULL; + rpcsvc_listener_t *next = NULL; + int ret = 0; + xlator_t *this = NULL; + + GF_ASSERT (ctx); + + rpc = ctx->listener; + ctx->listener = NULL; + + (void) rpcsvc_program_unregister(rpc, &glusterfs_mop_prog); + + list_for_each_entry_safe (listener, next, &rpc->listeners, list) { + rpcsvc_listener_destroy (listener); + } + + (void) rpcsvc_unregister_notify (rpc, glusterfs_rpcsvc_notify, THIS); + + GF_FREE (rpc); + + cmd_args = &ctx->cmd_args; + if (cmd_args->sock_file) { + ret = unlink (cmd_args->sock_file); + if (ret && (ENOENT == errno)) { + ret = 0; + } + } + + if (ret) { + this = THIS; + gf_log (this->name, GF_LOG_ERROR, "Failed to unlink linstener " + "socket %s, error: %s", cmd_args->sock_file, + strerror (errno)); + } + return ret; +} + int glusterfs_mgmt_init (glusterfs_ctx_t *ctx) { diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 7bc063b2fee..4ad47ec621a 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -198,6 +198,7 @@ int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx); int glusterfs_volumes_init (glusterfs_ctx_t *ctx); int glusterfs_mgmt_init (glusterfs_ctx_t *ctx); int glusterfs_listener_init (glusterfs_ctx_t *ctx); +int glusterfs_listener_stop (glusterfs_ctx_t *ctx); int create_fuse_mount (glusterfs_ctx_t *ctx) @@ -798,7 +799,7 @@ cleanup_and_exit (int signum) ctx->cleanup_started = 1; glusterfs_mgmt_pmap_signout (ctx); if (ctx->listener) { - ctx->listener = NULL; + (void) glusterfs_listener_stop (ctx); } /* Call fini() of FUSE xlator first: diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index b64c43f0e75..f87338315af 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1273,28 +1273,44 @@ rpcsvc_submit_message (rpcsvc_request_t *req, struct iovec *proghdr, int -rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog) +rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *program) { int ret = -1; - - if (!svc || !prog) { + rpcsvc_program_t *prog = NULL; + if (!svc || !program) { goto out; } - ret = rpcsvc_program_unregister_portmap (prog); + ret = rpcsvc_program_unregister_portmap (program); if (ret == -1) { gf_log (GF_RPCSVC, GF_LOG_ERROR, "portmap unregistration of" " program failed"); goto out; } + pthread_mutex_lock (&svc->rpclock); + { + list_for_each_entry (prog, &svc->programs, program) { + if ((prog->prognum == program->prognum) + && (prog->progver == program->progver)) { + break; + } + } + } + pthread_mutex_unlock (&svc->rpclock); + + if (prog == NULL) { + ret = -1; + goto out; + } + gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Program unregistered: %s, Num: %d," " Ver: %d, Port: %d", prog->progname, prog->prognum, prog->progver, prog->progport); pthread_mutex_lock (&svc->rpclock); { - list_del (&prog->program); + list_del_init (&prog->program); } pthread_mutex_unlock (&svc->rpclock); @@ -1302,8 +1318,8 @@ rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog) out: if (ret == -1) { gf_log (GF_RPCSVC, GF_LOG_ERROR, "Program unregistration failed" - ": %s, Num: %d, Ver: %d, Port: %d", prog->progname, - prog->prognum, prog->progver, prog->progport); + ": %s, Num: %d, Ver: %d, Port: %d", program->progname, + program->prognum, program->progver, program->progport); } return ret; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index ed7ba1d51e6..73770bb76a9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -2783,19 +2783,40 @@ glusterd_nodesvc_connect (char *server, char *socketpath) { dict_t *options = NULL; struct rpc_clnt *rpc = NULL; - ret = rpc_clnt_transport_unix_options_build (&options, socketpath); - if (ret) - goto out; - ret = glusterd_rpc_create (&rpc, options, - glusterd_nodesvc_rpc_notify, - server); - if (ret) - goto out; - (void) glusterd_nodesvc_set_rpc (server, rpc); + rpc = glusterd_nodesvc_get_rpc (server); + + if (rpc == NULL) { + ret = rpc_clnt_transport_unix_options_build (&options, + socketpath); + if (ret) + goto out; + ret = glusterd_rpc_create (&rpc, options, + glusterd_nodesvc_rpc_notify, + server); + if (ret) + goto out; + (void) glusterd_nodesvc_set_rpc (server, rpc); + } out: return ret; } +int32_t +glusterd_nodesvc_disconnect (char *server) +{ + struct rpc_clnt *rpc = NULL; + + rpc = glusterd_nodesvc_get_rpc (server); + + if (rpc) { + rpc_clnt_connection_cleanup (&rpc->conn); + rpc_clnt_unref (rpc); + (void)glusterd_nodesvc_set_rpc (server, NULL); + } + + return 0; +} + int32_t glusterd_nodesvc_start (char *server) { @@ -2909,6 +2930,31 @@ glusterd_is_nodesvc_running (char *server) return glusterd_is_service_running (pidfile, NULL); } +int32_t +glusterd_nodesvc_unlink_socket_file (char *server) +{ + int ret = 0; + char sockfpath[PATH_MAX] = {0,}; + char rundir[PATH_MAX] = {0,}; + glusterd_conf_t *priv = THIS->private; + + glusterd_get_nodesvc_rundir (server, priv->workdir, + rundir, sizeof (rundir)); + + glusterd_nodesvc_set_socket_filepath (rundir, priv->uuid, + sockfpath, sizeof (sockfpath)); + + ret = unlink (sockfpath); + if (ret && (ENOENT == errno)) { + ret = 0; + } else { + gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s" + " error: %s", sockfpath, strerror (errno)); + } + + return ret; +} + int32_t glusterd_nodesvc_stop (char *server, int sig) { @@ -2918,9 +2964,17 @@ glusterd_nodesvc_stop (char *server, int sig) if (!glusterd_is_nodesvc_running (server)) goto out; + + (void)glusterd_nodesvc_disconnect (server); + glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile, sizeof (pidfile)); ret = glusterd_service_stop (server, pidfile, sig, _gf_true); + + if (ret == 0) { + glusterd_nodesvc_set_running (server, _gf_false); + (void)glusterd_nodesvc_unlink_socket_file (server); + } out: return ret; } -- cgit