From 47fcbc4c055a7880d2926e918ae1e1f57c7db20d Mon Sep 17 00:00:00 2001 From: Mohammed Rafi KC Date: Mon, 24 Jun 2019 12:00:20 +0530 Subject: glusterd/svc: update pid of mux volumes from the shd process For a normal volume, we are updating the pid from a the process while we do a daemonization or at the end of the init if it is no-daemon mode. Along with updating the pid we also lock the file, to make sure that the process is running fine. With brick mux, we were updating the pidfile from gluterd after an attach/detach request. There are two problems with this approach. 1) We are not holding a pidlock for any file other than parent process. 2) There is a chance for possible race conditions with attach/detach. For example, shd start and a volume stop could race. Let's say we are starting an shd and it is attached to a volume. While we trying to link the pid file to the running process, this would have deleted by the thread that doing a volume stop. Backport of : https://review.gluster.org/#/c/glusterfs/+/22935/ >Change-Id: I29a00352102877ce09ea3f376ca52affceb5cf1a >Updates: bz#1722541 >Signed-off-by: Mohammed Rafi KC Change-Id: I29a00352102877ce09ea3f376ca52affceb5cf1a Updates: bz#1732668 Signed-off-by: Mohammed Rafi KC --- libglusterfs/src/glusterfs/glusterfs.h | 2 +- libglusterfs/src/glusterfs/libglusterfs-messages.h | 3 +- libglusterfs/src/graph.c | 154 ++++++++++++++++++++- 3 files changed, 151 insertions(+), 8 deletions(-) (limited to 'libglusterfs') diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 930d83cdfe6..18d8d2136f8 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -746,7 +746,7 @@ typedef struct { char vol_id[NAME_MAX + 1]; struct list_head volfile_list; glusterfs_graph_t *graph; - + FILE *pidfp; } gf_volfile_t; glusterfs_ctx_t * diff --git a/libglusterfs/src/glusterfs/libglusterfs-messages.h b/libglusterfs/src/glusterfs/libglusterfs-messages.h index ea2aa606470..7e0eebbe535 100644 --- a/libglusterfs/src/glusterfs/libglusterfs-messages.h +++ b/libglusterfs/src/glusterfs/libglusterfs-messages.h @@ -111,6 +111,7 @@ GLFS_MSGID( LG_MSG_PTHREAD_NAMING_FAILED, LG_MSG_SYSCALL_RETURNS_WRONG, LG_MSG_XXH64_TO_GFID_FAILED, LG_MSG_ASYNC_WARNING, LG_MSG_ASYNC_FAILURE, LG_MSG_GRAPH_CLEANUP_FAILED, LG_MSG_GRAPH_SETUP_FAILED, - LG_MSG_GRAPH_DETACH_STARTED, LG_MSG_GRAPH_ATTACH_FAILED); + LG_MSG_GRAPH_DETACH_STARTED, LG_MSG_GRAPH_ATTACH_FAILED, + LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED); #endif /* !_LG_MESSAGES_H_ */ diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c index 0b99e618191..51704560164 100644 --- a/libglusterfs/src/graph.c +++ b/libglusterfs/src/graph.c @@ -1473,6 +1473,21 @@ out: return parent_graph; } +int +glusterfs_svc_mux_pidfile_cleanup(gf_volfile_t *volfile_obj) +{ + if (!volfile_obj || !volfile_obj->pidfp) + return 0; + + gf_msg_trace("glusterfsd", 0, "pidfile %s cleanup", volfile_obj->vol_id); + + lockf(fileno(volfile_obj->pidfp), F_ULOCK, 0); + fclose(volfile_obj->pidfp); + volfile_obj->pidfp = NULL; + + return 0; +} + int glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj) { @@ -1509,6 +1524,7 @@ glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj) list_del_init(&volfile_obj->volfile_list); glusterfs_mux_xlator_unlink(parent_graph->top, xl); + glusterfs_svc_mux_pidfile_cleanup(volfile_obj); parent_graph->last_xl = glusterfs_get_last_xlator(parent_graph); parent_graph->xl_count -= graph->xl_count; parent_graph->leaf_count -= graph->leaf_count; @@ -1537,9 +1553,127 @@ out: return ret; } +int +glusterfs_svc_mux_pidfile_setup(gf_volfile_t *volfile_obj, const char *pid_file) +{ + int ret = -1; + FILE *pidfp = NULL; + + if (!pid_file || !volfile_obj) + goto out; + + if (volfile_obj->pidfp) { + ret = 0; + goto out; + } + pidfp = fopen(pid_file, "a+"); + if (!pidfp) { + goto out; + } + volfile_obj->pidfp = pidfp; + + ret = lockf(fileno(pidfp), F_TLOCK, 0); + if (ret) { + ret = 0; + goto out; + } +out: + return ret; +} + +int +glusterfs_svc_mux_pidfile_update(gf_volfile_t *volfile_obj, + const char *pid_file, pid_t pid) +{ + int ret = 0; + FILE *pidfp = NULL; + int old_pid; + + if (!volfile_obj->pidfp) { + ret = glusterfs_svc_mux_pidfile_setup(volfile_obj, pid_file); + if (ret == -1) + goto out; + } + pidfp = volfile_obj->pidfp; + ret = fscanf(pidfp, "%d", &old_pid); + if (ret <= 0) { + goto update; + } + if (old_pid == pid) { + ret = 0; + goto out; + } else { + gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, + "Old pid=%d found in pidfile %s. Cleaning the old pid and " + "Updating new pid=%d", + old_pid, pid_file, pid); + } +update: + ret = sys_ftruncate(fileno(pidfp), 0); + if (ret) { + gf_msg("glusterfsd", GF_LOG_ERROR, errno, + LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, + "pidfile %s truncation failed", pid_file); + goto out; + } + + ret = fprintf(pidfp, "%d\n", pid); + if (ret <= 0) { + gf_msg("glusterfsd", GF_LOG_ERROR, errno, + LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, "pidfile %s write failed", + pid_file); + goto out; + } + + ret = fflush(pidfp); + if (ret) { + gf_msg("glusterfsd", GF_LOG_ERROR, errno, + LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, "pidfile %s write failed", + pid_file); + goto out; + } +out: + return ret; +} + +int +glusterfs_update_mux_pid(dict_t *dict, gf_volfile_t *volfile_obj) +{ + char *file = NULL; + int ret = -1; + + GF_VALIDATE_OR_GOTO("graph", dict, out); + GF_VALIDATE_OR_GOTO("graph", volfile_obj, out); + + ret = dict_get_str(dict, "pidfile", &file); + if (ret < 0) { + gf_msg("mgmt", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED, + "Failed to get pidfile from dict for volfile_id=%s", + volfile_obj->vol_id); + } + + ret = glusterfs_svc_mux_pidfile_update(volfile_obj, file, getpid()); + if (ret < 0) { + ret = -1; + gf_msg("mgmt", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED, + "Failed to update " + "the pidfile for volfile_id=%s", + volfile_obj->vol_id); + + goto out; + } + + if (ret == 1) + gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, + "PID %d updated in pidfile=%s", getpid(), file); + ret = 0; +out: + return ret; +} int glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp, - char *volfile_id, char *checksum) + char *volfile_id, char *checksum, + dict_t *dict) { glusterfs_graph_t *graph = NULL; glusterfs_graph_t *parent_graph = NULL; @@ -1622,18 +1756,25 @@ glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp, ret = -1; goto out; } + volfile_obj->pidfp = NULL; + snprintf(volfile_obj->vol_id, sizeof(volfile_obj->vol_id), "%s", + volfile_id); + + if (strcmp(ctx->cmd_args.process_name, "glustershd") == 0) { + ret = glusterfs_update_mux_pid(dict, volfile_obj); + if (ret == -1) { + goto out; + } + } graph->used = 1; parent_graph->id++; list_add(&graph->list, &ctx->graphs); INIT_LIST_HEAD(&volfile_obj->volfile_list); volfile_obj->graph = graph; - snprintf(volfile_obj->vol_id, sizeof(volfile_obj->vol_id), "%s", - volfile_id); memcpy(volfile_obj->volfile_checksum, checksum, sizeof(volfile_obj->volfile_checksum)); list_add_tail(&volfile_obj->volfile_list, &ctx->volfile_list); - gf_log_dump_graph(fp, graph); graph = NULL; @@ -1661,7 +1802,8 @@ out: int glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx, - gf_volfile_t *volfile_obj, char *checksum) + gf_volfile_t *volfile_obj, char *checksum, + dict_t *dict) { glusterfs_graph_t *oldvolfile_graph = NULL; glusterfs_graph_t *newvolfile_graph = NULL; @@ -1710,7 +1852,7 @@ glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx, } volfile_obj = NULL; ret = glusterfs_process_svc_attach_volfp(ctx, newvolfile_fp, vol_id, - checksum); + checksum, dict); goto out; } -- cgit