mgmt/shd: Implement multiplexing in self heal daemon

Problem:

Shd daemon is per node, which means they create a graph
with all volumes on it. While this is a great for utilizing
resources, it is so good in terms of performance and managebility.

Because self-heal daemons doesn't have capability to automatically
reconfigure their graphs. So each time when any configurations
changes happens to the volumes(replicate/disperse), we need to restart
shd to bring the changes into the graph.

Because of this all on going heal for all other volumes has to be
stopped in the middle, and need to restart all over again.

Solution:

This changes makes shd as a per volume daemon, so that the graph
will be generated for each volumes.

When we want to start/reconfigure shd for a volume, we first search
for an existing shd running on the node, if there is none, we will
start a new process. If already a daemon is running for shd, then
we will simply detach a graph for a volume and reatach the updated
graph for the volume. This won't touch any of the on going operations
for any other volumes on the shd daemon.

Example of an shd graph when it is per volume

                           graph
                     -----------------------
                     |     debug-iostat    |
                     -----------------------
                    /         |             \
                   /          |              \
              ---------    ---------      ----------
              | AFR-1 |    | AFR-2 |      |  AFR-3 |
              --------     ---------      ----------

A running shd daemon with 3 volumes will be like-->

                           graph
                     -----------------------
                     |     debug-iostat    |
                     -----------------------
                    /           |           \
                   /            |            \
              ------------   ------------  ------------
              | volume-1 |   | volume-2 |  | volume-3 |
              ------------   ------------  ------------

Change-Id: Idcb2698be3eeb95beaac47125565c93370afbd99
fixes: bz#1659708
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>

1 files changed, 28 insertions, 2 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 0a2fffdbe63..eb89e3be93e 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -28,6 +28,7 @@
 #include "glusterd-sm.h"
 #include "glusterd-snapd-svc.h"
 #include "glusterd-tierd-svc.h"
+#include "glusterd-shd-svc.h"
 #include "glusterd-bitd-svc.h"
 #include "glusterd1-xdr.h"
 #include "protocol-common.h"
@@ -167,7 +168,6 @@ typedef struct {
     char workdir[VALID_GLUSTERD_PATHMAX];
     char rundir[VALID_GLUSTERD_PATHMAX];
     rpcsvc_t *rpc;
-    glusterd_svc_t shd_svc;
     glusterd_svc_t nfs_svc;
     glusterd_svc_t bitd_svc;
     glusterd_svc_t scrub_svc;
@@ -176,6 +176,7 @@ typedef struct {
     struct cds_list_head volumes;
     struct cds_list_head snapshots;   /*List of snap volumes */
     struct cds_list_head brick_procs; /* List of brick processes */
+    struct cds_list_head shd_procs;   /* List of shd processes */
     pthread_mutex_t xprt_lock;
     struct list_head xprt_list;
     pthread_mutex_t import_volumes;
@@ -216,6 +217,11 @@ typedef struct {
     gf_atomic_t blockers;
     uint32_t mgmt_v3_lock_timeout;
     gf_boolean_t restart_bricks;
+    pthread_mutex_t attach_lock; /* Lock can be per process or a common one */
+    pthread_mutex_t volume_lock; /* We release the big_lock from lot of places
+                                    which might lead the modification of volinfo
+                                    list.
+                                 */
 } glusterd_conf_t;
 
 typedef enum gf_brick_status {
@@ -495,6 +501,7 @@ struct glusterd_volinfo_ {
 
     glusterd_snapdsvc_t snapd;
     glusterd_tierdsvc_t tierd;
+    glusterd_shdsvc_t shd;
     glusterd_gfproxydsvc_t gfproxyd;
     int32_t quota_xattr_version;
     gf_boolean_t stage_deleted;         /* volume has passed staging
@@ -621,7 +628,6 @@ typedef enum {
 #define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR "/gluster/snaps"
 #define GLUSTERD_BITD_RUN_DIR "/bitd"
 #define GLUSTERD_SCRUB_RUN_DIR "/scrub"
-#define GLUSTERD_GLUSTERSHD_RUN_DIR "/glustershd"
 #define GLUSTERD_NFS_RUN_DIR "/nfs"
 #define GLUSTERD_QUOTAD_RUN_DIR "/quotad"
 #define GLUSTER_SHARED_STORAGE_BRICK_DIR GLUSTERD_DEFAULT_WORKDIR "/ss_brick"
@@ -677,6 +683,26 @@ typedef ssize_t (*gd_serialize_t)(struct iovec outmsg, void *args);
         }                                                                      \
     } while (0)
 
+#define GLUSTERD_GET_SHD_RUNDIR(path, volinfo, priv)                           \
+    do {                                                                       \
+        int32_t _shd_dir_len;                                                  \
+        _shd_dir_len = snprintf(path, PATH_MAX, "%s/shd/%s", priv->rundir,     \
+                                volinfo->volname);                             \
+        if ((_shd_dir_len < 0) || (_shd_dir_len >= PATH_MAX)) {                \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_SHD_PID_FILE(path, volinfo, priv)                         \
+    do {                                                                       \
+        int32_t _shd_pid_len;                                                  \
+        _shd_pid_len = snprintf(path, PATH_MAX, "%s/shd/%s-shd.pid",           \
+                                priv->rundir, volinfo->volname);               \
+        if ((_shd_pid_len < 0) || (_shd_pid_len >= PATH_MAX)) {                \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
 #define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv)                       \
     do {                                                                       \
         int32_t _vol_pid_len;                                                  \