posix: option to handle the shared bricks for statvfs()

Currently 'storage/posix' xlator has an option called option
`export-statfs-size no`, which exports zero as values for few
fields in `struct statvfs`. In a case of backend brick shared
between multiple brick processes, the values of these variables
should be `field_value / number-of-bricks-at-node`. This way,
even the issue of 'min-free-disk' etc at different layers would
also be handled properly when the statfs() sys call is made.

Fixes #241

Change-Id: I2e320e1fdcc819ab9173277ef3498201432c275f
Signed-off-by: Amar Tumballi <amarts@redhat.com>
Reviewed-on: https://review.gluster.org/17618
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

8 files changed, 97 insertions, 12 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 8d4ea13af95..c7b618745b3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -22,6 +22,7 @@
 #include "glusterd-server-quorum.h"
 #include "run.h"
 #include "glusterd-volgen.h"
+#include "syscall.h"
 #include <sys/signal.h>
 
 /* misc */
@@ -1322,6 +1323,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         xlator_t                     *this           = NULL;
         glusterd_conf_t              *conf           = NULL;
         gf_boolean_t                  is_valid_add_brick = _gf_false;
+        struct statvfs                brickstat = {0,};
 
         this = THIS;
         GF_ASSERT (this);
@@ -1396,6 +1398,21 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                 if (ret)
                         goto out;
 
+                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        ret = sys_statvfs (brickinfo->path, &brickstat);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, errno,
+                                        GD_MSG_STATVFS_FAILED,
+                                        "Failed to fetch disk utilization "
+                                        "from the brick (%s:%s). Please check the health of "
+                                        "the brick. Error code was %s",
+                                        brickinfo->hostname, brickinfo->path,
+                                        strerror (errno));
+
+                                goto out;
+                        }
+                        brickinfo->statfs_fsid = brickstat.f_fsid;
+                }
                 /* hot tier bricks are added to head of brick list */
                 if (dict_get (dict, "attach-tier")) {
                         cds_list_add (&brickinfo->brick_list, &volinfo->bricks);
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index 14424d36890..2caa16c2eda 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -4901,6 +4901,14 @@
  */
 #define GD_MSG_BRICKPROC_NEW_FAILED                (GLUSTERD_COMP_BASE + 606)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_STATVFS_FAILED  (GLUSTERD_COMP_BASE + 607)
+
 /*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 72b70f916c6..8eb301f040f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -352,6 +352,12 @@ gd_store_brick_snap_details_write (int fd, glusterd_brickinfo_t *brickinfo)
         snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
         ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
                                    value);
+        if (ret)
+                goto out;
+
+        memset (value, 0, sizeof (value));
+        snprintf (value, sizeof (value), "%lu", brickinfo->statfs_fsid);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_FSID, value);
 
 out:
         return ret;
@@ -2508,6 +2514,10 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
                                 strncpy (brickinfo->brick_id, value,
                                          sizeof (brickinfo->brick_id));
+                        } else if (!strncmp (key,
+                                             GLUSTERD_STORE_KEY_BRICK_FSID,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_FSID))) {
+                                gf_string2uint64 (value, &brickinfo->statfs_fsid);
                         } else {
                                 gf_msg (this->name, GF_LOG_ERROR, 0,
                                         GD_MSG_UNKNOWN_KEY, "Unknown key: %s",
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 3e31c965638..b515ca6c554 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -97,6 +97,7 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_BRICK_FSTYPE         "fs-type"
 #define GLUSTERD_STORE_KEY_BRICK_MNTOPTS        "mnt-opts"
 #define GLUSTERD_STORE_KEY_BRICK_ID             "brick-id"
+#define GLUSTERD_STORE_KEY_BRICK_FSID           "brick-fsid"
 
 #define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
 #define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 6ff11a2e050..f1627df688f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -186,6 +186,32 @@ out:
         return ret;
 }
 
+/* This is going to be a O(n^2) operation as we have to pick a brick,
+   make sure it belong to this machine, and compare another brick belonging
+   to this machine (if exists), is sharing the backend */
+static void
+gd_set_shared_brick_count (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t *brickinfo = NULL;
+        glusterd_brickinfo_t *trav = NULL;
+
+        cds_list_for_each_entry (brickinfo, &volinfo->bricks,
+                                 brick_list) {
+                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+                brickinfo->fs_share_count = 0;
+                cds_list_for_each_entry (trav, &volinfo->bricks,
+                                         brick_list) {
+                        if (!gf_uuid_compare (trav->uuid, MY_UUID) &&
+                            (trav->statfs_fsid == brickinfo->statfs_fsid)) {
+                                brickinfo->fs_share_count++;
+                        }
+                }
+        }
+
+        return;
+}
+
 int
 glusterd_volume_brick_for_each (glusterd_volinfo_t *volinfo, void *data,
                int (*fn) (glusterd_volinfo_t *, glusterd_brickinfo_t *,
@@ -195,6 +221,8 @@ glusterd_volume_brick_for_each (glusterd_volinfo_t *volinfo, void *data,
         glusterd_volinfo_t *dup_volinfo = NULL;
         int                ret          = 0;
 
+        gd_set_shared_brick_count (volinfo);
+
         if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
                 ret = _brick_for_each (volinfo, NULL, data, fn);
                 if (ret)
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 0a0668e9ea6..1ada7232f3e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1440,6 +1440,7 @@ static int
 brick_graph_add_posix (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
+        char            tmpstr[10] = {0,};
         int             ret = -1;
         gf_boolean_t    quota_enabled   = _gf_true;
         gf_boolean_t    trash_enabled   = _gf_false;
@@ -1491,6 +1492,9 @@ brick_graph_add_posix (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (quota_enabled || pgfid_feat || trash_enabled)
                 xlator_set_option (xl, "update-link-count-parent",
                                    "on");
+
+        snprintf (tmpstr, sizeof (tmpstr), "%d", brickinfo->fs_share_count);
+        ret = xlator_set_option (xl, "shared-brick-count", tmpstr);
 out:
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 7254e281497..b95b8a4e863 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2164,6 +2164,7 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                 *brick_mount_dir = NULL;
         char                  key[PATH_MAX]   = "";
         char                 *address_family_str = NULL;
+        struct statvfs        brickstat  = {0,};
 
         this = THIS;
         GF_ASSERT (this);
@@ -2405,24 +2406,35 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                                  sizeof(brickinfo->mount_dir));
                 }
 
-#ifdef HAVE_BD_XLATOR
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)
-                    && brickinfo->vg[0]) {
-                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        ret = sys_statvfs (brickinfo->path, &brickstat);
                         if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_INVALID_VG, "%s", msg);
+                                gf_log ("brick-op", GF_LOG_ERROR, "Failed to fetch disk"
+                                        " utilization from the brick (%s:%s). Please "
+                                        "check health of the brick. Error code was %s",
+                                        brickinfo->hostname, brickinfo->path,
+                                        strerror (errno));
                                 goto out;
                         }
+                        brickinfo->statfs_fsid = brickstat.f_fsid;
 
-                        /* if anyone of the brick does not have thin
-                           support, disable it for entire volume */
-                        caps &= brickinfo->caps;
-                } else {
-                                caps = 0;
-                }
+#ifdef HAVE_BD_XLATOR
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                                if (ret) {
+                                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                                GD_MSG_INVALID_VG, "%s", msg);
+                                        goto out;
+                                }
 
+                                /* if anyone of the brick does not have thin
+                                   support, disable it for entire volume */
+                                caps &= brickinfo->caps;
+                        } else {
+                                caps = 0;
+                        }
 #endif
+                }
 
                 cds_list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
                 brick = strtok_r (NULL, " \n", &saveptr);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index b2141853db4..3226ec24c0f 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -232,6 +232,11 @@ struct glusterd_brickinfo {
          */
         uint16_t           group;
         uuid_t             jbr_uuid;
+
+        /* Below are used for handling the case of multiple bricks sharing
+           the backend filesystem */
+        uint64_t           statfs_fsid;
+        uint32_t           fs_share_count;
 };
 
 typedef struct glusterd_brickinfo glusterd_brickinfo_t;