Log AFR quorum stats in io-stats translator.

Summary:
Add AFR quorum state to io-stats translator.

Sample output:

{
  "storage.gluster.nfsd.test-replicate-0.has-quorum": "1",
  "storage.gluster.nfsd.test-replicate-0.quorum-threshold": "1",
  "storage.gluster.nfsd.test-replicate-1.has-quorum": "1",
  "storage.gluster.nfsd.test-replicate-1.quorum-threshold": "1"
}

The quorum-threshold field shows the number of bricks that can be
lost while still maintaining quorum. Negative numbers indicate
that quorum has been lost and show the number of bricks that must
be brought online to restore quorum.

Additionally, I found that the code contained both
afr_have_quorum() and afr_has_quorum(), which were mostly
cut-n-pasted copies of each other, but with subtle differences.

Mercifully, afr_have_quorum() was totally unused, so I nuked it in
passing.

This is a port of D4089969 to 3.8.

Test Plan: Run, observe stats output. Kill brick, observe proper change. fb-smoke.

Reviewers: #posix_storage, sshreyas

Reviewed By: sshreyas

Subscribers: sshreyas

Differential Revision: https://phabricator.intern.facebook.com/D4089969

Change-Id: Ifddb351aebfe63998846bb52be8942415ce4c1a9
Reviewed-on: https://review.gluster.org/18188
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

7 files changed, 146 insertions, 58 deletions

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 59f3df19420..dbe18f2c243 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -192,6 +192,7 @@
 #define GF_AFR_DIRTY "trusted.afr.dirty"
 #define GF_XATTROP_ENTRY_OUT "glusterfs.xattrop-entry-delete"
 #define GF_XATTROP_PURGE_INDEX "glusterfs.xattrop-purge-index"
+#define GF_AFR_QUORUM_CHECK "glusterfs.afr.quorum-check"
 
 #define GF_GFIDLESS_LOOKUP "gfidless-lookup"
 /* replace-brick and pump related internal xattrs */
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 4c2343f8e9b..b34cc83b635 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -5373,44 +5373,6 @@ afr_set_low_priority (call_frame_t *frame)
         frame->root->pid = LOW_PRIO_PROC_PID;
 }
 
-
-gf_boolean_t
-afr_have_quorum (char *logname, afr_private_t *priv)
-{
-        unsigned int        quorum = 0;
-        unsigned int        up_children = 0;
-
-        GF_VALIDATE_OR_GOTO(logname,priv,out);
-
-        up_children = __afr_get_up_children_count (priv);
-        quorum = priv->quorum_count;
-        if (quorum != AFR_QUORUM_AUTO)
-                return up_children >= quorum;
-
-        quorum = priv->child_count / 2 + 1;
-        if (up_children >= quorum)
-                return _gf_true;
-
-        /*
-         * Special case for even numbers of nodes: if we have exactly half
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
-         */
-        if ((priv->child_count % 2) == 0) {
-                quorum = priv->child_count / 2;
-                if (up_children >= quorum) {
-                        if (priv->child_up[0]) {
-                                return _gf_true;
-                        }
-                }
-        }
-
-out:
-        return _gf_false;
-}
-
 void
 afr_priv_destroy (afr_private_t *priv)
 {
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index a917bc08ae0..3a6e581356f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1431,6 +1431,52 @@ afr_marker_populate_args (call_frame_t *frame, int type, int *gauge,
         return priv->child_count;
 }
 
+static int afr_get_quorum_state (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t  *priv = this->private;
+        dict_t         *dict              = NULL;
+        int             ret               = 0;
+        int             threshold;
+        gf_boolean_t    have_quorum;
+        char            key[128];
+
+        if (!priv) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        have_quorum = afr_check_quorum (priv->child_up, this, &threshold);
+
+        snprintf (key, sizeof (key),
+                        "%s.has-quorum",
+                        this->name);
+        if (dict_set_int32 (dict, key, have_quorum ? 1 : 0 )) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key),
+                        "%s.quorum-threshold",
+                        this->name);
+        if (dict_set_int32 (dict, key, (int32_t)threshold)) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+out:
+        AFR_STACK_UNWIND (getxattr, frame, ret, -ret, dict, NULL);
+        if (dict) {
+               dict_unref (dict);
+        }
+        return ret;
+}
+
 static int
 afr_handle_heal_xattrs (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         const char *heal_op)
@@ -1522,6 +1568,11 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 		goto out;
 	}
 
+        if (!strcmp (name, GF_AFR_QUORUM_CHECK)) {
+                afr_get_quorum_state (frame, this);
+                return 0;
+        }
+
         if (!strncmp (name, AFR_XATTR_PREFIX,
                       strlen (AFR_XATTR_PREFIX))) {
                 op_errno = ENODATA;
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 9b5063d8aa8..f1fbf20cb3f 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -672,7 +672,7 @@ afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
 }
 
 gf_boolean_t
-afr_has_quorum (unsigned char *subvols, xlator_t *this)
+afr_check_quorum (unsigned char *subvols, xlator_t *this, int *threshold)
 {
         unsigned int  quorum_count = 0;
         afr_private_t *priv  = NULL;
@@ -682,29 +682,33 @@ afr_has_quorum (unsigned char *subvols, xlator_t *this)
         up_children_count = AFR_COUNT (subvols, priv->child_count);
 
         if (priv->quorum_count == AFR_QUORUM_AUTO) {
-        /*
-         * Special case for even numbers of nodes in auto-quorum:
-         * if we have exactly half children up
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
-         */
-                if ((priv->child_count % 2 == 0) &&
-                    (up_children_count == (priv->child_count/2)))
-                        return subvols[0];
-        }
-
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                quorum_count = priv->child_count/2 + 1;
+                /*
+                 * Special case for even numbers of nodes in auto-quorum:
+                 * if we have exactly half children up
+                 * and that includes the first ("senior-most") node, then that counts
+                 * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
+                 * while preserving the critical property that there can only be one
+                 * such group.
+                 */
+                if (priv->child_count % 2 == 0 && subvols[0]) {
+                        quorum_count = priv->child_count / 2;
+                } else {
+                        quorum_count = priv->child_count / 2 + 1;
+                }
         } else {
                 quorum_count = priv->quorum_count;
         }
 
-        if (up_children_count >= quorum_count)
-                return _gf_true;
+        if (threshold) {
+                *threshold = (int)(up_children_count - quorum_count);
+        }
 
-        return _gf_false;
+        return up_children_count >= quorum_count;
+}
+
+gf_boolean_t
+afr_has_quorum (unsigned char *subvols, xlator_t *this) {
+        return afr_check_quorum (subvols, this, NULL);
 }
 
 static gf_boolean_t
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index dcdadbc84f4..535c82cbc55 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -49,6 +49,8 @@ int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
 int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
 int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
 call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
+gf_boolean_t
+afr_check_quorum (unsigned char *subvols, xlator_t *this, int *threshold);
 gf_boolean_t afr_has_quorum (unsigned char *subvols, xlator_t *this);
 gf_boolean_t afr_needs_changelog_update (afr_local_t *local);
 void afr_zero_fill_stat (afr_local_t *local);
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 0b5c095c3b4..c710b59a520 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -150,6 +150,7 @@ struct ios_conf {
         ios_sample_buf_t          *ios_sample_buf;
         struct dnscache           *dnscache;
         int32_t                   ios_dnscache_ttl_sec;
+        gf_boolean_t              iamnfsd;
 };
 
 
@@ -1609,6 +1610,58 @@ ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
         stats->started_at = *now;
 }
 
+static int io_stats_dump_quorum (xlator_t *this, struct ios_dump_args *args) {
+        FILE *logf = args->u.logfp;
+        loc_t root_loc = {0};
+        dict_t *dict = NULL;
+        xlator_list_t *child = NULL;
+        const char *leading_comma = "";
+
+        if (args->type != IOS_DUMP_TYPE_JSON_FILE) {
+                        return -EINVAL;
+        }
+
+        if (!this->itable->root) {
+                return -ENOENT;
+        }
+
+        // If we don't build a valid 'loc', dht_getxattr swallows our request
+        // instead of passing it down to AFR.
+        root_loc.path = "/";
+        root_loc.name = "";
+        root_loc.inode = inode_ref (this->itable->root);
+        gf_uuid_copy (root_loc.gfid, root_loc.inode->gfid);
+
+        ios_log (this, logf, "{");
+
+        for (child = this->children; child; child = child->next) {
+                dict = NULL;
+
+                syncop_getxattr (child->xlator, &root_loc, &dict,
+                                 GF_AFR_QUORUM_CHECK, NULL, NULL);
+
+                if (dict) {
+                        const data_pair_t *e;
+
+                        dict_for_each (dict, e) {
+                                ios_log (this, logf,
+                                         "%s\"storage.gluster.nfsd.%s\": \"%d\"",
+                                         leading_comma,
+                                         e->key, data_to_int32 (e->value));
+                                leading_comma = ",";
+                        }
+
+                        dict_unref (dict);
+                }
+        }
+
+        ios_log (this, logf, "}");
+
+        inode_unref (root_loc.inode);
+
+        return 0;
+}
+
 int
 io_stats_dump (xlator_t *this, struct ios_dump_args *args,
                gf1_cli_info_op op, gf_boolean_t is_peek)
@@ -1656,6 +1709,10 @@ io_stats_dump (xlator_t *this, struct ios_dump_args *args,
             op == GF_CLI_INFO_INCREMENTAL)
                 io_stats_dump_global (this, &incremental, &now, increment, args);
 
+        if (conf->iamnfsd) {
+                io_stats_dump_quorum (this, args);
+        }
+
         return 0;
 }
 
@@ -4020,6 +4077,8 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
+        GF_OPTION_INIT ("iam-nfs-daemon", conf->iamnfsd, bool, out);
+
         GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
         GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
@@ -4439,6 +4498,13 @@ struct volume_options options[] = {
                          "log messages that can be buffered for a time equal to"
                          " the value of the option brick-log-flush-timeout."
         },
+        {  .key = {"iam-nfs-daemon"},
+           .type = GF_OPTION_TYPE_BOOL,
+           .default_value = "off",
+           .description = "This option differentiates if the io-stats "
+                          "translator is running as part of an NFS daemon "
+                          "or not."
+        },
         { .key  = {NULL} },
 
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 1f087b43ab4..c6abfd87307 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -4949,8 +4949,10 @@ volgen_graph_set_iam_nfsd (const volgen_graph_t *graph)
 
         for (trav = first_of ((volgen_graph_t *)graph); trav;
                         trav = trav->next) {
-                if (strcmp (trav->type, "cluster/replicate") != 0)
+                if (strcmp (trav->type, "cluster/replicate") != 0 ||
+                    strcmp (trav->type, "debug/io-stats") != 0) {
                         continue;
+                }
 
                 ret = xlator_set_option (trav, "iam-nfs-daemon", "yes");
                 if (ret)