cluster/tier : Reset watermarks in tier

A node which contains only cold bricks and has detected that
the high watermark value has been breached on the hot tier will
never reset the watermark to the correct value. The promotion check
will thus always fail and no promotions will occur from that node.

> Change-Id: I0f0804744cd184c263acbea1ee50cd6010a49ec5
> BUG: 1303895
> Signed-off-by: N Balachandran <nbalacha@redhat.com>
> Reviewed-on: http://review.gluster.org/13341
> Smoke: Gluster Build System <jenkins@build.gluster.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
> Reviewed-by: Dan Lambright <dlambrig@redhat.com>
(cherry picked from commit 545f4ed2c7195a21210e6a055c27c1b7a115e18c)

Change-Id: Iba3aa9c57cf5828ab87140c2c8257146a8772836
BUG: 1306129
Signed-off-by: N Balachandran <nbalacha@redhat.com>
Reviewed-on: http://review.gluster.org/13411
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>

1 files changed, 36 insertions, 9 deletions

diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 8353cdafb60..92b5aa5f999 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -193,6 +193,9 @@ out:
 }
 
 
+/* Check and update the watermark every WM_INTERVAL seconds */
+#define WM_INTERVAL            5
+
 static int
 tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
 {
@@ -510,6 +513,21 @@ tier_migrate_using_query_file (void *_args)
                 if (!tier_do_migration (this, query_cbk_args->is_promotion, &root_loc)) {
                         gfdb_methods.gfdb_query_record_free (query_record);
                         query_record = NULL;
+
+                        /* We have crossed the high watermark. Stop processing
+                         * files if this is a promotion cycle so demotion gets
+                         * a chance to start if not already running*/
+
+                        if (query_cbk_args->is_promotion &&
+                            defrag->tier_conf.mode == TIER_WM_HI) {
+
+                                gf_msg (this->name, GF_LOG_INFO, 0,
+                                        DHT_MSG_LOG_TIER_STATUS,
+                                        "High watermark crossed during "
+                                        "promotion. Exiting "
+                                        "tier_migrate_using_query_file");
+                                break;
+                        }
                         continue;
                 }
 
@@ -597,8 +615,6 @@ tier_migrate_using_query_file (void *_args)
                         p_loc.inode = linked_inode;
 
 
-
-
                         /* Preparing File Inode */
                         gf_uuid_copy (loc.gfid, query_record->gfid);
                         loc.inode = inode_new (defrag->root_inode->table);
@@ -1639,6 +1655,8 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)
         xlator_t *xlator                        = NULL;
         gf_tier_conf_t *tier_conf               = NULL;
         loc_t root_loc                          = { 0 };
+        int check_watermark                     = 0;
+
 
         conf   = this->private;
 
@@ -1720,6 +1738,19 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)
                         goto out;
                 }
 
+                check_watermark++;
+
+                if (check_watermark >= WM_INTERVAL) {
+                        check_watermark = 0;
+                        ret = tier_check_watermark (this, &root_loc);
+                        if (ret != 0) {
+                                gf_msg (this->name, GF_LOG_CRITICAL, errno,
+                                        DHT_MSG_LOG_TIER_ERROR,
+                                        "Failed to get watermark");
+                                continue;
+                        }
+                }
+
                 freq_demote = tier_get_freq_demote (tier_conf);
 
                 is_demotion_triggered = (is_hot_list_empty) ? _gf_false :
@@ -1736,13 +1767,9 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)
                 if (!is_promotion_triggered && !is_demotion_triggered)
                         continue;
 
-                ret = tier_check_watermark (this, &root_loc);
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Failed to get watermark");
-                        goto out;
-                }
+                /* Check the statfs immediately after the processing threads
+                   return */
+                check_watermark = WM_INTERVAL;
 
                 ret_promotion = -1;
                 ret_demotion = -1;