quick-read: Integrate quick read with upcall and increase cache time

Fixes : #261
Co-author: Subha sree Mohankumar <smohanku@redhat.com>
Change-Id: Ie9dd94e86459123663b9b200d92940625ef68eab
Signed-off-by: Poornima G <pgurusid@redhat.com>

4 files changed, 204 insertions, 4 deletions

diff --git a/tests/basic/quick-read-with-upcall.t b/tests/basic/quick-read-with-upcall.t
new file mode 100644
index 00000000000..318e93a1bf0
--- /dev/null
+++ b/tests/basic/quick-read-with-upcall.t
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+ #. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a distributed volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1..2};
+TEST $CLI volume start $V0
+
+# Mount FUSE without selinux:
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+TEST glusterfs -s $H0 --volfile-id $V0 $M1;
+
+D0="test-message0";
+D1="test-message1";
+
+function write_to()
+{
+    local file="$1";
+    local data="$2";
+    echo "$data" > "$file";
+}
+
+
+TEST write_to "$M0/test.txt" "$D0"
+EXPECT "$D0" cat $M0/test.txt
+EXPECT "$D0" cat $M1/test.txt
+
+TEST write_to "$M0/test.txt" "$D1"
+EXPECT "$D1" cat $M0/test.txt
+EXPECT "$D0" cat $M1/test.txt
+
+sleep 1
+EXPECT "$D1" cat $M1/test.txt
+
+TEST $CLI volume set $V0 features.cache-invalidation on
+TEST $CLI volume set $V0 performance.qr-cache-timeout 60
+TEST $CLI volume set $V0 performance.md-cache-timeout 60
+
+TEST write_to "$M0/test1.txt" "$D0"
+EXPECT "$D0" cat $M0/test1.txt
+EXPECT "$D0" cat $M1/test1.txt
+
+TEST write_to "$M0/test1.txt" "$D1"
+EXPECT "$D1" cat $M0/test1.txt
+EXPECT "$D0" cat $M1/test1.txt
+
+sleep 1
+EXPECT "$D0" cat $M1/test1.txt
+
+sleep 60
+EXPECT "$D1" cat $M1/test1.txt
+
+TEST $CLI volume set $V0 performance.cache-invalidation on
+
+TEST write_to "$M0/test2.txt" "$D0"
+EXPECT "$D0" cat $M0/test2.txt
+EXPECT "$D0" cat $M1/test2.txt
+
+TEST write_to "$M0/test2.txt" "$D1"
+EXPECT "$D1" cat $M0/test2.txt
+EXPECT "$D1" cat $M1/test2.txt
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 8a3f6a49184..34c246ca03b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1701,6 +1701,18 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version = 1,
           .flags      = VOLOPT_FLAG_CLIENT_OPT
         },
+        { .key        = "performance.qr-cache-timeout",
+          .voltype    = "performance/quick-read",
+          .option     = "cache-timeout",
+          .op_version = 1,
+          .flags      = VOLOPT_FLAG_CLIENT_OPT
+        },
+         { .key        = "performance.cache-invalidation",
+           .voltype    = "performance/quick-read",
+           .option     = "cache-invalidation",
+           .op_version = GD_OP_VERSION_4_0_0,
+           .flags      = VOLOPT_FLAG_CLIENT_OPT
+        },
         { .key        = "performance.flush-behind",
           .voltype    = "performance/write-behind",
           .option     = "flush-behind",
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index e4f0885de2c..71875200dde 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -12,6 +12,7 @@
 #include "quick-read.h"
 #include "statedump.h"
 #include "quick-read-messages.h"
+#include "upcall-utils.h"
 
 qr_inode_t *qr_inode_ctx_get (xlator_t *this, inode_t *inode);
 void __qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode);
@@ -388,6 +389,9 @@ __qr_cache_is_fresh (xlator_t *this, qr_inode_t *qr_inode)
 
 	timersub (&now, &qr_inode->last_refresh, &diff);
 
+        if (qr_inode->last_refresh.tv_sec < priv->last_child_down)
+                return _gf_false;
+
 	if (diff.tv_sec >= conf->cache_timeout)
 		return _gf_false;
 
@@ -594,9 +598,13 @@ unlock:
 		iov.iov_base = iobuf->ptr;
 		iov.iov_len = op_ret;
 
+                GF_ATOMIC_INC (priv->qr_counter.cache_hit);
 		STACK_UNWIND_STRICT (readv, frame, op_ret, 0, &iov, 1,
 				     &buf, iobref, xdata);
-	}
+	} else {
+                GF_ATOMIC_INC (priv->qr_counter.cache_miss);
+        }
+
         if (iobuf)
                 iobuf_unref (iobuf);
 
@@ -812,6 +820,12 @@ qr_priv_dump (xlator_t *this)
 
         gf_proc_dump_write ("total_files_cached", "%d", file_count);
         gf_proc_dump_write ("total_cache_used", "%d", total_size);
+        gf_proc_dump_write ("cache-hit", "%"PRId64,
+                            priv->qr_counter.cache_hit);
+        gf_proc_dump_write ("cache-hit", "%"PRId64,
+                            priv->qr_counter.cache_miss);
+        gf_proc_dump_write ("cache-hit", "%"PRId64,
+                            priv->qr_counter.file_data_invals);
 
 out:
         return 0;
@@ -899,6 +913,9 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("cache-timeout", conf->cache_timeout, options, int32,
                           out);
 
+        GF_OPTION_RECONF ("cache-invalidation", conf->qr_invalidation, options,
+                          bool, out);
+
         GF_OPTION_RECONF ("cache-size", cache_size_new, options, size_uint64, out);
         if (!check_cache_size_ok (this, cache_size_new)) {
                 ret = -1;
@@ -1046,6 +1063,8 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("cache-timeout", conf->cache_timeout, int32, out);
 
+        GF_OPTION_INIT ("cache-invalidation", conf->qr_invalidation, bool, out);
+
         GF_OPTION_INIT ("cache-size", conf->cache_size, size_uint64, out);
         if (!check_cache_size_ok (this, conf->cache_size)) {
                 ret = -1;
@@ -1082,6 +1101,8 @@ init (xlator_t *this)
 
         ret = 0;
 
+        time (&priv->last_child_down);
+
         this->private = priv;
 out:
         if ((ret == -1) && priv) {
@@ -1133,6 +1154,90 @@ qr_conf_destroy (qr_conf_t *conf)
 
 
 void
+qr_update_child_down_time (xlator_t *this, time_t *now)
+{
+        qr_private_t *priv = NULL;
+
+        priv = this->private;
+
+        LOCK (&priv->lock);
+        {
+                priv->last_child_down = *now;
+        }
+        UNLOCK (&priv->lock);
+}
+
+
+static int
+qr_invalidate (xlator_t *this, void *data)
+{
+        struct gf_upcall                    *up_data    = NULL;
+        struct gf_upcall_cache_invalidation *up_ci      = NULL;
+        inode_t                             *inode      = NULL;
+        int                                  ret        = 0;
+        inode_table_t                       *itable     = NULL;
+        qr_private_t                        *priv       = NULL;
+
+        up_data = (struct gf_upcall *)data;
+
+        if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+                goto out;
+
+        priv = this->private;
+        up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+        if (up_ci && (up_ci->flags & UP_WRITE_FLAGS)) {
+                GF_ATOMIC_INC (priv->qr_counter.file_data_invals);
+                itable = ((xlator_t *)this->graph->top)->itable;
+                inode = inode_find (itable, up_data->gfid);
+                if (!inode) {
+                        ret = -1;
+                        goto out;
+                }
+                qr_inode_prune (this, inode);
+        }
+
+out:
+        if (inode)
+                inode_unref (inode);
+
+        return ret;
+}
+
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        int                  ret  = 0;
+        qr_private_t        *priv = NULL;
+        time_t               now  = 0;
+        qr_conf_t           *conf = NULL;
+
+        priv = this->private;
+        conf = &priv->conf;
+
+        switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+                time (&now);
+                qr_update_child_down_time (this, &now);
+                break;
+        case GF_EVENT_UPCALL:
+                if (conf->qr_invalidation)
+                        ret = qr_invalidate (this, data);
+                break;
+        default:
+                break;
+        }
+
+        if (default_notify (this, event, data) != 0)
+                ret = -1;
+
+        return ret;
+}
+
+
+void
 fini (xlator_t *this)
 {
         qr_private_t *priv = NULL;
@@ -1189,12 +1294,10 @@ struct volume_options options[] = {
           .default_value = "128MB",
           .op_version = {1},
           .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .description = "Size of the read cache."
+          .description = "Size of small file read cache."
         },
         { .key  = {"cache-timeout"},
           .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = 60,
           .default_value = "1",
         },
         { .key  = {"max-file-size"},
@@ -1203,5 +1306,11 @@ struct volume_options options[] = {
           .max  = 1 * GF_UNIT_KB * 1000,
           .default_value = "64KB",
         },
+        { .key = {"cache-invalidation"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false",
+          .description = "When \"on\", invalidates/updates the metadata cache,"
+                         " on receiving the cache-invalidation notifications",
+        },
         { .key  = {NULL} }
 };
diff --git a/xlators/performance/quick-read/src/quick-read.h b/xlators/performance/quick-read/src/quick-read.h
index 0697bf5fb14..415572e9c27 100644
--- a/xlators/performance/quick-read/src/quick-read.h
+++ b/xlators/performance/quick-read/src/quick-read.h
@@ -55,6 +55,7 @@ struct qr_conf {
         int32_t          cache_timeout;
         uint64_t         cache_size;
         int              max_pri;
+        gf_boolean_t     qr_invalidation;
         struct list_head priority_list;
 };
 typedef struct qr_conf qr_conf_t;
@@ -66,9 +67,18 @@ struct qr_inode_table {
 };
 typedef struct qr_inode_table qr_inode_table_t;
 
+struct qr_statistics {
+        gf_atomic_t cache_hit;
+        gf_atomic_t cache_miss;
+        gf_atomic_t file_data_invals; /* No. of invalidates received from upcall */
+};
+
 struct qr_private {
         qr_conf_t         conf;
         qr_inode_table_t  table;
+        time_t last_child_down;
+        gf_lock_t lock;
+        struct qr_statistics qr_counter;
 };
 typedef struct qr_private qr_private_t;