fuse: add --lru-limit option

The inode LRU mechanism is moot in fuse xlator (ie. there is no
limit for the LRU list), as fuse inodes are referenced from
kernel context, and thus they can only be dropped on request of
the kernel. This might results in a high number of passive
inodes which are useless for the glusterfs client, causing a
significant memory overhead.

This change tries to remedy this by extending the LRU semantics
and allowing to set a finite limit on the fuse inode LRU.

A brief history of problem:

When gluster's inode table was designed, fuse didn't have any
'invalidate' method, which means, userspace application could
never ask kernel to send a 'forget()' fop, instead had to wait
for kernel to send it based on kernel's parameters. Inode table
remembers the number of times kernel has cached the inode based
on the 'nlookup' parameter. And 'nlookup' field is not used by
no other entry points (like server-protocol, gfapi etc).

Hence the inode_table of fuse module always has to have lru-limit
as '0', which means no limit. GlusterFS always had to keep all
inodes in memory as kernel would have had a reference to it.
Again, the reason for this is, kernel's glusterfs inode reference
was pointer of 'inode_t' structure in glusterfs. As it is a
pointer, we could never free it (to prevent segfault, or memory
corruption).

Solution:

In the inode table, handle the prune case of inodes with 'nlookup'
differently, and call a 'invalidator' method, which in this case is
fuse_invalidate(), and it sends the request to kernel for getting
the forget request.

When the kernel sends the forget, it means, it has dropped all
the reference to the inode, and it will send the forget with the
'nlookup' parameter too. We just need to make sure to reduce the
'nlookup' value we have when we get forget. That automatically
cause the relevant prune to happen.

Credits: Csaba Henk, Xavier Hernandez, Raghavendra Gowdappa, Nithya B

fixes: bz#1560969
Change-Id: Ifee0737b23b12b1426c224ec5b8f591f487d83a2
Signed-off-by: Amar Tumballi <amarts@redhat.com>

1 files changed, 219 insertions, 35 deletions

diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 3bf32cfe442..b4a62897498 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -23,6 +23,100 @@
    move latest accessed dentry to list_head of inode
 */
 
+// clang-format off
+/*
+
+Details as per Xavi:
+
+ I think we should have 3 lists: active, lru and invalidate.
+
+We'll need 3 things: refs, nlookups and invalidate_sent flag. Any change of
+refs, invalidate_sent flag and moving from one list to another must be done
+atomically.
+
+With this information, these are the states that cause a transition:
+
+    refs nlookups inv_sent    op
+      1      0        0      unref  -> refs = 0, active--->destroy
+      1      1        0      unref  -> refs = 0, active--->lru
+      1      1        0     forget  -> nlookups = 0, active--->active
+     *0      1        0     forget  -> nlookups = 0, lru--->destroy
+     *0      1        1     forget  -> nlookups = 0, invalidate--->destroy
+      0      1        0       ref   -> refs = 1, lru--->active
+      0      1        1       ref   -> refs = 1, inv_sent = 0, invalidate--->active
+      0      1        0    overflow -> refs = 1, inv_sent = 1, lru--->invalidate
+      1      1        1      unref  -> refs = 0, invalidate--->invalidate
+      1      1        1     forget  -> nlookups = 0, inv_sent = 0, invalidate--->active
+
+(*) technically these combinations cannot happen because a forget sent by the
+kernel first calls ref() and then unref(). However it's equivalent.
+
+overflow means that lru list has grown beyond the limit and the inode needs to
+be invalidated. All other combinations do not cause a change in state or are not
+possible.
+
+Based on this, the code could be similar to this:
+
+    ref(inode, inv)
+    {
+        if (refs == 0) {
+            if (inv_sent) {
+                invalidate_count--;
+                inv_sent = 0;
+            } else {
+                lru_count--;
+            }
+            if (inv) {
+                inv_sent = 1;
+                invalidate_count++;
+                list_move(inode, invalidate);
+            } else {
+                active_count++;
+                list_move(inode, active);
+            }
+        }
+        refs++;
+    }
+
+    unref(inode, clear)
+    {
+        if (clear && inv_sent) {
+            // there is a case of fuse itself sending forget, without
+            // invalidate, after entry delete, like unlink(), rmdir().
+            inv_sent = 0;
+            invalidate_count--;
+            active_count++;
+            list_move(inode, active);
+        }
+        refs--;
+        if ((refs == 0) && !inv_sent) {
+            active_count--;
+            if (nlookups == 0) {
+                destroy(inode);
+            } else {
+                lru_count++;
+                list_move(inode, lru);
+            }
+        }
+    }
+
+    forget(inode)
+    {
+        ref(inode, false);
+        nlookups--;
+        unref(inode, true);
+    }
+
+    overflow(inode)
+    {
+        ref(inode, true);
+        invalidator(inode);
+        unref(inode, false);
+    }
+
+*/
+// clang-format on
+
 #define INODE_DUMP_LIST(head, key_buf, key_prefix, list_type)                  \
     {                                                                          \
         int i = 1;                                                             \
@@ -37,7 +131,7 @@
     }
 
 static inode_t *
-__inode_unref(inode_t *inode);
+__inode_unref(inode_t *inode, bool clear);
 
 static int
 inode_table_prune(inode_table_t *table);
@@ -132,7 +226,7 @@ __dentry_unset(dentry_t *dentry)
     dentry->name = NULL;
 
     if (dentry->parent) {
-        __inode_unref(dentry->parent);
+        __inode_unref(dentry->parent, false);
         dentry->parent = NULL;
     }
 
@@ -446,7 +540,7 @@ out:
 }
 
 static inode_t *
-__inode_unref(inode_t *inode)
+__inode_unref(inode_t *inode, bool clear)
 {
     int index = 0;
     xlator_t *this = NULL;
@@ -455,8 +549,6 @@ __inode_unref(inode_t *inode)
     if (!inode)
         return NULL;
 
-    this = THIS;
-
     /*
      * Root inode should always be in active list of inode table. So unrefs
      * on root inode are no-ops.
@@ -464,6 +556,13 @@ __inode_unref(inode_t *inode)
     if (__is_root_gfid(inode->gfid))
         return inode;
 
+    this = THIS;
+
+    if (clear && inode->invalidate_sent) {
+        inode->invalidate_sent = false;
+        inode->table->invalidate_size--;
+        __inode_activate(inode);
+    }
     GF_ASSERT(inode->ref);
 
     --inode->ref;
@@ -474,7 +573,7 @@ __inode_unref(inode_t *inode)
         inode->_ctx[index].ref--;
     }
 
-    if (!inode->ref) {
+    if (!inode->ref && !inode->invalidate_sent) {
         inode->table->active_size--;
 
         nlookup = GF_ATOMIC_GET(inode->nlookup);
@@ -488,7 +587,7 @@ __inode_unref(inode_t *inode)
 }
 
 static inode_t *
-__inode_ref(inode_t *inode)
+__inode_ref(inode_t *inode, bool is_invalidate)
 {
     int index = 0;
     xlator_t *this = NULL;
@@ -498,11 +597,6 @@ __inode_ref(inode_t *inode)
 
     this = THIS;
 
-    if (!inode->ref) {
-        inode->table->lru_size--;
-        __inode_activate(inode);
-    }
-
     /*
      * Root inode should always be in active list of inode table. So unrefs
      * on root inode are no-ops. If we do not allow unrefs but allow refs,
@@ -514,6 +608,22 @@ __inode_ref(inode_t *inode)
     if (__is_root_gfid(inode->gfid) && inode->ref)
         return inode;
 
+    if (!inode->ref) {
+        if (inode->invalidate_sent) {
+            inode->invalidate_sent = false;
+            inode->table->invalidate_size--;
+        } else {
+            inode->table->lru_size--;
+        }
+        if (is_invalidate) {
+            inode->invalidate_sent = true;
+            inode->table->invalidate_size++;
+            list_move_tail(&inode->list, &inode->table->invalidate);
+        } else {
+            __inode_activate(inode);
+        }
+    }
+
     inode->ref++;
 
     index = __inode_get_xl_index(inode, this);
@@ -537,7 +647,7 @@ inode_unref(inode_t *inode)
 
     pthread_mutex_lock(&table->lock);
     {
-        inode = __inode_unref(inode);
+        inode = __inode_unref(inode, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -558,7 +668,7 @@ inode_ref(inode_t *inode)
 
     pthread_mutex_lock(&table->lock);
     {
-        inode = __inode_ref(inode);
+        inode = __inode_ref(inode, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -592,7 +702,7 @@ __dentry_create(inode_t *inode, inode_t *parent, const char *name)
     }
 
     if (parent)
-        newd->parent = __inode_ref(parent);
+        newd->parent = __inode_ref(parent, false);
 
     list_add(&newd->inode_list, &inode->dentry_list);
     newd->inode = inode;
@@ -662,7 +772,7 @@ inode_new(inode_table_t *table)
     {
         inode = __inode_create(table);
         if (inode != NULL) {
-            __inode_ref(inode);
+            __inode_ref(inode, false);
         }
     }
     pthread_mutex_unlock(&table->lock);
@@ -769,7 +879,7 @@ inode_grep(inode_table_t *table, inode_t *parent, const char *name)
             inode = dentry->inode;
 
         if (inode)
-            __inode_ref(inode);
+            __inode_ref(inode, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -912,7 +1022,7 @@ inode_find(inode_table_t *table, uuid_t gfid)
     {
         inode = __inode_find(table, gfid);
         if (inode)
-            __inode_ref(inode);
+            __inode_ref(inode, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -1057,7 +1167,7 @@ inode_link(inode_t *inode, inode_t *parent, const char *name, struct iatt *iatt)
         linked_inode = __inode_link(inode, parent, name, iatt);
 
         if (linked_inode)
-            __inode_ref(linked_inode);
+            __inode_ref(linked_inode, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -1124,6 +1234,31 @@ inode_forget(inode_t *inode, uint64_t nlookup)
     return 0;
 }
 
+int
+inode_forget_with_unref(inode_t *inode, uint64_t nlookup)
+{
+    inode_table_t *table = NULL;
+
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
+
+    table = inode->table;
+
+    pthread_mutex_lock(&table->lock);
+    {
+        inode_forget_atomic(inode, nlookup);
+        __inode_unref(inode, true);
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    inode_table_prune(table);
+
+    return 0;
+}
+
 /*
  * Invalidate an inode. This is invoked when a translator decides that an
  * inode's cache is no longer valid. Any translator interested in taking action
@@ -1298,7 +1433,7 @@ inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
             parent = dentry->parent;
 
         if (parent)
-            __inode_ref(parent);
+            __inode_ref(parent, false);
     }
     pthread_mutex_unlock(&table->lock);
 
@@ -1480,6 +1615,8 @@ inode_table_prune(inode_table_t *table)
     inode_t *del = NULL;
     inode_t *tmp = NULL;
     inode_t *entry = NULL;
+    uint64_t nlookup = 0;
+    int64_t lru_size = 0;
 
     if (!table)
         return -1;
@@ -1488,7 +1625,11 @@ inode_table_prune(inode_table_t *table)
 
     pthread_mutex_lock(&table->lock);
     {
-        while (table->lru_limit && table->lru_size > (table->lru_limit)) {
+        if (!table->lru_limit)
+            goto purge_list;
+
+        lru_size = table->lru_size;
+        while (lru_size > (table->lru_limit)) {
             if (list_empty(&table->lru)) {
                 gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
                                  LG_MSG_INVALID_INODE_LIST,
@@ -1498,26 +1639,46 @@ inode_table_prune(inode_table_t *table)
                 break;
             }
 
+            lru_size--;
             entry = list_entry(table->lru.next, inode_t, list);
+            /* The logic of invalidation is required only if invalidator_fn
+               is present */
+            if (table->invalidator_fn) {
+                /* check for valid inode with 'nlookup' */
+                nlookup = GF_ATOMIC_GET(entry->nlookup);
+                if (nlookup) {
+                    __inode_ref(entry, true);
+                    tmp = entry;
+                    break;
+                }
+            }
 
             table->lru_size--;
             __inode_retire(entry);
-
             ret++;
         }
 
+    purge_list:
         list_splice_init(&table->purge, &purge);
         table->purge_size = 0;
     }
     pthread_mutex_unlock(&table->lock);
 
+    /* Pick 1 inode for invalidation */
+    if (tmp) {
+        xlator_t *old_THIS = THIS;
+        THIS = table->invalidator_xl;
+        table->invalidator_fn(table->invalidator_xl, tmp);
+        THIS = old_THIS;
+        inode_unref(tmp);
+    }
+
+    /* Just so that if purge list is handled too, then clear it off */
+    list_for_each_entry_safe(del, tmp, &purge, list)
     {
-        list_for_each_entry_safe(del, tmp, &purge, list)
-        {
-            list_del_init(&del->list);
-            inode_forget_atomic(del, 0);
-            __inode_destroy(del);
-        }
+        list_del_init(&del->list);
+        inode_forget_atomic(del, 0);
+        __inode_destroy(del);
     }
 
     return ret;
@@ -1545,9 +1706,12 @@ __inode_table_init_root(inode_table_t *table)
 }
 
 inode_table_t *
-inode_table_new(size_t lru_limit, xlator_t *xl)
+inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
+                             int32_t (*invalidator_fn)(xlator_t *, inode_t *),
+                             xlator_t *invalidator_xl)
 {
     inode_table_t *new = NULL;
+    uint32_t mem_pool_size = lru_limit;
     int ret = -1;
     int i = 0;
 
@@ -1559,20 +1723,20 @@ inode_table_new(size_t lru_limit, xlator_t *xl)
     new->ctxcount = xl->graph->xl_count + 1;
 
     new->lru_limit = lru_limit;
+    new->invalidator_fn = invalidator_fn;
+    new->invalidator_xl = invalidator_xl;
 
     new->hashsize = 14057; /* TODO: Random Number?? */
 
     /* In case FUSE is initing the inode table. */
-    if (lru_limit == 0)
-        lru_limit = DEFAULT_INODE_MEMPOOL_ENTRIES;
-
-    new->inode_pool = mem_pool_new(inode_t, lru_limit);
+    if (!mem_pool_size || (mem_pool_size > DEFAULT_INODE_MEMPOOL_ENTRIES))
+        mem_pool_size = DEFAULT_INODE_MEMPOOL_ENTRIES;
 
+    new->inode_pool = mem_pool_new(inode_t, mem_pool_size);
     if (!new->inode_pool)
         goto out;
 
-    new->dentry_pool = mem_pool_new(dentry_t, lru_limit);
-
+    new->dentry_pool = mem_pool_new(dentry_t, mem_pool_size);
     if (!new->dentry_pool)
         goto out;
 
@@ -1604,6 +1768,7 @@ inode_table_new(size_t lru_limit, xlator_t *xl)
     INIT_LIST_HEAD(&new->active);
     INIT_LIST_HEAD(&new->lru);
     INIT_LIST_HEAD(&new->purge);
+    INIT_LIST_HEAD(&new->invalidate);
 
     ret = gf_asprintf(&new->name, "%s/inode", xl->name);
     if (-1 == ret) {
@@ -1633,6 +1798,13 @@ out:
     return new;
 }
 
+inode_table_t *
+inode_table_new(uint32_t lru_limit, xlator_t *xl)
+{
+    /* Only fuse for now requires the inode table with invalidator */
+    return inode_table_with_invalidator(lru_limit, xl, NULL, NULL);
+}
+
 int
 inode_table_ctx_free(inode_table_t *table)
 {
@@ -1771,6 +1943,14 @@ inode_table_destroy(inode_table_t *inode_table)
             inode_table->lru_size--;
         }
 
+        /* Same logic for invalidate list */
+        while (!list_empty(&inode_table->invalidate)) {
+            trav = list_first_entry(&inode_table->invalidate, inode_t, list);
+            inode_forget_atomic(trav, 0);
+            __inode_retire(trav);
+            inode_table->invalidate_size--;
+        }
+
         while (!list_empty(&inode_table->active)) {
             trav = list_first_entry(&inode_table->active, inode_t, list);
             /* forget and unref the inode to retire and add it to
@@ -2280,6 +2460,7 @@ inode_dump(inode_t *inode, char *prefix)
         gf_proc_dump_write("fd-count", "%u", inode->fd_count);
         gf_proc_dump_write("active-fd-count", "%u", inode->active_fd_count);
         gf_proc_dump_write("ref", "%u", inode->ref);
+        gf_proc_dump_write("invalidate-sent", "%d", inode->invalidate_sent);
         gf_proc_dump_write("ia_type", "%d", inode->ia_type);
         if (inode->_ctx) {
             inode_ctx = GF_CALLOC(inode->table->ctxcount, sizeof(*inode_ctx),
@@ -2353,10 +2534,13 @@ inode_table_dump(inode_table_t *itable, char *prefix)
     gf_proc_dump_write(key, "%d", itable->lru_size);
     gf_proc_dump_build_key(key, prefix, "purge_size");
     gf_proc_dump_write(key, "%d", itable->purge_size);
+    gf_proc_dump_build_key(key, prefix, "invalidate_size");
+    gf_proc_dump_write(key, "%d", itable->invalidate_size);
 
     INODE_DUMP_LIST(&itable->active, key, prefix, "active");
     INODE_DUMP_LIST(&itable->lru, key, prefix, "lru");
     INODE_DUMP_LIST(&itable->purge, key, prefix, "purge");
+    INODE_DUMP_LIST(&itable->invalidate, key, prefix, "invalidate");
 
     pthread_mutex_unlock(&itable->lock);
 }