storage/posix: Added janitor thread.

The janitor thread deletes all files and directories in
the "/" GF_REPLICATE_TRASH_DIR directory. This directory
is used by replicate self-heal to dump files and
directories it deletes.

This is needed because letting replicate walk the directory
tree and delete a directory and all its children is too racy.
Instead, replicate self-heal only does an atomic rename(),
and the janitor thread takes care of actually deleting them.

Signed-off-by: Vikas Gorur <vikas@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 227 (replicate selfheal does not remove directory with contents in it)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=227

7 files changed, 137 insertions, 18 deletions

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 848e9ba7d..f9cc6152e 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -203,6 +203,8 @@ typedef enum {
 #define GF_SET_EPOCH_TIME     0x8 /* used by afr dir lookup selfheal */
 
 
+#define GF_REPLICATE_TRASH_DIR          ".trash"
+
 struct _xlator_cmdline_option {
 	struct list_head cmd_args;
 	char *volume;
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index fab60b66e..b48488526 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -361,7 +361,7 @@ afr_readdir_cbk (call_frame_t *frame, void *cookie,
                                                        child_index);
 
                         if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, AFR_TRASH_DIR)) {
+                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
                                 list_del_init (&entry->list);
                                 FREE (entry);
                         }
@@ -405,7 +405,7 @@ afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         entry->d_stat.st_ino = inum;
 
                         if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, AFR_TRASH_DIR)) {
+                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
                                 list_del_init (&entry->list);
                                 FREE (entry);
                         }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 50d11cc18..420b7155c 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -532,8 +532,8 @@ afr_sh_entry_expunge_rename_cbk (call_frame_t *expunge_frame, void *cookie,
 static void
 init_trash_loc (loc_t *trash_loc, inode_table_t *table)
 {
-        trash_loc->path   = strdup ("/" AFR_TRASH_DIR);
-        trash_loc->name   = AFR_TRASH_DIR;
+        trash_loc->path   = strdup ("/" GF_REPLICATE_TRASH_DIR);
+        trash_loc->name   = GF_REPLICATE_TRASH_DIR;
         trash_loc->parent = table->root;
         trash_loc->inode  = inode_new (table);
 }
@@ -545,9 +545,9 @@ make_trash_path (const char *path)
         char *c  = NULL;
         char *tp = NULL;
 
-        tp = CALLOC (strlen ("/" AFR_TRASH_DIR) + strlen (path) + 1, sizeof (char));
+        tp = CALLOC (strlen ("/" GF_REPLICATE_TRASH_DIR) + strlen (path) + 1, sizeof (char));
 
-        strcpy (tp, AFR_TRASH_DIR);
+        strcpy (tp, GF_REPLICATE_TRASH_DIR);
         strcat (tp, path);
 
         c = strchr (tp, '/') + 1;
@@ -615,7 +615,7 @@ afr_sh_entry_expunge_mkdir_cbk (call_frame_t *expunge_frame, void *cookie, xlato
 
         if (op_ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "mkdir of /" AFR_TRASH_DIR " failed on %s",
+                        "mkdir of /" GF_REPLICATE_TRASH_DIR " failed on %s",
                         priv->children[active_src]->name);
 
                 goto out;
@@ -624,7 +624,7 @@ afr_sh_entry_expunge_mkdir_cbk (call_frame_t *expunge_frame, void *cookie, xlato
         /* mkdir successful */
 
         trash_inode = inode_link (inode, expunge_local->loc.inode->table->root,
-                                  AFR_TRASH_DIR, buf);
+                                  GF_REPLICATE_TRASH_DIR, buf);
 
         afr_sh_entry_expunge_rename (expunge_frame, this, active_src,
                                      trash_inode);
@@ -662,7 +662,7 @@ afr_sh_entry_expunge_lookup_trash_cbk (call_frame_t *expunge_frame, void *cookie
                 init_trash_loc (&trash_loc, expunge_local->loc.inode->table);
 
                 gf_log (this->name, GF_LOG_TRACE,
-                        "creating directory " AFR_TRASH_DIR " on subvolume %s",
+                        "creating directory " GF_REPLICATE_TRASH_DIR " on subvolume %s",
                         priv->children[active_src]->name);
 
                 STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_mkdir_cbk,
@@ -677,7 +677,7 @@ afr_sh_entry_expunge_lookup_trash_cbk (call_frame_t *expunge_frame, void *cookie
 
         if (op_ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "lookup of /" AFR_TRASH_DIR " failed on %s",
+                        "lookup of /" GF_REPLICATE_TRASH_DIR " failed on %s",
                         priv->children[active_src]->name);
                 goto out;
         }
@@ -685,7 +685,7 @@ afr_sh_entry_expunge_lookup_trash_cbk (call_frame_t *expunge_frame, void *cookie
         /* lookup successful */
 
         trash_inode = inode_link (inode, expunge_local->loc.inode->table->root,
-                                  AFR_TRASH_DIR, buf);
+                                  GF_REPLICATE_TRASH_DIR, buf);
 
         afr_sh_entry_expunge_rename (expunge_frame, this, active_src,
                                      trash_inode);
@@ -713,7 +713,7 @@ afr_sh_entry_expunge_lookup_trash (call_frame_t *expunge_frame, xlator_t *this,
 
         root = expunge_local->loc.inode->table->root;
 
-        trash = inode_grep (root->table, root, AFR_TRASH_DIR);
+        trash = inode_grep (root->table, root, GF_REPLICATE_TRASH_DIR);
 
         if (trash) {
                 /* inode is in cache, so no need to mkdir */
@@ -728,7 +728,7 @@ afr_sh_entry_expunge_lookup_trash (call_frame_t *expunge_frame, xlator_t *this,
         init_trash_loc (&trash_loc, expunge_local->loc.inode->table);
 
 	gf_log (this->name, GF_LOG_TRACE,
-		"looking up /" AFR_TRASH_DIR " on %s",
+		"looking up /" GF_REPLICATE_TRASH_DIR " on %s",
                 priv->children[active_src]->name);
 
 	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_trash_cbk,
@@ -934,7 +934,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
 	if ((strcmp (name, ".") == 0)
 	    || (strcmp (name, "..") == 0)
             || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, AFR_TRASH_DIR) == 0))) {
+                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
 
                 gf_log (this->name, GF_LOG_TRACE,
 			"skipping inspection of %s under %s",
@@ -1927,7 +1927,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
 	if ((strcmp (entry->d_name, ".") == 0)
 	    || (strcmp (entry->d_name, "..") == 0)
             || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, AFR_TRASH_DIR) == 0))) {
+                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
 
 		gf_log (this->name, GF_LOG_TRACE,
 			"skipping inspection of %s under %s",
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 9448dc4e6..33cd70cd4 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -961,7 +961,7 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
 
 	frame->local = local;
 
-        if (!strcmp (loc->path, "/" AFR_TRASH_DIR)) {
+        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
                 op_errno = ENOENT;
                 goto out;
         }
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index bf07a89d7..c5db12a44 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -31,7 +31,6 @@
 #include "compat-errno.h"
 
 #define AFR_XATTR_PREFIX "trusted.afr"
-#define AFR_TRASH_DIR    ".trash"
 
 typedef struct _afr_private {
 	gf_lock_t lock;               /* to guard access to child_count, etc */
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 325c69634..1219620f1 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -29,6 +29,7 @@
 #include <sys/resource.h>
 #include <errno.h>
 #include <libgen.h>
+#include <pthread.h>
 #include <ftw.h>
 
 #ifndef GF_BSD_HOST_OS
@@ -50,7 +51,7 @@
 #include "syscall.h"
 #include "statedump.h"
 #include "locking.h"
-#include <libgen.h>
+#include "timer.h"
 
 #undef HAVE_SET_FSID
 #ifdef HAVE_SET_FSID
@@ -1322,6 +1323,95 @@ posix_mknod (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
+
+static int
+janitor_walker (const char *fpath, const struct stat *sb,
+                int typeflag, struct FTW *ftwbuf)
+{
+        switch (sb->st_mode & S_IFMT) {
+        case S_IFREG:
+        case S_IFBLK:
+        case S_IFLNK:
+        case S_IFCHR:
+        case S_IFIFO:
+        case S_IFSOCK:
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "unlinking %s", fpath);
+                unlink (fpath);
+                break;
+
+        case S_IFDIR:
+                if (ftwbuf->level) { /* don't remove top level dir */
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "removing directory %s", fpath);
+
+                        rmdir (fpath);
+                }
+                break;
+        }
+
+        return FTW_CONTINUE;
+}
+
+
+#define JANITOR_SLEEP_DURATION          2
+
+static void *
+posix_janitor_thread_proc (void *data)
+{
+        xlator_t *            this = NULL;
+        struct posix_private *priv = NULL;
+
+        this = data;
+        priv = this->private;
+
+        THIS = this;
+
+        while (1) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "janitor woke up, cleaning out /" GF_REPLICATE_TRASH_DIR);
+
+                nftw (priv->trash_path,
+                      janitor_walker,
+                      32,
+                      FTW_DEPTH | FTW_PHYS);
+
+                sleep (JANITOR_SLEEP_DURATION);
+        }
+
+        return NULL;
+}
+
+
+static void
+posix_spawn_janitor_thread (xlator_t *this)
+{
+        struct posix_private *priv = NULL;
+        int ret = 0;
+
+        priv = this->private;
+
+        LOCK (&priv->lock);
+        {
+                if (!priv->janitor_present) {
+                        ret = pthread_create (&priv->janitor, NULL,
+                                              posix_janitor_thread_proc, this);
+
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "spawning janitor thread failed: %s",
+                                        strerror (errno));
+                                goto unlock;
+                        }
+
+                        priv->janitor_present = _gf_true;
+                }
+        }
+unlock:
+        UNLOCK (&priv->lock);
+}
+
+
 int32_t
 posix_mkdir (call_frame_t *frame, xlator_t *this,
              loc_t *loc, mode_t mode)
@@ -1384,6 +1474,14 @@ posix_mkdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got mkdir %s, spawning janitor thread",
+                        loc->path);
+
+                posix_spawn_janitor_thread (this);
+        }
+
 #ifndef HAVE_SET_FSID
         op_ret = chown (real_path, frame->root->uid, gid);
         if (op_ret == -1) {
@@ -4738,6 +4836,21 @@ init (xlator_t *this)
         _private->base_path = strdup (dir_data->data);
         _private->base_path_length = strlen (_private->base_path);
 
+        _private->trash_path = CALLOC (1, _private->base_path_length
+                                       + strlen ("/")
+                                       + strlen (GF_REPLICATE_TRASH_DIR)
+                                       + 1);
+
+        if (!_private->trash_path) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory.");
+                ret = -1;
+                goto out;
+        }
+
+        strncpy (_private->trash_path, _private->base_path, _private->base_path_length);
+        strcat (_private->trash_path, "/" GF_REPLICATE_TRASH_DIR);
+
         LOCK_INIT (&_private->lock);
 
         ret = gethostname (_private->hostname, 256);
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index b16800c5c..0a99caeb6 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -114,6 +114,11 @@ struct posix_private {
 */
         uint64_t        gen_seq;
         gf_lock_t       gen_lock;
+
+/* janitor thread which cleans up /.trash (created by replicate) */
+        pthread_t       janitor;
+        gf_boolean_t    janitor_present;
+        char *          trash_path;
 };
 
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)