tier:unlink during migration

files deleted during promotion were not deleting as the
files are moving from hashed to non-hashed

On deleting a file that is undergoing promotion,
the unlink call is not sent to the dst file as the
hashed subvol == cached subvol. This causes
the file to reappear once the migration is complete.

This patch also fixes a problem with stale linkfile
deleting.

Backport of>
>Change-Id: I4b02a498218c9d8eeaa4556fa4219e91e7fa71e5
>BUG: 1282390
>Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
>Reviewed-on: http://review.gluster.org/12829
>Tested-by: NetBSD Build System <jenkins@build.gluster.org>
>Tested-by: Gluster Build System <jenkins@build.gluster.com>
>Reviewed-by: Dan Lambright <dlambrig@redhat.com>
>Tested-by: Dan Lambright <dlambrig@redhat.com>

(cherry picked from commit b5de382afa8c5777e455c7a376fc4f1f01d782d1)

Change-Id: I951adb4d929926bcd646dd7574f7a2d41d57479d
BUG: 1282388
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/12991
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>

5 files changed, 374 insertions, 75 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 242e1798753..f80924e818b 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1167,9 +1167,19 @@ dht_lookup_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
 int
 dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict) {
 
-        int ret = 0;
+        int                      ret = 0;
+        xlator_t                *this           = NULL;
+        char                    *linktoskip_key = NULL;
+
+        this    = THIS;
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
 
-        ret = dict_set_int32 (dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
+        if (dht_is_tier_xlator (this))
+                linktoskip_key = TIER_SKIP_NON_LINKTO_UNLINK;
+        else
+                linktoskip_key = DHT_SKIP_NON_LINKTO_UNLINK;
+
+        ret = dict_set_int32 (dict, linktoskip_key, 1);
 
         if (ret)
                 goto err;
@@ -2431,102 +2441,103 @@ err:
         return 0;
 }
 
-
 int
-dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev  = NULL;
+        dht_local_t     *local          = NULL;
+        call_frame_t    *prev           = NULL;
 
         local = frame->local;
         prev  = cookie;
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1) {
-                        local->op_ret   = -1;
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
                         local->op_errno = op_errno;
                         gf_msg_debug (this->name, op_errno,
-                                      "Unlink: subvolume %s returned -1",
-                                       prev->this->name);
+                                      "Unlink link: subvolume %s"
+                                      " returned -1",
+                                      prev->this->name);
                         goto unlock;
                 }
 
                 local->op_ret = 0;
-
-                local->postparent = *postparent;
-                local->preparent = *preparent;
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
         }
 unlock:
         UNLOCK (&frame->lock);
 
+
         DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+                          &local->preparent, &local->postparent, xdata);
 
         return 0;
 }
 
-
 int
-dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-
-        xlator_t *cached_subvol = NULL;
+        dht_local_t     *local          = NULL;
+        call_frame_t    *prev           = NULL;
+        xlator_t        *hashed_subvol  = NULL;
 
         local = frame->local;
         prev  = cookie;
 
         LOCK (&frame->lock);
         {
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == ENOTCONN))) {
-                        local->op_errno = op_errno;
+                if (op_ret == -1) {
+                        if (op_errno != ENOENT) {
+                                local->op_ret   = -1;
+                                local->op_errno = op_errno;
+                        } else {
+                                local->op_ret = 0;
+                        }
                         gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->this->name);
+                                      "Unlink: subvolume %s returned -1",
+                                       prev->this->name);
                         goto unlock;
                 }
 
                 local->op_ret = 0;
+
+                local->postparent = *postparent;
+                local->preparent = *preparent;
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
 
-        if (local->op_ret == -1)
-                goto err;
-
-        cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s",
-                              local->loc.path);
-                local->op_errno = EINVAL;
-                goto err;
+        if (!local->op_ret) {
+                hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+                if (hashed_subvol &&
+                hashed_subvol != local->cached_subvol) {
+                        /*
+                         * If hashed and cached are different, then we need
+                         * to unlink linkfile from hashed subvol if data
+                         * file is deleted successfully
+                         */
+                        STACK_WIND (frame, dht_unlink_linkfile_cbk,
+                                    hashed_subvol,
+                                    hashed_subvol->fops->unlink, &local->loc,
+                                    local->flags, xdata);
+                        return 0;
+                }
         }
 
-        STACK_WIND (frame, dht_unlink_cbk,
-                    cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc, local->flags, NULL);
-
-        return 0;
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, xdata);
 
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
         return 0;
 }
 
@@ -5625,7 +5636,6 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
             dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
         int          op_errno = -1;
         dht_local_t *local = NULL;
 
@@ -5640,15 +5650,6 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
                 goto err;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        /* Dont fail unlink if hashed_subvol is NULL which can be the result
-         * of layout anomaly */
-        if (!hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-        }
-
         cached_subvol = local->cached_subvol;
         if (!cached_subvol) {
                 gf_msg_debug (this->name, 0,
@@ -5658,15 +5659,9 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
         }
 
         local->flags = xflag;
-        if (hashed_subvol && hashed_subvol != cached_subvol) {
-                STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc,
-                            xflag, xdata);
-        } else {
-                STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc,
-                            xflag, xdata);
-        }
+        STACK_WIND (frame, dht_unlink_cbk,
+                    cached_subvol, cached_subvol->fops->unlink, loc,
+                    xflag, xdata);
 
         return 0;
 err:
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 5fa97a41881..31b8b6804b7 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1149,6 +1149,9 @@ dht_layout_missing_dirs (dht_layout_t *layout);
 int
 dht_refresh_layout (call_frame_t *frame);
 
+gf_boolean_t
+dht_is_tier_xlator (xlator_t *this);
+
 int
 dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
                                                  int32_t *op_errno);
diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c
index 129932c9ef9..0adf3387fe7 100644
--- a/xlators/cluster/dht/src/tier-common.c
+++ b/xlators/cluster/dht/src/tier-common.c
@@ -22,6 +22,302 @@
 #include "tier.h"
 
 int
+tier_unlink_nonhashed_linkfile_cbk (call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int op_ret, int op_errno,
+                                    struct iatt *preparent,
+                                    struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if ((op_ret == -1) && (op_errno != ENOENT)) {
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        gf_msg_debug (this->name, op_errno,
+                                      "Unlink link: subvolume %s"
+                                      " returned -1",
+                                      prev->this->name);
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (local->op_ret == -1)
+                goto err;
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
+
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
+                          NULL, NULL, NULL);
+        return 0;
+}
+
+int
+tier_unlink_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *preparent, dict_t *xdata,
+                        struct iatt *postparent)
+{
+        dht_local_t             *local             = NULL;
+        call_frame_t            *prev              = NULL;
+        dht_conf_t              *conf              = NULL;
+        xlator_t                *hot_subvol        = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+        conf = this->private;
+        hot_subvol = TIER_UNHASHED_SUBVOL;
+
+        if (!op_ret) {
+                /*
+                 * linkfile present on hot tier. unlinking the linkfile
+                 */
+                STACK_WIND (frame, tier_unlink_nonhashed_linkfile_cbk,
+                            hot_subvol, hot_subvol->fops->unlink,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
+        LOCK (&frame->lock);
+        {
+                if (op_errno == ENOENT) {
+                        local->op_ret   = 0;
+                        local->op_errno = op_errno;
+                } else {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                }
+                gf_msg_debug (this->name, op_errno,
+                              "Lookup : subvolume %s returned -1",
+                               prev->this->name);
+        }
+
+        UNLOCK (&frame->lock);
+
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, xdata);
+
+        return 0;
+}
+
+int
+tier_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                /* Ignore EINVAL for tier to ignore error when the file
+                        does not exist on the other tier  */
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == EINVAL))) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = op_ret;
+                        gf_msg_debug (this->name, op_errno,
+                                      "Unlink link: subvolume %s"
+                                      " returned -1",
+                                      prev->this->name);
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (local->op_ret == -1)
+                goto err;
+
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, xdata);
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
+                          NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+tier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t     *local          = NULL;
+        call_frame_t    *prev           = NULL;
+        struct iatt     *stbuf          = NULL;
+        dht_conf_t      *conf           = NULL;
+        int              ret            = -1;
+        xlator_t        *hot_tier       = NULL;
+        xlator_t        *cold_tier      = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+        conf  = this->private;
+
+        cold_tier = TIER_HASHED_SUBVOL;
+        hot_tier = TIER_UNHASHED_SUBVOL;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        if (op_errno == ENOENT) {
+                                local->op_ret = 0;
+                        } else {
+                                local->op_ret   = -1;
+                                local->op_errno = op_errno;
+                        }
+                        gf_msg_debug (this->name, op_errno,
+                                      "Unlink: subvolume %s returned -1"
+                                      " with errno = %d",
+                                       prev->this->name, op_errno);
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+
+                local->postparent = *postparent;
+                local->preparent = *preparent;
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (local->op_ret)
+                goto out;
+
+        if (cold_tier != local->cached_subvol) {
+                /*
+                 * File is present in hot tier, so there will be
+                 * a link file on cold tier, deleting the linkfile
+                 * from cold tier
+                 */
+                STACK_WIND (frame, tier_unlink_linkfile_cbk,
+                            cold_tier,
+                            cold_tier->fops->unlink, &local->loc,
+                            local->flags, xdata);
+                return 0;
+        }
+
+        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+        if (!ret && stbuf && ((IS_DHT_MIGRATION_PHASE2 (stbuf)) ||
+                      IS_DHT_MIGRATION_PHASE1 (stbuf))) {
+                /*
+                 * File is migrating from cold to hot tier.
+                 * Delete the destination linkfile.
+                 */
+                STACK_WIND (frame, tier_unlink_lookup_cbk,
+                            hot_tier,
+                            hot_tier->fops->lookup,
+                            &local->loc, NULL);
+                return 0;
+
+        }
+
+out:
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, xdata);
+
+        return 0;
+}
+
+int
+tier_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
+{
+        xlator_t                *cached_subvol = NULL;
+        xlator_t                *hashed_subvol = NULL;
+        dht_conf_t              *conf          = NULL;
+        int                      op_errno      = -1;
+        dht_local_t             *local         = NULL;
+        int                      ret           = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        conf = this->private;
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
+        if (!local) {
+                op_errno = ENOMEM;
+
+                goto err;
+        }
+
+        hashed_subvol = TIER_HASHED_SUBVOL;
+
+        cached_subvol = local->cached_subvol;
+        if (!cached_subvol) {
+                gf_msg_debug (this->name, 0,
+                              "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->flags = xflag;
+        if (hashed_subvol == cached_subvol) {
+                /*
+                 * File resides in cold tier. We need to stat
+                 * the file to see if it is being promoted.
+                 * If yes we need to delete the destination
+                 * file as well.
+                 */
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata) {
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                DHT_IATT_IN_XDATA_KEY, "yes");
+                        if (ret) {
+                                gf_msg_debug (this->name, 0,
+                                        "Failed to set dictionary key %s",
+                                         DHT_IATT_IN_XDATA_KEY);
+                        }
+                }
+        }
+
+        /*
+         * File is on hot tier, delete the data file first, then
+         * linkfile from cold.
+         */
+        STACK_WIND (frame, tier_unlink_cbk,
+                    cached_subvol, cached_subvol->fops->unlink, loc,
+                    xflag, xdata);
+        if (xdata)
+                dict_unref (xdata);
+        return 0;
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
 tier_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno, gf_dirent_t *orig_entries,
                  dict_t *xdata)
diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h
index db5bc8c9226..a9d98eab36d 100644
--- a/xlators/cluster/dht/src/tier-common.h
+++ b/xlators/cluster/dht/src/tier-common.h
@@ -10,6 +10,11 @@
 
 #ifndef _TIER_COMMON_H_
 #define _TIER_COMMON_H_
+/* Function definitions */
+
+int32_t
+tier_unlink (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, int xflag, dict_t *xdata);
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 92b5aa5f999..e883c1cba03 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -2348,7 +2348,7 @@ struct xlator_fops fops = {
         .readdirp    = tier_readdirp,
         .fsyncdir    = dht_fsyncdir,
         .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
+        .unlink      = tier_unlink,
         .link        = dht_link,
         .mkdir       = dht_mkdir,
         .rmdir       = dht_rmdir,