diff options
| author | Pranith Kumar K <pkarampu@redhat.com> | 2018-12-18 14:38:22 +0530 | 
|---|---|---|
| committer | Amar Tumballi <amarts@redhat.com> | 2018-12-28 10:46:00 +0000 | 
| commit | a12cadc1377ef51ad52defd1da91bf8f599e5786 (patch) | |
| tree | 3380715fe1dd17255d75af1f9486e54a04a23bfb | |
| parent | 8d38c5b7337e44ad9f282966c0aa2e99bd7da506 (diff) | |
cluster/afr: Refactor internal locking code to allow multiple inodelks
For implementing copy_file_range fop, AFR needs to perform two inodelks in the
same transaction. This patch brings in the necessary structure to make it
easier to do so.
Entry-locks in AFR were already taking multiple entry-locks on different inodes
with the respective basenames. This patch extends the logic in inodelks to use
the same lockee_t structure. This lead to removal of quite a lot of duplicate
code present in afr-lk-common.c as both the locks are doing same things except
'winding' part.
updates: #536
Change-Id: Ibfce7e3f260bb27b18645152ec680c33866fe0ae
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
| -rw-r--r-- | xlators/cluster/afr/src/afr-common.c | 9 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-dir-write.c | 143 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-lk-common.c | 814 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-messages.h | 2 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-transaction.c | 170 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr.h | 26 | 
6 files changed, 366 insertions, 798 deletions
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c index 32fa634def1..0f77607fee0 100644 --- a/xlators/cluster/afr/src/afr-common.c +++ b/xlators/cluster/afr/src/afr-common.c @@ -1794,11 +1794,9 @@ afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)      afr_matrix_cleanup(local->pending, priv->child_count); -    GF_FREE(local->internal_lock.locked_nodes); -      GF_FREE(local->internal_lock.lower_locked_nodes); -    afr_entry_lockee_cleanup(&local->internal_lock); +    afr_lockees_cleanup(&local->internal_lock);      GF_FREE(local->transaction.pre_op); @@ -5708,11 +5706,6 @@ afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)  {      int ret = -ENOMEM; -    lk->locked_nodes = GF_CALLOC(sizeof(*lk->locked_nodes), child_count, -                                 gf_afr_mt_char); -    if (NULL == lk->locked_nodes) -        goto out; -      lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),                                         child_count, gf_afr_mt_char);      if (NULL == lk->lower_locked_nodes) diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c index 486b06b162f..f2c8ef450e1 100644 --- a/xlators/cluster/afr/src/afr-dir-write.c +++ b/xlators/cluster/afr/src/afr-dir-write.c @@ -425,15 +425,11 @@ int  afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -475,16 +471,6 @@ afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -555,15 +541,11 @@ int  afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,            dev_t dev, mode_t umask, dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -598,16 +580,6 @@ afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -676,15 +648,11 @@ int  afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,            mode_t umask, dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -726,16 +694,6 @@ afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -804,15 +762,11 @@ int  afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,           dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -847,16 +801,6 @@ afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(newloc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -926,15 +870,11 @@ int  afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,              loc_t *loc, mode_t umask, dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -968,16 +908,6 @@ afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -1050,15 +980,10 @@ int  afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,             dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    int nlockee = 0; - -    priv = this->private;      transaction_frame = copy_frame(frame);      if (!transaction_frame) { @@ -1101,35 +1026,6 @@ afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(oldloc->path);      local->transaction.new_basename = AFR_BASENAME(newloc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = nlockee = 0; -    ret = afr_init_entry_lockee( -        &int_lock->lockee[nlockee], local, &local->transaction.new_parent_loc, -        local->transaction.new_basename, priv->child_count); -    if (ret) -        goto out; - -    nlockee++; -    ret = afr_init_entry_lockee(&int_lock->lockee[nlockee], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    nlockee++; -    if (local->newloc.inode && IA_ISDIR(local->newloc.inode->ia_type)) { -        ret = afr_init_entry_lockee(&int_lock->lockee[nlockee], local, -                                    &local->newloc, NULL, priv->child_count); -        if (ret) -            goto out; - -        nlockee++; -    } -    qsort(int_lock->lockee, nlockee, sizeof(*int_lock->lockee), -          afr_entry_lockee_cmp); -    int_lock->lockee_count = nlockee; -      ret = afr_transaction(transaction_frame, this,                            AFR_ENTRY_RENAME_TRANSACTION);      if (ret < 0) { @@ -1198,15 +1094,11 @@ int  afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,             dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    priv = this->private; -      transaction_frame = copy_frame(frame);      if (!transaction_frame)          goto out; @@ -1239,16 +1131,6 @@ afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[0], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    int_lock->lockee_count++;      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; @@ -1315,15 +1197,10 @@ int  afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,            dict_t *xdata)  { -    afr_private_t *priv = NULL;      afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL;      call_frame_t *transaction_frame = NULL;      int ret = -1;      int op_errno = ENOMEM; -    int nlockee = 0; - -    priv = this->private;      transaction_frame = copy_frame(frame);      if (!transaction_frame) @@ -1357,26 +1234,6 @@ afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,      local->transaction.main_frame = frame;      local->transaction.basename = AFR_BASENAME(loc->path); -    int_lock = &local->internal_lock; - -    int_lock->lockee_count = nlockee = 0; -    ret = afr_init_entry_lockee(&int_lock->lockee[nlockee], local, -                                &local->transaction.parent_loc, -                                local->transaction.basename, priv->child_count); -    if (ret) -        goto out; - -    nlockee++; -    ret = afr_init_entry_lockee(&int_lock->lockee[nlockee], local, &local->loc, -                                NULL, priv->child_count); -    if (ret) -        goto out; - -    nlockee++; -    qsort(int_lock->lockee, nlockee, sizeof(*int_lock->lockee), -          afr_entry_lockee_cmp); -    int_lock->lockee_count = nlockee; -      ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);      if (ret < 0) {          op_errno = -ret; diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c index a80eab0a349..4091671fa3f 100644 --- a/xlators/cluster/afr/src/afr-lk-common.c +++ b/xlators/cluster/afr/src/afr-lk-common.c @@ -22,11 +22,40 @@  #define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */  #define LOCKED_LOWER 0x2 /* for lower path */ +void +afr_lockee_cleanup(afr_lockee_t *lockee) +{ +    if (lockee->fd) { +        fd_unref(lockee->fd); +        lockee->fd = NULL; +    } else { +        loc_wipe(&lockee->loc); +    } + +    GF_FREE(lockee->basename); +    lockee->basename = NULL; +    GF_FREE(lockee->locked_nodes); +    lockee->locked_nodes = NULL; + +    return; +} + +void +afr_lockees_cleanup(afr_internal_lock_t *int_lock) +{ +    int i = 0; + +    for (i = 0; i < int_lock->lockee_count; i++) { +        afr_lockee_cleanup(&int_lock->lockee[i]); +    } + +    return; +}  int  afr_entry_lockee_cmp(const void *l1, const void *l2)  { -    const afr_entry_lockee_t *r1 = l1; -    const afr_entry_lockee_t *r2 = l2; +    const afr_lockee_t *r1 = l1; +    const afr_lockee_t *r2 = l2;      int ret = 0;      uuid_t gfid1 = {0};      uuid_t gfid2 = {0}; @@ -81,31 +110,14 @@ internal_lock_count(call_frame_t *frame, xlator_t *this)  }  int -afr_is_inodelk_transaction(afr_transaction_type type) +afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename, +                     int child_count)  { -    int ret = 0; - -    switch (type) { -        case AFR_DATA_TRANSACTION: -        case AFR_METADATA_TRANSACTION: -            ret = 1; -            break; - -        case AFR_ENTRY_RENAME_TRANSACTION: -        case AFR_ENTRY_TRANSACTION: -            ret = 0; -            break; -    } - -    return ret; -} - -int -afr_init_entry_lockee(afr_entry_lockee_t *lockee, afr_local_t *local, -                      loc_t *loc, char *basename, int child_count) -{ -    int ret = -1; +    int ret = -ENOMEM; +    afr_internal_lock_t *int_lock = &local->internal_lock; +    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count]; +    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);      loc_copy(&lockee->loc, loc);      lockee->basename = (basename) ? gf_strdup(basename) : NULL;      if (basename && !lockee->basename) @@ -119,28 +131,45 @@ afr_init_entry_lockee(afr_entry_lockee_t *lockee, afr_local_t *local,          goto out;      ret = 0; +    int_lock->lockee_count++;  out: +    if (ret) { +        afr_lockee_cleanup(lockee); +    }      return ret;  } -void -afr_entry_lockee_cleanup(afr_internal_lock_t *int_lock) +int +afr_add_inode_lockee(afr_local_t *local, int child_count)  { -    int i = 0; +    int ret = -ENOMEM; +    afr_internal_lock_t *int_lock = &local->internal_lock; +    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count]; -    for (i = 0; i < int_lock->lockee_count; i++) { -        loc_wipe(&int_lock->lockee[i].loc); -        if (int_lock->lockee[i].basename) -            GF_FREE(int_lock->lockee[i].basename); -        if (int_lock->lockee[i].locked_nodes) -            GF_FREE(int_lock->lockee[i].locked_nodes); +    if (local->fd) { +        lockee->fd = fd_ref(local->fd); +    } else { +        loc_copy(&lockee->loc, &local->loc);      } -    return; +    lockee->locked_count = 0; +    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes), +                                     gf_afr_mt_afr_node_character); + +    if (!lockee->locked_nodes) +        goto out; + +    ret = 0; +    int_lock->lockee_count++; +out: +    if (ret) { +        afr_lockee_cleanup(lockee); +    } +    return ret;  }  static int -initialize_entrylk_variables(call_frame_t *frame, xlator_t *this) +initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)  {      afr_local_t *local = NULL;      afr_internal_lock_t *int_lock = NULL; @@ -152,9 +181,10 @@ initialize_entrylk_variables(call_frame_t *frame, xlator_t *this)      local = frame->local;      int_lock = &local->internal_lock; -    int_lock->entrylk_lock_count = 0; +    int_lock->lock_count = 0;      int_lock->lock_op_ret = -1;      int_lock->lock_op_errno = 0; +    int_lock->lk_attempted_count = 0;      for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {          if (!int_lock->lockee[i].locked_nodes) @@ -167,28 +197,6 @@ initialize_entrylk_variables(call_frame_t *frame, xlator_t *this)      return 0;  } -static int -initialize_inodelk_variables(call_frame_t *frame, xlator_t *this) -{ -    afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL; -    afr_private_t *priv = NULL; - -    priv = this->private; -    local = frame->local; -    int_lock = &local->internal_lock; - -    int_lock->lock_count = 0; -    int_lock->lk_attempted_count = 0; -    int_lock->lock_op_ret = -1; -    int_lock->lock_op_errno = 0; - -    memset(int_lock->locked_nodes, 0, -           sizeof(*int_lock->locked_nodes) * priv->child_count); - -    return 0; -} -  int  afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)  { @@ -216,19 +224,74 @@ afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)      return call_count;  } -/* FIXME: What if UNLOCK fails */ +static void +afr_log_locks_failure(call_frame_t *frame, char *where, char *what, +                      int op_errno) +{ +    xlator_t *this = frame->this; +    gf_lkowner_t *lk_owner = &frame->root->lk_owner; +    afr_local_t *local = frame->local; +    const char *fop = NULL; +    char *gfid = NULL; +    const char *name = NULL; + +    fop = gf_fop_list[local->op]; + +    switch (local->transaction.type) { +        case AFR_ENTRY_RENAME_TRANSACTION: +        case AFR_ENTRY_TRANSACTION: +            switch (local->op) { +                case GF_FOP_LINK: +                    gfid = uuid_utoa(local->newloc.pargfid); +                    name = local->newloc.name; +                    break; +                default: +                    gfid = uuid_utoa(local->loc.pargfid); +                    name = local->loc.name; +                    break; +            } +            gf_msg(this->name, GF_LOG_WARNING, op_errno, +                   AFR_MSG_INTERNAL_LKS_FAILED, +                   "Unable to do entry %s with lk-owner:%s on %s " +                   "while attempting %s on {pgfid:%s, name:%s}.", +                   what, lkowner_utoa(lk_owner), where, fop, gfid, name); +            break; +        case AFR_DATA_TRANSACTION: +        case AFR_METADATA_TRANSACTION: +            gfid = uuid_utoa(local->inode->gfid); +            gf_msg(this->name, GF_LOG_WARNING, op_errno, +                   AFR_MSG_INTERNAL_LKS_FAILED, +                   "Unable to do inode %s with lk-owner:%s on %s " +                   "while attempting %s on gfid:%s.", +                   what, lkowner_utoa(lk_owner), where, fop, gfid); +            break; +    } +} +  static int32_t  afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,                        int32_t op_ret, int32_t op_errno, dict_t *xdata)  {      afr_local_t *local = NULL; +    afr_private_t *priv = NULL;      afr_internal_lock_t *int_lock = NULL; +    int lockee_num = 0;      int call_count = 0; +    int child_index = 0;      int ret = 0;      local = frame->local;      int_lock = &local->internal_lock; +    priv = this->private; +    lockee_num = (int)((long)cookie) / priv->child_count; +    child_index = (int)((long)cookie) % priv->child_count; + +    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) { +        afr_log_locks_failure(frame, priv->children[child_index]->name, +                              "unlock", op_errno); +    } +    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;      if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)          ret = afr_write_subvol_reset(frame, this); @@ -239,7 +302,6 @@ afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,      UNLOCK(&frame->lock);      if (call_count == 0) { -        gf_msg_trace(this->name, 0, "All internal locks unlocked");          int_lock->lock_cbk(frame, this);      } @@ -247,165 +309,108 @@ afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,  }  void -afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock, -                     int32_t child_index) -{ -    int_lock->locked_nodes[child_index] &= LOCKED_NO; -} - -static int32_t -afr_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                       int32_t op_ret, int32_t op_errno, dict_t *xdata) -{ -    afr_local_t *local = NULL; -    afr_internal_lock_t *int_lock = NULL; -    int32_t child_index = (long)cookie; -    afr_private_t *priv = NULL; - -    local = frame->local; -    int_lock = &local->internal_lock; - -    priv = this->private; - -    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) { -        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL, -               "path=%s gfid=%s: unlock failed on subvolume %s " -               "with lock owner %s", -               local->loc.path, loc_gfid_utoa(&(local->loc)), -               priv->children[child_index]->name, -               lkowner_utoa(&frame->root->lk_owner)); -    } - -    afr_update_uninodelk(local, int_lock, child_index); - -    afr_unlock_common_cbk(frame, cookie, this, op_ret, op_errno, xdata); - -    return 0; -} - -static int -afr_unlock_inodelk(call_frame_t *frame, xlator_t *this) +afr_internal_lock_wind(call_frame_t *frame, +                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *, +                                      int32_t, int32_t, dict_t *), +                       void *cookie, int child, int lockee_num, +                       gf_boolean_t blocking, gf_boolean_t unlock)  { -    afr_internal_lock_t *int_lock = NULL; -    afr_local_t *local = NULL; -    afr_private_t *priv = NULL; +    afr_local_t *local = frame->local; +    xlator_t *this = frame->this; +    afr_private_t *priv = this->private; +    afr_internal_lock_t *int_lock = &local->internal_lock; +    entrylk_cmd cmd = ENTRYLK_LOCK_NB; +    int32_t cmd1 = F_SETLK;      struct gf_flock flock = {          0,      }; -    int call_count = 0; -    int i = 0; - -    local = frame->local; -    int_lock = &local->internal_lock; -    priv = this->private; - -    flock.l_start = int_lock->flock.l_start; -    flock.l_len = int_lock->flock.l_len; -    flock.l_type = F_UNLCK; - -    call_count = afr_locked_nodes_count(int_lock->locked_nodes, -                                        priv->child_count); - -    int_lock->lk_call_count = call_count; - -    if (!call_count) { -        GF_ASSERT(!local->transaction.do_eager_unlock); -        gf_msg_trace(this->name, 0, "No internal locks unlocked"); - -        int_lock->lock_cbk(frame, this); -        goto out; -    } -    for (i = 0; i < priv->child_count; i++) { -        if ((int_lock->locked_nodes[i] & LOCKED_YES) != LOCKED_YES) -            continue; - -        if (local->fd) { -            STACK_WIND_COOKIE( -                frame, afr_unlock_inodelk_cbk, (void *)(long)i, -                priv->children[i], priv->children[i]->fops->finodelk, -                int_lock->domain, local->fd, F_SETLK, &flock, NULL); -        } else { -            STACK_WIND_COOKIE( -                frame, afr_unlock_inodelk_cbk, (void *)(long)i, -                priv->children[i], priv->children[i]->fops->inodelk, -                int_lock->domain, &local->loc, F_SETLK, &flock, NULL); -        } +    switch (local->transaction.type) { +        case AFR_ENTRY_TRANSACTION: +        case AFR_ENTRY_RENAME_TRANSACTION: +            if (unlock) { +                cmd = ENTRYLK_UNLOCK; +            } else if (blocking) { /*Doesn't make sense to have blocking +                                      unlock*/ +                cmd = ENTRYLK_LOCK; +            } -        if (!--call_count) +            if (local->fd) { +                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child], +                                  priv->children[child]->fops->fentrylk, +                                  int_lock->domain, +                                  int_lock->lockee[lockee_num].fd, +                                  int_lock->lockee[lockee_num].basename, cmd, +                                  ENTRYLK_WRLCK, NULL); +            } else { +                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child], +                                  priv->children[child]->fops->entrylk, +                                  int_lock->domain, +                                  &int_lock->lockee[lockee_num].loc, +                                  int_lock->lockee[lockee_num].basename, cmd, +                                  ENTRYLK_WRLCK, NULL); +            }              break; -    } -out: -    return 0; -} - -static int32_t -afr_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                       int32_t op_ret, int32_t op_errno, dict_t *xdata) -{ -    afr_local_t *local = NULL; -    afr_private_t *priv = NULL; -    afr_internal_lock_t *int_lock = NULL; -    int32_t child_index = 0; -    int lockee_no = 0; - -    priv = this->private; -    lockee_no = (int)((long)cookie) / priv->child_count; -    child_index = (int)((long)cookie) % priv->child_count; -    local = frame->local; -    int_lock = &local->internal_lock; +        case AFR_DATA_TRANSACTION: +        case AFR_METADATA_TRANSACTION: +            flock = int_lock->lockee[lockee_num].flock; +            if (unlock) { +                flock.l_type = F_UNLCK; +            } else if (blocking) { /*Doesn't make sense to have blocking +                                      unlock*/ +                cmd1 = F_SETLKW; +            } -    if (op_ret < 0) { -        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_ENTRY_UNLOCK_FAIL, -               "%s: unlock failed on %s", local->loc.path, -               priv->children[child_index]->name); +            if (local->fd) { +                STACK_WIND_COOKIE( +                    frame, cbk, cookie, priv->children[child], +                    priv->children[child]->fops->finodelk, int_lock->domain, +                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL); +            } else { +                STACK_WIND_COOKIE( +                    frame, cbk, cookie, priv->children[child], +                    priv->children[child]->fops->inodelk, int_lock->domain, +                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL); +            } +            break;      } - -    int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO; -    afr_unlock_common_cbk(frame, cookie, this, op_ret, op_errno, NULL); - -    return 0;  }  static int -afr_unlock_entrylk(call_frame_t *frame, xlator_t *this) +afr_unlock_now(call_frame_t *frame, xlator_t *this)  {      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL;      afr_private_t *priv = NULL;      int call_count = 0; -    int index = 0; -    int lockee_no = 0; -    int copies = 0; +    int child_index = 0; +    int lockee_num = 0;      int i = -1;      local = frame->local;      int_lock = &local->internal_lock;      priv = this->private; -    copies = priv->child_count;      call_count = afr_lockee_locked_nodes_count(int_lock);      int_lock->lk_call_count = call_count;      if (!call_count) { +        GF_ASSERT(!local->transaction.do_eager_unlock);          gf_msg_trace(this->name, 0, "No internal locks unlocked");          int_lock->lock_cbk(frame, this);          goto out;      }      for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) { -        lockee_no = i / copies; -        index = i % copies; -        if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) { -            STACK_WIND_COOKIE( -                frame, afr_unlock_entrylk_cbk, (void *)(long)i, -                priv->children[index], priv->children[index]->fops->entrylk, -                int_lock->domain, &int_lock->lockee[lockee_no].loc, -                int_lock->lockee[lockee_no].basename, ENTRYLK_UNLOCK, -                ENTRYLK_WRLCK, NULL); - +        lockee_num = i / priv->child_count; +        child_index = i % priv->child_count; +        if (int_lock->lockee[lockee_num].locked_nodes[child_index] & +            LOCKED_YES) { +            afr_internal_lock_wind(frame, afr_unlock_common_cbk, +                                   (void *)(long)i, child_index, lockee_num, +                                   _gf_false, _gf_true);              if (!--call_count)                  break;          } @@ -415,18 +420,6 @@ out:      return 0;  } -int32_t -afr_unlock_now(call_frame_t *frame, xlator_t *this) -{ -    afr_local_t *local = frame->local; - -    if (afr_is_inodelk_transaction(local->transaction.type)) -        afr_unlock_inodelk(frame, this); -    else -        afr_unlock_entrylk(frame, this); -    return 0; -} -  static int32_t  afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,               int32_t op_errno, dict_t *xdata) @@ -436,14 +429,14 @@ afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,      afr_private_t *priv = NULL;      int cky = (long)cookie;      int child_index = 0; -    int lockee_no = 0; +    int lockee_num = 0;      priv = this->private;      local = frame->local;      int_lock = &local->internal_lock;      child_index = ((int)cky) % priv->child_count; -    lockee_no = ((int)cky) / priv->child_count; +    lockee_num = ((int)cky) / priv->child_count;      LOCK(&frame->lock);      { @@ -470,23 +463,16 @@ afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,          afr_unlock_now(frame, this);      } else {          if (op_ret == 0) { -            if (local->transaction.type == AFR_ENTRY_TRANSACTION || -                local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) { -                int_lock->lockee[lockee_no] -                    .locked_nodes[child_index] |= LOCKED_YES; -                int_lock->lockee[lockee_no].locked_count++; -                int_lock->entrylk_lock_count++; -            } else { -                int_lock->locked_nodes[child_index] |= LOCKED_YES; -                int_lock->lock_count++; - -                if (local->transaction.type == AFR_DATA_TRANSACTION) { -                    LOCK(&local->inode->lock); -                    { -                        local->inode_ctx->lock_count++; -                    } -                    UNLOCK(&local->inode->lock); +            int_lock->lockee[lockee_num] +                .locked_nodes[child_index] |= LOCKED_YES; +            int_lock->lockee[lockee_num].locked_count++; +            int_lock->lock_count++; +            if (local->transaction.type == AFR_DATA_TRANSACTION) { +                LOCK(&local->inode->lock); +                { +                    local->inode_ctx->lock_count++;                  } +                UNLOCK(&local->inode->lock);              }          }          afr_lock_blocking(frame, this, cky + 1); @@ -495,30 +481,6 @@ afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,      return 0;  } -static int32_t -afr_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                         int32_t op_ret, int32_t op_errno, dict_t *xdata) -{ -    afr_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata); -    return 0; -} - -static int32_t -afr_blocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                         int32_t op_ret, int32_t op_errno, dict_t *xdata) -{ -    afr_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata); -    return 0; -} - -static gf_boolean_t -afr_is_entrylk(afr_transaction_type trans_type) -{ -    if (afr_is_inodelk_transaction(trans_type)) -        return _gf_false; -    return _gf_true; -} -  static gf_boolean_t  _is_lock_wind_needed(afr_local_t *local, int child_index)  { @@ -528,40 +490,12 @@ _is_lock_wind_needed(afr_local_t *local, int child_index)      return _gf_true;  } -static void -afr_log_entry_locks_failure(xlator_t *this, afr_local_t *local, -                            afr_internal_lock_t *int_lock) -{ -    const char *fop = NULL; -    char *pargfid = NULL; -    const char *name = NULL; - -    fop = gf_fop_list[local->op]; - -    switch (local->op) { -        case GF_FOP_LINK: -            pargfid = uuid_utoa(local->newloc.pargfid); -            name = local->newloc.name; -            break; -        default: -            pargfid = uuid_utoa(local->loc.pargfid); -            name = local->loc.name; -            break; -    } - -    gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_BLOCKING_LKS_FAILED, -           "Unable to obtain sufficient blocking entry locks on at least " -           "one child while attempting %s on {pgfid:%s, name:%s}.", -           fop, pargfid, name); -} -  static gf_boolean_t  is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)  {      afr_local_t *local = NULL;      afr_private_t *priv = NULL;      afr_internal_lock_t *int_lock = NULL; -    gf_boolean_t is_entrylk = _gf_false;      int child = 0;      int nlockee = 0;      int lockee_count = 0; @@ -571,42 +505,26 @@ is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)      priv = this->private;      int_lock = &local->internal_lock;      lockee_count = int_lock->lockee_count; -    is_entrylk = afr_is_entrylk(local->transaction.type); - -    if (!is_entrylk) { -        if (int_lock->lock_count == 0) { -            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_BLOCKING_LKS_FAILED, -                   "Unable to obtain " -                   "blocking inode lock on even one child for " -                   "gfid:%s.", -                   uuid_utoa(local->inode->gfid)); -            return _gf_false; -        } else { -            /*inodelk succeeded on at least one child. */ -            return _gf_true; -        } -    } else { -        if (int_lock->entrylk_lock_count == 0) { -            afr_log_entry_locks_failure(this, local, int_lock); -            return _gf_false; -        } -        /* For FOPS that take multiple sets of locks (mkdir, rename), -         * there must be at least one brick on which the locks from -         * all lock sets were successful. */ -        for (child = 0; child < priv->child_count; child++) { -            ret = _gf_true; -            for (nlockee = 0; nlockee < lockee_count; nlockee++) { -                if (!(int_lock->lockee[nlockee].locked_nodes[child] & -                      LOCKED_YES)) -                    ret = _gf_false; -            } -            if (ret) -                return ret; +    if (int_lock->lock_count == 0) { +        afr_log_locks_failure(frame, "any subvolume", "lock", +                              int_lock->lock_op_errno); +        return _gf_false; +    } +    /* For FOPS that take multiple sets of locks (mkdir, rename), +     * there must be at least one brick on which the locks from +     * all lock sets were successful. */ +    for (child = 0; child < priv->child_count; child++) { +        ret = _gf_true; +        for (nlockee = 0; nlockee < lockee_count; nlockee++) { +            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES)) +                ret = _gf_false;          } -        if (!ret) -            afr_log_entry_locks_failure(this, local, int_lock); +        if (ret) +            return ret;      } +    if (!ret) +        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);      return ret;  } @@ -617,27 +535,16 @@ afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL;      afr_private_t *priv = NULL; -    struct gf_flock flock = { -        0, -    };      uint64_t ctx = 0;      int ret = 0;      int child_index = 0; -    int lockee_no = 0; -    gf_boolean_t is_entrylk = _gf_false; +    int lockee_num = 0;      local = frame->local;      int_lock = &local->internal_lock;      priv = this->private;      child_index = cookie % priv->child_count; -    lockee_no = cookie / priv->child_count; -    is_entrylk = afr_is_entrylk(local->transaction.type); - -    if (!is_entrylk) { -        flock.l_start = int_lock->flock.l_start; -        flock.l_len = int_lock->flock.l_len; -        flock.l_type = int_lock->flock.l_type; -    } +    lockee_num = cookie / priv->child_count;      if (local->fd) {          ret = fd_ctx_get(local->fd, this, &ctx); @@ -681,52 +588,8 @@ afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)          return 0;      } -    switch (local->transaction.type) { -        case AFR_DATA_TRANSACTION: -        case AFR_METADATA_TRANSACTION: - -            if (local->fd) { -                STACK_WIND_COOKIE( -                    frame, afr_blocking_inodelk_cbk, (void *)(long)child_index, -                    priv->children[child_index], -                    priv->children[child_index]->fops->finodelk, -                    int_lock->domain, local->fd, F_SETLKW, &flock, NULL); - -            } else { -                STACK_WIND_COOKIE( -                    frame, afr_blocking_inodelk_cbk, (void *)(long)child_index, -                    priv->children[child_index], -                    priv->children[child_index]->fops->inodelk, -                    int_lock->domain, &local->loc, F_SETLKW, &flock, NULL); -            } - -            break; - -        case AFR_ENTRY_RENAME_TRANSACTION: -        case AFR_ENTRY_TRANSACTION: -            /*Accounting for child_index increments on 'down' -             *and 'fd-less' children */ - -            if (local->fd) { -                STACK_WIND_COOKIE(frame, afr_blocking_entrylk_cbk, -                                  (void *)(long)cookie, -                                  priv->children[child_index], -                                  priv->children[child_index]->fops->fentrylk, -                                  int_lock->domain, local->fd, -                                  int_lock->lockee[lockee_no].basename, -                                  ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL); -            } else { -                STACK_WIND_COOKIE( -                    frame, afr_blocking_entrylk_cbk, (void *)(long)cookie, -                    priv->children[child_index], -                    priv->children[child_index]->fops->entrylk, -                    int_lock->domain, &int_lock->lockee[lockee_no].loc, -                    int_lock->lockee[lockee_no].basename, ENTRYLK_LOCK, -                    ENTRYLK_WRLCK, NULL); -            } - -            break; -    } +    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie, +                           child_index, lockee_num, _gf_true, _gf_false);      return 0;  } @@ -743,20 +606,10 @@ afr_blocking_lock(call_frame_t *frame, xlator_t *this)      local = frame->local;      int_lock = &local->internal_lock; -    switch (local->transaction.type) { -        case AFR_DATA_TRANSACTION: -        case AFR_METADATA_TRANSACTION: -            initialize_inodelk_variables(frame, this); -            break; - -        case AFR_ENTRY_RENAME_TRANSACTION: -        case AFR_ENTRY_TRANSACTION: -            up_count = AFR_COUNT(local->child_up, priv->child_count); -            int_lock->lk_call_count = int_lock->lk_expected_count = -                (int_lock->lockee_count * up_count); -            initialize_entrylk_variables(frame, this); -            break; -    } +    up_count = AFR_COUNT(local->child_up, priv->child_count); +    int_lock->lk_call_count = int_lock->lk_expected_count = +        (int_lock->lockee_count * up_count); +    initialize_internal_lock_variables(frame, this);      afr_lock_blocking(frame, this, 0); @@ -764,171 +617,20 @@ afr_blocking_lock(call_frame_t *frame, xlator_t *this)  }  static int32_t -afr_nonblocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                            int32_t op_ret, int32_t op_errno, dict_t *xdata) +afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, +                         int32_t op_ret, int32_t op_errno, dict_t *xdata)  {      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL;      int call_count = 0; -    int child_index = (long)cookie; -    int copies = 0; -    int index = 0; -    int lockee_no = 0; -    afr_private_t *priv = NULL; - -    priv = this->private; - -    copies = priv->child_count; -    index = child_index % copies; -    lockee_no = child_index / copies; - -    local = frame->local; -    int_lock = &local->internal_lock; - -    LOCK(&frame->lock); -    { -        if (op_ret < 0) { -            if (op_errno == ENOSYS) { -                /* return ENOTSUP */ -                gf_msg(this->name, GF_LOG_ERROR, ENOSYS, -                       AFR_MSG_LOCK_XLATOR_NOT_LOADED, -                       "subvolume does not support " -                       "locking. please load features/locks" -                       " xlator on server"); -                local->op_ret = op_ret; -                int_lock->lock_op_ret = op_ret; - -                int_lock->lock_op_errno = op_errno; -                local->op_errno = op_errno; -            } -        } else if (op_ret == 0) { -            int_lock->lockee[lockee_no].locked_nodes[index] |= LOCKED_YES; -            int_lock->lockee[lockee_no].locked_count++; -            int_lock->entrylk_lock_count++; -        } - -        call_count = --int_lock->lk_call_count; -    } -    UNLOCK(&frame->lock); - -    if (call_count == 0) { -        gf_msg_trace(this->name, 0, "Last locking reply received"); -        /* all locks successful. Proceed to call FOP */ -        if (int_lock->entrylk_lock_count == int_lock->lk_expected_count) { -            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk"); -            int_lock->lock_op_ret = 0; -            int_lock->lock_cbk(frame, this); -        } -        /* Not all locks were successful. Unlock and try locking -           again, this time with serially blocking locks */ -        else { -            gf_msg_trace(this->name, 0, -                         "%d servers locked. Trying again " -                         "with blocking calls", -                         int_lock->lock_count); - -            afr_unlock_now(frame, this); -        } -    } - -    return 0; -} - -int -afr_nonblocking_entrylk(call_frame_t *frame, xlator_t *this) -{ -    afr_internal_lock_t *int_lock = NULL; -    afr_local_t *local = NULL; +    int child_index = 0; +    int lockee_num = 0;      afr_private_t *priv = NULL; -    afr_fd_ctx_t *fd_ctx = NULL; -    int copies = 0; -    int index = 0; -    int lockee_no = 0; -    int32_t call_count = 0; -    int i = 0; -    local = frame->local; -    int_lock = &local->internal_lock;      priv = this->private; -    copies = priv->child_count; -    initialize_entrylk_variables(frame, this); - -    if (local->fd) { -        fd_ctx = afr_fd_ctx_get(local->fd, this); -        if (!fd_ctx) { -            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED, -                   "unable to get fd ctx for fd=%p", local->fd); - -            local->op_ret = -1; -            int_lock->lock_op_ret = -1; -            local->op_errno = EINVAL; -            int_lock->lock_op_errno = EINVAL; - -            afr_unlock_now(frame, this); -            return -1; -        } - -        call_count = int_lock->lockee_count * internal_lock_count(frame, this); -        int_lock->lk_call_count = call_count; -        int_lock->lk_expected_count = call_count; - -        if (!call_count) { -            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON, -                   "fd not open on any subvolumes. aborting."); -            afr_unlock_now(frame, this); -            goto out; -        } - -        /* Send non-blocking entrylk calls only on up children -           and where the fd has been opened */ -        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) { -            index = i % copies; -            lockee_no = i / copies; -            if (local->child_up[index]) { -                STACK_WIND_COOKIE(frame, afr_nonblocking_entrylk_cbk, -                                  (void *)(long)i, priv->children[index], -                                  priv->children[index]->fops->fentrylk, -                                  this->name, local->fd, -                                  int_lock->lockee[lockee_no].basename, -                                  ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL); -                if (!--call_count) -                    break; -            } -        } -    } else { -        call_count = int_lock->lockee_count * internal_lock_count(frame, this); -        int_lock->lk_call_count = call_count; -        int_lock->lk_expected_count = call_count; - -        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) { -            index = i % copies; -            lockee_no = i / copies; -            if (local->child_up[index]) { -                STACK_WIND_COOKIE(frame, afr_nonblocking_entrylk_cbk, -                                  (void *)(long)i, priv->children[index], -                                  priv->children[index]->fops->entrylk, -                                  this->name, &int_lock->lockee[lockee_no].loc, -                                  int_lock->lockee[lockee_no].basename, -                                  ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL); - -                if (!--call_count) -                    break; -            } -        } -    } -out: -    return 0; -} - -int32_t -afr_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, -                            int32_t op_ret, int32_t op_errno, dict_t *xdata) -{ -    afr_internal_lock_t *int_lock = NULL; -    afr_local_t *local = NULL; -    int call_count = 0; -    int child_index = (long)cookie; +    child_index = ((long)cookie) % priv->child_count; +    lockee_num = ((long)cookie) / priv->child_count;      local = frame->local;      int_lock = &local->internal_lock; @@ -953,11 +655,14 @@ afr_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,                         " xlator on server");                  local->op_ret = op_ret;                  int_lock->lock_op_ret = op_ret; +                  int_lock->lock_op_errno = op_errno;                  local->op_errno = op_errno;              } -        } else { -            int_lock->locked_nodes[child_index] |= LOCKED_YES; +        } else if (op_ret == 0) { +            int_lock->lockee[lockee_num] +                .locked_nodes[child_index] |= LOCKED_YES; +            int_lock->lockee[lockee_num].locked_count++;              int_lock->lock_count++;          } @@ -966,7 +671,7 @@ afr_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,      UNLOCK(&frame->lock);      if (call_count == 0) { -        gf_msg_trace(this->name, 0, "Last inode locking reply received"); +        gf_msg_trace(this->name, 0, "Last locking reply received");          /* all locks successful. Proceed to call FOP */          if (int_lock->lock_count == int_lock->lk_expected_count) {              gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk"); @@ -977,8 +682,8 @@ afr_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,             again, this time with serially blocking locks */          else {              gf_msg_trace(this->name, 0, -                         "%d servers locked. " -                         "Trying again with blocking calls", +                         "%d servers locked. Trying again " +                         "with blocking calls",                           int_lock->lock_count);              afr_unlock_now(frame, this); @@ -989,12 +694,14 @@ afr_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,  }  int -afr_nonblocking_inodelk(call_frame_t *frame, xlator_t *this) +afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)  {      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL;      afr_private_t *priv = NULL;      afr_fd_ctx_t *fd_ctx = NULL; +    int child = 0; +    int lockee_num = 0;      int32_t call_count = 0;      int i = 0;      int ret = 0; @@ -1003,7 +710,7 @@ afr_nonblocking_inodelk(call_frame_t *frame, xlator_t *this)      int_lock = &local->internal_lock;      priv = this->private; -    initialize_inodelk_variables(frame, this); +    initialize_internal_lock_variables(frame, this);      if (local->fd) {          fd_ctx = afr_fd_ctx_get(local->fd, this); @@ -1022,36 +729,29 @@ afr_nonblocking_inodelk(call_frame_t *frame, xlator_t *this)          }      } -    call_count = internal_lock_count(frame, this); +    call_count = int_lock->lockee_count * internal_lock_count(frame, this);      int_lock->lk_call_count = call_count;      int_lock->lk_expected_count = call_count;      if (!call_count) { -        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN, -               "All bricks are down, aborting."); +        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON, +               "fd not open on any subvolumes. aborting.");          afr_unlock_now(frame, this);          goto out;      } -    /* Send non-blocking inodelk calls only on up children +    /* Send non-blocking lock calls only on up children         and where the fd has been opened */ -    for (i = 0; i < priv->child_count; i++) { -        if (!local->child_up[i]) -            continue; - -        if (local->fd) { -            STACK_WIND_COOKIE( -                frame, afr_nonblocking_inodelk_cbk, (void *)(long)i, -                priv->children[i], priv->children[i]->fops->finodelk, -                int_lock->domain, local->fd, F_SETLK, &int_lock->flock, NULL); -        } else { -            STACK_WIND_COOKIE( -                frame, afr_nonblocking_inodelk_cbk, (void *)(long)i, -                priv->children[i], priv->children[i]->fops->inodelk, -                int_lock->domain, &local->loc, F_SETLK, &int_lock->flock, NULL); +    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) { +        child = i % priv->child_count; +        lockee_num = i / priv->child_count; +        if (local->child_up[child]) { +            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk, +                                   (void *)(long)i, child, lockee_num, +                                   _gf_false, _gf_false); +            if (!--call_count) +                break;          } -        if (!--call_count) -            break;      }  out:      return ret; diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h index baf1a6c291f..c9c99270e98 100644 --- a/xlators/cluster/afr/src/afr-messages.h +++ b/xlators/cluster/afr/src/afr-messages.h @@ -30,7 +30,7 @@ GLFS_MSGID(AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET,             AFR_MSG_GFID_NULL, AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,             AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,             AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED, -           AFR_MSG_BLOCKING_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO, +           AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,             AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,             AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR,             AFR_MSG_SELF_HEAL_INFO, AFR_MSG_READ_SUBVOL_ERROR, diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c index 8d97ff53253..f0cb7ab34f3 100644 --- a/xlators/cluster/afr/src/afr-transaction.c +++ b/xlators/cluster/afr/src/afr-transaction.c @@ -797,21 +797,9 @@ afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)  unsigned char *  afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)  { -    unsigned char *locked_nodes = NULL; -    switch (type) { -        case AFR_DATA_TRANSACTION: -        case AFR_METADATA_TRANSACTION: -            locked_nodes = int_lock->locked_nodes; -            break; - -        case AFR_ENTRY_TRANSACTION: -        case AFR_ENTRY_RENAME_TRANSACTION: -            /*Because same set of subvols participate in all lockee -             * entities*/ -            locked_nodes = int_lock->lockee[0].locked_nodes; -            break; -    } -    return locked_nodes; +    /*Because same set of subvols participate in all lockee +     * entities*/ +    return int_lock->lockee[0].locked_nodes;  }  int @@ -2057,31 +2045,7 @@ err:  }  int -afr_post_nonblocking_inodelk_cbk(call_frame_t *frame, xlator_t *this) -{ -    afr_internal_lock_t *int_lock = NULL; -    afr_local_t *local = NULL; - -    local = frame->local; -    int_lock = &local->internal_lock; - -    /* Initiate blocking locks if non-blocking has failed */ -    if (int_lock->lock_op_ret < 0) { -        gf_msg_debug(this->name, 0, -                     "Non blocking inodelks failed. Proceeding to blocking"); -        int_lock->lock_cbk = afr_internal_lock_finish; -        afr_blocking_lock(frame, this); -    } else { -        gf_msg_debug(this->name, 0, -                     "Non blocking inodelks done. Proceeding to FOP"); -        afr_internal_lock_finish(frame, this); -    } - -    return 0; -} - -int -afr_post_nonblocking_entrylk_cbk(call_frame_t *frame, xlator_t *this) +afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)  {      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL; @@ -2092,12 +2056,12 @@ afr_post_nonblocking_entrylk_cbk(call_frame_t *frame, xlator_t *this)      /* Initiate blocking locks if non-blocking has failed */      if (int_lock->lock_op_ret < 0) {          gf_msg_debug(this->name, 0, -                     "Non blocking entrylks failed. Proceeding to blocking"); +                     "Non blocking locks failed. Proceeding to blocking");          int_lock->lock_cbk = afr_internal_lock_finish;          afr_blocking_lock(frame, this);      } else {          gf_msg_debug(this->name, 0, -                     "Non blocking entrylks done. Proceeding to FOP"); +                     "Non blocking locks done. Proceeding to FOP");          afr_internal_lock_finish(frame, this);      } @@ -2115,7 +2079,7 @@ afr_post_blocking_rename_cbk(call_frame_t *frame, xlator_t *this)      int_lock = &local->internal_lock;      if (int_lock->lock_op_ret < 0) { -        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_BLOCKING_LKS_FAILED, +        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INTERNAL_LKS_FAILED,                 "Blocking entrylks failed.");          afr_transaction_done(frame, this); @@ -2146,25 +2110,26 @@ afr_post_lower_unlock_cbk(call_frame_t *frame, xlator_t *this)  }  int -afr_set_transaction_flock(xlator_t *this, afr_local_t *local) +afr_set_transaction_flock(xlator_t *this, afr_local_t *local, +                          afr_lockee_t *lockee)  { -    afr_internal_lock_t *int_lock = NULL;      afr_private_t *priv = NULL; +    struct gf_flock *flock = NULL; -    int_lock = &local->internal_lock;      priv = this->private; +    flock = &lockee->flock;      if ((priv->arbiter_count || local->transaction.eager_lock_on ||           priv->full_lock) &&          local->transaction.type == AFR_DATA_TRANSACTION) {          /*Lock entire file to avoid network split brains.*/ -        int_lock->flock.l_len = 0; -        int_lock->flock.l_start = 0; +        flock->l_len = 0; +        flock->l_start = 0;      } else { -        int_lock->flock.l_len = local->transaction.len; -        int_lock->flock.l_start = local->transaction.start; +        flock->l_len = local->transaction.len; +        flock->l_start = local->transaction.start;      } -    int_lock->flock.l_type = F_WRLCK; +    flock->l_type = F_WRLCK;      return 0;  } @@ -2174,26 +2139,21 @@ afr_lock(call_frame_t *frame, xlator_t *this)  {      afr_internal_lock_t *int_lock = NULL;      afr_local_t *local = NULL; +    int i = 0;      local = frame->local;      int_lock = &local->internal_lock; +    int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;      int_lock->domain = this->name;      switch (local->transaction.type) {          case AFR_DATA_TRANSACTION:          case AFR_METADATA_TRANSACTION: -            afr_set_transaction_flock(this, local); - -            int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk; - -            afr_nonblocking_inodelk(frame, this); -            break; - -        case AFR_ENTRY_RENAME_TRANSACTION: +            for (i = 0; i < int_lock->lockee_count; i++) { +                afr_set_transaction_flock(this, local, &int_lock->lockee[i]); +            } -            int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk; -            afr_nonblocking_entrylk(frame, this);              break;          case AFR_ENTRY_TRANSACTION: @@ -2202,11 +2162,11 @@ afr_lock(call_frame_t *frame, xlator_t *this)                  int_lock->lk_loc = &local->transaction.parent_loc;              else                  GF_ASSERT(local->fd); - -            int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk; -            afr_nonblocking_entrylk(frame, this); +            break; +        case AFR_ENTRY_RENAME_TRANSACTION:              break;      } +    afr_lock_nonblocking(frame, this);      return 0;  } @@ -2277,17 +2237,19 @@ afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)  /* }}} */  static void  afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src, -                      xlator_t *this) +                      xlator_t *this, int lockee_num)  {      afr_private_t *priv = this->private; +    afr_lockee_t *sl = &src->lockee[lockee_num]; +    afr_lockee_t *dl = &dst->lockee[lockee_num];      dst->domain = src->domain; -    dst->flock.l_len = src->flock.l_len; -    dst->flock.l_start = src->flock.l_start; -    dst->flock.l_type = src->flock.l_type; -    dst->lock_count = src->lock_count; -    memcpy(dst->locked_nodes, src->locked_nodes, -           priv->child_count * sizeof(*dst->locked_nodes)); +    dl->flock.l_len = sl->flock.l_len; +    dl->flock.l_start = sl->flock.l_start; +    dl->flock.l_type = sl->flock.l_type; +    dl->locked_count = sl->locked_count; +    memcpy(dl->locked_nodes, sl->locked_nodes, +           priv->child_count * sizeof(*dl->locked_nodes));  }  void @@ -2308,7 +2270,7 @@ __afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)          if (conflict && !list_empty(&lock->owners))              return;          afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock, -                              each->transaction.frame->this); +                              each->transaction.frame->this, 0);          list_move_tail(&each->transaction.wait_list, shared);          list_add_tail(&each->transaction.owner_list, &lock->owners);      } @@ -2785,7 +2747,7 @@ __afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,              *timer_local = list_entry(lock->post_op.next, afr_local_t,                                        transaction.owner_list);              afr_copy_inodelk_vars(&local->internal_lock, -                                  &(*timer_local)->internal_lock, this); +                                  &(*timer_local)->internal_lock, this, 0);              lock->delay_timer = NULL;              *do_pre_op = _gf_true;              list_add_tail(&local->transaction.owner_list, &lock->owners); @@ -2802,7 +2764,7 @@ __afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,          owner_local = list_entry(lock->owners.next, afr_local_t,                                   transaction.owner_list);          afr_copy_inodelk_vars(&local->internal_lock, -                              &owner_local->internal_lock, this); +                              &owner_local->internal_lock, this, 0);          *take_lock = _gf_false;          *do_pre_op = _gf_true;      } @@ -2872,6 +2834,62 @@ fail:  }  int +afr_transaction_lockee_init(call_frame_t *frame) +{ +    afr_local_t *local = frame->local; +    afr_internal_lock_t *int_lock = &local->internal_lock; +    afr_private_t *priv = frame->this->private; +    int ret = 0; + +    switch (local->transaction.type) { +        case AFR_DATA_TRANSACTION: +        case AFR_METADATA_TRANSACTION: +            ret = afr_add_inode_lockee(local, priv->child_count); +            break; + +        case AFR_ENTRY_TRANSACTION: +        case AFR_ENTRY_RENAME_TRANSACTION: +            ret = afr_add_entry_lockee(local, &local->transaction.parent_loc, +                                       local->transaction.basename, +                                       priv->child_count); +            if (ret) { +                goto out; +            } +            if (local->op == GF_FOP_RENAME) { +                ret = afr_add_entry_lockee( +                    local, &local->transaction.new_parent_loc, +                    local->transaction.new_basename, priv->child_count); +                if (ret) { +                    goto out; +                } + +                if (local->newloc.inode && +                    IA_ISDIR(local->newloc.inode->ia_type)) { +                    ret = afr_add_entry_lockee(local, &local->newloc, NULL, +                                               priv->child_count); +                    if (ret) { +                        goto out; +                    } +                } +            } else if (local->op == GF_FOP_RMDIR) { +                ret = afr_add_entry_lockee(local, &local->loc, NULL, +                                           priv->child_count); +                if (ret) { +                    goto out; +                } +            } + +            if (int_lock->lockee_count > 1) { +                qsort(int_lock->lockee, int_lock->lockee_count, +                      sizeof(*int_lock->lockee), afr_entry_lockee_cmp); +            } +            break; +    } +out: +    return ret; +} + +int  afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)  {      afr_local_t *local = NULL; @@ -2904,6 +2922,10 @@ afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)      if (ret < 0)          goto out; +    ret = afr_transaction_lockee_init(frame); +    if (ret) +        goto out; +      if (type != AFR_METADATA_TRANSACTION) {          goto txn_start;      } diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h index 9c12d2d9ac0..cdcc6027fdb 100644 --- a/xlators/cluster/afr/src/afr.h +++ b/xlators/cluster/afr/src/afr.h @@ -288,12 +288,14 @@ afr_index_from_ia_type(ia_type_t type)  }  typedef struct { +    struct gf_flock flock;      loc_t loc; +    fd_t *fd;      char *basename;      unsigned char *locked_nodes;      int locked_count; -} afr_entry_lockee_t; +} afr_lockee_t;  int  afr_entry_lockee_cmp(const void *l1, const void *l2); @@ -302,20 +304,17 @@ typedef struct {      loc_t *lk_loc;      int lockee_count; -    afr_entry_lockee_t lockee[AFR_LOCKEE_COUNT_MAX]; +    afr_lockee_t lockee[AFR_LOCKEE_COUNT_MAX]; -    struct gf_flock flock;      const char *lk_basename;      const char *lower_basename;      const char *higher_basename;      char lower_locked;      char higher_locked; -    unsigned char *locked_nodes;      unsigned char *lower_locked_nodes;      int32_t lock_count; -    int32_t entrylk_lock_count;      int32_t lk_call_count;      int32_t lk_expected_count; @@ -980,11 +979,14 @@ int  xattr_is_equal(dict_t *this, char *key1, data_t *value1, void *data);  int -afr_init_entry_lockee(afr_entry_lockee_t *lockee, afr_local_t *local, -                      loc_t *loc, char *basename, int child_count); +afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename, +                     int child_count); + +int +afr_add_inode_lockee(afr_local_t *local, int child_count);  void -afr_entry_lockee_cleanup(afr_internal_lock_t *int_lock); +afr_lockees_cleanup(afr_internal_lock_t *int_lock);  int  afr_attempt_lock_recovery(xlator_t *this, int32_t child_index); @@ -1002,10 +1004,7 @@ int32_t  afr_unlock(call_frame_t *frame, xlator_t *this);  int -afr_nonblocking_entrylk(call_frame_t *frame, xlator_t *this); - -int -afr_nonblocking_inodelk(call_frame_t *frame, xlator_t *this); +afr_lock_nonblocking(call_frame_t *frame, xlator_t *this);  int  afr_blocking_lock(call_frame_t *frame, xlator_t *this); @@ -1272,9 +1271,6 @@ afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame);  void  afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock,                       int32_t child_index); -int -afr_is_inodelk_transaction(afr_transaction_type type); -  afr_fd_ctx_t *  __afr_fd_ctx_get(fd_t *fd, xlator_t *this);  | 
