From 12cfc1983fa1521d6da94ad488145182cf97db2c Mon Sep 17 00:00:00 2001 From: Sunil Kumar Acharya Date: Fri, 28 Apr 2017 18:09:01 +0530 Subject: cluster/ec: Improve performance with xattrop update Existing EC code updates the xattr on the subvolume in a sequential pattern resulting in very poor performance. With this fix EC now updates the xattr on the subvolume in parallel which improves the xattr update performance. BUG: 1445663 Change-Id: I3fc40d66db0b88875ca96a9fa01002ba386c0486 Signed-off-by: Sunil Kumar Acharya --- libglusterfs/src/cluster-syncop.c | 108 ++++++++++++++++---------------- libglusterfs/src/cluster-syncop.h | 49 +++++++++++++++ libglusterfs/src/dict.c | 25 +++++++- xlators/cluster/ec/src/ec-heal.c | 126 ++++++++++++++++++++++++++++++-------- 4 files changed, 228 insertions(+), 80 deletions(-) diff --git a/libglusterfs/src/cluster-syncop.c b/libglusterfs/src/cluster-syncop.c index 36945d69379..b7f4dfe7701 100644 --- a/libglusterfs/src/cluster-syncop.c +++ b/libglusterfs/src/cluster-syncop.c @@ -84,9 +84,9 @@ syncbarrier_wake (&__local->barrier); \ } while (0) -static int -fop_success_fill (default_args_cbk_t *replies, int numsubvols, - unsigned char *success) +int32_t +cluster_fop_success_fill (default_args_cbk_t *replies, int numsubvols, + unsigned char *success) { int i = 0; int count = 0; @@ -221,7 +221,6 @@ cluster_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, return 0; } - int32_t cluster_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, inode_t *inode, @@ -233,7 +232,6 @@ cluster_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this, return 0; } - int32_t cluster_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, struct iatt *buf, @@ -587,7 +585,7 @@ cluster_fgetxattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fgetxattr, fd, name, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -598,7 +596,7 @@ cluster_fsetxattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsetxattr, fd, dict, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -609,7 +607,7 @@ cluster_setxattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, setxattr, loc, dict, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -619,7 +617,7 @@ cluster_statfs (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, statfs, loc, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -630,7 +628,7 @@ cluster_fsyncdir (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsyncdir, fd, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -641,7 +639,7 @@ cluster_opendir (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, opendir, loc, fd, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -651,7 +649,7 @@ cluster_fstat (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fstat, fd, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -662,7 +660,7 @@ cluster_fsync (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsync, fd, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -672,7 +670,7 @@ cluster_flush (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, flush, fd, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -684,7 +682,7 @@ cluster_writev (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, writev, fd, vector, count, off, flags, iobref, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -695,7 +693,7 @@ cluster_readv (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readv, fd, size, offset, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -707,7 +705,7 @@ cluster_open (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, open, loc, flags, fd, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -718,7 +716,7 @@ cluster_create (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, create, loc, flags, mode, umask, fd, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -729,7 +727,7 @@ cluster_link (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, link, oldloc, newloc, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -740,7 +738,7 @@ cluster_rename (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rename, oldloc, newloc, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -752,7 +750,7 @@ cluster_symlink (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, symlink, linkpath, loc, umask, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -763,7 +761,7 @@ cluster_rmdir (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rmdir, loc, flags, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -774,7 +772,7 @@ cluster_unlink (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, unlink, loc, xflag, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int @@ -785,7 +783,7 @@ cluster_mkdir (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, mkdir, loc, mode, umask, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -797,7 +795,7 @@ cluster_mknod (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, mknod, loc, mode, rdev, umask, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -808,7 +806,7 @@ cluster_readlink (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readlink, loc, size, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -820,7 +818,7 @@ cluster_access (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, access, loc, mask, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -831,7 +829,7 @@ cluster_ftruncate (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, ftruncate, fd, offset, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -842,7 +840,7 @@ cluster_getxattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, getxattr, loc, name, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -854,7 +852,7 @@ cluster_xattrop (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, xattrop, loc, flags, dict, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -865,7 +863,7 @@ cluster_fxattrop (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fxattrop, fd, flags, dict, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -876,7 +874,7 @@ cluster_removexattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, removexattr, loc, name, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -887,7 +885,7 @@ cluster_fremovexattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fremovexattr, fd, name, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -898,7 +896,7 @@ cluster_lk (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, lk, fd, cmd, lock, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -910,7 +908,7 @@ cluster_rchecksum (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, rchecksum, fd, offset, len, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -922,7 +920,7 @@ cluster_readdir (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readdir, fd, size, off, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -934,7 +932,7 @@ cluster_readdirp (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, readdirp, fd, size, off, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -945,7 +943,7 @@ cluster_setattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, setattr, loc, stbuf, valid, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -956,7 +954,7 @@ cluster_truncate (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, truncate, loc, offset, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -966,7 +964,7 @@ cluster_stat (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, stat, loc, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -976,7 +974,7 @@ cluster_lookup (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, lookup, loc, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -987,7 +985,7 @@ cluster_fsetattr (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, fsetattr, fd, stbuf, valid, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -998,7 +996,7 @@ cluster_fallocate (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fallocate, fd, keep_size, offset, len, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -1009,7 +1007,7 @@ cluster_discard (xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, discard, fd, offset, len, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int32_t @@ -1020,7 +1018,7 @@ cluster_zerofill(xlator_t **subvols, unsigned char *on, int numsubvols, { FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, zerofill, fd, offset, len, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } @@ -1030,7 +1028,7 @@ cluster_ipc (xlator_t **subvols, unsigned char *on, int numsubvols, call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) { FOP_ONLIST (subvols, on, numsubvols, replies, output, frame, ipc, op, xdata); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int @@ -1055,7 +1053,7 @@ cluster_uninodelk (xlator_t **subvols, unsigned char *locked_on, int numsubvols, loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int @@ -1077,7 +1075,7 @@ cluster_tryinodelk (xlator_t **subvols, unsigned char *on, int numsubvols, &loc, F_SETLK, &flock, NULL); loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, locked_on); + return cluster_fop_success_fill (replies, numsubvols, locked_on); } int @@ -1103,7 +1101,8 @@ cluster_inodelk (xlator_t **subvols, unsigned char *on, int numsubvols, for (i = 0; i < numsubvols; i++) { if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) { - fop_success_fill (replies, numsubvols, locked_on); + cluster_fop_success_fill (replies, numsubvols, + locked_on); cluster_uninodelk (subvols, locked_on, numsubvols, replies, output, frame, this, dom, inode, off, size); @@ -1115,7 +1114,7 @@ cluster_inodelk (xlator_t **subvols, unsigned char *on, int numsubvols, } loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, locked_on); + return cluster_fop_success_fill (replies, numsubvols, locked_on); } @@ -1137,7 +1136,7 @@ cluster_unentrylk (xlator_t **subvols, unsigned char *locked_on, int numsubvols, loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, output); + return cluster_fop_success_fill (replies, numsubvols, output); } int @@ -1155,7 +1154,7 @@ cluster_tryentrylk (xlator_t **subvols, unsigned char *on, int numsubvols, NULL); loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, locked_on); + return cluster_fop_success_fill (replies, numsubvols, locked_on); } int @@ -1177,7 +1176,8 @@ cluster_entrylk (xlator_t **subvols, unsigned char *on, int numsubvols, for (i = 0; i < numsubvols; i++) { if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) { - fop_success_fill (replies, numsubvols, locked_on); + cluster_fop_success_fill (replies, numsubvols, + locked_on); cluster_unentrylk (subvols, locked_on, numsubvols, replies, output, frame, this, dom, inode, name); @@ -1189,5 +1189,5 @@ cluster_entrylk (xlator_t **subvols, unsigned char *on, int numsubvols, } loc_wipe (&loc); - return fop_success_fill (replies, numsubvols, locked_on); + return cluster_fop_success_fill (replies, numsubvols, locked_on); } diff --git a/libglusterfs/src/cluster-syncop.h b/libglusterfs/src/cluster-syncop.h index 7e17faae0e0..ff9387acace 100644 --- a/libglusterfs/src/cluster-syncop.h +++ b/libglusterfs/src/cluster-syncop.h @@ -20,6 +20,46 @@ #include "default-args.h" #include "syncop.h" +/********************************************************************* + * + * PARALLEL_FOP_ONLIST: + * Performs file operations in parallel on bricks. + * This macro expects a helper function(func) to implement the + * functionality. + * + ********************************************************************/ +#define PARALLEL_FOP_ONLIST(subvols, on, numsubvols, replies, frame, \ + func, args ...) \ +do { \ + int __i = 0; \ + int __count = 0; \ + cluster_local_t __local = {0,}; \ + void *__old_local = frame->local; \ + \ + __local.replies = replies; \ + cluster_replies_wipe (replies, numsubvols); \ + for (__i = 0; __i < numsubvols; __i++) \ + INIT_LIST_HEAD (&replies[__i].entries.list); \ + if (syncbarrier_init (&__local.barrier)) \ + break; \ + frame->local = &__local; \ + for (__i = 0; __i < numsubvols; __i++) { \ + if (on[__i]) { \ + __count++; \ + } \ + } \ + __local.barrier.waitfor = __count; \ + for (__i = 0; __i < numsubvols; __i++) { \ + if (on[__i]) { \ + func (frame, subvols[__i], __i, ## args); \ + } \ + } \ + syncbarrier_wait (&__local.barrier, __count); \ + syncbarrier_destroy (&__local.barrier); \ + frame->local = __old_local; \ + STACK_RESET (frame->root); \ +} while (0) + typedef struct cluster_local_ { default_args_cbk_t *replies; syncbarrier_t barrier; @@ -160,4 +200,13 @@ cluster_fsetattr (xlator_t **subvols, unsigned char *on, int numsubvols, void cluster_replies_wipe (default_args_cbk_t *replies, int num_subvols); + +int32_t +cluster_fop_success_fill (default_args_cbk_t *replies, int numsubvols, + unsigned char *success); + +int32_t +cluster_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, + dict_t *xdata); #endif /* !_CLUSTER_SYNCOP_H */ diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c index c4f3fb71de3..243c92985a8 100644 --- a/libglusterfs/src/dict.c +++ b/libglusterfs/src/dict.c @@ -2321,7 +2321,15 @@ err: return ret; } - +/******************************************************************** + * + * dict_set_bin_common: + * This is the common function to set key and its value in + * dictionary. Flag(is_static) should be set appropriately based + * on the type of memory type used for value(*ptr). If flag is set + * to false value(*ptr) will be freed using GF_FREE() on destroy. + * + *******************************************************************/ static int dict_set_bin_common (dict_t *this, char *key, void *ptr, size_t size, gf_boolean_t is_static) @@ -2353,13 +2361,26 @@ err: return ret; } +/******************************************************************** + * + * dict_set_bin: + * Set key and its value in the dictionary. This function should + * be called if the value is stored in dynamic memory. + * + *******************************************************************/ int dict_set_bin (dict_t *this, char *key, void *ptr, size_t size) { return dict_set_bin_common (this, key, ptr, size, _gf_false); } - +/******************************************************************** + * + * dict_set_static_bin: + * Set key and its value in the dictionary. This function should + * be called if the value is stored in static memory. + * + *******************************************************************/ int dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size) { diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c index fae31778532..a6de3eee439 100644 --- a/xlators/cluster/ec/src/ec-heal.c +++ b/xlators/cluster/ec/src/ec-heal.c @@ -215,6 +215,25 @@ ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data, return 0; } +/******************************************************************** + * ec_wind_xattrop_parallel: + * Helper function to update the extended attributes + * in parallel. + * + *******************************************************************/ +void +ec_wind_xattrop_parallel (call_frame_t *frame, xlator_t *subvol, + int child_index, loc_t *loc, + gf_xattrop_flags_t flags, dict_t **dict, + dict_t *xdata) +{ + gf_msg_debug ("EC", 0, "WIND: on child %d ", child_index); + STACK_WIND_COOKIE (frame, cluster_xattrop_cbk, + (void *)(uintptr_t) child_index, + subvol, subvol->fops->xattrop, loc, + flags, dict[child_index], xdata); +} + int32_t ec_heal_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, @@ -391,19 +410,36 @@ ec_adjust_versions (call_frame_t *frame, ec_t *ec, ec_txn_t type, { int i = 0; int ret = 0; - dict_t *xattr = NULL; + int call_count = 0; + dict_t **xattr = NULL; int op_ret = 0; loc_t loc = {0}; gf_boolean_t erase_dirty = _gf_false; - uint64_t versions_xattr[2] = {0}; - uint64_t dirty_xattr[2] = {0}; + uint64_t *versions_xattr = NULL; + uint64_t *dirty_xattr = NULL; uint64_t allzero[2] = {0}; + unsigned char *on = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + /* Allocate the required memory */ loc.inode = inode_ref (inode); gf_uuid_copy (loc.gfid, inode->gfid); - xattr = dict_new (); - if (!xattr) + on = alloca0 (ec->nodes); + output = alloca0 (ec->nodes); + EC_REPLIES_ALLOC (replies, ec->nodes); + xattr = GF_CALLOC (ec->nodes, sizeof (*xattr), gf_common_mt_pointer); + if (!xattr) { + op_ret = -ENOMEM; goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict_new (); + if (!xattr[i]) { + op_ret = -ENOMEM; + goto out; + } + } /* dirty xattr represents if the file/dir needs heal. Unless all the * copies are healed, don't erase it */ @@ -413,45 +449,87 @@ ec_adjust_versions (call_frame_t *frame, ec_t *ec, ec_txn_t type, else op_ret = -ENOTCONN; + /* Populate the xattr array */ for (i = 0; i < ec->nodes; i++) { if (!sources[i] && !healed_sinks[i]) continue; + versions_xattr = GF_CALLOC (EC_VERSION_SIZE, + sizeof(*versions_xattr), + gf_common_mt_pointer); + if (!versions_xattr) { + op_ret = -ENOMEM; + continue; + } + versions_xattr[type] = hton64(versions[source] - versions[i]); - ret = dict_set_static_bin (xattr, EC_XATTR_VERSION, - versions_xattr, - sizeof (versions_xattr)); + ret = dict_set_bin (xattr[i], EC_XATTR_VERSION, + versions_xattr, + (sizeof (*versions_xattr) * EC_VERSION_SIZE) + ); if (ret < 0) { - op_ret = -ENOTCONN; + op_ret = -ENOMEM; continue; } if (erase_dirty) { + dirty_xattr = GF_CALLOC (EC_VERSION_SIZE, + sizeof(*dirty_xattr), + gf_common_mt_pointer); + if (!dirty_xattr) { + op_ret = -ENOMEM; + continue; + } + dirty_xattr[type] = hton64(-dirty[i]); - ret = dict_set_static_bin (xattr, EC_XATTR_DIRTY, - dirty_xattr, - sizeof (dirty_xattr)); + ret = dict_set_bin (xattr[i], EC_XATTR_DIRTY, + dirty_xattr, + (sizeof(*dirty_xattr) * + EC_VERSION_SIZE) + ); if (ret < 0) { - op_ret = -ENOTCONN; + op_ret = -ENOMEM; continue; } } - if ((memcmp (versions_xattr, allzero, sizeof (allzero)) == 0) && - (memcmp (dirty_xattr, allzero, sizeof (allzero)) == 0)) - continue; + if (memcmp (versions_xattr, allzero, + (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) { - ret = syncop_xattrop (ec->xl_list[i], &loc, - GF_XATTROP_ADD_ARRAY64, xattr, NULL, - NULL); - if (ret < 0) { - op_ret = -ret; - continue; + if (!erase_dirty) { + continue; + } + + if (memcmp (dirty_xattr, allzero, (sizeof (*dirty_xattr) + * EC_VERSION_SIZE)) == 0) { + continue; + } } + + on[i] = 1; + call_count++; + } + + /* Update the bricks with xattr */ + if (call_count) { + PARALLEL_FOP_ONLIST (ec->xl_list, on, ec->nodes, replies, + frame, ec_wind_xattrop_parallel, + &loc, GF_XATTROP_ADD_ARRAY64, xattr, NULL); + ret = cluster_fop_success_fill (replies, ec->nodes, output); + } + + if (ret < call_count) { + op_ret = -ENOTCONN; + goto out; } out: - if (xattr) - dict_unref (xattr); + /* Cleanup */ + for (i = 0; i < ec->nodes; i++) { + if (xattr[i]) + dict_unref (xattr[i]); + } + GF_FREE (xattr); + cluster_replies_wipe (replies, ec->nodes); loc_wipe (&loc); return op_ret; } -- cgit