diff options
Diffstat (limited to 'xlators/cluster/ec/src')
43 files changed, 34763 insertions, 20912 deletions
diff --git a/xlators/cluster/ec/src/Makefile.am b/xlators/cluster/ec/src/Makefile.am index e2a9330a944..406a636bbc2 100644 --- a/xlators/cluster/ec/src/Makefile.am +++ b/xlators/cluster/ec/src/Makefile.am @@ -12,9 +12,13 @@ ec_sources += ec-dir-write.c ec_sources += ec-inode-read.c ec_sources += ec-inode-write.c ec_sources += ec-combine.c -ec_sources += ec-gf.c ec_sources += ec-method.c +ec_sources += ec-galois.c +ec_sources += ec-code.c +ec_sources += ec-code-c.c +ec_sources += ec-gf8.c ec_sources += ec-heal.c +ec_sources += ec-heald.c ec_headers := ec.h ec_headers += ec-mem-types.h @@ -23,20 +27,50 @@ ec_headers += ec-data.h ec_headers += ec-fops.h ec_headers += ec-common.h ec_headers += ec-combine.h -ec_headers += ec-gf.h ec_headers += ec-method.h +ec_headers += ec-galois.h +ec_headers += ec-code.h +ec_headers += ec-code-c.h +ec_headers += ec-gf8.h +ec_headers += ec-heald.h +ec_headers += ec-messages.h +ec_headers += ec-types.h + +if ENABLE_EC_DYNAMIC_INTEL + ec_sources += ec-code-intel.c + ec_headers += ec-code-intel.h +endif + +if ENABLE_EC_DYNAMIC_X64 + ec_sources += ec-code-x64.c + ec_headers += ec-code-x64.h +endif + +if ENABLE_EC_DYNAMIC_SSE + ec_sources += ec-code-sse.c + ec_headers += ec-code-sse.h +endif + +if ENABLE_EC_DYNAMIC_AVX + ec_sources += ec-code-avx.c + ec_headers += ec-code-avx.h +endif ec_ext_sources = $(top_builddir)/xlators/lib/src/libxlator.c ec_ext_headers = $(top_builddir)/xlators/lib/src/libxlator.h -ec_la_LDFLAGS = -module -avoid-version +ec_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) ec_la_SOURCES = $(ec_sources) $(ec_headers) $(ec_ext_sources) $(ec_ext_headers) ec_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -AM_CPPFLAGS = $(GF_CPPFLAGS) +AM_CPPFLAGS = $(GF_CPPFLAGS) AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src +AM_CPPFLAGS += -I$(top_srcdir)/rpc/rpc-lib/src +AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src +AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src +AM_CPPFLAGS += -DGLUSTERFS_LIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\" AM_CFLAGS = -Wall $(GF_CFLAGS) diff --git a/xlators/cluster/ec/src/ec-code-avx.c b/xlators/cluster/ec/src/ec-code-avx.c new file mode 100644 index 00000000000..70afaa00f54 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-avx.c @@ -0,0 +1,109 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_avx_prolog(ec_code_builder_t *builder) +{ + builder->loop = builder->address; +} + +static void +ec_code_avx_epilog(ec_code_builder_t *builder) +{ + ec_code_intel_op_add_i2r(builder, 32, REG_DX); + ec_code_intel_op_add_i2r(builder, 32, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_avx_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_mov_m2avx( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_mov_m2avx(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_avx_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + ec_code_intel_op_mov_avx2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_avx_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_mov_avx2avx(builder, src, dst); +} + +static void +ec_code_avx_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_xor_avx2avx(builder, src, dst); +} + +static void +ec_code_avx_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2) +{ + ec_code_intel_op_mov_avx2avx(builder, src1, dst); + ec_code_intel_op_xor_avx2avx(builder, src2, dst); +} + +static void +ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_xor_m2avx( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_xor_m2avx(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_avx_needed_flags[] = {"avx2", NULL}; + +ec_code_gen_t ec_code_gen_avx = {.name = "avx", + .flags = ec_code_avx_needed_flags, + .width = 32, + .prolog = ec_code_avx_prolog, + .epilog = ec_code_avx_epilog, + .load = ec_code_avx_load, + .store = ec_code_avx_store, + .copy = ec_code_avx_copy, + .xor2 = ec_code_avx_xor2, + .xor3 = ec_code_avx_xor3, + .xorm = ec_code_avx_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-avx.h b/xlators/cluster/ec/src/ec-code-avx.h new file mode 100644 index 00000000000..fdca4ad2c8f --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-avx.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_AVX_H__ +#define __EC_CODE_AVX_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_avx; + +#endif /* __EC_CODE_AVX_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-c.c b/xlators/cluster/ec/src/ec-code-c.c new file mode 100644 index 00000000000..acdc665c2cf --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-c.c @@ -0,0 +1,11679 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <inttypes.h> +#include <string.h> + +#include "ec-method.h" +#include "ec-code-c.h" + +#define WIDTH (EC_METHOD_WORD_SIZE / sizeof(uint64_t)) + +static void +gf8_muladd_00(void *out, void *in) +{ + memcpy(out, in, EC_METHOD_WORD_SIZE * 8); +} + +static void +gf8_muladd_01(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + out_ptr[0] ^= in_ptr[0]; + out_ptr[WIDTH] ^= in_ptr[WIDTH]; + out_ptr[WIDTH * 2] ^= in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] ^= in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] ^= in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] ^= in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] ^= in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] ^= in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_02(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in7; + out1 = in0; + out7 = in6; + out5 = in4; + out6 = in5; + out3 = in2 ^ in7; + out4 = in3 ^ in7; + out2 = in1 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_03(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in7; + tmp0 = in2 ^ in7; + out1 = in0 ^ in1; + out7 = in6 ^ in7; + out5 = in4 ^ in5; + out6 = in5 ^ in6; + out4 = in3 ^ in4 ^ in7; + out2 = tmp0 ^ in1; + out3 = tmp0 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_04(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in6; + out1 = in7; + out7 = in5; + out6 = in4; + tmp0 = in6 ^ in7; + out2 = in0 ^ in6; + out5 = in3 ^ in7; + out3 = tmp0 ^ in1; + out4 = tmp0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_05(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in6; + out1 = in1 ^ in7; + out7 = in5 ^ in7; + out6 = in4 ^ in6; + out2 = out0 ^ in2; + out3 = out1 ^ in3 ^ in6; + out5 = out7 ^ in3; + out4 = out6 ^ in2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_06(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in6 ^ in7; + tmp0 = in1 ^ in6; + out1 = in0 ^ in7; + out7 = in5 ^ in6; + out6 = in4 ^ in5; + out4 = in2 ^ in3 ^ in6; + out5 = in3 ^ in4 ^ in7; + out3 = tmp0 ^ in2; + out2 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_07(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in5 ^ in6; + tmp2 = in0 ^ in7; + tmp3 = tmp0 ^ in3; + out6 = tmp1 ^ in4; + out7 = tmp1 ^ in7; + out0 = tmp2 ^ in6; + out1 = tmp2 ^ in1; + out3 = tmp3 ^ in1; + out4 = tmp3 ^ in4; + out5 = out4 ^ out7 ^ in2; + out2 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_08(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5; + out1 = in6; + out7 = in4; + out6 = in3 ^ in7; + out3 = in0 ^ in5 ^ in6; + out5 = in2 ^ in6 ^ in7; + out2 = in5 ^ in7; + out4 = out2 ^ in1 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_09(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in5; + tmp0 = in3 ^ in6; + out1 = in1 ^ in6; + out7 = in4 ^ in7; + out2 = in2 ^ in5 ^ in7; + out3 = tmp0 ^ out0; + out6 = tmp0 ^ in7; + out4 = out1 ^ out7 ^ in5; + out5 = out2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5 ^ in7; + out1 = in0 ^ in6; + out7 = in4 ^ in6; + out2 = in1 ^ in5; + out6 = out0 ^ in3; + out3 = out0 ^ out1 ^ in2; + out5 = out7 ^ in2 ^ in7; + out4 = out2 ^ in3 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in0 ^ in6; + tmp2 = in4 ^ in7; + out0 = in0 ^ in5 ^ in7; + out2 = tmp0 ^ in1; + out1 = tmp1 ^ in1; + out6 = tmp1 ^ out0 ^ in3; + out7 = tmp2 ^ in6; + out4 = tmp2 ^ out6 ^ in1; + out3 = out6 ^ in0 ^ in2; + out5 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5 ^ in6; + out1 = in6 ^ in7; + out7 = in4 ^ in5; + tmp0 = in1 ^ in5; + tmp1 = in0 ^ in7; + out5 = in2 ^ in3 ^ in6; + out6 = in3 ^ in4 ^ in7; + out2 = tmp1 ^ out0; + out4 = tmp0 ^ in2; + out3 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + tmp1 = in5 ^ in6; + out1 = in1 ^ in6 ^ in7; + out7 = tmp0 ^ in7; + out4 = tmp0 ^ in1 ^ in2; + out0 = tmp1 ^ in0; + tmp2 = tmp1 ^ in3; + out6 = tmp2 ^ out7; + out2 = out0 ^ in2 ^ in7; + out3 = out0 ^ out1 ^ in3; + out5 = tmp2 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in5; + tmp2 = in5 ^ in6; + out1 = in0 ^ in6 ^ in7; + out3 = tmp0 ^ tmp1; + out2 = tmp0 ^ tmp2; + tmp3 = tmp1 ^ in3; + out7 = tmp2 ^ in4; + out0 = tmp2 ^ in7; + out4 = tmp3 ^ in1 ^ in7; + out5 = tmp3 ^ out7; + out6 = out0 ^ out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in6 ^ in7; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in5; + out1 = tmp1 ^ in0; + out7 = tmp2 ^ in4; + out0 = tmp2 ^ in0; + out6 = out7 ^ in3; + out5 = out6 ^ in2 ^ in7; + tmp3 = tmp1 ^ out0 ^ in2; + out4 = tmp1 ^ out5; + out2 = tmp3 ^ in6; + out3 = tmp3 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_10(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4; + out1 = in5; + out7 = in3 ^ in7; + tmp0 = in6 ^ in7; + out2 = in4 ^ in6; + tmp1 = out2 ^ in5; + out6 = tmp0 ^ in2; + out3 = tmp0 ^ tmp1; + out5 = out2 ^ out3 ^ in1; + out4 = tmp1 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_11(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3; + out0 = in0 ^ in4; + out1 = in1 ^ in5; + out6 = in2 ^ in7; + out4 = in0 ^ in5 ^ in6; + out5 = in1 ^ in6 ^ in7; + out2 = in2 ^ in4 ^ in6; + out3 = in3 ^ in4 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_12(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in7; + out1 = in0 ^ in5; + out3 = in2 ^ in4 ^ in5; + tmp0 = out0 ^ in6; + out2 = tmp0 ^ in1; + tmp1 = tmp0 ^ in3; + out6 = tmp0 ^ out3; + out5 = out2 ^ in5; + out7 = tmp1 ^ in4; + out4 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_13(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3 ^ in6; + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in7; + out6 = in2 ^ in5 ^ in7; + out4 = tmp0 ^ out7 ^ in7; + out1 = tmp0 ^ in1; + out0 = tmp1 ^ in0; + out5 = tmp1 ^ in1 ^ in6; + out3 = tmp1 ^ out6 ^ in3; + out2 = out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_14(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in6; + out1 = in5 ^ in7; + out2 = in0 ^ in4; + tmp0 = out0 ^ in5; + out7 = out1 ^ in3; + tmp1 = out1 ^ in2; + out3 = tmp0 ^ in1; + out6 = tmp0 ^ tmp1; + out4 = tmp1 ^ out2; + out5 = out3 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_15(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3 ^ in5; + tmp0 = in0 ^ in4; + out1 = in1 ^ in5 ^ in7; + out5 = in1 ^ in3 ^ in6; + out0 = tmp0 ^ in6; + out2 = tmp0 ^ in2; + out3 = out5 ^ in4 ^ in5; + out6 = out2 ^ in0 ^ in7; + out4 = tmp0 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_16(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in7; + tmp2 = in2 ^ in3 ^ in4; + out1 = tmp0 ^ in7; + out4 = tmp0 ^ tmp2; + out0 = tmp1 ^ in6; + tmp3 = tmp1 ^ in1; + out6 = out0 ^ in2 ^ in5; + out2 = tmp3 ^ in0; + out3 = out6 ^ in1; + out7 = tmp2 ^ out6; + out5 = tmp3 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_17(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in4; + out4 = tmp0 ^ in0 ^ in3; + out7 = tmp1 ^ in5; + tmp3 = tmp1 ^ in1; + out6 = tmp2 ^ in7; + out5 = tmp3 ^ in4; + out3 = tmp3 ^ out6; + out0 = out3 ^ out4 ^ in1; + out2 = out3 ^ out7 ^ in0; + out1 = tmp2 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_18(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in5; + out1 = in5 ^ in6; + tmp0 = in4 ^ in7; + out5 = in1 ^ in2 ^ in5; + out6 = in2 ^ in3 ^ in6; + out2 = tmp0 ^ out1; + out7 = tmp0 ^ in3; + tmp1 = tmp0 ^ in0; + out3 = tmp1 ^ in6; + out4 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_19(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in2; + out7 = in3 ^ in4; + tmp0 = in0 ^ in7; + out6 = in2 ^ in3; + out1 = in1 ^ in5 ^ in6; + out0 = in0 ^ in4 ^ in5; + out4 = tmp0 ^ in1; + tmp1 = tmp0 ^ in6; + out2 = tmp1 ^ out0 ^ in2; + out3 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + tmp1 = in5 ^ in6; + tmp2 = tmp0 ^ in1; + out0 = tmp0 ^ in7; + out1 = tmp1 ^ in0; + tmp3 = tmp1 ^ in3; + out5 = tmp2 ^ in2; + out2 = tmp2 ^ in6; + out7 = tmp3 ^ out0; + out6 = tmp3 ^ in2; + out4 = tmp3 ^ out2 ^ in0; + out3 = tmp0 ^ out1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in2 ^ in5; + tmp2 = in3 ^ in6; + out5 = tmp0 ^ in1; + tmp3 = tmp0 ^ in0; + out6 = tmp1 ^ in3; + out0 = tmp1 ^ tmp3 ^ in7; + out7 = tmp2 ^ in4; + tmp4 = out5 ^ in6; + out3 = tmp2 ^ tmp3; + out2 = tmp4 ^ in5; + out4 = tmp4 ^ out3; + out1 = tmp3 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in4 ^ in6; + tmp2 = in5 ^ in7; + out6 = tmp0 ^ tmp1; + out0 = tmp1 ^ in5; + out1 = tmp2 ^ in6; + tmp3 = tmp2 ^ in1; + tmp4 = tmp2 ^ in4; + out2 = tmp4 ^ in0; + out7 = tmp4 ^ in3; + out5 = tmp0 ^ tmp3; + out3 = tmp3 ^ out2; + out4 = out3 ^ in2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in3 ^ in4; + tmp3 = in2 ^ in7; + out3 = tmp0 ^ tmp1; + out5 = tmp0 ^ tmp3; + tmp4 = tmp1 ^ in5; + out6 = tmp2 ^ in2; + out7 = tmp2 ^ in5; + out2 = tmp3 ^ tmp4; + out4 = out3 ^ out6 ^ in6; + out0 = tmp4 ^ in6; + out1 = out2 ^ out4 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in1; + out3 = tmp1 ^ tmp2; + out2 = tmp2 ^ in5; + out4 = out3 ^ in3 ^ in6; + tmp3 = out4 ^ in7; + out6 = tmp3 ^ out2 ^ in4; + out7 = tmp1 ^ out6; + out0 = out7 ^ in3; + out1 = tmp0 ^ out0; + out5 = tmp3 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + tmp1 = tmp0 ^ in5; + out7 = tmp1 ^ in3; + out0 = tmp1 ^ in0 ^ in7; + out6 = out7 ^ in2 ^ in6; + out1 = out0 ^ in1 ^ in4; + out4 = out0 ^ out6 ^ in1; + out3 = tmp0 ^ out4; + out2 = out4 ^ out7 ^ in7; + out5 = out3 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_20(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4; + out0 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in6 ^ in7; + out2 = out0 ^ in5; + out4 = tmp0 ^ in5; + out3 = tmp0 ^ tmp1; + out7 = tmp1 ^ in2; + out6 = tmp1 ^ in1 ^ in5; + out5 = out2 ^ out3 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_21(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in4; + tmp0 = in4 ^ in6; + out4 = in3 ^ in5; + out7 = in2 ^ in6; + out0 = in0 ^ in3 ^ in7; + out6 = in1 ^ in5 ^ in7; + out3 = tmp0 ^ in7; + out5 = tmp0 ^ in0; + out2 = out4 ^ in2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_22(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3; + out1 = in0 ^ in4; + out7 = in2 ^ in7; + out4 = in4 ^ in5 ^ in7; + out5 = in0 ^ in5 ^ in6; + out6 = in1 ^ in6 ^ in7; + out3 = in2 ^ in3 ^ in4 ^ in6; + out2 = in1 ^ in3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_23(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in2; + out0 = in0 ^ in3; + out4 = in5 ^ in7; + out5 = in0 ^ in6; + out6 = in1 ^ in7; + out3 = in2 ^ in4 ^ in6; + out1 = in0 ^ in1 ^ in4; + out2 = out4 ^ out6 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_24(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4 ^ in7; + tmp0 = in3 ^ in4; + out0 = in3 ^ in6 ^ in7; + out3 = tmp0 ^ in1; + tmp1 = out0 ^ in5; + out6 = tmp1 ^ out3; + out2 = tmp1 ^ in0; + out7 = tmp1 ^ in2 ^ in3; + out5 = out2 ^ in4; + out4 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_25(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in4; + tmp0 = in2 ^ in5; + out1 = out3 ^ in7; + out7 = tmp0 ^ in6; + out6 = out1 ^ in5; + out4 = out7 ^ in3 ^ in7; + out2 = out4 ^ in0; + out0 = tmp0 ^ out2; + out5 = out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_26(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in6; + tmp0 = in4 ^ in7; + out7 = in2 ^ in5 ^ in7; + tmp1 = out0 ^ in0 ^ in5; + out1 = tmp0 ^ in0; + tmp2 = tmp0 ^ in6; + out2 = tmp1 ^ in1; + out5 = tmp1 ^ in7; + out6 = tmp2 ^ in1; + out4 = tmp2 ^ out7; + out3 = out0 ^ out6 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_27(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in2 ^ in5; + out0 = in0 ^ in3 ^ in6; + out6 = in1 ^ in4 ^ in7; + out4 = out7 ^ in6; + out2 = out0 ^ out7 ^ in1; + out5 = out0 ^ in7; + out1 = out6 ^ in0; + out3 = out6 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_28(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3; + out1 = in4 ^ in6; + out0 = in3 ^ in5 ^ in7; + tmp0 = out1 ^ in7; + tmp1 = out0 ^ in4; + out7 = tmp0 ^ in2; + tmp2 = tmp0 ^ in1; + out3 = tmp1 ^ in0; + out6 = tmp1 ^ tmp2; + out4 = tmp2 ^ in3; + out5 = out3 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_29(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in3; + tmp0 = in1 ^ in3; + tmp1 = in4 ^ in6; + tmp2 = in0 ^ in4 ^ in7; + out6 = tmp0 ^ in5; + out4 = tmp0 ^ in6 ^ in7; + out1 = tmp1 ^ in1; + out7 = tmp1 ^ in2; + out3 = tmp2 ^ in5; + out5 = tmp2 ^ in2; + out0 = out3 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in5; + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + out7 = in2 ^ in4 ^ in7; + out3 = tmp1 ^ out0 ^ in2; + out2 = tmp0 ^ in7; + out6 = tmp0 ^ in6; + out1 = tmp1 ^ in6; + out5 = tmp1 ^ out7 ^ in5; + out4 = out1 ^ in0 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in6; + out7 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in2 ^ in7; + out6 = in1 ^ in3; + out1 = out4 ^ in0 ^ in4; + out3 = tmp0 ^ out7; + out0 = tmp0 ^ in3; + out5 = tmp1 ^ in0; + out2 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in2 ^ in3 ^ in4; + tmp2 = tmp0 ^ in6; + out4 = tmp1 ^ in1; + out5 = tmp1 ^ in0 ^ in5; + tmp3 = tmp2 ^ in4; + out6 = tmp2 ^ out4; + out7 = tmp3 ^ in7; + out2 = tmp3 ^ out5; + out3 = out6 ^ in0; + out0 = tmp1 ^ out7; + out1 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out4 = tmp0 ^ in1; + tmp1 = tmp0 ^ in0; + out2 = tmp1 ^ in6; + out5 = tmp1 ^ in4; + tmp2 = out2 ^ in2; + tmp3 = tmp2 ^ in5; + out0 = tmp3 ^ in7; + out7 = tmp3 ^ out5; + out6 = out4 ^ out7 ^ in6; + out3 = tmp2 ^ out6; + out1 = out0 ^ out6 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in7; + out0 = in3 ^ in5 ^ in6; + tmp1 = tmp0 ^ in0; + tmp2 = tmp0 ^ in2; + out1 = tmp1 ^ in6; + out4 = tmp2 ^ in1; + out7 = tmp2 ^ in5; + out3 = out0 ^ out4 ^ in0; + out2 = out3 ^ out7 ^ in7; + out6 = tmp1 ^ out2; + out5 = tmp1 ^ out7 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in5; + out4 = in1 ^ in2 ^ in7; + out6 = in1 ^ in3 ^ in4; + out5 = tmp0 ^ in2; + tmp2 = tmp0 ^ in6; + out7 = tmp1 ^ in4; + out0 = tmp2 ^ in5; + out2 = tmp2 ^ out4; + out1 = tmp2 ^ out6 ^ in7; + out3 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_30(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4 ^ in5; + tmp0 = in3 ^ in6; + tmp1 = in4 ^ in7; + out6 = in1 ^ in2 ^ in5; + out3 = tmp0 ^ in5; + out4 = tmp0 ^ in0; + out7 = tmp0 ^ in2; + out0 = tmp1 ^ in3; + out2 = tmp1 ^ out3; + out5 = tmp1 ^ in0 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_31(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in5 ^ in6; + tmp0 = in4 ^ in5; + tmp1 = in0 ^ in3 ^ in4; + tmp2 = out3 ^ in2; + out1 = tmp0 ^ in1; + out0 = tmp1 ^ in7; + out4 = tmp1 ^ in6; + out6 = tmp2 ^ in1; + out2 = tmp2 ^ out0 ^ in0; + out5 = out1 ^ in0 ^ in7; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_32(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in4; + out7 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in0 ^ in7; + out6 = in1 ^ in2; + out1 = in0 ^ in4 ^ in5; + out2 = tmp0 ^ out0 ^ in1; + out3 = tmp0 ^ out7 ^ in7; + out4 = tmp1 ^ in6; + out5 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_33(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in1 ^ in5; + out6 = in1 ^ in2 ^ in6; + out7 = tmp0 ^ in7; + out0 = tmp1 ^ in3; + out1 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in7; + tmp4 = tmp2 ^ in4 ^ in6; + out5 = tmp3 ^ in0; + out3 = tmp3 ^ out6; + out4 = tmp4 ^ out5; + out2 = tmp0 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_34(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in4 ^ in5; + tmp2 = tmp0 ^ in1; + tmp3 = tmp0 ^ in6; + out1 = tmp1 ^ in7; + tmp4 = tmp1 ^ in2; + out5 = tmp2 ^ in0; + out3 = tmp2 ^ out1; + out0 = tmp3 ^ in7; + out7 = tmp3 ^ tmp4; + out6 = tmp4 ^ in1; + out2 = out3 ^ out5 ^ in3; + out4 = tmp4 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_35(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in5 ^ in7; + out7 = tmp0 ^ tmp1 ^ in3; + out3 = tmp1 ^ in1; + out1 = out3 ^ in4; + tmp2 = out1 ^ in7; + out5 = tmp2 ^ in0 ^ in3; + out6 = tmp0 ^ tmp2; + out0 = out3 ^ out5 ^ in6; + out4 = tmp0 ^ out0; + out2 = out4 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_36(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in2; + tmp0 = in1 ^ in3; + out0 = in3 ^ in4 ^ in6; + out6 = in1 ^ in2 ^ in4; + out5 = tmp0 ^ in0; + tmp1 = out5 ^ in5; + out2 = tmp1 ^ in4; + out3 = tmp1 ^ out4; + out1 = tmp0 ^ out2 ^ in7; + out7 = out3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_37(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in4; + tmp2 = tmp0 ^ in6; + out3 = tmp0 ^ in5; + out4 = tmp1 ^ in0; + out6 = tmp2 ^ in4; + out1 = out3 ^ out4 ^ in7; + tmp3 = out4 ^ in1 ^ in3; + out7 = tmp3 ^ out1; + out2 = tmp3 ^ in5; + out5 = tmp1 ^ out2; + out0 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_38(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in3; + tmp0 = in3 ^ in4; + tmp1 = in5 ^ in7; + tmp2 = out3 ^ in1; + out2 = tmp0 ^ in6; + out0 = tmp0 ^ tmp1; + out4 = tmp1 ^ tmp2; + out7 = out2 ^ in2; + out1 = out2 ^ in3 ^ in5; + out6 = out4 ^ in0 ^ in2; + out5 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_39(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0; + tmp0 = in1 ^ in5; + tmp1 = tmp0 ^ in4; + out1 = tmp1 ^ in6; + out5 = out1 ^ in0 ^ in2; + tmp2 = tmp0 ^ out5; + out2 = tmp2 ^ in0 ^ in3; + out7 = out2 ^ in7; + out6 = tmp1 ^ out7; + out4 = tmp2 ^ out6; + out0 = out4 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in0 ^ in2; + tmp2 = in3 ^ in4; + tmp3 = in1 ^ in6; + tmp4 = in3 ^ in7; + out4 = tmp0 ^ in5; + out5 = tmp1 ^ tmp3; + out3 = tmp1 ^ tmp4; + out0 = tmp2 ^ in5; + out7 = tmp2 ^ in2; + tmp5 = tmp3 ^ in4; + out2 = tmp4 ^ tmp5; + out1 = tmp5 ^ out4; + out6 = tmp0 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in3; + out3 = tmp1 ^ in0; + out6 = tmp1 ^ tmp2; + out2 = out6 ^ in4; + out7 = tmp0 ^ out2; + out0 = out3 ^ out7 ^ in5; + out5 = out0 ^ out2 ^ in7; + out1 = tmp2 ^ out0; + out4 = out1 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in7; + tmp2 = in1 ^ in6 ^ in7; + out2 = tmp0 ^ in4; + out3 = tmp0 ^ tmp2; + out4 = tmp1 ^ out3 ^ in5; + out5 = tmp2 ^ out2 ^ in2; + out1 = out4 ^ out5 ^ in6; + out0 = out1 ^ in3; + out7 = tmp1 ^ out0; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = tmp0 ^ in3; + out2 = tmp1 ^ in4; + tmp2 = out2 ^ in5; + out4 = tmp2 ^ in1 ^ in6; + out5 = out4 ^ in7; + out6 = out5 ^ in0; + out7 = out6 ^ in1; + out0 = tmp0 ^ out7; + out1 = tmp1 ^ out5; + out3 = tmp2 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = tmp0 ^ in4; + out0 = tmp1 ^ in6; + out7 = tmp1 ^ in2; + out6 = out7 ^ in1 ^ in5 ^ in7; + out2 = out6 ^ in0 ^ in2; + out4 = out0 ^ out6 ^ in0; + out5 = tmp0 ^ out4; + out3 = out5 ^ in7; + out1 = out3 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + out3 = tmp0 ^ in2 ^ in6; + tmp1 = out3 ^ in5 ^ in7; + out4 = tmp1 ^ in4; + out5 = tmp1 ^ in3; + out1 = out4 ^ in2; + out7 = out1 ^ out3 ^ in3; + out2 = tmp0 ^ out7 ^ in5; + tmp2 = out2 ^ in0; + out6 = tmp2 ^ in6; + out0 = tmp1 ^ tmp2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_40(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in6 ^ in7; + out4 = tmp0 ^ in2; + out5 = tmp0 ^ in5; + out0 = tmp1 ^ in2; + out7 = tmp1 ^ in1 ^ in5; + out2 = out0 ^ in4; + out3 = out2 ^ out5 ^ in7; + out6 = out3 ^ out4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_41(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in6 ^ in7; + out5 = in3 ^ in4; + out1 = in1 ^ in3 ^ in7; + out6 = in0 ^ in4 ^ in5; + out3 = tmp0 ^ in2; + out7 = tmp0 ^ in1; + out2 = tmp1 ^ in4; + out0 = tmp1 ^ in0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_42(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in6; + out5 = in3 ^ in5; + out1 = in0 ^ in3 ^ in7; + out7 = in1 ^ in5 ^ in7; + out4 = in2 ^ in4 ^ in7; + out6 = in0 ^ in4 ^ in6; + out2 = out0 ^ in1 ^ in4; + out3 = out5 ^ in6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_43(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in3; + out7 = in1 ^ in5; + out4 = in2 ^ in7; + out6 = in0 ^ in4; + out0 = in0 ^ in2 ^ in6; + out3 = in5 ^ in6 ^ in7; + out2 = in1 ^ in4 ^ in6; + out1 = in0 ^ in1 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_44(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3; + out0 = in2 ^ in7; + tmp0 = in4 ^ in7; + out7 = in1 ^ in6 ^ in7; + out6 = in0 ^ in5 ^ in6; + out4 = tmp0 ^ in3 ^ in6; + out3 = out0 ^ in1 ^ in3 ^ in5; + out2 = out0 ^ in0 ^ in4; + out5 = tmp0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_45(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in3; + out7 = in1 ^ in6; + out5 = in4 ^ in7; + out6 = in0 ^ in5; + out0 = in0 ^ in2 ^ in7; + out4 = in3 ^ in6 ^ in7; + out2 = out5 ^ in0; + out3 = out0 ^ out6 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_46(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2; + out1 = in0 ^ in3; + out7 = in1 ^ in7; + out4 = in4 ^ in6; + out5 = in5 ^ in7; + out6 = in0 ^ in6; + out3 = in1 ^ in3 ^ in5; + out2 = out4 ^ out6 ^ in1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_47(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in6; + out7 = in1; + out5 = in7; + out6 = in0; + tmp0 = in0 ^ in1; + out3 = in1 ^ in5; + out0 = in0 ^ in2; + out1 = tmp0 ^ in3; + out2 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_48(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out1 = in3 ^ in6 ^ in7; + out3 = tmp0 ^ in0; + out0 = tmp0 ^ out1 ^ in5; + tmp1 = out0 ^ in4; + out2 = tmp1 ^ in7; + out5 = tmp1 ^ in3; + out4 = out5 ^ in1; + out7 = tmp0 ^ out4; + out6 = tmp1 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_49(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in2; + tmp0 = in2 ^ in5; + out2 = in4 ^ in5 ^ in6; + tmp1 = tmp0 ^ out2 ^ in3; + out7 = out2 ^ in1; + out5 = tmp1 ^ in7; + out4 = out5 ^ out7 ^ in6; + out1 = tmp0 ^ out4; + out6 = out1 ^ out7 ^ in0; + out0 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in3 ^ in7; + out0 = tmp0 ^ in5; + out3 = tmp1 ^ in0; + out5 = tmp1 ^ out0; + out4 = out0 ^ in1 ^ in4; + out1 = out3 ^ in6; + out2 = out4 ^ in7; + out6 = out1 ^ in4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in7; + tmp0 = in1 ^ in5; + tmp1 = in2 ^ in6; + tmp2 = out3 ^ in3; + out7 = tmp0 ^ in4; + out4 = tmp0 ^ tmp1; + tmp3 = tmp1 ^ in0; + out6 = tmp2 ^ in4; + out5 = tmp2 ^ tmp3; + out1 = tmp2 ^ in1 ^ in6; + out2 = out7 ^ in6 ^ in7; + out0 = tmp3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in6; + tmp0 = in2 ^ in5; + tmp1 = out1 ^ in5 ^ in7; + out0 = tmp0 ^ in7; + tmp2 = tmp0 ^ in4; + out6 = tmp1 ^ in0; + out2 = tmp2 ^ in0; + out5 = tmp2 ^ in6; + out3 = tmp0 ^ out6 ^ in1; + out7 = out0 ^ out5 ^ in1; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out4 = in1 ^ in3 ^ in5; + tmp2 = tmp0 ^ in7; + out2 = tmp0 ^ in4; + out1 = tmp1 ^ in3; + out7 = tmp1 ^ in4; + out0 = tmp2 ^ in2; + out6 = tmp2 ^ in3; + out5 = out7 ^ in1 ^ in2; + out3 = tmp1 ^ out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in5; + out7 = in1 ^ in4 ^ in7; + out1 = in0 ^ in3 ^ in6; + out5 = out0 ^ in6; + out4 = out7 ^ in5; + out3 = out1 ^ in1; + out6 = out1 ^ in7; + out2 = out4 ^ in0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2 ^ in6; + out7 = in1 ^ in4; + out3 = in0 ^ in1 ^ in6; + out4 = in1 ^ in5 ^ in7; + out0 = in0 ^ in2 ^ in5; + out6 = in0 ^ in3 ^ in7; + out1 = out3 ^ in3; + out2 = out4 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_50(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in7; + tmp0 = in3 ^ in5; + out0 = out2 ^ in4 ^ in6; + out1 = tmp0 ^ in7; + tmp1 = tmp0 ^ in6; + out3 = out0 ^ in3; + out7 = tmp1 ^ in1; + tmp2 = tmp1 ^ in0; + out5 = out3 ^ in1 ^ in2; + out4 = tmp2 ^ in2; + out6 = tmp2 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_51(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in7; + out3 = in2 ^ in4 ^ in6 ^ in7; + out0 = out3 ^ in0; + out6 = out0 ^ in5; + out4 = out6 ^ in3 ^ in7; + out1 = out0 ^ out4 ^ in1; + out7 = out1 ^ in6; + out5 = out7 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_52(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in2; + tmp0 = in2 ^ in4; + tmp1 = in3 ^ in5; + tmp2 = in3 ^ in6; + tmp3 = in0 ^ in7; + out0 = tmp0 ^ in6; + out6 = tmp0 ^ tmp3; + out7 = tmp1 ^ in1; + out1 = tmp1 ^ tmp3; + out3 = tmp2 ^ in4; + out5 = tmp2 ^ in1 ^ in7; + out4 = tmp2 ^ out1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_53(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1; + out3 = in4 ^ in6; + out0 = out3 ^ in0 ^ in2; + out6 = out0 ^ in7; + out4 = out6 ^ in5; + out7 = out0 ^ out4 ^ in1 ^ in3; + out1 = out7 ^ in0; + out5 = out7 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_54(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in5; + tmp0 = in1 ^ in3; + tmp1 = in2 ^ in4; + tmp2 = in0 ^ in7; + out5 = in1 ^ in4 ^ in6; + out4 = tmp2 ^ out1; + out7 = tmp0 ^ in6; + out3 = tmp0 ^ tmp1; + out0 = tmp1 ^ in7; + tmp3 = tmp2 ^ in2; + out2 = tmp3 ^ in6; + out6 = tmp3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_55(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in1 ^ in4; + tmp2 = in6 ^ in7; + out7 = tmp0 ^ tmp2; + out1 = tmp0 ^ in5; + out3 = tmp1 ^ in2; + out5 = tmp1 ^ in5 ^ in6; + out2 = tmp2 ^ in0; + out4 = out5 ^ out7 ^ in0; + out6 = out2 ^ in2 ^ in5; + out0 = out5 ^ out6 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_56(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in4; + tmp0 = in0 ^ in2; + out4 = in0 ^ in5; + out7 = in1 ^ in3; + out5 = in1 ^ in6; + out6 = tmp0 ^ in7; + out2 = tmp0 ^ out5; + out1 = out4 ^ in3; + out3 = out7 ^ in4 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_57(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in7; + out0 = in0 ^ in2 ^ in4; + out5 = in1 ^ in5 ^ in6; + out4 = tmp0 ^ in4; + out1 = tmp0 ^ in1 ^ in3; + out2 = tmp0 ^ out5; + out3 = tmp1 ^ in4; + out7 = tmp1 ^ in3; + out6 = tmp1 ^ out2 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_58(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in5; + tmp0 = in2 ^ in3 ^ in4; + out5 = tmp0 ^ in1; + out6 = tmp0 ^ in0 ^ in5; + out3 = out6 ^ in7; + tmp1 = out2 ^ out5; + out7 = tmp1 ^ in6; + out4 = tmp1 ^ out3 ^ in3; + out0 = out4 ^ out7 ^ in0; + out1 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_59(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in5; + tmp0 = in0 ^ in5 ^ in7; + out3 = tmp0 ^ in2 ^ in4; + out0 = out3 ^ in6; + tmp1 = out0 ^ in7; + out6 = tmp1 ^ in3; + out5 = out6 ^ in0 ^ in1 ^ in6; + out4 = tmp0 ^ out5; + out1 = tmp1 ^ out4; + out7 = out1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in5; + out5 = tmp0 ^ in3; + out4 = tmp0 ^ in0; + tmp2 = tmp1 ^ in4; + out2 = tmp1 ^ in1 ^ in7; + out7 = tmp2 ^ out5; + out6 = out4 ^ out7 ^ in5; + out0 = tmp2 ^ in6; + out1 = out0 ^ out6 ^ in7; + out3 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in1 ^ in5; + out5 = tmp0 ^ tmp2; + tmp3 = tmp1 ^ in6; + out3 = tmp1 ^ in5; + out2 = tmp2 ^ in7; + tmp4 = out3 ^ in2; + out7 = out2 ^ in3 ^ in4; + out0 = tmp4 ^ in6; + out6 = tmp0 ^ tmp3; + out4 = tmp2 ^ tmp4; + out1 = tmp3 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in6; + tmp1 = in0 ^ in2 ^ in5; + out1 = tmp0 ^ in5; + tmp2 = tmp0 ^ in1; + out2 = tmp1 ^ in6; + out6 = tmp1 ^ in3; + out4 = tmp2 ^ in0; + out7 = tmp2 ^ in4; + out3 = tmp1 ^ out7; + out0 = out3 ^ out4 ^ in7; + out5 = out0 ^ in1 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in0 ^ in6; + out2 = tmp1 ^ in5; + tmp2 = out2 ^ in3; + out6 = tmp2 ^ in2; + out1 = tmp0 ^ tmp2; + tmp3 = out1 ^ in4 ^ in5; + out4 = tmp3 ^ in0; + out7 = tmp3 ^ in7; + tmp4 = out4 ^ out6; + out5 = tmp4 ^ in7; + out0 = tmp0 ^ out5; + out3 = tmp1 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in3 ^ in5; + tmp2 = in1 ^ in7; + out7 = in1 ^ in3 ^ in4; + out0 = tmp0 ^ in4; + tmp3 = tmp1 ^ in0; + out5 = tmp2 ^ in2; + out1 = tmp3 ^ in6; + out6 = tmp0 ^ tmp3; + tmp4 = tmp2 ^ out1; + out3 = tmp4 ^ in4; + out4 = tmp1 ^ tmp4; + out2 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in5; + tmp1 = in0 ^ in6; + tmp2 = tmp0 ^ in7; + tmp3 = tmp1 ^ in3; + out2 = tmp1 ^ tmp2; + out5 = tmp2 ^ in2; + out6 = tmp3 ^ in2; + out3 = out2 ^ in4; + out4 = out3 ^ in5; + out1 = tmp0 ^ tmp3; + out7 = tmp3 ^ out4; + out0 = out4 ^ out5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_60(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in5; + tmp0 = in3 ^ in6; + out1 = in3 ^ in4 ^ in7; + out7 = out4 ^ in1; + tmp1 = out4 ^ in4; + out0 = tmp0 ^ in2; + out5 = tmp0 ^ in0; + out2 = tmp0 ^ tmp1; + out3 = tmp1 ^ in7; + out6 = out3 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_61(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + out4 = tmp0 ^ in4; + tmp1 = out4 ^ in3; + out3 = tmp1 ^ in7; + out2 = tmp1 ^ in2 ^ in6; + out1 = tmp0 ^ out3 ^ in1; + out0 = out2 ^ out4 ^ in0; + out7 = tmp1 ^ out1; + out6 = out0 ^ out1 ^ in2; + out5 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_62(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4 ^ in5; + tmp0 = in0 ^ in3 ^ in4; + out1 = tmp0 ^ in7; + out5 = tmp0 ^ in6; + tmp1 = out1 ^ in0; + tmp2 = tmp1 ^ out3; + out4 = tmp2 ^ in2; + tmp3 = tmp2 ^ in1; + out0 = out4 ^ in5 ^ in6; + out7 = tmp3 ^ out0; + out6 = tmp0 ^ tmp3; + out2 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_63(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in7; + out3 = tmp0 ^ in5; + tmp2 = out3 ^ in6; + out4 = out3 ^ in2 ^ in7; + out5 = tmp2 ^ in0; + tmp3 = out5 ^ in3; + out0 = tmp3 ^ out4; + out2 = tmp1 ^ tmp2; + out6 = tmp1 ^ tmp3; + tmp4 = tmp0 ^ out2; + out1 = tmp4 ^ out5; + out7 = tmp4 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_64(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in3; + out1 = in3 ^ in4; + out7 = in1 ^ in2; + tmp0 = in4 ^ in5; + tmp1 = in0 ^ in7; + out4 = in5 ^ in6 ^ in7; + out2 = tmp0 ^ out0 ^ in0; + out3 = tmp0 ^ out7 ^ in6; + out5 = tmp1 ^ in6; + out6 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_65(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in4 ^ in5; + tmp2 = in6 ^ in7; + out7 = in1 ^ in2 ^ in7; + out1 = in1 ^ in3 ^ in4; + out0 = tmp0 ^ in2; + out2 = tmp0 ^ tmp1; + out4 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in0; + out3 = out4 ^ out7 ^ in3; + out5 = tmp3 ^ in5; + out6 = tmp3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_66(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in3; + tmp2 = in0 ^ in4; + out7 = tmp0 ^ in6; + out0 = tmp1 ^ in7; + out1 = tmp2 ^ in3; + tmp3 = tmp2 ^ in6; + tmp4 = out1 ^ in5; + out5 = tmp3 ^ in7; + out4 = tmp3 ^ tmp4; + out2 = tmp0 ^ tmp4 ^ in7; + out6 = tmp1 ^ out2 ^ in4; + out3 = tmp3 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_67(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in7; + out1 = tmp1 ^ in4; + out0 = tmp2 ^ in2; + tmp3 = out1 ^ in7; + out2 = tmp3 ^ in5; + out3 = out2 ^ in0 ^ in6; + out7 = tmp1 ^ out0 ^ in6; + out5 = tmp1 ^ out3; + out4 = tmp2 ^ out5; + out6 = tmp3 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_68(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in2 ^ in3 ^ in5; + tmp2 = tmp0 ^ in1; + tmp3 = tmp0 ^ in6; + out0 = tmp1 ^ in6; + out6 = tmp2 ^ in0; + out7 = tmp1 ^ tmp2; + out1 = tmp3 ^ in7; + out2 = out1 ^ in2; + out4 = tmp2 ^ out2; + out3 = out4 ^ out6 ^ in3; + out5 = tmp3 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_69(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in6 ^ in7; + out2 = tmp0 ^ in3 ^ in4; + out1 = out2 ^ in1; + out3 = out2 ^ in0 ^ in2; + out4 = out1 ^ in2 ^ in3; + out6 = out1 ^ in0 ^ in7; + out7 = out4 ^ in5 ^ in6; + out5 = out4 ^ out6 ^ in5; + out0 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + out3 = in0 ^ in4 ^ in6; + tmp1 = tmp0 ^ in3; + out4 = tmp1 ^ in1; + tmp2 = tmp1 ^ in7; + out2 = out4 ^ in4; + out0 = tmp2 ^ in5; + out5 = tmp2 ^ out3; + out7 = out2 ^ in3 ^ in5; + out1 = tmp0 ^ out5; + out6 = tmp1 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + out2 = tmp0 ^ in1 ^ in3; + out4 = out2 ^ in2; + tmp1 = out2 ^ in0; + out7 = out4 ^ in3 ^ in5 ^ in7; + out1 = tmp1 ^ in7; + out3 = tmp1 ^ in1; + out6 = tmp1 ^ in5; + out0 = tmp1 ^ out7 ^ in6; + out5 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1; + tmp0 = in2 ^ in3; + out5 = in0 ^ in2; + out1 = in3 ^ in4 ^ in6; + tmp1 = out5 ^ in1; + out0 = tmp0 ^ in5; + out6 = tmp0 ^ tmp1; + out3 = tmp1 ^ in4; + out7 = out3 ^ in0; + out2 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in4; + tmp0 = in0 ^ in2; + tmp1 = out4 ^ in3; + out7 = out4 ^ in2 ^ in7; + out5 = tmp0 ^ in5; + out3 = tmp0 ^ tmp1; + out1 = tmp1 ^ in6; + out0 = out5 ^ in3; + out2 = out3 ^ out7 ^ in4; + out6 = out1 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + out4 = tmp0 ^ in7; + out6 = tmp0 ^ in0 ^ in5; + out5 = tmp1 ^ in2; + tmp2 = tmp1 ^ in3; + out3 = tmp2 ^ out4; + out1 = tmp2 ^ in6; + out2 = tmp0 ^ out5; + out0 = out2 ^ out3 ^ in5; + out7 = out1 ^ out2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in7; + tmp1 = tmp0 ^ in4; + tmp2 = tmp0 ^ in0 ^ in2; + out4 = tmp1 ^ in1; + out0 = tmp2 ^ in5; + out3 = out4 ^ in0; + out2 = out3 ^ in7; + out1 = out2 ^ in6; + out6 = out1 ^ in4 ^ in5; + out7 = tmp2 ^ out1; + out5 = tmp1 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_70(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2; + tmp0 = in2 ^ in4; + out2 = in2 ^ in3 ^ in5; + tmp1 = tmp0 ^ in6; + tmp2 = out2 ^ in7; + out0 = tmp1 ^ in3; + out4 = tmp1 ^ in0; + out7 = tmp2 ^ in1; + out6 = out4 ^ in1; + out5 = out7 ^ in0 ^ in2; + out1 = tmp0 ^ tmp2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_71(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in5; + out3 = in2 ^ in3; + tmp0 = in0 ^ in2; + tmp1 = out2 ^ in1; + out4 = tmp0 ^ in6; + tmp2 = tmp0 ^ in1; + out7 = tmp1 ^ in2; + out1 = tmp1 ^ in4 ^ in7; + out0 = out4 ^ in3 ^ in4; + out6 = tmp2 ^ in4; + out5 = tmp2 ^ out3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_72(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in7; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in3 ^ in7; + out1 = tmp1 ^ in5; + out5 = out1 ^ in1; + tmp2 = tmp0 ^ out5; + out2 = tmp2 ^ in2; + out7 = out2 ^ in6; + out6 = tmp1 ^ out7; + out4 = tmp2 ^ out6; + out0 = out4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_73(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in7; + out2 = out3 ^ in1 ^ in5; + out1 = out2 ^ in0 ^ in4; + out5 = out1 ^ in5; + out6 = out1 ^ out3 ^ in2; + out0 = out2 ^ out6 ^ in6; + out7 = out0 ^ out1 ^ in3; + out4 = out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_74(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in2 ^ in6; + out4 = in0 ^ in4 ^ in7; + out5 = in0 ^ in1 ^ in5; + out0 = tmp0 ^ in2; + out1 = tmp0 ^ in5; + out3 = tmp1 ^ in7; + out6 = tmp1 ^ in0; + out2 = tmp1 ^ out5 ^ in3; + out7 = out3 ^ in3 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_75(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in7; + tmp0 = in1 ^ in3; + out5 = in0 ^ in1; + out7 = tmp0 ^ in2; + tmp1 = tmp0 ^ in4; + out6 = out5 ^ in2; + tmp2 = out7 ^ in6; + out1 = tmp1 ^ in5; + out0 = tmp1 ^ out6; + out3 = tmp2 ^ in7; + out2 = tmp2 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_76(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in6; + tmp0 = in0 ^ in5; + tmp1 = in3 ^ in7; + tmp2 = tmp0 ^ in4; + tmp3 = tmp1 ^ in2; + out5 = tmp2 ^ in1; + out1 = tmp2 ^ in3; + out0 = tmp3 ^ in4; + out4 = out1 ^ in5; + out7 = tmp3 ^ out3; + out2 = tmp0 ^ out7; + out6 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_77(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in3; + tmp0 = in1 ^ in4; + tmp1 = in1 ^ in6; + tmp2 = out4 ^ in5; + out5 = tmp0 ^ in0; + out1 = tmp0 ^ tmp2; + out3 = tmp1 ^ in3; + out2 = tmp1 ^ tmp2 ^ in7; + out7 = out3 ^ in2; + tmp3 = out7 ^ in6; + out6 = tmp2 ^ tmp3; + out0 = tmp3 ^ out5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_78(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in7; + tmp2 = in0 ^ in5 ^ in6; + out2 = tmp1 ^ in3; + out3 = tmp2 ^ in2; + out5 = out3 ^ in1 ^ in3; + out0 = tmp0 ^ out3 ^ in4; + out1 = tmp1 ^ out0; + out4 = out1 ^ out5 ^ in5; + out7 = tmp0 ^ out4; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_79(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in5; + tmp2 = tmp1 ^ in2; + out4 = tmp2 ^ in0 ^ in7; + tmp3 = out4 ^ in5; + out5 = tmp3 ^ out2 ^ in6; + out7 = tmp0 ^ tmp2; + out6 = tmp0 ^ tmp3; + out3 = tmp1 ^ out5; + out0 = out3 ^ in4; + out1 = tmp3 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + out2 = tmp0 ^ in3; + tmp1 = out2 ^ in4; + out4 = tmp1 ^ in0 ^ in5; + out5 = out4 ^ in6; + out6 = out5 ^ in7; + out7 = out6 ^ in0; + out0 = out7 ^ in1; + out1 = tmp0 ^ out6; + out3 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in3; + tmp0 = in0 ^ in5; + out4 = tmp0 ^ out2 ^ in2; + tmp1 = out4 ^ in4; + out6 = tmp1 ^ in7; + out5 = tmp1 ^ in5 ^ in6; + out0 = out6 ^ in1 ^ in6; + tmp2 = out0 ^ in2; + out1 = tmp2 ^ in1; + out3 = tmp2 ^ in4; + out7 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = tmp0 ^ in4; + out0 = tmp1 ^ in2; + out1 = tmp1 ^ in6; + out7 = out0 ^ in1 ^ in5 ^ in7; + out5 = out1 ^ out7 ^ in0; + out3 = out5 ^ in6; + out6 = tmp0 ^ out5; + out2 = out6 ^ in1; + out4 = out2 ^ out7 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = tmp0 ^ in3; + tmp2 = tmp0 ^ in6; + out7 = tmp1 ^ in4; + tmp3 = tmp2 ^ in0; + out5 = tmp3 ^ in7; + out4 = tmp3 ^ in2 ^ in5; + out2 = tmp1 ^ out5; + out6 = tmp2 ^ out2; + out0 = out4 ^ out7 ^ in6; + out1 = tmp3 ^ out0; + out3 = out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in0 ^ in5; + out1 = tmp0 ^ tmp1 ^ in6; + out3 = tmp1 ^ in1; + out4 = out1 ^ in1 ^ in7; + tmp2 = out4 ^ in3; + out5 = tmp2 ^ in2; + out6 = tmp0 ^ out5; + out7 = tmp1 ^ out4 ^ in2; + out2 = out6 ^ in5 ^ in7; + out0 = tmp2 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in7; + tmp1 = tmp0 ^ in3 ^ in5; + tmp2 = tmp1 ^ in0; + out0 = tmp2 ^ in4; + out6 = tmp2 ^ in1; + out3 = tmp0 ^ out6; + tmp3 = out3 ^ in6; + out1 = tmp3 ^ in4; + out2 = tmp3 ^ in5; + out4 = tmp3 ^ in7; + out5 = tmp1 ^ out1; + out7 = out0 ^ out4 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_80(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in4 ^ in5; + out1 = in2 ^ in6 ^ in7; + out5 = tmp0 ^ in4; + tmp2 = tmp0 ^ in1; + out6 = tmp1 ^ in3; + out7 = tmp1 ^ in0 ^ in6; + out4 = tmp2 ^ in7; + out3 = tmp2 ^ out6; + out2 = out3 ^ out5 ^ in6; + out0 = out2 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_81(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + tmp1 = tmp0 ^ in3; + out6 = tmp1 ^ in5; + out5 = out6 ^ in2 ^ in6; + out3 = out5 ^ in1; + out2 = tmp0 ^ out3; + out1 = out3 ^ out6 ^ in7; + out4 = tmp1 ^ out1; + out7 = out2 ^ out4 ^ in0; + out0 = out7 ^ in1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_82(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in2; + tmp0 = in6 ^ in7; + out5 = in2 ^ in3; + out6 = in3 ^ in4; + out7 = in0 ^ in4 ^ in5; + out0 = in1 ^ in5 ^ in6; + out1 = tmp0 ^ in0 ^ in2; + out2 = tmp0 ^ in3 ^ in5; + out3 = tmp0 ^ out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_83(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in5; + tmp2 = in3 ^ in6; + out4 = in1 ^ in2 ^ in4; + out0 = tmp0 ^ in5 ^ in6; + out5 = tmp1 ^ in3; + tmp3 = tmp1 ^ in7; + out6 = tmp2 ^ in4; + out2 = tmp2 ^ tmp3; + tmp4 = tmp3 ^ out4; + out1 = tmp3 ^ out0; + out3 = tmp4 ^ in3; + out7 = tmp0 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_84(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in6; + out6 = in3 ^ in5; + out0 = in1 ^ in5 ^ in7; + out7 = in0 ^ in4 ^ in6; + out4 = in1 ^ in3 ^ in6; + out5 = in2 ^ in4 ^ in7; + out2 = out6 ^ in0 ^ in1; + out3 = out5 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_85(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in4; + out1 = tmp0 ^ in2; + out6 = tmp1 ^ in5; + out4 = tmp2 ^ in3; + tmp3 = out1 ^ out6; + out2 = tmp3 ^ in0; + out3 = tmp2 ^ tmp3 ^ in7; + out7 = out2 ^ out3 ^ in1; + out5 = tmp1 ^ out3; + out0 = tmp2 ^ out7 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_86(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in3; + out7 = in0 ^ in4; + out0 = in1 ^ in5; + out5 = in2 ^ in7; + out3 = in4 ^ in5 ^ in6; + out1 = in0 ^ in2 ^ in6; + out4 = in1 ^ in6 ^ in7; + out2 = in0 ^ in3 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_87(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in3 ^ in6; + tmp0 = in0 ^ in1; + out7 = in0 ^ in4 ^ in7; + out5 = in2 ^ in5 ^ in7; + out3 = out6 ^ in4 ^ in5; + out0 = tmp0 ^ in5; + tmp1 = tmp0 ^ in6; + out2 = out5 ^ in0 ^ in3; + out1 = tmp1 ^ in2; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_88(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in7; + tmp0 = in5 ^ in6; + out0 = in1 ^ in6 ^ in7; + out6 = in4 ^ in5 ^ in7; + out3 = out0 ^ out1 ^ in0 ^ in4; + out7 = tmp0 ^ in0; + tmp1 = tmp0 ^ in3; + out2 = out0 ^ in3; + out4 = tmp1 ^ in2; + out5 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_89(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in6; + out1 = tmp1 ^ in1; + out7 = tmp2 ^ in5; + out0 = tmp2 ^ in1; + out2 = out1 ^ in3 ^ in6; + out6 = out7 ^ in0 ^ in4; + out5 = out6 ^ in3; + out3 = tmp0 ^ out2 ^ in4; + out4 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in6; + out7 = in0 ^ in5; + out2 = in3 ^ in6; + out6 = in4 ^ in7; + out1 = in0 ^ in2 ^ in7; + out3 = out0 ^ out6 ^ in0; + out4 = out1 ^ out7 ^ in6; + out5 = out2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in3 ^ in6; + tmp2 = in5 ^ in7; + tmp3 = tmp0 ^ in7; + out0 = tmp0 ^ in6; + out2 = tmp1 ^ in2; + out5 = tmp1 ^ tmp2; + out7 = tmp2 ^ in0; + tmp4 = tmp3 ^ in4; + out1 = tmp3 ^ in2; + out6 = tmp4 ^ out0; + out4 = out6 ^ in2 ^ in5; + out3 = tmp1 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2; + out0 = in1 ^ in7; + out7 = in0 ^ in6; + out5 = in4 ^ in6; + out6 = in5 ^ in7; + out2 = out0 ^ in0 ^ in3; + out3 = out5 ^ out7 ^ in2 ^ in7; + out4 = out6 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in2; + tmp0 = in6 ^ in7; + out0 = in0 ^ in1 ^ in7; + out5 = in4 ^ in5 ^ in6; + out6 = tmp0 ^ in5; + out7 = tmp0 ^ in0; + out4 = tmp0 ^ out5 ^ in3; + out2 = out0 ^ in2 ^ in3; + out3 = out2 ^ in1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1; + out4 = in5; + out7 = in0; + out5 = in6; + out6 = in7; + out3 = in0 ^ in4; + out1 = in0 ^ in2; + out2 = in0 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in1; + tmp0 = in0 ^ in3; + out4 = in4 ^ in5; + out7 = in0 ^ in7; + out5 = in5 ^ in6; + out6 = in6 ^ in7; + out1 = out0 ^ in2; + out2 = tmp0 ^ in2; + out3 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_90(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in6 ^ in7; + out3 = tmp0 ^ in7; + out1 = tmp1 ^ in5; + tmp2 = out1 ^ in4; + out6 = tmp2 ^ in3; + out5 = out6 ^ in1; + out4 = out5 ^ in0; + out0 = tmp0 ^ tmp2; + out7 = tmp0 ^ out4; + out2 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_91(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = tmp0 ^ in3 ^ in5; + out2 = tmp1 ^ in1; + out6 = tmp1 ^ in7; + tmp2 = out2 ^ in5 ^ in7; + out3 = tmp2 ^ in4; + out5 = tmp2 ^ in6; + out1 = tmp1 ^ out5 ^ in2; + tmp3 = out1 ^ in0; + out4 = tmp3 ^ in3; + out0 = tmp0 ^ tmp3; + out7 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_92(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1; + tmp0 = in4 ^ in5; + tmp1 = tmp0 ^ in1; + out2 = tmp0 ^ in3 ^ in7; + out0 = tmp1 ^ in6; + out7 = out2 ^ in0; + out4 = out0 ^ in0 ^ in2; + out5 = out4 ^ out7 ^ in5; + out6 = tmp1 ^ out5; + out1 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_93(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in3; + tmp0 = in2 ^ in7; + tmp1 = out3 ^ in6; + tmp2 = tmp0 ^ in4; + out5 = tmp0 ^ tmp1; + out6 = tmp2 ^ in3; + out2 = out6 ^ in5; + out0 = out2 ^ out5 ^ in0; + out7 = tmp1 ^ out0; + out1 = tmp2 ^ out0; + out4 = out1 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_94(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in6; + tmp0 = in1 ^ in4 ^ in5; + out1 = out3 ^ in5; + out5 = tmp0 ^ out3; + out0 = tmp0 ^ in7; + out4 = tmp0 ^ in0 ^ in3; + out6 = out1 ^ in3 ^ in7; + out2 = out4 ^ in6; + out7 = out0 ^ out2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_95(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out3 = tmp0 ^ in6; + tmp1 = tmp0 ^ in7; + tmp2 = out3 ^ in0; + out6 = tmp1 ^ in5; + tmp3 = tmp2 ^ in4; + out7 = tmp3 ^ in2; + tmp4 = tmp3 ^ in5; + out2 = tmp4 ^ in1; + tmp5 = out2 ^ in6; + out0 = tmp1 ^ tmp5; + out1 = tmp5 ^ out7; + out4 = tmp2 ^ out1; + out5 = tmp4 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_96(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in6 ^ in7; + tmp0 = in1 ^ in5; + tmp1 = in5 ^ in6; + out6 = out3 ^ in2 ^ in3; + out0 = tmp0 ^ in4; + tmp2 = tmp1 ^ in2; + out4 = out0 ^ in0 ^ in7; + out1 = tmp2 ^ in0; + out5 = tmp2 ^ in1; + out7 = tmp0 ^ out4 ^ in3; + out2 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_97(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in2 ^ in6; + out3 = in3 ^ in6 ^ in7; + out7 = tmp0 ^ in3; + tmp2 = tmp0 ^ in5; + out5 = tmp1 ^ in1; + out6 = tmp1 ^ out3; + out0 = tmp2 ^ in1; + out2 = tmp2 ^ out3 ^ in2; + tmp3 = out0 ^ in4; + out4 = tmp3 ^ in7; + out1 = tmp1 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_98(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in7; + tmp1 = in1 ^ in4 ^ in7; + out1 = tmp0 ^ in2; + out0 = tmp1 ^ in6; + out2 = tmp1 ^ in3; + out6 = out0 ^ out1 ^ in1; + out5 = tmp0 ^ out2; + out3 = tmp1 ^ out6 ^ in0; + out7 = out0 ^ out5 ^ in0; + out4 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_99(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + out5 = in1 ^ in3 ^ in4; + out6 = in2 ^ in4 ^ in5; + out4 = tmp0 ^ in2; + tmp1 = tmp0 ^ in6; + tmp2 = out5 ^ in7; + out7 = tmp1 ^ in5; + out0 = tmp1 ^ tmp2; + out2 = tmp2 ^ in2; + out3 = out0 ^ out6 ^ in3; + out1 = tmp1 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out5 = in1 ^ in3 ^ in5; + tmp2 = tmp0 ^ in7; + out3 = tmp0 ^ tmp1; + out0 = tmp1 ^ in4; + out7 = tmp2 ^ in3; + out1 = tmp2 ^ in2; + out6 = out0 ^ in1 ^ in2; + out4 = out1 ^ in4 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in3; + tmp0 = in3 ^ in5; + out6 = in2 ^ in4; + out4 = in0 ^ in2 ^ in7; + out7 = tmp0 ^ in0; + out2 = out6 ^ in3; + out1 = out4 ^ in1 ^ in5; + out3 = out7 ^ in1 ^ in6; + out0 = tmp0 ^ out3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in5; + tmp0 = in0 ^ in3 ^ in6; + out3 = out1 ^ in0; + out6 = out1 ^ in6; + out7 = tmp0 ^ in7; + out4 = out7 ^ in4; + out2 = out4 ^ in1; + out0 = tmp0 ^ out2; + out5 = out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in2 ^ in5; + tmp0 = in0 ^ in3; + out5 = in1 ^ in4 ^ in7; + out1 = out6 ^ in1; + out3 = tmp0 ^ out6; + out7 = tmp0 ^ in6; + out0 = out5 ^ in0; + out4 = out7 ^ in7; + out2 = out5 ^ out7 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in4; + tmp0 = in0 ^ in5; + out6 = in2 ^ in6; + out7 = in0 ^ in3 ^ in7; + out4 = in0 ^ in4 ^ in6; + out5 = in1 ^ in5 ^ in7; + out1 = tmp0 ^ in2; + out3 = tmp0 ^ in7; + out2 = out4 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in2; + out7 = in0 ^ in3; + tmp0 = in0 ^ in1; + out4 = in0 ^ in6; + out5 = in1 ^ in7; + out1 = tmp0 ^ in2 ^ in5; + out2 = out7 ^ in2 ^ in4 ^ in6; + out3 = out7 ^ in5 ^ in7; + out0 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + out2 = tmp0 ^ in7; + tmp1 = tmp0 ^ in5; + out6 = out2 ^ in3 ^ in4; + out0 = tmp1 ^ in3; + tmp2 = out0 ^ in2; + out3 = tmp2 ^ in7; + tmp3 = tmp2 ^ in1; + out5 = tmp3 ^ in0; + out4 = tmp3 ^ out6; + out7 = out5 ^ out6 ^ in1; + out1 = tmp1 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in4; + out4 = tmp1 ^ in7; + out7 = tmp2 ^ in0; + out6 = tmp2 ^ out4 ^ in3; + out3 = out4 ^ in6; + out2 = out3 ^ in5; + out1 = out2 ^ in4; + out5 = out1 ^ out6 ^ in0; + out0 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in6; + tmp0 = in1 ^ in3 ^ in5; + out3 = tmp0 ^ in6; + out4 = tmp0 ^ in2 ^ in4; + out0 = out3 ^ in7; + out6 = out0 ^ in4; + out1 = out0 ^ out4 ^ in0; + out7 = out1 ^ in5; + out5 = out7 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in6; + out3 = in1 ^ in5 ^ in6; + tmp0 = out2 ^ in0; + out4 = out2 ^ out3 ^ in3; + tmp1 = tmp0 ^ in4; + out0 = tmp0 ^ out4 ^ in7; + out5 = tmp1 ^ in3; + out7 = tmp1 ^ in5; + out1 = tmp1 ^ in1 ^ in7; + out6 = tmp1 ^ out0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in2 ^ in4; + tmp2 = in2 ^ in5; + tmp3 = in0 ^ in7; + out0 = tmp0 ^ in5; + out6 = tmp0 ^ in6 ^ in7; + out1 = tmp1 ^ in6; + out7 = tmp1 ^ tmp3; + out3 = tmp2 ^ in3; + tmp4 = tmp2 ^ out1; + out2 = tmp3 ^ in1; + out5 = tmp4 ^ out7; + out4 = tmp4 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in5; + tmp0 = in1 ^ in6; + tmp1 = in0 ^ in1; + tmp2 = in2 ^ in4; + out6 = in1 ^ in3 ^ in7; + out4 = tmp0 ^ in5; + out1 = tmp0 ^ tmp2; + out0 = tmp1 ^ in3 ^ in5; + out2 = tmp1 ^ in2 ^ in7; + out7 = tmp2 ^ in0; + out5 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0; + out3 = in3 ^ in5 ^ in7; + out1 = in0 ^ in2 ^ in4 ^ in6; + out0 = out3 ^ in1; + out7 = out1 ^ in7; + out6 = out0 ^ in6; + out5 = out7 ^ in5; + out4 = out6 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in2; + out3 = in5 ^ in7; + out7 = out2 ^ in4 ^ in6; + out6 = out3 ^ in1 ^ in3; + out1 = out7 ^ in1; + out5 = out7 ^ in7; + out0 = out6 ^ in0; + out4 = out6 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in1 ^ in6; + tmp2 = in0 ^ in2 ^ in7; + out1 = tmp0 ^ in7; + out4 = tmp0 ^ in6; + out0 = tmp1 ^ in3; + out2 = tmp1 ^ in5; + out6 = tmp1 ^ in4; + out7 = tmp2 ^ in5; + out3 = tmp2 ^ out0 ^ in6; + out5 = out7 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in6; + out6 = in1 ^ in4; + out7 = in0 ^ in2 ^ in5; + out5 = in0 ^ in3 ^ in7; + out2 = out4 ^ in1 ^ in5; + out1 = out6 ^ in2 ^ in7; + out0 = out2 ^ out7 ^ in3; + out3 = out1 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in1 ^ in3; + tmp2 = in6 ^ in7; + out1 = tmp0 ^ in4 ^ in7; + out3 = tmp1 ^ in0; + out0 = tmp1 ^ tmp2; + out2 = tmp2 ^ in5; + out7 = tmp0 ^ out2; + out6 = out1 ^ out7 ^ in1; + out5 = out0 ^ out6 ^ in0; + out4 = out5 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in1; + tmp0 = in1 ^ in4; + tmp1 = in0 ^ in7; + out6 = tmp0 ^ in5; + out1 = tmp0 ^ tmp1 ^ in2; + out5 = tmp1 ^ in3 ^ in4; + out0 = tmp0 ^ out5 ^ in6; + out4 = out0 ^ out3 ^ in2; + out2 = out4 ^ in3 ^ in5; + out7 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in3; + out1 = in2 ^ in4; + tmp0 = in0 ^ in2; + out4 = in4 ^ in7; + out5 = in0 ^ in5; + out6 = in1 ^ in6; + out7 = tmp0 ^ in7; + out3 = tmp0 ^ in3 ^ in6; + out2 = out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in7; + out5 = in0; + out6 = in1; + out7 = in0 ^ in2; + out0 = in0 ^ in1 ^ in3; + out2 = out7 ^ in1 ^ in5; + out1 = in1 ^ in2 ^ in4; + out3 = out7 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3 ^ in4; + tmp0 = in0 ^ in4; + tmp1 = in0 ^ in7; + out0 = in1 ^ in3 ^ in7; + out1 = tmp0 ^ in2; + out5 = tmp0 ^ in5; + tmp2 = tmp1 ^ in6; + out2 = tmp1 ^ in5; + out3 = tmp2 ^ in3; + out7 = tmp2 ^ in2; + out6 = tmp2 ^ out2 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3; + tmp0 = in0 ^ in7; + out5 = in0 ^ in4; + out6 = in1 ^ in5; + out7 = in0 ^ in2 ^ in6; + out0 = tmp0 ^ in1 ^ in3; + out3 = tmp0 ^ in6; + out2 = tmp0 ^ in2 ^ in5; + out1 = out5 ^ in1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + tmp1 = in3 ^ in6; + out2 = tmp0 ^ in7; + tmp2 = tmp0 ^ tmp1; + out0 = tmp2 ^ in5; + out3 = tmp2 ^ in2; + out6 = out3 ^ in6; + tmp3 = out6 ^ in0 ^ in1; + out7 = tmp3 ^ in5; + out5 = tmp3 ^ out2; + out1 = out0 ^ out5 ^ in0; + out4 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + out2 = tmp0 ^ in2 ^ in7; + tmp1 = out2 ^ in6; + out1 = tmp1 ^ in5; + out3 = tmp1 ^ in7; + out4 = tmp1 ^ in0; + out6 = out3 ^ in3; + out0 = out6 ^ in0 ^ in2 ^ in5; + out5 = tmp1 ^ out0 ^ in1; + out7 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4; + tmp0 = in4 ^ in7; + tmp1 = in1 ^ in3 ^ in6; + out3 = tmp0 ^ tmp1; + tmp2 = tmp1 ^ in0; + out0 = out3 ^ in5; + out4 = tmp2 ^ in2; + tmp3 = out4 ^ in6; + out5 = tmp0 ^ tmp3; + out1 = tmp3 ^ out0; + tmp4 = out1 ^ in7; + out7 = tmp4 ^ in3; + out6 = tmp2 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out3 = tmp1 ^ in4 ^ in7; + tmp2 = tmp0 ^ out3; + out0 = tmp2 ^ in3; + out1 = tmp2 ^ in2; + out5 = out0 ^ in2 ^ in6; + out7 = tmp1 ^ out5; + out4 = out7 ^ in1 ^ in5 ^ in7; + out6 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in1; + out5 = out4 ^ in2; + tmp0 = out4 ^ in4; + out6 = out5 ^ in0 ^ in3; + out7 = tmp0 ^ out6; + out2 = tmp0 ^ in6 ^ in7; + out3 = out7 ^ in0 ^ in7; + out0 = out5 ^ out7 ^ in5; + out1 = out0 ^ out6 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in4; + out4 = tmp0 ^ in4; + out3 = tmp1 ^ in7; + tmp2 = out4 ^ in5; + out7 = out3 ^ in0 ^ in3; + out0 = tmp2 ^ in3; + out2 = tmp0 ^ out3 ^ in6; + out5 = tmp1 ^ tmp2; + out6 = out2 ^ out7 ^ in2; + out1 = tmp0 ^ out0 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in4; + tmp0 = in1 ^ in2; + tmp1 = in0 ^ in4; + tmp2 = in3 ^ in5; + tmp3 = out3 ^ in1 ^ in7; + out5 = tmp0 ^ tmp1; + out6 = tmp0 ^ tmp2; + out2 = tmp1 ^ in6; + out4 = tmp1 ^ tmp3; + out0 = tmp3 ^ in5; + out1 = out2 ^ in2 ^ in5; + out7 = tmp2 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4; + tmp0 = in0 ^ in4; + out2 = tmp0 ^ in2 ^ in6; + tmp1 = out2 ^ in7; + out1 = out2 ^ in1 ^ in5; + out7 = tmp1 ^ in3; + out5 = out1 ^ in6; + out6 = tmp0 ^ out1 ^ in3; + out0 = tmp1 ^ out6; + out4 = out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + tmp1 = in2 ^ in5; + out2 = tmp0 ^ in5; + out4 = tmp1 ^ in0; + tmp2 = tmp1 ^ in7; + out6 = tmp2 ^ out2; + out7 = out4 ^ in3; + out1 = tmp2 ^ in4; + out3 = tmp0 ^ out7; + out0 = out3 ^ out4 ^ in6; + out5 = out0 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in4 ^ in5; + out4 = tmp0 ^ tmp1; + tmp2 = tmp0 ^ in3 ^ in7; + out3 = out4 ^ in1; + out7 = tmp2 ^ in5; + out2 = out3 ^ in0; + out1 = out2 ^ in7; + out6 = out1 ^ in5 ^ in6; + out0 = tmp2 ^ out6; + out5 = tmp1 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in7; + out2 = tmp0 ^ in4; + tmp1 = out2 ^ in2; + out1 = tmp1 ^ in0; + out6 = tmp1 ^ in1; + out4 = out1 ^ in3 ^ in4; + tmp2 = out4 ^ out6; + out7 = out4 ^ in6 ^ in7; + out5 = tmp2 ^ in6; + out3 = tmp0 ^ tmp2; + out0 = out6 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in4 ^ in5 ^ in7; + tmp0 = out2 ^ in1; + out4 = out2 ^ in0 ^ in3; + out1 = tmp0 ^ in0; + out6 = tmp0 ^ in6; + out3 = out1 ^ in2; + tmp1 = out4 ^ out6 ^ in4; + out0 = tmp1 ^ in7; + out5 = tmp1 ^ in5; + out7 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in2 ^ in4; + out0 = in1 ^ in3 ^ in4; + out6 = in1 ^ in2 ^ in7; + out7 = tmp0 ^ in3; + out5 = tmp0 ^ out6 ^ in6; + out1 = tmp1 ^ in5; + tmp2 = out1 ^ out5 ^ in1; + out3 = tmp2 ^ in3; + out4 = tmp1 ^ tmp2; + out2 = tmp2 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in1 ^ in4; + out0 = tmp0 ^ tmp1; + out7 = tmp0 ^ in2 ^ in7; + out1 = tmp1 ^ in2 ^ in5; + tmp2 = out1 ^ in0; + out2 = tmp2 ^ in6; + out3 = out2 ^ in1 ^ in7; + out4 = out3 ^ in2; + out5 = tmp1 ^ out4; + out6 = tmp2 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3 ^ in6; + out4 = tmp0 ^ in5; + out7 = tmp0 ^ in2; + out3 = out4 ^ in4; + out1 = out3 ^ out7 ^ in0; + out2 = out3 ^ in3 ^ in7; + out0 = out2 ^ out4 ^ in1; + out5 = tmp0 ^ out0; + out6 = out1 ^ out5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + out3 = tmp0 ^ in5 ^ in6; + out4 = out3 ^ in3; + tmp1 = out3 ^ in7; + out2 = tmp1 ^ in2; + out5 = tmp1 ^ in1; + tmp2 = out2 ^ in5; + out7 = tmp2 ^ in3 ^ in4; + tmp3 = tmp0 ^ out5; + out0 = tmp3 ^ out4; + out1 = tmp2 ^ tmp3; + out6 = tmp3 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2 ^ in5; + tmp0 = in1 ^ in4; + tmp1 = in3 ^ in6; + out0 = out5 ^ in1; + out4 = tmp0 ^ in7; + out3 = tmp0 ^ tmp1; + out1 = tmp1 ^ in2; + out6 = tmp1 ^ in0; + out7 = out4 ^ in0; + out2 = out4 ^ out5 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2; + tmp0 = in0 ^ in1; + out4 = in1 ^ in7; + out6 = in0 ^ in3; + out3 = in1 ^ in4 ^ in6; + tmp1 = tmp0 ^ in2; + out7 = tmp0 ^ in4; + out0 = tmp1 ^ in5; + out1 = tmp1 ^ out6 ^ in6; + out2 = out6 ^ out7 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in3 ^ in4; + tmp0 = in0 ^ in3 ^ in6; + out5 = in2 ^ in4 ^ in5; + tmp1 = out4 ^ in7; + out1 = tmp0 ^ in2; + out6 = tmp0 ^ in5; + out2 = out5 ^ in3; + out7 = tmp0 ^ tmp1; + out3 = tmp1 ^ in2 ^ in6; + out0 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in3; + tmp0 = in0 ^ in2; + tmp1 = in3 ^ in5; + out5 = in2 ^ in4; + tmp2 = tmp0 ^ out4; + out2 = tmp1 ^ in4; + out6 = tmp1 ^ in0; + out0 = tmp1 ^ tmp2 ^ in7; + out1 = tmp2 ^ in6; + out7 = out1 ^ out5 ^ in3; + out3 = tmp0 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in7; + out3 = tmp0 ^ in4; + tmp1 = tmp0 ^ in2; + out1 = tmp1 ^ in6; + out5 = tmp1 ^ in5; + out4 = out1 ^ out3 ^ in1; + out0 = out4 ^ in4 ^ in5; + out2 = out0 ^ out3 ^ in0; + out7 = out1 ^ out2 ^ in7; + out6 = tmp1 ^ out0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4 ^ in7; + tmp0 = in3 ^ in7; + out4 = in1 ^ in2 ^ in6; + out6 = in0 ^ in3 ^ in4; + out5 = tmp0 ^ in2; + out1 = tmp0 ^ out4; + out0 = out4 ^ in0 ^ in5; + out2 = out0 ^ out5 ^ in4; + out7 = tmp0 ^ out2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in6; + tmp1 = in1 ^ in7; + tmp2 = tmp0 ^ in0; + tmp3 = tmp0 ^ tmp1; + tmp4 = tmp2 ^ in4; + out0 = tmp3 ^ in2; + out6 = tmp4 ^ in3; + out2 = out6 ^ in2; + out7 = tmp1 ^ tmp4; + out3 = tmp2 ^ out2; + tmp5 = out3 ^ in5; + out5 = tmp5 ^ in7; + out4 = tmp3 ^ tmp5; + out1 = tmp4 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in4; + tmp0 = in3 ^ in5; + tmp1 = out3 ^ in7; + out6 = tmp0 ^ in0 ^ in4; + out5 = tmp1 ^ in3; + out2 = out6 ^ in6; + out7 = out2 ^ in1 ^ in3; + out0 = tmp1 ^ out7; + out1 = tmp0 ^ out0; + out4 = out1 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in2; + out1 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in0 ^ in7; + out2 = out1 ^ in1 ^ in4; + out4 = tmp0 ^ in4; + out5 = tmp0 ^ in7; + out6 = tmp1 ^ in6; + out7 = tmp1 ^ in1; + out3 = out2 ^ in0 ^ in2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in5 ^ in6; + out7 = in0 ^ in1; + tmp0 = in1 ^ in3; + out5 = in6 ^ in7; + out6 = in0 ^ in7; + out0 = out7 ^ in2; + out3 = out7 ^ in4 ^ in5; + out1 = tmp0 ^ in2; + out2 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in6; + out0 = tmp1 ^ in1; + tmp3 = tmp1 ^ in3; + out6 = tmp2 ^ in5; + out7 = tmp2 ^ in1; + out2 = tmp3 ^ in4; + out5 = out6 ^ in0 ^ in4; + out4 = out5 ^ in3; + out1 = tmp0 ^ tmp3; + out3 = tmp3 ^ out5 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in7; + tmp1 = in5 ^ in7; + out7 = in0 ^ in1 ^ in6; + out5 = tmp0 ^ in6; + out2 = tmp0 ^ in3; + out6 = tmp1 ^ in0; + out4 = tmp1 ^ in3 ^ in6; + tmp2 = out5 ^ out7 ^ in2; + out1 = tmp2 ^ out2; + out0 = tmp2 ^ in4; + out3 = tmp2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = in1 ^ in6; + out1 = in2 ^ in3 ^ in7; + out5 = tmp0 ^ in6; + out0 = tmp1 ^ in2; + tmp2 = out5 ^ in0 ^ in7; + out3 = tmp2 ^ in4; + out6 = tmp0 ^ out3; + out7 = tmp1 ^ tmp2 ^ in3; + tmp3 = out1 ^ out6; + out4 = tmp2 ^ tmp3; + out2 = tmp3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in3 ^ in6; + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in7; + out6 = in0 ^ in4 ^ in7; + out2 = tmp0 ^ out5 ^ in4; + out7 = tmp0 ^ in5; + out0 = tmp0 ^ in2 ^ in6; + out4 = tmp1 ^ in5; + out1 = tmp1 ^ in1 ^ in3; + out3 = out6 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = tmp0 ^ in3; + out4 = tmp1 ^ in4; + tmp2 = out4 ^ in6; + out3 = tmp2 ^ in0; + out5 = tmp2 ^ in2; + out2 = out3 ^ in5 ^ in7; + out6 = tmp1 ^ out2; + out7 = out2 ^ out4 ^ in1; + out1 = tmp2 ^ out6; + out0 = tmp0 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in6; + tmp1 = in0 ^ in1 ^ in5; + out4 = in2 ^ in3 ^ in5; + out5 = tmp0 ^ in4; + out7 = tmp1 ^ in6; + out1 = tmp1 ^ out4 ^ in7; + tmp2 = out5 ^ in0; + out2 = tmp2 ^ in7; + out3 = tmp2 ^ out4; + out6 = tmp0 ^ out2 ^ in5; + out0 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in1 ^ in4; + tmp2 = in2 ^ in5; + out7 = tmp0 ^ tmp1; + out0 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in3; + out1 = tmp3 ^ in6; + tmp4 = out1 ^ in1; + out2 = tmp4 ^ in7; + out3 = out2 ^ in2; + out4 = tmp0 ^ out3; + out5 = tmp3 ^ out3; + out6 = tmp4 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5 ^ in6; + tmp1 = tmp0 ^ in1; + out1 = tmp1 ^ in2; + out2 = tmp1 ^ in7; + out3 = out2 ^ in3; + out5 = out3 ^ in2; + tmp2 = out3 ^ in0; + out4 = tmp2 ^ in4; + out7 = tmp0 ^ out4; + out6 = tmp2 ^ out1 ^ in6; + out0 = out2 ^ out6 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in6; + out2 = tmp0 ^ in2 ^ in3; + out1 = out2 ^ in0; + out3 = out2 ^ in1; + out4 = out1 ^ in1 ^ in2; + out6 = out1 ^ in6 ^ in7; + out7 = out4 ^ in4 ^ in5; + out5 = out4 ^ out6 ^ in4; + out0 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in5 ^ in6; + tmp0 = out2 ^ in2; + tmp1 = tmp0 ^ in1; + out1 = tmp1 ^ in0; + out3 = tmp1 ^ in3; + out4 = out1 ^ in2 ^ in4; + tmp2 = out4 ^ in5; + out7 = tmp2 ^ in7; + out0 = tmp0 ^ out7; + tmp3 = out0 ^ in0; + out5 = tmp3 ^ in6; + out6 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in5; + tmp0 = in1 ^ in5; + tmp1 = tmp0 ^ in2; + out4 = tmp1 ^ in0; + tmp2 = tmp1 ^ in6; + out2 = out4 ^ in3 ^ in7; + out0 = tmp2 ^ in4; + out5 = tmp2 ^ out3; + out1 = tmp0 ^ out5 ^ in7; + out6 = tmp0 ^ out2 ^ in4; + out7 = tmp1 ^ out6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in5; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in1 ^ in5; + out4 = tmp1 ^ in2; + out0 = out4 ^ in6; + tmp2 = tmp0 ^ out0; + out5 = tmp2 ^ in3; + out1 = out5 ^ in7; + out6 = tmp1 ^ out1; + out7 = tmp2 ^ out6; + out2 = out7 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2 ^ in4 ^ in6; + out5 = tmp0 ^ in3; + out0 = tmp0 ^ in5 ^ in7; + out3 = out0 ^ out5 ^ in2; + tmp1 = out3 ^ in0; + out1 = tmp1 ^ in6; + out2 = tmp1 ^ in7; + out4 = tmp1 ^ in1; + out6 = tmp1 ^ in4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + out3 = in2 ^ in5 ^ in7; + out2 = tmp0 ^ in5; + tmp1 = tmp0 ^ out3 ^ in1; + out1 = tmp1 ^ in6; + out4 = tmp1 ^ in4; + tmp2 = out1 ^ in4; + out6 = tmp2 ^ in1; + out7 = tmp2 ^ in2; + out0 = tmp2 ^ in3; + out5 = tmp2 ^ in0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0; + out5 = in1; + tmp0 = in1 ^ in2; + out6 = in0 ^ in2; + out0 = tmp0 ^ in4; + tmp1 = tmp0 ^ in3; + out7 = tmp1 ^ out6; + out2 = tmp1 ^ in6; + out3 = out7 ^ in7; + out1 = tmp1 ^ in1 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in4; + out5 = in1 ^ in5; + out2 = in1 ^ in3 ^ in6; + out3 = in0 ^ in1 ^ in7; + out6 = in0 ^ in2 ^ in6; + out0 = out4 ^ in1 ^ in2; + out1 = out5 ^ in2 ^ in3; + out7 = out3 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in4; + tmp0 = in2 ^ in7; + tmp1 = in0 ^ in2 ^ in3; + out0 = tmp0 ^ out5; + out4 = tmp0 ^ tmp1; + out2 = tmp0 ^ in3 ^ in6; + out1 = tmp1 ^ in5; + out3 = tmp1 ^ in1; + out6 = out1 ^ in3; + out7 = out3 ^ in2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in1 ^ in5; + tmp2 = in3 ^ in7; + out3 = tmp0 ^ in2; + out5 = tmp1 ^ in4; + out6 = tmp1 ^ out3 ^ in6; + out2 = tmp2 ^ in6; + tmp3 = tmp2 ^ in4; + tmp4 = out3 ^ in3; + out4 = tmp3 ^ in0; + out1 = tmp4 ^ in5; + out0 = tmp3 ^ tmp4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in0 ^ in3; + out6 = tmp0 ^ in4; + tmp2 = tmp0 ^ in7; + out3 = tmp1 ^ in6; + tmp3 = tmp1 ^ in1; + out1 = tmp1 ^ tmp2 ^ in5; + out4 = tmp2 ^ in6; + out2 = tmp3 ^ in2; + out7 = tmp3 ^ in5; + out5 = tmp2 ^ out2; + out0 = out2 ^ out3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in6; + out2 = in0 ^ in1 ^ in3; + out6 = out3 ^ in2 ^ in4; + out7 = out2 ^ in5 ^ in7; + out0 = out6 ^ in1; + out4 = out6 ^ in7; + out5 = out7 ^ in0; + out1 = out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3 ^ in6; + tmp1 = in3 ^ in4 ^ in7; + out4 = tmp0 ^ in0; + out5 = tmp1 ^ in1; + out3 = out4 ^ in7; + out2 = out3 ^ in6; + out1 = out2 ^ in5; + out6 = tmp1 ^ out1; + out0 = tmp0 ^ out5; + out7 = out0 ^ out1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in3 ^ in7; + tmp0 = out2 ^ in1 ^ in5; + out1 = tmp0 ^ in2; + out7 = tmp0 ^ in6; + out5 = tmp0 ^ in0 ^ in4; + tmp1 = out1 ^ out5 ^ in6; + out4 = tmp1 ^ in3; + out6 = tmp1 ^ in5; + tmp2 = tmp1 ^ in7; + out0 = tmp2 ^ in1; + out3 = tmp2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in7; + tmp0 = in2 ^ in4; + out4 = out3 ^ in3 ^ in5; + out2 = tmp0 ^ in1; + tmp1 = tmp0 ^ in6; + out0 = out4 ^ in2; + out6 = out4 ^ in0; + out1 = tmp1 ^ in3; + out5 = tmp1 ^ in0; + out7 = out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in4; + tmp0 = in1 ^ in7; + out3 = tmp0 ^ in3; + tmp1 = out3 ^ in5; + out4 = tmp1 ^ in4; + tmp2 = tmp1 ^ in0; + out0 = tmp2 ^ in2; + out6 = tmp2 ^ in6; + tmp3 = out0 ^ out4 ^ in6; + out5 = tmp3 ^ in5; + out7 = tmp0 ^ tmp3; + out1 = tmp2 ^ out5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in2; + out4 = in1 ^ in5; + out2 = in2 ^ in4 ^ in7; + out5 = in0 ^ in2 ^ in6; + out0 = out3 ^ in3 ^ in5; + out7 = out3 ^ in0 ^ in4; + out6 = out2 ^ out7 ^ in3; + out1 = out5 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4 ^ in7; + tmp0 = in1 ^ in3; + out3 = tmp0 ^ in2; + tmp1 = out3 ^ in0; + out0 = tmp1 ^ in5; + tmp2 = tmp1 ^ in4; + out1 = tmp2 ^ in6; + tmp3 = tmp2 ^ in3; + out7 = tmp3 ^ in7; + out6 = out1 ^ out2 ^ in2; + tmp4 = tmp0 ^ out0; + out5 = tmp4 ^ in6; + out4 = tmp3 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in6; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in2 ^ in6; + out2 = tmp1 ^ in1; + out7 = out2 ^ in5; + tmp2 = tmp0 ^ out7; + out4 = tmp2 ^ in3; + out0 = out4 ^ in7; + out6 = tmp1 ^ out0; + out5 = tmp2 ^ out6; + out1 = out5 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in6; + tmp0 = in0 ^ in1; + tmp1 = in5 ^ in7; + out2 = tmp0 ^ in4 ^ in6; + tmp2 = tmp1 ^ out2; + out6 = tmp2 ^ in3; + out7 = tmp2 ^ in2; + out0 = out6 ^ in2 ^ in4; + out5 = out6 ^ in1 ^ in2; + out1 = tmp0 ^ out5 ^ in5; + out4 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in6 ^ in7; + out2 = out3 ^ in0 ^ in4; + out4 = out3 ^ in1 ^ in5; + out1 = out2 ^ in3; + out7 = out2 ^ out4 ^ in2; + out0 = out4 ^ in3 ^ in7; + out5 = out1 ^ in4; + out6 = out0 ^ out2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out3 = tmp0 ^ in6 ^ in7; + tmp1 = out3 ^ in0; + out5 = tmp1 ^ in5; + tmp2 = tmp1 ^ in4; + tmp3 = out5 ^ in7; + out1 = tmp2 ^ in1; + out0 = tmp3 ^ in1; + out6 = out1 ^ in2; + out2 = tmp0 ^ tmp2; + tmp4 = tmp3 ^ out6; + out4 = tmp4 ^ in6; + out7 = tmp4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3 ^ in6; + tmp0 = in4 ^ in7; + out1 = in2 ^ in3 ^ in4; + out5 = tmp0 ^ in0; + tmp1 = tmp0 ^ in1; + tmp2 = tmp1 ^ in5; + out0 = tmp1 ^ out1; + out2 = tmp2 ^ in2; + out6 = tmp2 ^ out5; + tmp3 = out6 ^ in6; + out3 = tmp3 ^ in7; + out7 = tmp3 ^ in2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in6; + out4 = tmp1 ^ in4; + out6 = tmp2 ^ in5; + out7 = tmp2 ^ in2 ^ in7; + out3 = out6 ^ in3 ^ in7; + out0 = tmp1 ^ out7; + out2 = out3 ^ out4 ^ in0; + out5 = tmp0 ^ out2; + out1 = out0 ^ out5 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in6 ^ in7; + out5 = in0 ^ in7; + out6 = in0 ^ in1; + out0 = in1 ^ in2 ^ in3; + out2 = in2 ^ in4 ^ in5; + out7 = out6 ^ in2; + out1 = out0 ^ out6 ^ in4; + out3 = out7 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4 ^ in5; + tmp0 = in0 ^ in1; + out4 = in4 ^ in6 ^ in7; + out5 = in0 ^ in5 ^ in7; + out6 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out0 = tmp1 ^ in3; + out7 = tmp1 ^ in7; + out1 = out0 ^ in4; + out3 = out0 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in5; + out4 = in2 ^ in3 ^ in7; + out5 = in0 ^ in3 ^ in4; + out6 = out3 ^ in1 ^ in4; + out1 = out4 ^ in4; + out0 = out4 ^ in1 ^ in6; + out2 = out0 ^ out5 ^ in5; + out7 = out2 ^ in4 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_ED(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in3 ^ in5; + out4 = tmp0 ^ in3 ^ in7; + out3 = tmp1 ^ in0; + out1 = out4 ^ in1; + out5 = out3 ^ in4; + out7 = out1 ^ out5 ^ in6; + out2 = tmp0 ^ out7; + out0 = tmp1 ^ out7; + out6 = out2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2; + tmp0 = in0 ^ in1; + out5 = in0 ^ in3; + tmp1 = tmp0 ^ in2; + out6 = tmp0 ^ in4; + tmp2 = tmp1 ^ out5; + out7 = tmp1 ^ in5; + out1 = tmp2 ^ out6 ^ in7; + out0 = tmp2 ^ in6; + tmp3 = out7 ^ in1; + out3 = tmp3 ^ in7; + out2 = tmp3 ^ in4 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in6; + out5 = tmp0 ^ in3; + out2 = tmp0 ^ tmp1; + out6 = tmp1 ^ in0 ^ in1; + out3 = out5 ^ in2 ^ in7; + out7 = out3 ^ in1 ^ in3; + out0 = out4 ^ out6 ^ in3; + out1 = tmp1 ^ out0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in4 ^ in5; + out2 = tmp0 ^ in6; + out3 = tmp1 ^ in1; + tmp2 = tmp1 ^ in7; + out1 = out2 ^ out3 ^ in3; + tmp3 = tmp0 ^ tmp2; + out0 = tmp3 ^ in3; + out5 = tmp3 ^ in0; + out4 = out1 ^ out5 ^ in4; + out7 = out4 ^ in2; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in6; + tmp0 = in3 ^ in5; + out3 = tmp0 ^ in1 ^ in4; + tmp1 = out3 ^ in2; + out1 = tmp1 ^ in6; + tmp2 = tmp1 ^ in0; + tmp3 = out1 ^ in5; + out0 = tmp2 ^ in7; + out6 = tmp2 ^ in4; + out7 = tmp3 ^ in0; + out5 = tmp0 ^ out0; + out4 = tmp3 ^ out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + out2 = in2 ^ in6 ^ in7; + tmp1 = tmp0 ^ in1; + tmp2 = tmp1 ^ in2; + out0 = tmp2 ^ in3; + out3 = tmp2 ^ in7; + out5 = out3 ^ in0 ^ in4; + tmp3 = tmp0 ^ out5; + out7 = tmp3 ^ in3; + out4 = tmp3 ^ out2; + out1 = out0 ^ out4 ^ in4; + out6 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in6 ^ in7; + tmp0 = in0 ^ in1; + out4 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out5 = tmp1 ^ in7; + out6 = tmp1 ^ in3; + out7 = out6 ^ in4; + out0 = out7 ^ in5; + out1 = out0 ^ in6; + out3 = out0 ^ in0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in1 ^ in2; + tmp0 = out2 ^ in3; + out4 = tmp0 ^ in4; + out5 = out4 ^ in5; + out6 = out5 ^ in6; + out7 = out6 ^ in7; + out0 = out7 ^ in0; + out1 = out0 ^ in1; + out3 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in1; + tmp0 = out2 ^ in2; + out4 = tmp0 ^ in3; + out5 = out4 ^ in4; + out6 = out5 ^ in5; + out7 = out6 ^ in6; + out0 = out7 ^ in7; + out1 = out0 ^ in0; + out3 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + out2 = tmp0 ^ in2; + out4 = out2 ^ in1 ^ in4; + out7 = out4 ^ in3 ^ in5; + out5 = out7 ^ in4 ^ in7; + out0 = tmp0 ^ out7 ^ in6; + tmp1 = out0 ^ in1; + out6 = out0 ^ in0 ^ in5; + out3 = tmp1 ^ in3; + out1 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in7; + tmp0 = out2 ^ in1; + out4 = tmp0 ^ in2; + out5 = out4 ^ in3 ^ in7; + out6 = out5 ^ in4; + out7 = out6 ^ in5; + out0 = out7 ^ in6; + out1 = out0 ^ in7; + out3 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in3 ^ in5; + tmp2 = tmp0 ^ in6; + out4 = tmp0 ^ tmp1; + out1 = tmp1 ^ in2 ^ in4; + out3 = tmp2 ^ in1; + out5 = out3 ^ in5; + out7 = out1 ^ out5 ^ in7; + out6 = tmp1 ^ out7; + out0 = tmp2 ^ out7; + out2 = out6 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = in0 ^ in6; + out4 = tmp0 ^ in0; + tmp2 = tmp1 ^ in4; + tmp3 = tmp1 ^ in2; + out5 = tmp2 ^ in1; + out3 = out5 ^ in3; + tmp4 = tmp3 ^ out3; + out1 = tmp4 ^ in5; + out0 = tmp4 ^ in0 ^ in7; + out6 = tmp0 ^ out0 ^ in4; + out7 = tmp2 ^ tmp4; + out2 = tmp3 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = tmp0 ^ in2; + tmp2 = tmp0 ^ in5; + tmp3 = tmp1 ^ in7; + out5 = tmp2 ^ in6; + out6 = tmp3 ^ in6; + out7 = tmp3 ^ in3; + out3 = out6 ^ in4; + out2 = tmp1 ^ out5; + out4 = out2 ^ out3 ^ in1; + out0 = out4 ^ out7 ^ in5; + out1 = tmp2 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in5 ^ in6; + tmp0 = in0 ^ in1; + out4 = in0 ^ in5 ^ in7; + out5 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out6 = tmp1 ^ in7; + out7 = tmp1 ^ in3; + out0 = out7 ^ in4; + out1 = out0 ^ in5; + out3 = out0 ^ in6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in0 ^ in7; + out2 = tmp0 ^ tmp1 ^ in5; + out3 = tmp1 ^ in4; + tmp2 = out2 ^ in6; + out6 = tmp2 ^ in4; + out7 = tmp2 ^ in3; + out4 = out6 ^ in1 ^ in3; + tmp3 = out4 ^ in0; + out1 = tmp3 ^ in6; + out0 = tmp3 ^ in1 ^ in5; + out5 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in7; + out2 = tmp0 ^ tmp1; + out6 = out2 ^ in2 ^ in4; + tmp2 = out6 ^ in0; + out1 = tmp2 ^ in3; + out0 = tmp0 ^ out1 ^ in6; + out5 = out0 ^ in2; + tmp3 = out5 ^ in1; + out3 = tmp3 ^ in6; + out7 = tmp2 ^ tmp3; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + out2 = tmp0 ^ in5; + out3 = tmp0 ^ in4; + tmp1 = out3 ^ in6; + out4 = tmp1 ^ in5; + tmp2 = tmp1 ^ in1; + out6 = tmp2 ^ in7; + tmp3 = tmp2 ^ in0; + out0 = tmp3 ^ in3; + tmp4 = out0 ^ out4 ^ in7; + out5 = tmp4 ^ in6; + out7 = tmp4 ^ in2; + out1 = tmp3 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in5; + tmp0 = in4 ^ in7; + tmp1 = out2 ^ in2; + out4 = tmp1 ^ in6; + out7 = tmp1 ^ in1 ^ in3; + out1 = tmp0 ^ out7; + tmp2 = out1 ^ in5; + out6 = tmp2 ^ in3; + tmp3 = tmp2 ^ in7; + out0 = tmp3 ^ in6; + out3 = tmp3 ^ in1; + out5 = tmp0 ^ out0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void (*gf8_muladd[])(void *out, void *in) = { + gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, gf8_muladd_04, + gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, gf8_muladd_08, gf8_muladd_09, + gf8_muladd_0A, gf8_muladd_0B, gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, + gf8_muladd_0F, gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13, + gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, gf8_muladd_18, + gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, gf8_muladd_1C, gf8_muladd_1D, + gf8_muladd_1E, gf8_muladd_1F, gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, + gf8_muladd_23, gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27, + gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, gf8_muladd_2C, + gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, gf8_muladd_30, gf8_muladd_31, + gf8_muladd_32, gf8_muladd_33, gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, + gf8_muladd_37, gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B, + gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, gf8_muladd_40, + gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, gf8_muladd_44, gf8_muladd_45, + gf8_muladd_46, gf8_muladd_47, gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, + gf8_muladd_4B, gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F, + gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, gf8_muladd_54, + gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, gf8_muladd_58, gf8_muladd_59, + gf8_muladd_5A, gf8_muladd_5B, gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, + gf8_muladd_5F, gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63, + gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, gf8_muladd_68, + gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, gf8_muladd_6C, gf8_muladd_6D, + gf8_muladd_6E, gf8_muladd_6F, gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, + gf8_muladd_73, gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77, + gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, gf8_muladd_7C, + gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, gf8_muladd_80, gf8_muladd_81, + gf8_muladd_82, gf8_muladd_83, gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, + gf8_muladd_87, gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B, + gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, gf8_muladd_90, + gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, gf8_muladd_94, gf8_muladd_95, + gf8_muladd_96, gf8_muladd_97, gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, + gf8_muladd_9B, gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F, + gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, gf8_muladd_A4, + gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, gf8_muladd_A8, gf8_muladd_A9, + gf8_muladd_AA, gf8_muladd_AB, gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, + gf8_muladd_AF, gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3, + gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, gf8_muladd_B8, + gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, gf8_muladd_BC, gf8_muladd_BD, + gf8_muladd_BE, gf8_muladd_BF, gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, + gf8_muladd_C3, gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7, + gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, gf8_muladd_CC, + gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, gf8_muladd_D0, gf8_muladd_D1, + gf8_muladd_D2, gf8_muladd_D3, gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, + gf8_muladd_D7, gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB, + gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, gf8_muladd_E0, + gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, gf8_muladd_E4, gf8_muladd_E5, + gf8_muladd_E6, gf8_muladd_E7, gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, + gf8_muladd_EB, gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF, + gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, gf8_muladd_F4, + gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, gf8_muladd_F8, gf8_muladd_F9, + gf8_muladd_FA, gf8_muladd_FB, gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, + gf8_muladd_FF}; + +static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = { + 0, +}; + +void +ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count) +{ + uint32_t i, last, tmp; + + last = 1; + for (i = count; i > 0; i--) { + if (values[i - 1] != 0) { + tmp = values[i - 1]; + values[i - 1] = ec_gf_div(gf, tmp, last); + last = tmp; + } + } +} + +void +ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values, + uint32_t count) +{ + src += offset; + gf8_muladd_00(dst, src); + while (--count > 0) { + src += EC_METHOD_CHUNK_SIZE; + gf8_muladd[*values](dst, src); + values++; + } +} + +void +ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values, + uint32_t count) +{ + uint32_t i, last, tmp; + + i = 0; + while ((last = *values++) == 0) { + i++; + } + gf8_muladd_00(dst, src[i++] + offset); + while (i < count) { + tmp = *values++; + if (tmp != 0) { + gf8_muladd[last](dst, src[i] + offset); + last = tmp; + } + i++; + } + gf8_muladd[last](dst, zero); +} diff --git a/xlators/cluster/ec/src/ec-code-c.h b/xlators/cluster/ec/src/ec-code-c.h new file mode 100644 index 00000000000..42b5a064eb8 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-c.h @@ -0,0 +1,27 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_C_H__ +#define __EC_CODE_C_H__ + +#include "ec-types.h" + +void +ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count); + +void +ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values, + uint32_t count); + +void +ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values, + uint32_t count); + +#endif /* __EC_CODE_C_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-intel.c b/xlators/cluster/ec/src/ec-code-intel.c new file mode 100644 index 00000000000..f1c4e13e321 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-intel.c @@ -0,0 +1,594 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <inttypes.h> +#include <string.h> +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_intel_init(ec_code_intel_t *intel) +{ + memset(intel, 0, sizeof(ec_code_intel_t)); +} + +static void +ec_code_intel_prefix(ec_code_intel_t *intel, uint8_t prefix) +{ + intel->prefix.data[intel->prefix.bytes++] = prefix; +} + +static void +ec_code_intel_rex(ec_code_intel_t *intel, gf_boolean_t w) +{ + gf_boolean_t present = _gf_false; + + if (w) { + intel->rex.w = 1; + present = _gf_true; + } + if (intel->modrm.present) { + if (intel->modrm.reg > 7) { + intel->modrm.reg &= 7; + intel->rex.r = 1; + present = _gf_true; + } + if (intel->sib.present) { + if (intel->sib.index > 7) { + intel->sib.index &= 7; + intel->rex.x = 1; + present = _gf_true; + } + if (intel->sib.base > 7) { + intel->sib.base &= 7; + intel->rex.b = 1; + present = _gf_true; + } + } else if (intel->modrm.rm > 7) { + intel->modrm.rm &= 7; + intel->rex.b = 1; + present = _gf_true; + } + } else if (intel->reg > 7) { + intel->reg &= 7; + intel->rex.b = 1; + present = _gf_true; + } + intel->rex.present = present; +} + +static void +ec_code_intel_vex(ec_code_intel_t *intel, gf_boolean_t w, gf_boolean_t l, + ec_code_vex_opcode_t opcode, ec_code_vex_prefix_t prefix, + uint32_t reg) +{ + ec_code_intel_rex(intel, w); + if (((intel->rex.w == 1) || (intel->rex.x == 0) || (intel->rex.b == 0)) || + ((opcode != VEX_OPCODE_NONE) && (opcode != VEX_OPCODE_0F))) { + intel->rex.present = _gf_false; + + intel->vex.bytes = 3; + intel->vex.data[0] = 0xC4; + intel->vex.data[1] = ((intel->rex.r << 7) | (intel->rex.x << 6) | + (intel->rex.b << 5) | opcode) ^ + 0xE0; + intel->vex.data[2] = (intel->rex.w << 7) | ((~reg & 0x0F) << 3) | + (l ? 0x04 : 0x00) | prefix; + } else { + intel->vex.bytes = 2; + intel->vex.data[0] = 0xC5; + intel->vex.data[1] = (intel->rex.r << 7) | ((~reg & 0x0F) << 3) | + (l ? 0x04 : 0x00) | prefix; + } +} + +static void +ec_code_intel_modrm_reg(ec_code_intel_t *intel, uint32_t rm, uint32_t reg) +{ + intel->modrm.present = _gf_true; + intel->modrm.mod = 3; + intel->modrm.rm = rm; + intel->modrm.reg = reg; +} + +static void +ec_code_intel_modrm_mem(ec_code_intel_t *intel, uint32_t reg, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + if (index == REG_SP) { + intel->invalid = _gf_true; + return; + } + if ((index != REG_NULL) && (scale != 1) && (scale != 2) && (scale != 4) && + (scale != 8)) { + intel->invalid = _gf_true; + return; + } + scale >>= 1; + if (scale == 4) { + scale = 3; + } + + intel->modrm.present = _gf_true; + intel->modrm.reg = reg; + + intel->offset.value = offset; + if ((offset == 0) && (base != REG_BP)) { + intel->modrm.mod = 0; + intel->offset.bytes = 0; + } else if ((offset >= -128) && (offset <= 127)) { + intel->modrm.mod = 1; + intel->offset.bytes = 1; + } else { + intel->modrm.mod = 2; + intel->offset.bytes = 4; + } + + intel->modrm.rm = base; + if ((index != REG_NULL) || (base == REG_SP)) { + intel->modrm.rm = 4; + intel->sib.present = _gf_true; + intel->sib.index = index; + if (index == REG_NULL) { + intel->sib.index = 4; + } + intel->sib.scale = scale; + intel->sib.base = base; + if (base == REG_NULL) { + intel->sib.base = 5; + intel->modrm.mod = 0; + intel->offset.bytes = 4; + } + } else if (base == REG_NULL) { + intel->modrm.mod = 0; + intel->modrm.rm = 5; + intel->offset.bytes = 4; + } +} + +static void +ec_code_intel_op_1(ec_code_intel_t *intel, uint8_t opcode, uint32_t reg) +{ + intel->reg = reg; + intel->opcode.bytes = 1; + intel->opcode.data[0] = opcode; +} + +static void +ec_code_intel_op_2(ec_code_intel_t *intel, uint8_t opcode1, uint8_t opcode2, + uint32_t reg) +{ + intel->reg = reg; + intel->opcode.bytes = 2; + intel->opcode.data[0] = opcode1; + intel->opcode.data[1] = opcode2; +} + +static void +ec_code_intel_immediate_1(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 1; + intel->immediate.value = value; +} + +static void +ec_code_intel_immediate_2(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 2; + intel->immediate.value = value; +} + +static void +ec_code_intel_immediate_4(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 4; + intel->immediate.value = value; +} + +static void +ec_code_intel_emit(ec_code_builder_t *builder, ec_code_intel_t *intel) +{ + uint8_t insn[15]; + uint32_t i, count; + + if (intel->invalid) { + ec_code_error(builder, EINVAL); + return; + } + + count = 0; + for (i = 0; i < intel->prefix.bytes; i++) { + insn[count++] = intel->prefix.data[i]; + } + for (i = 0; i < intel->vex.bytes; i++) { + insn[count++] = intel->vex.data[i]; + } + if (intel->rex.present) { + insn[count++] = 0x40 | (intel->rex.w << 3) | (intel->rex.r << 2) | + (intel->rex.x << 1) | (intel->rex.b << 0); + } + for (i = 0; i < intel->opcode.bytes; i++) { + insn[count++] = intel->opcode.data[i]; + } + if (intel->modrm.present) { + insn[count++] = (intel->modrm.mod << 6) | (intel->modrm.reg << 3) | + (intel->modrm.rm << 0); + if (intel->sib.present) { + insn[count++] = (intel->sib.scale << 6) | (intel->sib.index << 3) | + (intel->sib.base << 0); + } + } + for (i = 0; i < intel->offset.bytes; i++) { + insn[count++] = intel->offset.data[i]; + } + for (i = 0; i < intel->immediate.bytes; i++) { + insn[count++] = intel->immediate.data[i]; + } + + ec_code_emit(builder, insn, count); +} + +void +ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_op_1(&intel, 0x50 | (reg & 7), reg); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_op_1(&intel, 0x58 | (reg & 7), reg); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if (size == 0) { + ec_code_intel_op_1(&intel, 0xC3, 0); + } else { + ec_code_intel_immediate_2(&intel, size); + ec_code_intel_op_1(&intel, 0xC2, 0); + } + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, dst, src); + ec_code_intel_op_1(&intel, 0x89, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x89, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x8B, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, dst, src); + ec_code_intel_op_1(&intel, 0x31, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x33, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value, + ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if ((value >= -128) && (value < 128)) { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0x83, 0); + ec_code_intel_immediate_1(&intel, value); + } else { + if (reg == REG_AX) { + ec_code_intel_op_1(&intel, 0x05, reg); + } else { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0x81, 0); + } + ec_code_intel_immediate_4(&intel, value); + } + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value, + ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if (reg == REG_AX) { + ec_code_intel_op_1(&intel, 0xA9, reg); + } else { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0xF7, 0); + } + ec_code_intel_immediate_4(&intel, value); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address) +{ + ec_code_intel_t intel; + int32_t rel; + + ec_code_intel_init(&intel); + + rel = address - builder->address - 2; + if ((rel >= -128) && (rel < 128)) { + ec_code_intel_op_1(&intel, 0x75, 0); + ec_code_intel_immediate_1(&intel, rel); + } else { + rel -= 4; + ec_code_intel_op_2(&intel, 0x0F, 0x85, 0); + ec_code_intel_immediate_4(&intel, rel); + } + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0x7F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_1(&intel, 0x6F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x7F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x6F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_1(&intel, 0xEF, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + dst); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0xEF, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + dst); + + ec_code_intel_emit(builder, &intel); +} diff --git a/xlators/cluster/ec/src/ec-code-intel.h b/xlators/cluster/ec/src/ec-code-intel.h new file mode 100644 index 00000000000..3fa4a174765 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-intel.h @@ -0,0 +1,191 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_INTEL_H__ +#define __EC_CODE_INTEL_H__ + +#include "ec-code.h" + +#define VEX_REG_NONE 0 + +enum _ec_code_intel_reg; +typedef enum _ec_code_intel_reg ec_code_intel_reg_t; + +enum _ec_code_vex_prefix; +typedef enum _ec_code_vex_prefix ec_code_vex_prefix_t; + +enum _ec_code_vex_opcode; +typedef enum _ec_code_vex_opcode ec_code_vex_opcode_t; + +struct _ec_code_intel_buffer; +typedef struct _ec_code_intel_buffer ec_code_intel_buffer_t; + +struct _ec_code_intel_sib; +typedef struct _ec_code_intel_sib ec_code_intel_sib_t; + +struct _ec_code_intel_modrm; +typedef struct _ec_code_intel_modrm ec_code_intel_modrm_t; + +struct _ec_code_intel_rex; +typedef struct _ec_code_intel_rex ec_code_intel_rex_t; + +struct _ec_code_intel; +typedef struct _ec_code_intel ec_code_intel_t; + +enum _ec_code_intel_reg { + REG_NULL = -1, + REG_AX, + REG_CX, + REG_DX, + REG_BX, + REG_SP, + REG_BP, + REG_SI, + REG_DI, + REG_8, + REG_9, + REG_10, + REG_11, + REG_12, + REG_13, + REG_14, + REG_15 +}; + +enum _ec_code_vex_prefix { + VEX_PREFIX_NONE = 0, + VEX_PREFIX_66, + VEX_PREFIX_F3, + VEX_PREFIX_F2 +}; + +enum _ec_code_vex_opcode { + VEX_OPCODE_NONE = 0, + VEX_OPCODE_0F, + VEX_OPCODE_0F_38, + VEX_OPCODE_0F_3A +}; + +struct _ec_code_intel_buffer { + uint32_t bytes; + union { + uint8_t data[4]; + uint32_t value; + }; +}; + +struct _ec_code_intel_sib { + gf_boolean_t present; + uint32_t base; + uint32_t index; + uint32_t scale; +}; + +struct _ec_code_intel_modrm { + gf_boolean_t present; + uint32_t mod; + uint32_t rm; + uint32_t reg; +}; + +struct _ec_code_intel_rex { + gf_boolean_t present; + uint32_t w; + uint32_t r; + uint32_t x; + uint32_t b; +}; + +struct _ec_code_intel { + gf_boolean_t invalid; + ec_code_intel_buffer_t prefix; + ec_code_intel_buffer_t opcode; + ec_code_intel_buffer_t offset; + ec_code_intel_buffer_t immediate; + ec_code_intel_buffer_t vex; + ec_code_intel_rex_t rex; + ec_code_intel_modrm_t modrm; + ec_code_intel_sib_t sib; + uint32_t reg; +}; + +void +ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg); +void +ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg); +void +ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size); + +void +ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst); +void +ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst); +void +ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst); +void +ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst); +void +ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value, + ec_code_intel_reg_t reg); +void +ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value, + ec_code_intel_reg_t reg); +void +ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address); + +void +ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); +void +ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); + +void +ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); +void +ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); + +#endif /* __EC_CODE_INTEL_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-sse.c b/xlators/cluster/ec/src/ec-code-sse.c new file mode 100644 index 00000000000..e11e7ff8400 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-sse.c @@ -0,0 +1,101 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_sse_prolog(ec_code_builder_t *builder) +{ + builder->loop = builder->address; +} + +static void +ec_code_sse_epilog(ec_code_builder_t *builder) +{ + ec_code_intel_op_add_i2r(builder, 16, REG_DX); + ec_code_intel_op_add_i2r(builder, 16, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_sse_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_mov_m2sse( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_mov_m2sse(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_sse_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + ec_code_intel_op_mov_sse2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_sse_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_mov_sse2sse(builder, src, dst); +} + +static void +ec_code_sse_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_xor_sse2sse(builder, src, dst); +} + +static void +ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_xor_m2sse( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_xor_m2sse(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_sse_needed_flags[] = {"sse2", NULL}; + +ec_code_gen_t ec_code_gen_sse = {.name = "sse", + .flags = ec_code_sse_needed_flags, + .width = 16, + .prolog = ec_code_sse_prolog, + .epilog = ec_code_sse_epilog, + .load = ec_code_sse_load, + .store = ec_code_sse_store, + .copy = ec_code_sse_copy, + .xor2 = ec_code_sse_xor2, + .xor3 = NULL, + .xorm = ec_code_sse_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-sse.h b/xlators/cluster/ec/src/ec-code-sse.h new file mode 100644 index 00000000000..f1acbcf894b --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-sse.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_SSE_H__ +#define __EC_CODE_SSE_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_sse; + +#endif /* __EC_CODE_SSE_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-x64.c b/xlators/cluster/ec/src/ec-code-x64.c new file mode 100644 index 00000000000..26565b4493f --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-x64.c @@ -0,0 +1,144 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static ec_code_intel_reg_t ec_code_x64_regmap[] = { + REG_AX, REG_CX, REG_BP, REG_8, REG_9, REG_10, + REG_11, REG_12, REG_13, REG_14, REG_15}; + +static void +ec_code_x64_prolog(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_intel_op_push_r(builder, REG_BP); + if (!builder->linear) { + ec_code_intel_op_push_r(builder, REG_BX); + } + if (builder->regs > 11) { + ec_code_error(builder, EINVAL); + return; + } + for (i = 7; i < builder->regs; i++) { + ec_code_intel_op_push_r(builder, ec_code_x64_regmap[i]); + } + + builder->loop = builder->address; +} + +static void +ec_code_x64_epilog(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_intel_op_add_i2r(builder, 8, REG_DX); + ec_code_intel_op_add_i2r(builder, 8, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + if (builder->regs > 11) { + ec_code_error(builder, EINVAL); + return; + } + for (i = builder->regs; i > 7; i--) { + ec_code_intel_op_pop_r(builder, ec_code_x64_regmap[i - 1]); + } + if (!builder->linear) { + ec_code_intel_op_pop_r(builder, REG_BX); + } + ec_code_intel_op_pop_r(builder, REG_BP); + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_x64_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + dst = ec_code_x64_regmap[dst]; + + if (builder->linear) { + ec_code_intel_op_mov_m2r( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_BX); + builder->base = idx; + } + ec_code_intel_op_mov_m2r(builder, REG_BX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_x64_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_mov_r2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_x64_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + dst = ec_code_x64_regmap[dst]; + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_mov_r2r(builder, src, dst); +} + +static void +ec_code_x64_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + dst = ec_code_x64_regmap[dst]; + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_xor_r2r(builder, src, dst); +} + +static void +ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + dst = ec_code_x64_regmap[dst]; + + if (builder->linear) { + ec_code_intel_op_xor_m2r( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_BX); + builder->base = idx; + } + ec_code_intel_op_xor_m2r(builder, REG_BX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_x64_needed_flags[] = {NULL}; + +ec_code_gen_t ec_code_gen_x64 = {.name = "x64", + .flags = ec_code_x64_needed_flags, + .width = sizeof(uint64_t), + .prolog = ec_code_x64_prolog, + .epilog = ec_code_x64_epilog, + .load = ec_code_x64_load, + .store = ec_code_x64_store, + .copy = ec_code_x64_copy, + .xor2 = ec_code_x64_xor2, + .xor3 = NULL, + .xorm = ec_code_x64_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-x64.h b/xlators/cluster/ec/src/ec-code-x64.h new file mode 100644 index 00000000000..bd8174e4bf5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-x64.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_X64_H__ +#define __EC_CODE_X64_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_x64; + +#endif /* __EC_CODE_X64_H__ */ diff --git a/xlators/cluster/ec/src/ec-code.c b/xlators/cluster/ec/src/ec-code.c new file mode 100644 index 00000000000..03162ae05a9 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code.c @@ -0,0 +1,1060 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <ctype.h> + +#include <glusterfs/syscall.h> + +#include "ec-mem-types.h" +#include "ec-code.h" +#include "ec-messages.h" +#include "ec-code-c.h" +#include "ec-helpers.h" + +#ifdef USE_EC_DYNAMIC_X64 +#include "ec-code-x64.h" +#endif + +#ifdef USE_EC_DYNAMIC_SSE +#include "ec-code-sse.h" +#endif + +#ifdef USE_EC_DYNAMIC_AVX +#include "ec-code-avx.h" +#endif + +#define EC_CODE_SIZE (1024 * 64) +#define EC_CODE_ALIGN 4096 + +#define EC_CODE_CHUNK_MIN_SIZE 512 + +#define EC_PROC_BUFFER_SIZE 4096 + +#define PROC_CPUINFO "/proc/cpuinfo" + +struct _ec_code_proc; +typedef struct _ec_code_proc ec_code_proc_t; + +struct _ec_code_proc { + int32_t fd; + gf_boolean_t eof; + gf_boolean_t error; + gf_boolean_t skip; + ssize_t size; + ssize_t pos; + char buffer[EC_PROC_BUFFER_SIZE]; +}; + +static ec_code_gen_t *ec_code_gen_table[] = { +#ifdef USE_EC_DYNAMIC_AVX + &ec_code_gen_avx, +#endif +#ifdef USE_EC_DYNAMIC_SSE + &ec_code_gen_sse, +#endif +#ifdef USE_EC_DYNAMIC_X64 + &ec_code_gen_x64, +#endif + NULL}; + +static void +ec_code_arg_set(ec_code_arg_t *arg, uint32_t value) +{ + arg->value = value; +} + +static void +ec_code_arg_assign(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; + + if (builder->regs <= reg) { + builder->regs = reg + 1; + } +} + +static void +ec_code_arg_use(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; +} + +static void +ec_code_arg_update(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; +} + +static ec_code_op_t * +ec_code_op_next(ec_code_builder_t *builder) +{ + ec_code_op_t *op; + + op = &builder->ops[builder->count++]; + memset(op, 0, sizeof(ec_code_op_t)); + + return op; +} + +static void +ec_code_load(ec_code_builder_t *builder, uint32_t bit, uint32_t offset) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_LOAD; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[bit]); + ec_code_arg_set(&op->arg2, offset); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_store(ec_code_builder_t *builder, uint32_t reg, uint32_t bit) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_STORE; + ec_code_arg_use(builder, op, &op->arg1, builder->map[reg]); + ec_code_arg_set(&op->arg2, 0); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_COPY; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src]); + ec_code_arg_set(&op->arg3, 0); +} + +static void +ec_code_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XOR2; + ec_code_arg_update(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src]); + ec_code_arg_set(&op->arg3, 0); +} + +static void +ec_code_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2) +{ + ec_code_op_t *op; + + if (builder->code->gen->xor3 == NULL) { + ec_code_copy(builder, dst, src1); + ec_code_xor2(builder, dst, src2); + + return; + } + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XOR3; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src1]); + ec_code_arg_use(builder, op, &op->arg3, builder->map[src2]); +} + +static void +ec_code_xorm(ec_code_builder_t *builder, uint32_t bit, uint32_t offset) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XORM; + ec_code_arg_update(builder, op, &op->arg1, builder->map[bit]); + ec_code_arg_set(&op->arg2, offset); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_dup(ec_code_builder_t *builder, ec_gf_op_t *op) +{ + switch (op->op) { + case EC_GF_OP_COPY: + ec_code_copy(builder, op->arg1, op->arg2); + break; + case EC_GF_OP_XOR2: + ec_code_xor2(builder, op->arg1, op->arg2); + break; + case EC_GF_OP_XOR3: + ec_code_xor3(builder, op->arg1, op->arg2, op->arg3); + break; + default: + break; + } +} + +static void +ec_code_gf_load(ec_code_builder_t *builder, uint32_t offset) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_load(builder, i, offset); + } +} + +static void +ec_code_gf_load_xor(ec_code_builder_t *builder, uint32_t offset) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_xorm(builder, i, offset); + } +} + +static void +ec_code_gf_store(ec_code_builder_t *builder) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_store(builder, i, i); + } +} + +static void +ec_code_gf_clear(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_xor2(builder, 0, 0); + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_store(builder, 0, i); + } +} + +static void +ec_code_gf_mul(ec_code_builder_t *builder, uint32_t value) +{ + ec_gf_mul_t *mul; + ec_gf_op_t *op; + uint32_t map[EC_GF_MAX_REGS]; + int32_t i; + + mul = builder->code->gf->table[value]; + for (op = mul->ops; op->op != EC_GF_OP_END; op++) { + ec_code_dup(builder, op); + } + + for (i = 0; i < mul->regs; i++) { + map[i] = builder->map[mul->map[i]]; + } + memcpy(builder->map, map, sizeof(uint32_t) * mul->regs); +} + +static ec_code_builder_t * +ec_code_prepare(ec_code_t *code, uint32_t count, uint32_t width, + gf_boolean_t linear) +{ + ec_code_builder_t *builder; + uint32_t i; + + count *= code->gf->bits + code->gf->max_ops; + count += code->gf->bits; + builder = GF_MALLOC( + sizeof(ec_code_builder_t) + sizeof(ec_code_op_t) * count, + ec_mt_ec_code_builder_t); + if (builder == NULL) { + return EC_ERR(ENOMEM); + } + + builder->address = 0; + builder->code = code; + builder->size = 0; + builder->count = 0; + builder->regs = 0; + builder->error = 0; + builder->bits = code->gf->bits; + builder->width = width; + builder->data = NULL; + builder->linear = linear; + builder->base = -1; + + for (i = 0; i < EC_GF_MAX_REGS; i++) { + builder->map[i] = i; + } + + return builder; +} + +static size_t +ec_code_space_size(void) +{ + return (sizeof(ec_code_space_t) + 15) & ~15; +} + +static size_t +ec_code_chunk_size(void) +{ + return (sizeof(ec_code_chunk_t) + 15) & ~15; +} + +static ec_code_chunk_t * +ec_code_chunk_from_space(ec_code_space_t *space) +{ + return (ec_code_chunk_t *)((uintptr_t)space + ec_code_space_size()); +} + +static void * +ec_code_to_executable(ec_code_space_t *space, void *addr) +{ + return (void *)((uintptr_t)addr - (uintptr_t)space + + (uintptr_t)space->exec); +} + +static void * +ec_code_from_executable(ec_code_space_t *space, void *addr) +{ + return (void *)((uintptr_t)addr - (uintptr_t)space->exec + + (uintptr_t)space); +} + +static void * +ec_code_func_from_chunk(ec_code_chunk_t *chunk, void **exec) +{ + void *addr; + + addr = (void *)((uintptr_t)chunk + ec_code_chunk_size()); + + *exec = ec_code_to_executable(chunk->space, addr); + + return addr; +} + +static ec_code_chunk_t * +ec_code_chunk_from_func(ec_code_func_linear_t func) +{ + ec_code_chunk_t *chunk; + + chunk = (ec_code_chunk_t *)((uintptr_t)func - ec_code_chunk_size()); + + return ec_code_from_executable(chunk->space, chunk); +} + +static ec_code_chunk_t * +ec_code_chunk_split(ec_code_chunk_t *chunk, size_t size) +{ + ec_code_chunk_t *extra; + ssize_t avail; + + avail = chunk->size - size - ec_code_chunk_size(); + if (avail > 0) { + extra = (ec_code_chunk_t *)((uintptr_t)chunk + chunk->size - avail); + extra->space = chunk->space; + extra->size = avail; + list_add(&extra->list, &chunk->list); + chunk->size = size; + } + list_del_init(&chunk->list); + + return chunk; +} + +static gf_boolean_t +ec_code_chunk_touch(ec_code_chunk_t *prev, ec_code_chunk_t *next) +{ + uintptr_t end; + + end = (uintptr_t)prev + ec_code_chunk_size() + prev->size; + return (end == (uintptr_t)next); +} + +static ec_code_space_t * +ec_code_space_create(ec_code_t *code, size_t size) +{ + char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX"; + ec_code_space_t *space; + void *exec; + int32_t fd, err; + + /* We need to create memory areas to store the generated dynamic code. + * Obviously these areas need to be written to be able to create the + * code and they also need to be executable to execute it. + * + * However it's a bad practice to have a memory region that is both + * writable *and* executable. In fact, selinux forbids this and causes + * attempts to do so to fail (unless specifically configured). + * + * To solve the problem we'll use two distinct memory areas mapped to + * the same physical storage. One of the memory areas will have write + * permission, and the other will have execute permission. Both areas + * will have the same contents. The physical storage will be a regular + * file that will be mmapped to both areas. + */ + + /* We need to create a temporary file as the backend storage for the + * memory mapped areas. */ + /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */ + fd = mkstemp(path); + if (fd < 0) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to create a temporary file for the ec dynamic " + "code"); + space = EC_ERR(err); + goto done; + } + /* Once created we don't need to keep it in the file system. It will + * still exist until we close the last file descriptor or unmap the + * memory areas bound to the file. */ + sys_unlink(path); + + size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1); + if (sys_ftruncate(fd, size) < 0) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to resize the file for the ec dynamic code"); + space = EC_ERR(err); + goto done_close; + } + + /* This creates an executable memory area to be able to run the + * generated fragments of code. */ + exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); + if (exec == MAP_FAILED) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to map the executable area for the ec dynamic " + "code"); + space = EC_ERR(err); + goto done_close; + } + /* It's not important to check the return value of mlock(). If it fails + * everything will continue to work normally. */ + mlock(exec, size); + + /* This maps a read/write memory area to be able to create the dynamici + * code. */ + space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (space == MAP_FAILED) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to map the writable area for the ec dynamic " + "code"); + space = EC_ERR(err); + + munmap(exec, size); + + goto done_close; + } + + space->exec = exec; + space->size = size; + space->code = code; + list_add_tail(&space->list, &code->spaces); + INIT_LIST_HEAD(&space->chunks); + +done_close: + /* If everything has succeeded, we already have the memory areas + * mapped. We don't need the file descriptor anymore because the + * backend storage will be there until the mmap()'d regions are + * unmapped. */ + sys_close(fd); +done: + return space; +} + +static void +ec_code_space_destroy(ec_code_space_t *space) +{ + list_del_init(&space->list); + + munmap(space->exec, space->size); + munmap(space, space->size); +} + +static void +ec_code_chunk_merge(ec_code_chunk_t *chunk) +{ + ec_code_chunk_t *item, *tmp; + + list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list) + { + if ((uintptr_t)item > (uintptr_t)chunk) { + list_add_tail(&chunk->list, &item->list); + if (ec_code_chunk_touch(chunk, item)) { + chunk->size += item->size + ec_code_chunk_size(); + list_del_init(&item->list); + } + + goto check; + } + if (ec_code_chunk_touch(item, chunk)) { + item->size += chunk->size + ec_code_chunk_size(); + list_del_init(&item->list); + chunk = item; + } + } + list_add_tail(&chunk->list, &chunk->space->chunks); + +check: + if (chunk->size == + chunk->space->size - ec_code_space_size() - ec_code_chunk_size()) { + ec_code_space_destroy(chunk->space); + } +} + +static ec_code_chunk_t * +ec_code_space_alloc(ec_code_t *code, size_t size) +{ + ec_code_space_t *space; + ec_code_chunk_t *chunk; + size_t map_size; + + /* To minimize fragmentation, we only allocate chunks of sizes multiples + * of EC_CODE_CHUNK_MIN_SIZE. */ + size = ((size + ec_code_chunk_size() + EC_CODE_CHUNK_MIN_SIZE - 1) & + ~(EC_CODE_CHUNK_MIN_SIZE - 1)) - + ec_code_chunk_size(); + list_for_each_entry(space, &code->spaces, list) + { + list_for_each_entry(chunk, &space->chunks, list) + { + if (chunk->size >= size) { + goto out; + } + } + } + + map_size = EC_CODE_SIZE - ec_code_space_size() - ec_code_chunk_size(); + if (map_size < size) { + map_size = size; + } + space = ec_code_space_create(code, map_size); + if (EC_IS_ERR(space)) { + return (ec_code_chunk_t *)space; + } + + chunk = ec_code_chunk_from_space(space); + chunk->size = map_size - ec_code_space_size() - ec_code_chunk_size(); + list_add(&chunk->list, &space->chunks); + +out: + chunk->space = space; + + return ec_code_chunk_split(chunk, size); +} + +static ec_code_chunk_t * +ec_code_alloc(ec_code_t *code, uint32_t size) +{ + ec_code_chunk_t *chunk; + + LOCK(&code->lock); + + chunk = ec_code_space_alloc(code, size); + + UNLOCK(&code->lock); + + return chunk; +} + +static void +ec_code_free(ec_code_chunk_t *chunk) +{ + gf_lock_t *lock; + + lock = &chunk->space->code->lock; + LOCK(lock); + + ec_code_chunk_merge(chunk); + + UNLOCK(lock); +} + +static int32_t +ec_code_write(ec_code_builder_t *builder) +{ + ec_code_gen_t *gen; + ec_code_op_t *op; + uint32_t i; + + builder->error = 0; + builder->size = 0; + builder->address = 0; + builder->base = -1; + + gen = builder->code->gen; + gen->prolog(builder); + for (i = 0; i < builder->count; i++) { + op = &builder->ops[i]; + switch (op->op) { + case EC_GF_OP_LOAD: + gen->load(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + case EC_GF_OP_STORE: + gen->store(builder, op->arg1.value, op->arg3.value); + break; + case EC_GF_OP_COPY: + gen->copy(builder, op->arg1.value, op->arg2.value); + break; + case EC_GF_OP_XOR2: + gen->xor2(builder, op->arg1.value, op->arg2.value); + break; + case EC_GF_OP_XOR3: + gen->xor3(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + case EC_GF_OP_XORM: + gen->xorm(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + default: + break; + } + } + gen->epilog(builder); + + return builder->error; +} + +static void * +ec_code_compile(ec_code_builder_t *builder) +{ + ec_code_chunk_t *chunk; + void *func; + int32_t err; + + err = ec_code_write(builder); + if (err != 0) { + return EC_ERR(err); + } + + chunk = ec_code_alloc(builder->code, builder->size); + if (EC_IS_ERR(chunk)) { + return chunk; + } + builder->data = ec_code_func_from_chunk(chunk, &func); + + err = ec_code_write(builder); + if (err != 0) { + ec_code_free(chunk); + + return EC_ERR(err); + } + + GF_FREE(builder); + + return func; +} + +ec_code_t * +ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen) +{ + ec_code_t *code; + + code = GF_MALLOC(sizeof(ec_code_t), ec_mt_ec_code_t); + if (code == NULL) { + return EC_ERR(ENOMEM); + } + memset(code, 0, sizeof(ec_code_t)); + INIT_LIST_HEAD(&code->spaces); + LOCK_INIT(&code->lock); + + code->gf = gf; + code->gen = gen; + + return code; +} + +void +ec_code_destroy(ec_code_t *code) +{ + if (!list_empty(&code->spaces)) { + } + + LOCK_DESTROY(&code->lock); + + GF_FREE(code); +} + +static uint32_t +ec_code_value_next(uint32_t *values, uint32_t count, uint32_t *offset) +{ + uint32_t i, next; + + next = 0; + for (i = *offset + 1; i < count; i++) { + next = values[i]; + if (next != 0) { + break; + } + } + *offset = i; + + return next; +} + +static void * +ec_code_build_dynamic(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count, gf_boolean_t linear) +{ + ec_code_builder_t *builder; + uint32_t offset, val, next; + + builder = ec_code_prepare(code, count, width, linear); + if (EC_IS_ERR(builder)) { + return builder; + } + + offset = -1; + next = ec_code_value_next(values, count, &offset); + if (next != 0) { + ec_code_gf_load(builder, offset); + do { + val = next; + next = ec_code_value_next(values, count, &offset); + if (next != 0) { + ec_code_gf_mul(builder, ec_gf_div(code->gf, val, next)); + ec_code_gf_load_xor(builder, offset); + } + } while (next != 0); + ec_code_gf_mul(builder, val); + ec_code_gf_store(builder); + } else { + ec_code_gf_clear(builder); + } + + return ec_code_compile(builder); +} + +static void * +ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values, uint32_t count, + gf_boolean_t linear) +{ + void *func; + + if (code->gen != NULL) { + func = ec_code_build_dynamic(code, width, values, count, linear); + if (!EC_IS_ERR(func)) { + return func; + } + + gf_msg_debug(THIS->name, GF_LOG_DEBUG, + "Unable to generate dynamic code. Falling back " + "to precompiled code"); + + /* The dynamic code generation shouldn't fail in normal + * conditions, but if it fails at some point, it's very + * probable that it will fail again, so we completely disable + * dynamic code generation. */ + code->gen = NULL; + } + + ec_code_c_prepare(code->gf, values, count); + + if (linear) { + return ec_code_c_linear; + } + + return ec_code_c_interleaved; +} + +ec_code_func_linear_t +ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count) +{ + return (ec_code_func_linear_t)ec_code_build(code, width, values, count, + _gf_true); +} + +ec_code_func_interleaved_t +ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count) +{ + return (ec_code_func_interleaved_t)ec_code_build(code, width, values, count, + _gf_false); +} + +void +ec_code_release(ec_code_t *code, ec_code_func_t *func) +{ + if ((func->linear != ec_code_c_linear) && + (func->interleaved != ec_code_c_interleaved)) { + ec_code_free(ec_code_chunk_from_func(func->linear)); + } +} + +void +ec_code_error(ec_code_builder_t *builder, int32_t error) +{ + if (builder->error == 0) { + gf_msg(THIS->name, GF_LOG_ERROR, error, EC_MSG_DYN_CODEGEN_FAILED, + "Failed to generate dynamic code"); + builder->error = error; + } +} + +void +ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count) +{ + if (builder->error != 0) { + return; + } + + if (builder->data != NULL) { + memcpy(builder->data + builder->size, bytes, count); + } + + builder->size += count; + builder->address += count; +} + +static char * +ec_code_proc_trim_left(char *text, ssize_t *length) +{ + ssize_t len; + + for (len = *length; (len > 0) && isspace(*text); len--) { + text++; + } + *length = len; + + return text; +} + +static char * +ec_code_proc_trim_right(char *text, ssize_t *length, char sep) +{ + char *last; + ssize_t len; + + len = *length; + + last = text; + for (len = *length; (len > 0) && (*text != sep); len--) { + if (!isspace(*text)) { + last = text + 1; + } + text++; + } + *last = 0; + *length = len; + + return text; +} + +static char * +ec_code_proc_line_parse(ec_code_proc_t *file, ssize_t *length) +{ + char *text, *end; + ssize_t len; + + len = file->size - file->pos; + text = ec_code_proc_trim_left(file->buffer + file->pos, &len); + end = ec_code_proc_trim_right(text, &len, '\n'); + if (len == 0) { + if (!file->eof) { + if (text == file->buffer) { + file->size = file->pos = 0; + file->skip = _gf_true; + } else { + file->size = file->pos = end - text; + memmove(file->buffer, text, file->pos + 1); + } + len = sys_read(file->fd, file->buffer + file->pos, + sizeof(file->buffer) - file->pos - 1); + if (len > 0) { + file->size += len; + } + file->error = len < 0; + file->eof = len <= 0; + + return NULL; + } + file->size = file->pos = 0; + } else { + file->pos = end - file->buffer + 1; + } + + *length = end - text; + + if (file->skip) { + file->skip = _gf_false; + text = NULL; + } + + return text; +} + +static char * +ec_code_proc_line(ec_code_proc_t *file, ssize_t *length) +{ + char *text; + + text = NULL; + while (!file->eof) { + text = ec_code_proc_line_parse(file, length); + if (text != NULL) { + break; + } + } + + return text; +} + +static char * +ec_code_proc_split(char *text, ssize_t *length, char sep) +{ + text = ec_code_proc_trim_right(text, length, sep); + if (*length == 0) { + return NULL; + } + (*length)--; + text++; + + return ec_code_proc_trim_left(text, length); +} + +static uint32_t +ec_code_cpu_check(uint32_t idx, char *list, uint32_t count) +{ + ec_code_gen_t *gen; + char **ptr; + char *table[count + 1]; + uint32_t i; + + for (i = 0; i < count; i++) { + table[i] = list; + list += strlen(list) + 1; + } + + gen = ec_code_gen_table[idx]; + while (gen != NULL) { + for (ptr = gen->flags; *ptr != NULL; ptr++) { + for (i = 0; i < count; i++) { + if (strcmp(*ptr, table[i]) == 0) { + break; + } + } + if (i >= count) { + gen = ec_code_gen_table[++idx]; + break; + } + } + if (*ptr == NULL) { + break; + } + } + + return idx; +} + +ec_code_gen_t * +ec_code_detect(xlator_t *xl, const char *def) +{ + ec_code_proc_t file; + ec_code_gen_t *gen = NULL; + char *line, *data, *list; + ssize_t length; + uint32_t count, base, select; + + if (strcmp(def, "none") == 0) { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE, + "Not using any cpu extensions"); + + return NULL; + } + + file.fd = sys_open(PROC_CPUINFO, O_RDONLY, 0); + if (file.fd < 0) { + goto out; + } + file.size = file.pos = 0; + file.eof = file.error = file.skip = _gf_false; + + select = 0; + if (strcmp(def, "auto") != 0) { + while (ec_code_gen_table[select] != NULL) { + if (strcmp(ec_code_gen_table[select]->name, def) == 0) { + break; + } + select++; + } + if (ec_code_gen_table[select] == NULL) { + gf_msg(xl->name, GF_LOG_WARNING, EINVAL, EC_MSG_EXTENSION_UNKNOWN, + "CPU extension '%s' is not known. Not using any cpu " + "extensions", + def); + + return NULL; + } + } else { + def = NULL; + } + + while ((line = ec_code_proc_line(&file, &length)) != NULL) { + data = ec_code_proc_split(line, &length, ':'); + if ((data != NULL) && (strcmp(line, "flags") == 0)) { + list = data; + count = 0; + while ((data != NULL) && (*data != 0)) { + count++; + data = ec_code_proc_split(data, &length, ' '); + } + base = select; + select = ec_code_cpu_check(select, list, count); + if ((base != select) && (def != NULL)) { + gf_msg(xl->name, GF_LOG_WARNING, ENOTSUP, + EC_MSG_EXTENSION_UNSUPPORTED, + "CPU extension '%s' is not supported", def); + def = NULL; + } + } + } + + if (file.error) { + gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_EXTENSION_FAILED, + "Unable to determine supported CPU extensions. Not using any " + "cpu extensions"); + + gen = NULL; + } else { + gen = ec_code_gen_table[select]; + if (gen == NULL) { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE, + "Not using any cpu extensions"); + } else { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION, + "Using '%s' CPU extensions", gen->name); + } + } + + sys_close(file.fd); + +out: + return gen; +} diff --git a/xlators/cluster/ec/src/ec-code.h b/xlators/cluster/ec/src/ec-code.h new file mode 100644 index 00000000000..75fb35d93e3 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code.h @@ -0,0 +1,44 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_H__ +#define __EC_CODE_H__ + +#include <glusterfs/xlator.h> +#include <glusterfs/list.h> + +#include "ec-types.h" +#include "ec-galois.h" + +ec_code_gen_t * +ec_code_detect(xlator_t *xl, const char *def); + +ec_code_t * +ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen); + +void +ec_code_destroy(ec_code_t *code); + +ec_code_func_linear_t +ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count); +ec_code_func_interleaved_t +ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count); +void +ec_code_release(ec_code_t *code, ec_code_func_t *func); + +void +ec_code_error(ec_code_builder_t *builder, int32_t error); + +void +ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count); + +#endif /* __EC_CODE_H__ */ diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c index 0f883d03fac..703a30e2485 100644 --- a/xlators/cluster/ec/src/ec-combine.c +++ b/xlators/cluster/ec/src/ec-combine.c @@ -11,379 +11,471 @@ #include <fnmatch.h> #include "libxlator.h" -#include "byte-order.h" +#include <glusterfs/byte-order.h> -#include "ec-data.h" +#include "ec-types.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" +#include "ec-messages.h" +#include <glusterfs/quota-common-utils.h> #define EC_QUOTA_PREFIX "trusted.glusterfs.quota." +#define EC_MISSING_DATA ((data_t *)1ULL) + struct _ec_dict_info; typedef struct _ec_dict_info ec_dict_info_t; struct _ec_dict_combine; typedef struct _ec_dict_combine ec_dict_combine_t; -struct _ec_dict_info -{ - dict_t * dict; - int32_t count; +struct _ec_dict_info { + dict_t *dict; + int32_t count; }; -struct _ec_dict_combine -{ - ec_cbk_data_t * cbk; - int32_t which; +struct _ec_dict_combine { + ec_cbk_data_t *cbk; + int32_t which; }; -void ec_iatt_time_merge(uint32_t * dst_sec, uint32_t * dst_nsec, - uint32_t src_sec, uint32_t src_nsec) +int32_t +ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + int valid = 0; + + if (!fop || !dst || !src) + return 0; + + switch (fop->id) { + case GF_FOP_REMOVEXATTR: + case GF_FOP_FREMOVEXATTR: + case GF_FOP_SETXATTR: + case GF_FOP_FSETXATTR: + return 1; + + case GF_FOP_SYMLINK: + case GF_FOP_LINK: + case GF_FOP_CREATE: + case GF_FOP_MKNOD: + case GF_FOP_MKDIR: + valid = 3; + break; + case GF_FOP_UNLINK: + case GF_FOP_RMDIR: + case GF_FOP_SETATTR: + case GF_FOP_FSETATTR: + case GF_FOP_TRUNCATE: + case GF_FOP_FTRUNCATE: + case GF_FOP_WRITE: + case GF_FOP_FALLOCATE: + case GF_FOP_DISCARD: + case GF_FOP_ZEROFILL: + valid = 2; + break; + case GF_FOP_RENAME: + valid = 5; + break; + default: + gf_msg_callingfn(fop->xl->name, GF_LOG_WARNING, EINVAL, + EC_MSG_INVALID_FOP, "Invalid fop %d", fop->id); + return 0; + break; + } + + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of '%s'", + gf_fop_list[fop->id]); + return 0; + } + return 1; +} + +void +ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec, int64_t src_sec, + uint32_t src_nsec) { if ((*dst_sec < src_sec) || - ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) - { + ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) { *dst_sec = src_sec; *dst_nsec = src_nsec; } } -int32_t ec_iatt_combine(struct iatt * dst, struct iatt * src, int32_t count) +static gf_boolean_t +ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt) { + uint64_t ino; int32_t i; - for (i = 0; i < count; i++) - { - if ((dst->ia_ino != src->ia_ino) || - (dst->ia_uid != src->ia_uid) || - (dst->ia_gid != src->ia_gid) || - (((dst->ia_type == IA_IFBLK) || (dst->ia_type == IA_IFCHR)) && - (dst->ia_rdev != src->ia_rdev)) || - ((dst->ia_type == IA_IFREG) && (dst->ia_size != src->ia_size)) || - (st_mode_from_ia(dst->ia_prot, dst->ia_type) != - st_mode_from_ia(src->ia_prot, src->ia_type)) || - (uuid_compare(dst->ia_gfid, src->ia_gfid) != 0)) - { - gf_log(THIS->name, GF_LOG_WARNING, - "Failed to combine iatt (inode: %lu-%lu, links: %u-%u, " - "uid: %u-%u, gid: %u-%u, rdev: %lu-%lu, size: %lu-%lu, " - "mode: %o-%o)", - dst->ia_ino, src->ia_ino, dst->ia_nlink, src->ia_nlink, - dst->ia_uid, src->ia_uid, dst->ia_gid, src->ia_gid, - dst->ia_rdev, src->ia_rdev, dst->ia_size, src->ia_size, - st_mode_from_ia(dst->ia_prot, dst->ia_type), - st_mode_from_ia(src->ia_prot, dst->ia_type)); + /* Only the top level fop will have fop->locks filled. */ + while (fop->parent != NULL) { + fop = fop->parent; + } + + /* Lookups are special requests always done without locks taken but they + * require to be able to identify differences between bricks. Special + * handling of these differences is already done in lookup specific code + * so we shouldn't ignore any difference here and consider all iatt + * structures as trusted. */ + if (fop->id == GF_FOP_LOOKUP) { + return _gf_true; + } + + /* Check if the iatt references an inode locked by the current fop */ + for (i = 0; i < fop->lock_count; i++) { + ino = gfid_to_ino(fop->locks[i].lock->loc.inode->gfid); + if (iatt->ia_ino == ino) { + return _gf_true; + } + } + + return _gf_false; +} + +int32_t +ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src, + int32_t count) +{ + int32_t i; + gf_boolean_t failed = _gf_false; + + for (i = 0; i < count; i++) { + /* Check for basic fields. These fields must be equal always, even if + * the inode is not locked because in these cases the parent inode + * will be locked and differences in these fields require changes in + * the parent directory. */ + if ((dst[i].ia_ino != src[i].ia_ino) || + (((dst[i].ia_type == IA_IFBLK) || (dst[i].ia_type == IA_IFCHR)) && + (dst[i].ia_rdev != src[i].ia_rdev)) || + (gf_uuid_compare(dst[i].ia_gfid, src[i].ia_gfid) != 0)) { + failed = _gf_true; + } + /* Check for not so stable fields. These fields can change if the + * inode is not locked. */ + if (!failed && ((dst[i].ia_uid != src[i].ia_uid) || + (dst[i].ia_gid != src[i].ia_gid) || + (st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type) != + st_mode_from_ia(src[i].ia_prot, src[i].ia_type)))) { + if (ec_iatt_is_trusted(fop, dst)) { + /* If the iatt contains information from an inode that is + * locked, these differences are real problems, so we need to + * report them. Otherwise we ignore them and don't care which + * data is returned. */ + failed = _gf_true; + } else { + gf_msg_debug(fop->xl->name, 0, + "Ignoring iatt differences because inode is not " + "locked"); + } + } + if (failed) { + gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_IATT_COMBINE_FAIL, + "Failed to combine iatt (inode: %" PRIu64 "-%" PRIu64 + ", " + "links: %u-%u, uid: %u-%u, gid: %u-%u, " + "rdev: %" PRIu64 "-%" PRIu64 ", size: %" PRIu64 "-%" PRIu64 + ", " + "mode: %o-%o), %s", + dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink, + src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid, dst[i].ia_gid, + src[i].ia_gid, dst[i].ia_rdev, src[i].ia_rdev, + dst[i].ia_size, src[i].ia_size, + st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type), + st_mode_from_ia(src[i].ia_prot, dst[i].ia_type), + ec_msg_str(fop)); return 0; } } - while (count-- > 0) - { - dst->ia_blocks += src->ia_blocks; - if (dst->ia_blksize < src->ia_blksize) - { - dst->ia_blksize = src->ia_blksize; + while (count-- > 0) { + dst[count].ia_blocks += src[count].ia_blocks; + if (dst[count].ia_blksize < src[count].ia_blksize) { + dst[count].ia_blksize = src[count].ia_blksize; } - ec_iatt_time_merge(&dst->ia_atime, &dst->ia_atime_nsec, src->ia_atime, - src->ia_atime_nsec); - ec_iatt_time_merge(&dst->ia_mtime, &dst->ia_mtime_nsec, src->ia_mtime, - src->ia_mtime_nsec); - ec_iatt_time_merge(&dst->ia_ctime, &dst->ia_ctime_nsec, src->ia_ctime, - src->ia_ctime_nsec); + ec_iatt_time_merge(&dst[count].ia_atime, &dst[count].ia_atime_nsec, + src[count].ia_atime, src[count].ia_atime_nsec); + ec_iatt_time_merge(&dst[count].ia_mtime, &dst[count].ia_mtime_nsec, + src[count].ia_mtime, src[count].ia_mtime_nsec); + ec_iatt_time_merge(&dst[count].ia_ctime, &dst[count].ia_ctime_nsec, + src[count].ia_ctime, src[count].ia_ctime_nsec); } return 1; } -void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count, - int32_t answers) +void +ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers) { uint64_t blocks; - while (count-- > 0) - { + while (count-- > 0) { blocks = iatt[count].ia_blocks * ec->fragments + answers - 1; blocks /= answers; iatt[count].ia_blocks = blocks; } } -int32_t ec_dict_data_compare(dict_t * dict, char * key, data_t * value, - void * arg) +gf_boolean_t +ec_xattr_match(dict_t *dict, char *key, data_t *value, void *arg) { - ec_dict_info_t * info = arg; - data_t * data; - - data = dict_get(info->dict, key); - if (data == NULL) - { - gf_log("ec", GF_LOG_DEBUG, "key '%s' found only on one dict", key); - - return -1; + if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) || + (strcmp(key, GET_LINK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) { + return _gf_false; } - info->count--; + return _gf_true; +} +gf_boolean_t +ec_value_ignore(char *key) +{ if ((strcmp(key, GF_CONTENT_KEY) == 0) || (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) || (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) || (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) || (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) || - (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0) || - (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) || - (fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) || + (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) || + (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) || + (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) || + (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) || (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) || - (XATTR_IS_NODE_UUID(key))) - { - return 0; + (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) || + (XATTR_IS_NODE_UUID(key))) { + return _gf_true; } - if ((data->len != value->len) || - (memcmp(data->data, value->data, data->len) != 0)) - { - gf_log("ec", GF_LOG_DEBUG, "key '%s' is different (size: %u, %u)", - key, data->len, value->len); - - return -1; - } - - return 0; + return _gf_false; } -int32_t ec_dict_data_show(dict_t * dict, char * key, data_t * value, - void * arg) +int32_t +ec_dict_compare(dict_t *dict1, dict_t *dict2) { - if (dict_get(arg, key) == NULL) - { - gf_log("ec", GF_LOG_DEBUG, "key '%s' found only on one dict", key); - } - + if (are_dicts_equal(dict1, dict2, ec_xattr_match, ec_value_ignore)) + return 1; return 0; } -int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2) +static uint32_t +ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key, + gf_boolean_t global) { - ec_dict_info_t info; - dict_t * dict; - - if (dict1 != NULL) - { - info.dict = dict1; - info.count = dict1->count; - dict = dict2; - } - else if (dict2 != NULL) - { - info.dict = dict2; - info.count = dict2->count; - dict = dict1; - } - else - { - return 1; - } + ec_t *ec = cbk->fop->xl->private; + ec_cbk_data_t *ans = NULL; + dict_t *dict = NULL; + data_t *data; + uint32_t count; + int32_t i; - if (dict != NULL) - { - if (dict_foreach(dict, ec_dict_data_compare, &info) != 0) - { - return 0; + for (i = 0; i < ec->nodes; i++) { + /* We initialize the list with EC_MISSING_DATA if we are + * returning a global list or the current subvolume belongs + * to the group of the accepted answer. Note that if some + * subvolume is known to be down before issuing the request, + * we won't have any answer from it, so we set here the + * appropriate default value. */ + if (global || ((cbk->mask & (1ULL << i)) != 0)) { + list[i] = EC_MISSING_DATA; + } else { + list[i] = NULL; } } - if (info.count != 0) + count = 0; + list_for_each_entry(ans, &cbk->fop->answer_list, answer_list) { - dict_foreach(info.dict, ec_dict_data_show, dict); - } - - return (info.count == 0); -} - -int32_t ec_dict_list(data_t ** list, int32_t * count, ec_cbk_data_t * cbk, - int32_t which, char * key) -{ - ec_cbk_data_t * ans; - dict_t * dict; - int32_t i, max; - - max = *count; - i = 0; - for (ans = cbk; ans != NULL; ans = ans->next) - { - if (i >= max) - { - gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Unexpected number of " - "dictionaries"); - - return 0; - } - - dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict; - list[i] = dict_get(dict, key); - if (list[i] == NULL) - { - gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Unexpected missing " - "dictionary entry"); - - return 0; + if (global || ((cbk->mask & ans->mask) != 0)) { + dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict; + data = dict_get(dict, key); + if (data != NULL) { + list[ans->idx] = data; + count++; + } } - - i++; } - *count = i; - - return 1; + return count; } -char * ec_concat_prepare(xlator_t * xl, char ** sep, char ** post, - const char * fmt, va_list args) +int32_t +ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post, + const char *fmt, va_list args) { - char * str, * tmp; + char *tmp; int32_t len; - len = gf_vasprintf(&str, fmt, args); - if (len < 0) - { - return NULL; + len = gf_vasprintf(str, fmt, args); + if (len < 0) { + return -ENOMEM; } - tmp = strchr(str, '{'); - if (tmp == NULL) - { + tmp = strchr(*str, '{'); + if (tmp == NULL) { goto out; } *tmp++ = 0; *sep = tmp; tmp = strchr(tmp, '}'); - if (tmp == NULL) - { + if (tmp == NULL) { goto out; } *tmp++ = 0; *post = tmp; - return str; + return 0; out: - gf_log(xl->name, GF_LOG_ERROR, "Invalid concat format"); + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_FORMAT, + "Invalid concat format"); - GF_FREE(str); + GF_FREE(*str); - return NULL; + return -EINVAL; } -int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk, - int32_t which, char * key, ...) +static int32_t +ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key, + const char *def, gf_boolean_t global, const char *fmt, ...) { - data_t * data[cbk->count]; - char * str = NULL, * pre = NULL, * sep, * post; - dict_t * dict; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + char *str = NULL, *pre = NULL, *sep, *post; + dict_t *dict; va_list args; - int32_t i, num, len, prelen, postlen, seplen, tmp; - int32_t ret = -1; + int32_t i, num, len, deflen, prelen, postlen, seplen, tmp; + int32_t err; - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) - { - return -1; - } + ec_dict_list(data, cbk, which, key, global); - va_start(args, key); - pre = ec_concat_prepare(cbk->fop->xl, &sep, &post, fmt, args); + va_start(args, fmt); + err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args); va_end(args); - if (pre == NULL) - { - return -1; + if (err != 0) { + return err; } prelen = strlen(pre); seplen = strlen(sep); postlen = strlen(post); - len = prelen + (num - 1) * seplen + postlen + 1; - for (i = 0; i < num; i++) - { - len += data[i]->len - 1; + deflen = 0; + if (def != NULL) { + deflen = strlen(def); } + len = prelen + postlen + 1; + num = -1; + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (def == NULL) { + continue; + } + len += deflen; + } else { + len += data[i]->len - 1; + } + if (num >= 0) { + len += seplen; + } + num++; + } + + err = -ENOMEM; + str = GF_MALLOC(len, gf_common_mt_char); - if (str == NULL) - { + if (str == NULL) { goto out; } memcpy(str, pre, prelen); len = prelen; - for (i = 0; i < num; i++) - { - if (i > 0) { + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (deflen == 0) { + continue; + } + tmp = deflen; + memcpy(str + len, def, tmp); + } else { + tmp = data[i]->len - 1; + memcpy(str + len, data[i]->data, tmp); + } + len += tmp; + if (i < num) { memcpy(str + len, sep, seplen); len += seplen; } - tmp = data[i]->len - 1; - memcpy(str + len, data[i]->data, tmp); - len += tmp; } memcpy(str + len, post, postlen + 1); dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if (dict_set_dynstr(dict, key, str) != 0) - { + if (new_key) { + key = new_key; + } + err = dict_set_dynstr(dict, key, str); + if (err != 0) { goto out; } str = NULL; - ret = 0; - out: GF_FREE(str); GF_FREE(pre); - return ret; + return err; } -int32_t ec_dict_data_merge(ec_cbk_data_t * cbk, int32_t which, char * key) +int32_t +ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t * data[cbk->count]; - dict_t * dict, * lockinfo, * tmp; - char * ptr = NULL; - int32_t i, num, len; - int32_t ret = -1; - - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) - { - return -1; - } + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict, *lockinfo, *tmp = NULL; + char *ptr = NULL; + int32_t i, len; + int32_t err; + + ec_dict_list(data, cbk, which, key, _gf_false); lockinfo = dict_new(); - if (lockinfo == NULL) - { - return -1; + if (lockinfo == NULL) { + return -ENOMEM; } - if (dict_unserialize(data[0]->data, data[0]->len, &lockinfo) != 0) - { - goto out; - } + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } - for (i = 1; i < num; i++) - { tmp = dict_new(); - if (tmp == NULL) - { + if (tmp == NULL) { + err = -ENOMEM; + + goto out; + } + err = dict_unserialize(data[i]->data, data[i]->len, &tmp); + if (err != 0) { goto out; } - if ((dict_unserialize(data[i]->data, data[i]->len, &tmp) != 0) || - (dict_copy(tmp, lockinfo) == NULL)) - { - dict_unref(tmp); + if (dict_copy(tmp, lockinfo) == NULL) { + err = -ENOMEM; goto out; } @@ -391,154 +483,190 @@ int32_t ec_dict_data_merge(ec_cbk_data_t * cbk, int32_t which, char * key) dict_unref(tmp); } - len = dict_serialized_length(lockinfo); - if (len < 0) - { - goto out; - } - ptr = GF_MALLOC(len, gf_common_mt_char); - if (ptr == NULL) - { - goto out; - } - if (dict_serialize(lockinfo, ptr) != 0) - { + tmp = NULL; + + err = dict_allocate_and_serialize(lockinfo, (char **)&ptr, + (unsigned int *)&len); + if (err != 0) { goto out; } + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if (dict_set_dynptr(dict, key, ptr, len) != 0) - { + err = dict_set_dynptr(dict, key, ptr, len); + if (err != 0) { goto out; } ptr = NULL; - ret = 0; - out: GF_FREE(ptr); dict_unref(lockinfo); + if (tmp != NULL) { + dict_unref(tmp); + } - return ret; + return err; } -int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key) +int32_t +ec_dict_data_uuid(ec_cbk_data_t *cbk, int32_t which, char *key) { - ec_cbk_data_t * ans, * min; - dict_t * src, * dst; - data_t * data; + ec_cbk_data_t *ans, *min; + dict_t *src, *dst; + data_t *data; min = cbk; - for (ans = cbk->next; ans != NULL; ans = ans->next) - { - if (ans->idx < min->idx) - { + for (ans = cbk->next; ans != NULL; ans = ans->next) { + if (ans->idx < min->idx) { min = ans; } } - if (min != cbk) - { + if (min != cbk) { src = (which == EC_COMBINE_XDATA) ? min->xdata : min->dict; dst = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; data = dict_get(src, key); - if (data == NULL) - { - return -1; + if (data == NULL) { + return -ENOENT; } - if (dict_set(dst, key, data) != 0) - { - return -1; + if (dict_set(dst, key, data) != 0) { + return -ENOMEM; } } return 0; } -int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key) +int32_t +ec_dict_data_iatt(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t * data[cbk->count]; - dict_t * dict; - int32_t i, num; - uint32_t max, tmp; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + struct iatt *stbuf, *tmp; + int32_t i, ret; - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) - { - return -1; - } + ec_dict_list(data, cbk, which, key, _gf_false); - if (num <= 1) - { - return 0; + stbuf = NULL; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + tmp = data_to_iatt(data[i], key); + if (tmp == NULL) { + ret = -EINVAL; + goto out; + } + if (stbuf == NULL) { + stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char); + if (stbuf == NULL) { + ret = -ENOMEM; + goto out; + } + *stbuf = *tmp; + } else { + if (!ec_iatt_combine(cbk->fop, stbuf, tmp, 1)) { + ret = -EINVAL; + goto out; + } + } } - max = data_to_uint32(data[0]); - for (i = 1; i < num; i++) - { - tmp = data_to_uint32(data[i]); - if (max < tmp) - { - max = tmp; + if ((stbuf != NULL) && (stbuf->ia_type == IA_IFREG)) { + ec_iatt_rebuild(ec, stbuf, 1, cbk->count); + /* TODO: not sure if an iatt could come in xdata from a fop that takes + * no locks. */ + if (!ec_get_inode_size(cbk->fop, cbk->fop->locks[0].lock->loc.inode, + &stbuf->ia_size)) { + ret = -EINVAL; + goto out; } } dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if (dict_set_uint32(dict, key, max) != 0) - { - return -1; + ret = dict_set_iatt(dict, key, stbuf, false); + if (ret >= 0) { + stbuf = NULL; } - return 0; +out: + GF_FREE(stbuf); + + return ret; } -int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key) +int32_t +ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t *data[cbk->count]; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; dict_t *dict; - int32_t i, num; - uint64_t max, tmp; + int32_t i; + uint32_t max, tmp; - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) { - return -1; - } + ec_dict_list(data, cbk, which, key, _gf_false); - if (num <= 1) { - return 0; - } + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } - max = data_to_uint64(data[0]); - for (i = 1; i < num; i++) { - tmp = data_to_uint64(data[i]); + tmp = data_to_uint32(data[i]); if (max < tmp) { max = tmp; } } dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if (dict_set_uint64(dict, key, max) != 0) { - return -1; - } - - return 0; + return dict_set_uint32(dict, key, max); } -int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) +int32_t +ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t *data[cbk->count]; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; dict_t *dict; - ec_t *ec; - int32_t i, num; + int32_t i; uint64_t max, tmp; - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) { - return -1; + ec_dict_list(data, cbk, which, key, _gf_false); + + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + + tmp = data_to_uint64(data[i]); + if (max < tmp) { + max = tmp; + } } - if (num == 0) { + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + return dict_set_uint64(dict, key, max); +} + +int32_t +ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict = NULL; + int32_t i = 0; + quota_meta_t size = { + 0, + }; + quota_meta_t max_size = { + 0, + }; + + if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) { return 0; } @@ -547,77 +675,83 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) * bricks and we can receive slightly different values. If that's the * case, we take the maximum of all received values. */ - max = ntoh64(*(uint64_t *)data_to_ptr(data[0])); - for (i = 1; i < num; i++) { - tmp = ntoh64(*(uint64_t *)data_to_ptr(data[i])); - if (max < tmp) { - max = tmp; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) || + (quota_data_to_meta(data[i], &size) < 0)) { + continue; } + + if (size.size > max_size.size) + max_size.size = size.size; + if (size.file_count > max_size.file_count) + max_size.file_count = size.file_count; + if (size.dir_count > max_size.dir_count) + max_size.dir_count = size.dir_count; } - ec = cbk->fop->xl->private; - max *= ec->fragments; + max_size.size *= ec->fragments; dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if (ec_dict_set_number(dict, key, max) != 0) { - return -1; - } - - return 0; + return quota_dict_set_meta(dict, key, &max_size, IA_IFDIR); } -int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key) +int32_t +ec_dict_data_stime(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t * data[cbk->count]; - dict_t * dict; - int32_t i, num; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i, err; - num = cbk->count; - if (!ec_dict_list(data, &num, cbk, which, key)) - { - return -1; - } + ec_dict_list(data, cbk, which, key, _gf_false); dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - for (i = 1; i < num; i++) - { - if (gf_get_max_stime(cbk->fop->xl, dict, key, data[i]) != 0) - { - gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "STIME combination " - "failed"); + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]); + if (err != 0) { + gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed"); - return -1; + return err; } } return 0; } -int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value, - void * arg) +int32_t +ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg) { - ec_dict_combine_t * data = arg; + ec_dict_combine_t *data = arg; if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) || - (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) - { - return ec_dict_data_concat("(<EC:%s> { })", data->cbk, data->which, - key, data->cbk->fop->xl->name); + (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) { + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, _gf_false, "(<EC:%s> { })", + data->cbk->fop->xl->name); } - if (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0) - { - return ec_dict_data_concat("{\n}", data->cbk, data->which, key); + if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) { + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, "{\n}"); } - if (strncmp(key, GF_XATTR_LOCKINFO_KEY, - strlen(GF_XATTR_LOCKINFO_KEY)) == 0) - { + if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) { return ec_dict_data_merge(data->cbk, data->which, key); } - if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) - { + if (strcmp(key, GET_LINK_COUNT) == 0) { + return ec_dict_data_max32(data->cbk, data->which, key); + } + + if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) { + return ec_dict_data_max32(data->cbk, data->which, key); + } + if ((strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0)) { return ec_dict_data_max32(data->cbk, data->which, key); } @@ -625,17 +759,22 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value, return ec_dict_data_quota(data->cbk, data->which, key); } /* Ignore all other quota attributes */ - if (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) { + if (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) { return 0; } - if (XATTR_IS_NODE_UUID(key)) - { - return ec_dict_data_uuid(data->cbk, data->which, key); + if (XATTR_IS_NODE_UUID(key)) { + if (data->cbk->fop->int32) { + /* List of node uuid is requested */ + return ec_dict_data_concat(data->cbk, data->which, key, + GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR, + _gf_true, "{ }"); + } else { + return ec_dict_data_uuid(data->cbk, data->which, key); + } } - if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) - { + if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) { return ec_dict_data_stime(data->cbk, data->which, key); } @@ -643,71 +782,74 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value, return ec_dict_data_max64(data->cbk, data->which, key); } + if (strcmp(key, GF_PRESTAT) == 0 || strcmp(key, GF_POSTSTAT) == 0) { + return ec_dict_data_iatt(data->cbk, data->which, key); + } + return 0; } -int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which) +int32_t +ec_dict_combine(ec_cbk_data_t *cbk, int32_t which) { - dict_t * dict; + dict_t *dict = NULL; ec_dict_combine_t data; + int32_t err = 0; data.cbk = cbk; data.which = which; dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - if ((dict != NULL) && - (dict_foreach(dict, ec_dict_data_combine, &data) != 0)) - { - gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Dictionary combination " - "failed"); + if (dict != NULL) { + err = dict_foreach(dict, ec_dict_data_combine, &data); + if (err != 0) { + gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_DICT_COMBINE_FAIL, "Dictionary combination failed"); - return 0; + return err; + } } - return 1; + return 0; } -int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count, - struct iovec * src_vector, int32_t src_count) +int32_t +ec_vector_compare(struct iovec *dst_vector, int32_t dst_count, + struct iovec *src_vector, int32_t src_count) { int32_t dst_size = 0, src_size = 0; - if (dst_count > 0) - { + if (dst_count > 0) { dst_size = iov_length(dst_vector, dst_count); } - if (src_count > 0) - { + if (src_count > 0) { src_size = iov_length(src_vector, src_count); } return (dst_size == src_size); } -int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src) +int32_t +ec_flock_compare(struct gf_flock *dst, struct gf_flock *src) { - if ((dst->l_type != src->l_type) || - (dst->l_whence != src->l_whence) || - (dst->l_start != src->l_start) || - (dst->l_len != src->l_len) || + if ((dst->l_type != src->l_type) || (dst->l_whence != src->l_whence) || + (dst->l_start != src->l_start) || (dst->l_len != src->l_len) || (dst->l_pid != src->l_pid) || - !is_same_lkowner(&dst->l_owner, &src->l_owner)) - { + !is_same_lkowner(&dst->l_owner, &src->l_owner)) { return 0; } return 1; } -void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src) +void +ec_statvfs_combine(struct statvfs *dst, struct statvfs *src) { - if (dst->f_bsize < src->f_bsize) - { + if (dst->f_bsize < src->f_bsize) { dst->f_bsize = src->f_bsize; } - if (dst->f_frsize < src->f_frsize) - { + if (dst->f_frsize < src->f_frsize) { dst->f_blocks *= dst->f_frsize; dst->f_blocks /= src->f_frsize; @@ -718,9 +860,7 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src) dst->f_bavail /= src->f_frsize; dst->f_frsize = src->f_frsize; - } - else if (dst->f_frsize > src->f_frsize) - { + } else if (dst->f_frsize > src->f_frsize) { src->f_blocks *= src->f_frsize; src->f_blocks /= dst->f_frsize; @@ -730,143 +870,126 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src) src->f_bavail *= src->f_frsize; src->f_bavail /= dst->f_frsize; } - if (dst->f_blocks > src->f_blocks) - { + if (dst->f_blocks > src->f_blocks) { dst->f_blocks = src->f_blocks; } - if (dst->f_bfree > src->f_bfree) - { + if (dst->f_bfree > src->f_bfree) { dst->f_bfree = src->f_bfree; } - if (dst->f_bavail > src->f_bavail) - { + if (dst->f_bavail > src->f_bavail) { dst->f_bavail = src->f_bavail; } - if (dst->f_files < src->f_files) - { + if (dst->f_files < src->f_files) { dst->f_files = src->f_files; } - if (dst->f_ffree > src->f_ffree) - { + if (dst->f_ffree > src->f_ffree) { dst->f_ffree = src->f_ffree; } - if (dst->f_favail > src->f_favail) - { + if (dst->f_favail > src->f_favail) { dst->f_favail = src->f_favail; } - if (dst->f_namemax > src->f_namemax) - { + if (dst->f_namemax > src->f_namemax) { dst->f_namemax = src->f_namemax; } - if (dst->f_flag != src->f_flag) - { - gf_log(THIS->name, GF_LOG_DEBUG, "Mismatching file system flags " - "(%lX, %lX)", - dst->f_flag, src->f_flag); + if (dst->f_flag != src->f_flag) { + gf_msg_debug(THIS->name, 0, + "Mismatching file system flags " + "(%lX, %lX)", + dst->f_flag, src->f_flag); } dst->f_flag &= src->f_flag; } -int32_t ec_combine_check(ec_cbk_data_t * dst, ec_cbk_data_t * src, - ec_combine_f combine) +int32_t +ec_combine_check(ec_cbk_data_t *dst, ec_cbk_data_t *src, ec_combine_f combine) { - ec_fop_data_t * fop = dst->fop; + ec_fop_data_t *fop = dst->fop; - if (dst->op_ret != src->op_ret) - { - gf_log(fop->xl->name, GF_LOG_DEBUG, "Mismatching return code in " - "answers of '%s': %d <-> %d", - ec_fop_name(fop->id), dst->op_ret, src->op_ret); + if (dst->op_ret != src->op_ret) { + gf_msg_debug(fop->xl->name, 0, + "Mismatching return code in " + "answers of '%s': %d <-> %d", + ec_fop_name(fop->id), dst->op_ret, src->op_ret); return 0; } - if (dst->op_ret < 0) - { - if (dst->op_errno != src->op_errno) - { - gf_log(fop->xl->name, GF_LOG_DEBUG, "Mismatching errno code in " - "answers of '%s': %d <-> %d", - ec_fop_name(fop->id), dst->op_errno, src->op_errno); + if (dst->op_ret < 0) { + if (dst->op_errno != src->op_errno) { + gf_msg_debug(fop->xl->name, 0, + "Mismatching errno code in " + "answers of '%s': %d <-> %d", + ec_fop_name(fop->id), dst->op_errno, src->op_errno); return 0; } } - if (!ec_dict_compare(dst->xdata, src->xdata)) - { - gf_log(fop->xl->name, GF_LOG_WARNING, "Mismatching xdata in answers " - "of '%s'", + if (!ec_dict_compare(dst->xdata, src->xdata)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_XDATA_MISMATCH, + "Mismatching xdata in answers " + "of '%s'", ec_fop_name(fop->id)); return 0; } - if ((dst->op_ret >= 0) && (combine != NULL)) - { + if ((dst->op_ret >= 0) && (combine != NULL)) { return combine(fop, dst, src); } return 1; } -void ec_combine(ec_cbk_data_t * cbk, ec_combine_f combine) +void +ec_combine(ec_cbk_data_t *newcbk, ec_combine_f combine) { - ec_fop_data_t * fop = cbk->fop; - ec_cbk_data_t * ans = NULL, * tmp = NULL; - struct list_head * item = NULL; - int32_t needed = 0, resume = 0; + ec_fop_data_t *fop = newcbk->fop; + ec_cbk_data_t *cbk = NULL, *tmp = NULL; + struct list_head *item = NULL; + int32_t needed = 0; char str[32]; LOCK(&fop->lock); + fop->received |= newcbk->mask; + item = fop->cbk_list.prev; - list_for_each_entry(ans, &fop->cbk_list, list) + list_for_each_entry(cbk, &fop->cbk_list, list) { - if (ec_combine_check(cbk, ans, combine)) - { - cbk->count += ans->count; - cbk->mask |= ans->mask; - - item = ans->list.prev; - while (item != &fop->cbk_list) - { + if (ec_combine_check(newcbk, cbk, combine)) { + newcbk->count += cbk->count; + newcbk->mask |= cbk->mask; + + item = cbk->list.prev; + while (item != &fop->cbk_list) { tmp = list_entry(item, ec_cbk_data_t, list); - if (tmp->count >= cbk->count) - { + if (tmp->count >= newcbk->count) { break; } item = item->prev; } - list_del(&ans->list); + list_del(&cbk->list); - cbk->next = ans; + newcbk->next = cbk; break; } } - list_add(&cbk->list, item); + list_add(&newcbk->list, item); ec_trace("ANSWER", fop, "combine=%s[%d]", - ec_bin(str, sizeof(str), cbk->mask, 0), cbk->count); + ec_bin(str, sizeof(str), newcbk->mask, 0), newcbk->count); - if ((cbk->count == fop->expected) && (fop->answer == NULL)) { - fop->answer = cbk; - - resume = 1; + cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list); + if ((fop->mask ^ fop->remaining) == fop->received) { + needed = fop->minimum - cbk->count; } - ans = list_entry(fop->cbk_list.next, ec_cbk_data_t, list); - needed = fop->minimum - ans->count - fop->winds + 1; - UNLOCK(&fop->lock); if (needed > 0) { - ec_dispatch_next(fop, cbk->idx); - } else if (resume) { - ec_update_bad(fop, cbk->mask); - - ec_resume(fop, 0); + ec_dispatch_next(fop, newcbk->idx); } } diff --git a/xlators/cluster/ec/src/ec-combine.h b/xlators/cluster/ec/src/ec-combine.h index 360844ccd71..1010cc3be26 100644 --- a/xlators/cluster/ec/src/ec-combine.h +++ b/xlators/cluster/ec/src/ec-combine.h @@ -11,24 +11,34 @@ #ifndef __EC_COMBINE_H__ #define __EC_COMBINE_H__ -#define EC_COMBINE_DICT 0 +#define EC_COMBINE_DICT 0 #define EC_COMBINE_XDATA 1 -typedef int32_t (* ec_combine_f)(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src); - -void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count, - int32_t answers); - -int32_t ec_iatt_combine(struct iatt * dst, struct iatt * src, int32_t count); -int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2); -int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count, - struct iovec * src_vector, int32_t src_count); -int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src); -void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src); - -int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which); - -void ec_combine(ec_cbk_data_t * cbk, ec_combine_f combine); - +typedef int32_t (*ec_combine_f)(ec_fop_data_t *fop, ec_cbk_data_t *dst, + ec_cbk_data_t *src); + +void +ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers); + +int32_t +ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src, + int32_t count); +int32_t +ec_dict_compare(dict_t *dict1, dict_t *dict2); +int32_t +ec_vector_compare(struct iovec *dst_vector, int32_t dst_count, + struct iovec *src_vector, int32_t src_count); +int32_t +ec_flock_compare(struct gf_flock *dst, struct gf_flock *src); +void +ec_statvfs_combine(struct statvfs *dst, struct statvfs *src); + +int32_t +ec_dict_combine(ec_cbk_data_t *cbk, int32_t which); + +void +ec_combine(ec_cbk_data_t *cbk, ec_combine_f combine); + +int32_t +ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src); #endif /* __EC_COMBINE_H__ */ diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c index 92f7657cd75..b955efd8c2d 100644 --- a/xlators/cluster/ec/src/ec-common.c +++ b/xlators/cluster/ec/src/ec-common.c @@ -8,222 +8,382 @@ cases as published by the Free Software Foundation. */ -#include "byte-order.h" +#include <glusterfs/byte-order.h> +#include <glusterfs/hashfn.h> #include "ec-mem-types.h" -#include "ec-data.h" +#include "ec-types.h" #include "ec-helpers.h" #include "ec-combine.h" #include "ec-common.h" #include "ec-fops.h" #include "ec-method.h" #include "ec.h" +#include "ec-messages.h" -int32_t ec_child_valid(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +#define EC_INVALID_INDEX UINT32_MAX + +void +ec_update_fd_status(fd_t *fd, xlator_t *xl, int idx, int32_t ret_status) { - return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1); + ec_fd_t *fd_ctx; + + if (fd == NULL) + return; + + LOCK(&fd->lock); + { + fd_ctx = __ec_fd_get(fd, xl); + if (fd_ctx) { + if (ret_status >= 0) + fd_ctx->fd_status[idx] = EC_FD_OPENED; + else + fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED; + } + } + UNLOCK(&fd->lock); } -int32_t ec_child_next(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +static uintptr_t +ec_fd_ctx_need_open(fd_t *fd, xlator_t *this, uintptr_t mask) { - while (!ec_child_valid(ec, fop, idx)) + int i = 0; + int count = 0; + ec_t *ec = NULL; + ec_fd_t *fd_ctx = NULL; + uintptr_t need_open = 0; + + ec = this->private; + + fd_ctx = ec_fd_get(fd, this); + if (!fd_ctx) + return count; + + LOCK(&fd->lock); { - if (++idx >= ec->nodes) - { - idx = 0; - } - if (idx == fop->first) - { - return -1; + for (i = 0; i < ec->nodes; i++) { + if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) && + ((ec->xl_up & (1 << i)) != 0) && ((mask & (1 << i)) != 0)) { + fd_ctx->fd_status[i] = EC_FD_OPENING; + need_open |= (1 << i); + count++; + } } } + UNLOCK(&fd->lock); - return idx; + /* If fd needs to open on minimum number of nodes + * then ignore fixing the fd as it has been + * requested from heal operation. + */ + if (count >= ec->fragments) { + need_open = 0; + } + + return need_open; } -uintptr_t ec_inode_good(inode_t * inode, xlator_t * xl) +static gf_boolean_t +ec_is_fd_fixable(fd_t *fd) { - ec_inode_t * ctx; - uintptr_t bad = 0; + if (!fd || !fd->inode) + return _gf_false; + else if (fd_is_anonymous(fd)) + return _gf_false; + else if (gf_uuid_is_null(fd->inode->gfid)) + return _gf_false; - ctx = ec_inode_get(inode, xl); - if (ctx != NULL) - { - bad = ctx->bad; + return _gf_true; +} + +static void +ec_fix_open(ec_fop_data_t *fop, uintptr_t mask) +{ + uintptr_t need_open = 0; + int ret = 0; + int32_t flags = 0; + loc_t loc = { + 0, + }; + + if (!ec_is_fd_fixable(fop->fd)) + goto out; + + /* Evaluate how many remote fd's to be opened */ + need_open = ec_fd_ctx_need_open(fop->fd, fop->xl, mask); + if (need_open == 0) { + goto out; + } + + loc.inode = inode_ref(fop->fd->inode); + gf_uuid_copy(loc.gfid, fop->fd->inode->gfid); + ret = loc_path(&loc, NULL); + if (ret < 0) { + goto out; } - return ~bad; + flags = fop->fd->flags & (~(O_TRUNC | O_APPEND | O_CREAT | O_EXCL)); + if (IA_IFDIR == fop->fd->inode->ia_type) { + ec_opendir(fop->frame, fop->xl, need_open, + EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, + &fop->loc[0], fop->fd, NULL); + } else { + ec_open(fop->frame, fop->xl, need_open, + EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, &loc, + flags, fop->fd, NULL); + } + +out: + loc_wipe(&loc); } -uintptr_t ec_fd_good(fd_t * fd, xlator_t * xl) +static off_t +ec_range_end_get(off_t fl_start, uint64_t fl_size) { - ec_fd_t * ctx; - uintptr_t bad = 0; - - ctx = ec_fd_get(fd, xl); - if (ctx != NULL) - { - bad = ctx->bad; + if (fl_size > 0) { + if (fl_size >= EC_RANGE_FULL) { + /* Infinity */ + fl_start = LLONG_MAX; + } else { + fl_start += fl_size - 1; + if (fl_start < 0) { + /* Overflow */ + fl_start = LLONG_MAX; + } + } } - return ~bad; + return fl_start; } -uintptr_t ec_update_inode(ec_fop_data_t * fop, inode_t * inode, uintptr_t good, - uintptr_t bad) +static gf_boolean_t +ec_is_range_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2) { - ec_inode_t * ctx = NULL; + return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start)); +} - if (inode != NULL) - { - LOCK(&inode->lock); +static gf_boolean_t +ec_lock_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2) +{ + ec_t *ec = l1->fop->xl->private; - ctx = __ec_inode_get(inode, fop->xl); - if (ctx != NULL) - { - ctx->bad &= ~good; - bad |= ctx->bad; - ctx->bad = bad; - } + /* Fops like access/stat won't have to worry what the other fops are + * modifying as the fop is wound only to one brick. So it can be + * executed in parallel*/ + if (l1->fop->minimum == EC_MINIMUM_ONE || + l2->fop->minimum == EC_MINIMUM_ONE) + return _gf_false; - UNLOCK(&inode->lock); + if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) && + (l2->fop->flags & EC_FLAG_LOCK_SHARED)) + return _gf_false; + + if (!ec->parallel_writes) { + return _gf_true; } - return bad; + return ec_is_range_conflict(l1, l2); } -uintptr_t ec_update_fd(ec_fop_data_t * fop, fd_t * fd, uintptr_t good, - uintptr_t bad) +uint32_t +ec_select_first_by_read_policy(ec_t *ec, ec_fop_data_t *fop) { - ec_fd_t * ctx = NULL; + if (ec->read_policy == EC_ROUND_ROBIN) { + return ec->idx; + } else if (ec->read_policy == EC_GFID_HASH) { + if (fop->use_fd) { + return SuperFastHash((char *)fop->fd->inode->gfid, + sizeof(fop->fd->inode->gfid)) % + ec->nodes; + } else { + if (gf_uuid_is_null(fop->loc[0].gfid)) + loc_gfid(&fop->loc[0], fop->loc[0].gfid); + return SuperFastHash((char *)fop->loc[0].gfid, + sizeof(fop->loc[0].gfid)) % + ec->nodes; + } + } + return 0; +} - LOCK(&fd->lock); +static gf_boolean_t +ec_child_valid(ec_t *ec, ec_fop_data_t *fop, uint32_t idx) +{ + return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1); +} - ctx = __ec_fd_get(fd, fop->xl); - if (ctx != NULL) - { - ctx->bad &= ~good; - bad |= ctx->bad; - ctx->bad = bad; +static uint32_t +ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx) +{ + while (!ec_child_valid(ec, fop, idx)) { + if (++idx >= ec->nodes) { + idx = 0; + } + if (idx == fop->first) { + return EC_INVALID_INDEX; + } } - UNLOCK(&fd->lock); - - return bad; + return idx; } -int32_t ec_heal_report(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, uintptr_t mask, - uintptr_t good, uintptr_t bad, dict_t * xdata) +int32_t +ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good, + uintptr_t bad, uint32_t pending, dict_t *xdata) { if (op_ret < 0) { - gf_log(this->name, GF_LOG_WARNING, "Heal failed (error %d)", - op_errno); + gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL, + "Heal failed"); } else { if ((mask & ~good) != 0) { - gf_log(this->name, GF_LOG_INFO, "Heal succeeded on %d/%d " - "subvolumes", - ec_bits_count(mask & ~(good | bad)), - ec_bits_count(mask & ~good)); + gf_msg(this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_SUCCESS, + "Heal succeeded on %d/%d " + "subvolumes", + gf_bits_count(mask & ~(good | bad)), + gf_bits_count(mask & ~good)); } } return 0; } -int32_t ec_fop_needs_heal(ec_fop_data_t *fop) +static uintptr_t +ec_fop_needs_name_heal(ec_fop_data_t *fop) +{ + ec_t *ec = NULL; + ec_cbk_data_t *cbk = NULL; + ec_cbk_data_t *enoent_cbk = NULL; + + ec = fop->xl->private; + if (fop->id != GF_FOP_LOOKUP) + return 0; + + if (!fop->loc[0].name || strlen(fop->loc[0].name) == 0) + return 0; + + list_for_each_entry(cbk, &fop->cbk_list, list) + { + if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) { + enoent_cbk = cbk; + break; + } + } + + if (!enoent_cbk) + return 0; + + return ec->xl_up & ~enoent_cbk->mask; +} + +int32_t +ec_fop_needs_heal(ec_fop_data_t *fop) { ec_t *ec = fop->xl->private; + if (fop->lock_count == 0) { + /* + * if fop->lock_count is zero that means it saw version mismatch + * without any locks so it can't be trusted. If we launch a heal + * based on this it will lead to INODELKs which will affect I/O + * performance. Considering self-heal-daemon and operations on + * the inode from client which take locks can still trigger the + * heal we can choose to not attempt a heal when fop->lock_count + * is zero. + */ + return 0; + } return (ec->xl_up & ~(fop->remaining | fop->good)) != 0; } -void ec_check_status(ec_fop_data_t * fop) +void +ec_check_status(ec_fop_data_t *fop) { - ec_t * ec = fop->xl->private; + ec_t *ec = fop->xl->private; int32_t partial = 0; + char str1[32], str2[32], str3[32], str4[32], str5[32]; - if (fop->answer->op_ret >= 0) { - if ((fop->id == GF_FOP_LOOKUP) || - (fop->id == GF_FOP_STAT) || (fop->id == GF_FOP_FSTAT)) { + if (!ec_fop_needs_name_heal(fop) && !ec_fop_needs_heal(fop)) { + return; + } + + if (fop->answer && fop->answer->op_ret >= 0) { + if ((fop->id == GF_FOP_LOOKUP) || (fop->id == GF_FOP_STAT) || + (fop->id == GF_FOP_FSTAT)) { partial = fop->answer->iatt[0].ia_type == IA_IFDIR; } else if (fop->id == GF_FOP_OPENDIR) { partial = 1; } } - if (!ec_fop_needs_heal(fop)) { - return; - } - - gf_log(fop->xl->name, GF_LOG_WARNING, "Operation failed on some " - "subvolumes (up=%lX, mask=%lX, " - "remaining=%lX, good=%lX, bad=%lX)", - ec->xl_up, fop->mask, fop->remaining, fop->good, fop->bad); - - if (fop->use_fd) - { + gf_msg( + fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS, + "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, " + "remaining=%s, good=%s, bad=%s," + "(Least significant bit represents first client/brick of subvol), %s)", + gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes, + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes), + ec_bin(str4, sizeof(str4), fop->good, ec->nodes), + ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good), + ec->nodes), + ec_msg_str(fop)); + if (fop->use_fd) { if (fop->fd != NULL) { ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, fop->fd, partial, NULL); } - } - else - { + } else { ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, &fop->loc[0], partial, NULL); - if (fop->loc[1].inode != NULL) - { + if (fop->loc[1].inode != NULL) { ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, &fop->loc[1], partial, NULL); } } } -void ec_update_bad(ec_fop_data_t * fop, uintptr_t good) +void +ec_update_good(ec_fop_data_t *fop, uintptr_t good) { - ec_t *ec = fop->xl->private; - uintptr_t bad; - - bad = ec->xl_up & ~(fop->remaining | good); - fop->bad |= bad; - fop->good |= good; - - if (fop->parent == NULL) - { - if ((fop->flags & EC_FLAG_UPDATE_LOC_PARENT) != 0) - { - ec_update_inode(fop, fop->loc[0].parent, good, bad); - } - if ((fop->flags & EC_FLAG_UPDATE_LOC_INODE) != 0) - { - ec_update_inode(fop, fop->loc[0].inode, good, bad); - } - ec_update_inode(fop, fop->loc[1].inode, good, bad); - if ((fop->flags & EC_FLAG_UPDATE_FD_INODE) != 0) - { - ec_update_inode(fop, fop->fd->inode, good, bad); - } - if ((fop->flags & EC_FLAG_UPDATE_FD) != 0) - { - ec_update_fd(fop, fop->fd, good, bad); - } + fop->good = good; + /* Fops that are executed only on one brick do not have enough information + * to decide if healing is needed or not. */ + if ((fop->expected != 1) && (fop->parent == NULL)) { ec_check_status(fop); } } +void +ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop) +{ + /* Fops that are executed only on one brick do not have enough information + * to update the global mask of good bricks. */ + if (fop->expected == 1) { + return; + } + + /* When updating the good mask of the lock, we only take into consideration + * those bits corresponding to the bricks where the fop has been executed. + * Bad bricks are removed from good_mask, but once marked as bad it's never + * set to good until the lock is released and reacquired */ + + lock->good_mask &= fop->good | fop->remaining; +} -void __ec_fop_set_error(ec_fop_data_t * fop, int32_t error) +void +__ec_fop_set_error(ec_fop_data_t *fop, int32_t error) { - if ((error != 0) && (fop->error == 0)) - { + if ((error != 0) && (fop->error == 0)) { fop->error = error; } } -void ec_fop_set_error(ec_fop_data_t * fop, int32_t error) +void +ec_fop_set_error(ec_fop_data_t *fop, int32_t error) { LOCK(&fop->lock); @@ -232,17 +392,63 @@ void ec_fop_set_error(ec_fop_data_t * fop, int32_t error) UNLOCK(&fop->lock); } -void ec_sleep(ec_fop_data_t *fop) +gf_boolean_t +ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro) +{ + if ((error != 0) && (cbk->op_ret >= 0)) { + /* If cbk->op_errno was 0, it means that the fop succeeded and this + * error has happened while processing the answer. If the operation was + * read-only, there's no problem (i.e. we simply return the generated + * error code). However if it caused a modification, we must return EIO + * to indicate that the operation has been partially executed. */ + cbk->op_errno = ro ? error : EIO; + cbk->op_ret = -1; + + ec_fop_set_error(cbk->fop, cbk->op_errno); + } + + return (cbk->op_ret < 0); +} + +ec_cbk_data_t * +ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro) +{ + ec_cbk_data_t *cbk; + int32_t err; + + cbk = fop->answer; + if (cbk == NULL) { + ec_fop_set_error(fop, EIO); + + return NULL; + } + + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); + } + + err = ec_dict_combine(cbk, EC_COMBINE_XDATA); + if (ec_cbk_set_error(cbk, -err, ro)) { + return NULL; + } + + return cbk; +} + +void +ec_sleep(ec_fop_data_t *fop) { LOCK(&fop->lock); + GF_ASSERT(fop->refs > 0); fop->refs++; fop->jobs++; UNLOCK(&fop->lock); } -int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume) +int32_t +ec_check_complete(ec_fop_data_t *fop, ec_resume_f resume) { int32_t error = -1; @@ -250,14 +456,11 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume) GF_ASSERT(fop->resume == NULL); - if (fop->jobs != 0) - { + if (--fop->jobs != 0) { ec_trace("WAIT", fop, "resume=%p", resume); fop->resume = resume; - } - else - { + } else { error = fop->error; fop->error = 0; } @@ -267,22 +470,8 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume) return error; } -void ec_wait_winds(ec_fop_data_t * fop) -{ - LOCK(&fop->lock); - - if (fop->winds > 0) - { - fop->jobs++; - fop->refs++; - - fop->flags |= EC_FLAG_WAITING_WINDS; - } - - UNLOCK(&fop->lock); -} - -void ec_resume(ec_fop_data_t * fop, int32_t error) +void +ec_resume(ec_fop_data_t *fop, int32_t error) { ec_resume_f resume = NULL; @@ -290,16 +479,13 @@ void ec_resume(ec_fop_data_t * fop, int32_t error) __ec_fop_set_error(fop, error); - if (--fop->jobs == 0) - { + if (--fop->jobs == 0) { resume = fop->resume; fop->resume = NULL; - if (resume != NULL) - { + if (resume != NULL) { ec_trace("RESUME", fop, "error=%d", error); - if (fop->error != 0) - { + if (fop->error != 0) { error = fop->error; } fop->error = 0; @@ -308,31 +494,50 @@ void ec_resume(ec_fop_data_t * fop, int32_t error) UNLOCK(&fop->lock); - if (resume != NULL) - { + if (resume != NULL) { resume(fop, error); } ec_fop_data_release(fop); } -void ec_resume_parent(ec_fop_data_t * fop, int32_t error) +void +ec_resume_parent(ec_fop_data_t *fop) { - ec_fop_data_t * parent; + ec_fop_data_t *parent; + int32_t error = 0; parent = fop->parent; - if (parent != NULL) - { + if (parent != NULL) { + if ((fop->fop_flags & EC_FOP_NO_PROPAGATE_ERROR) == 0) { + error = fop->error; + } ec_trace("RESUME_PARENT", fop, "error=%u", error); fop->parent = NULL; ec_resume(parent, error); } } -void ec_complete(ec_fop_data_t * fop) +gf_boolean_t +ec_is_recoverable_error(int32_t op_errno) +{ + switch (op_errno) { + case ENOTCONN: + case ESTALE: + case ENOENT: + case EBADFD: /*Opened fd but brick is disconnected*/ + case EIO: /*Backend-fs crash like XFS/ext4 etc*/ + return _gf_true; + } + return _gf_false; +} + +void +ec_complete(ec_fop_data_t *fop) { - ec_cbk_data_t * cbk = NULL; + ec_cbk_data_t *cbk = NULL; int32_t resume = 0, update = 0; + int healing_count = 0; LOCK(&fop->lock); @@ -342,8 +547,10 @@ void ec_complete(ec_fop_data_t * fop) if (fop->answer == NULL) { if (!list_empty(&fop->cbk_list)) { cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list); - if ((cbk->count >= fop->minimum) && - ((cbk->op_ret >= 0) || (cbk->op_errno != ENOTCONN))) { + healing_count = gf_bits_count(cbk->mask & fop->healing); + /* fop shouldn't be treated as success if it is not + * successful on at least fop->minimum good copies*/ + if ((cbk->count - healing_count) >= fop->minimum) { fop->answer = cbk; update = 1; @@ -352,69 +559,144 @@ void ec_complete(ec_fop_data_t * fop) resume = 1; } - else if ((fop->flags & EC_FLAG_WAITING_WINDS) != 0) - { - resume = 1; - } } UNLOCK(&fop->lock); - /* ec_update_bad() locks inode->lock. This may cause deadlocks with - fop->lock when used in another order. Since ec_update_bad() will not + /* ec_update_good() locks inode->lock. This may cause deadlocks with + fop->lock when used in another order. Since ec_update_good() will not be called more than once for each fop, it can be called from outside the fop->lock locked region. */ if (update) { - ec_update_bad(fop, cbk->mask); + ec_update_good(fop, cbk->mask); } - if (resume) - { + if (resume) { ec_resume(fop, 0); } ec_fop_data_release(fop); } -int32_t ec_child_select(ec_fop_data_t * fop) +/* There could be already granted locks sitting on the bricks, unlock for which + * must be wound at all costs*/ +static gf_boolean_t +ec_must_wind(ec_fop_data_t *fop) { - ec_t * ec = fop->xl->private; - uintptr_t mask = 0; - int32_t first = 0, num = 0; + if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) || + (fop->id == GF_FOP_LK)) { + if (fop->flock.l_type == F_UNLCK) + return _gf_true; + } else if ((fop->id == GF_FOP_ENTRYLK) || (fop->id == GF_FOP_FENTRYLK)) { + if (fop->entrylk_cmd == ENTRYLK_UNLOCK) + return _gf_true; + } - fop->mask &= ec->node_mask; + return _gf_false; +} - mask = ec->xl_up; - if (fop->parent == NULL) - { - if (fop->loc[0].inode != NULL) { - mask &= ec_inode_good(fop->loc[0].inode, fop->xl); - } - if (fop->loc[1].inode != NULL) { - mask &= ec_inode_good(fop->loc[1].inode, fop->xl); +static gf_boolean_t +ec_internal_op(ec_fop_data_t *fop) +{ + if (ec_must_wind(fop)) + return _gf_true; + if (fop->id == GF_FOP_XATTROP) + return _gf_true; + if (fop->id == GF_FOP_FXATTROP) + return _gf_true; + if (fop->id == GF_FOP_OPEN) + return _gf_true; + return _gf_false; +} + +char * +ec_msg_str(ec_fop_data_t *fop) +{ + loc_t *loc1 = NULL; + loc_t *loc2 = NULL; + char gfid1[64] = {0}; + char gfid2[64] = {0}; + ec_fop_data_t *parent = fop->parent; + + if (fop->errstr) + return fop->errstr; + if (!fop->use_fd) { + loc1 = &fop->loc[0]; + loc2 = &fop->loc[1]; + + if (fop->id == GF_FOP_RENAME) { + gf_asprintf(&fop->errstr, + "FOP : '%s' failed on '%s' and '%s' with gfids " + "%s and %s respectively. Parent FOP: %s", + ec_fop_name(fop->id), loc1->path, loc2->path, + uuid_utoa_r(loc1->gfid, gfid1), + uuid_utoa_r(loc2->gfid, gfid2), + parent ? ec_fop_name(parent->id) : "No Parent"); + } else { + gf_asprintf( + &fop->errstr, + "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), loc1->path, + uuid_utoa_r(loc1->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); } - if (fop->fd != NULL) { - if (fop->fd->inode != NULL) { - mask &= ec_inode_good(fop->fd->inode, fop->xl); - } - mask &= ec_fd_good(fop->fd, fop->xl); + } else { + gf_asprintf( + &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); + } + return fop->errstr; +} + +static void +ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need, + int32_t loglevel) +{ + ec_t *ec = fop->xl->private; + char str1[32], str2[32], str3[32]; + + gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT, + "Insufficient available children for this request: " + "Have : %d, Need : %u : Child UP : %s " + "Mask: %s, Healing : %s : %s ", + have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->healing, ec->nodes), + ec_msg_str(fop)); +} + +static int32_t +ec_child_select(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + int32_t first = 0, num = 0; + + ec_fop_cleanup(fop); + + fop->mask &= ec->node_mask; + /* Wind the fop on same subvols as parent for any internal extra fops like + * head/tail read in case of writev fop. Unlocks shouldn't do this because + * unlock should go on all subvols where lock is performed*/ + if (fop->parent && !ec_internal_op(fop)) { + fop->mask &= (fop->parent->mask & ~fop->parent->healing); + if (ec_is_data_fop(fop->id)) { + fop->healing |= fop->parent->healing; } } - if ((fop->mask & ~mask) != 0) - { - gf_log(fop->xl->name, GF_LOG_WARNING, "Executing operation with " - "some subvolumes unavailable " - "(%lX)", fop->mask & ~mask); - fop->mask &= mask; + if ((fop->mask & ~ec->xl_up) != 0) { + gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_EXEC_UNAVAIL, + "Executing operation with " + "some subvolumes unavailable. (%" PRIXPTR "). %s ", + fop->mask & ~ec->xl_up, ec_msg_str(fop)); + fop->mask &= ec->xl_up; } - switch (fop->minimum) - { + switch (fop->minimum) { case EC_MINIMUM_ALL: - fop->minimum = ec_bits_count(fop->mask); - if (fop->minimum >= ec->fragments) - { + fop->minimum = gf_bits_count(fop->mask); + if (fop->minimum >= ec->fragments) { break; } case EC_MINIMUM_MIN: @@ -424,41 +706,51 @@ int32_t ec_child_select(ec_fop_data_t * fop) fop->minimum = 1; } - first = ec->idx; - if (++first >= ec->nodes) - { - first = 0; + if (ec->read_policy == EC_ROUND_ROBIN) { + first = ec->idx; + if (++first >= ec->nodes) { + first = 0; + } + ec->idx = first; } - ec->idx = first; + num = gf_bits_count(fop->mask); + /*Unconditionally wind on healing subvolumes*/ + fop->mask |= fop->healing; fop->remaining = fop->mask; + fop->received = 0; ec_trace("SELECT", fop, ""); - num = ec_bits_count(fop->mask); - if ((num < fop->minimum) && (num < ec->fragments)) - { - gf_log(ec->xl->name, GF_LOG_ERROR, "Insufficient available childs " - "for this request (have %d, need " - "%d)", num, fop->minimum); - + if ((num < fop->minimum) && (num < ec->fragments)) { + ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR); return 0; } - ec_sleep(fop); + if (!fop->parent && fop->lock_count && + (fop->locks[0].update[EC_DATA_TXN] || + fop->locks[0].update[EC_METADATA_TXN])) { + if (ec->quorum_count && (num < ec->quorum_count)) { + ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR); + return 0; + } + } return 1; } -int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx) +void +ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx) { - ec_t * ec = fop->xl->private; + uint32_t i = EC_INVALID_INDEX; + ec_t *ec = fop->xl->private; LOCK(&fop->lock); - idx = ec_child_next(ec, fop, idx); - if (idx >= 0) - { + i = ec_child_next(ec, fop, idx); + if (i < EC_MAX_NODES) { + idx = i; + fop->remaining ^= 1ULL << idx; ec_trace("EXECUTE", fop, "idx=%d", idx); @@ -469,20 +761,18 @@ int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx) UNLOCK(&fop->lock); - if (idx >= 0) - { + if (i < EC_MAX_NODES) { fop->wind(ec, fop, idx); } - - return idx; } -void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask) +void +ec_dispatch_mask(ec_fop_data_t *fop, uintptr_t mask) { - ec_t * ec = fop->xl->private; + ec_t *ec = fop->xl->private; int32_t count, idx; - count = ec_bits_count(mask); + count = gf_bits_count(mask); LOCK(&fop->lock); @@ -496,10 +786,8 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask) UNLOCK(&fop->lock); idx = 0; - while (mask != 0) - { - if ((mask & 1) != 0) - { + while (mask != 0) { + if ((mask & 1) != 0) { fop->wind(ec, fop, idx); } idx++; @@ -507,1016 +795,2042 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask) } } -void ec_dispatch_start(ec_fop_data_t * fop) +void +ec_dispatch_start(ec_fop_data_t *fop) { fop->answer = NULL; fop->good = 0; - fop->bad = 0; INIT_LIST_HEAD(&fop->cbk_list); - if (fop->lock_count > 0) - { + if (fop->lock_count > 0) { ec_owner_copy(fop->frame, &fop->req_frame->root->lk_owner); } } -void ec_dispatch_one(ec_fop_data_t * fop) +void +ec_dispatch_one(ec_fop_data_t *fop) { - ec_t * ec = fop->xl->private; - ec_dispatch_start(fop); - if (ec_child_select(fop)) - { + if (ec_child_select(fop)) { + ec_sleep(fop); + fop->expected = 1; - fop->first = ec->idx; + fop->first = ec_select_first_by_read_policy(fop->xl->private, fop); ec_dispatch_next(fop, fop->first); } } -int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret, - int32_t op_errno) +gf_boolean_t +ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk) { - if ((op_ret < 0) && (op_errno == ENOTCONN)) - { - return (ec_dispatch_next(fop, idx) >= 0); + ec_cbk_data_t *tmp; + + tmp = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + *cbk = tmp; + } + if ((tmp != NULL) && (tmp->op_ret < 0) && + ec_is_recoverable_error(tmp->op_errno)) { + GF_ASSERT(fop->mask & (1ULL << tmp->idx)); + fop->mask ^= (1ULL << tmp->idx); + if (fop->mask) { + return _gf_true; + } } - return 0; + return _gf_false; } -void ec_dispatch_inc(ec_fop_data_t * fop) +void +ec_dispatch_inc(ec_fop_data_t *fop) { ec_dispatch_start(fop); - if (ec_child_select(fop)) - { - fop->expected = ec_bits_count(fop->remaining); + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = gf_bits_count(fop->remaining); fop->first = 0; ec_dispatch_next(fop, 0); } } -void ec_dispatch_all(ec_fop_data_t * fop) +void +ec_dispatch_all(ec_fop_data_t *fop) { ec_dispatch_start(fop); - if (ec_child_select(fop)) - { - fop->expected = ec_bits_count(fop->remaining); + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = gf_bits_count(fop->remaining); fop->first = 0; ec_dispatch_mask(fop, fop->remaining); } } -void ec_dispatch_min(ec_fop_data_t * fop) +void +ec_dispatch_min(ec_fop_data_t *fop) { - ec_t * ec = fop->xl->private; + ec_t *ec = fop->xl->private; uintptr_t mask; - int32_t idx, count; + uint32_t idx; + int32_t count; ec_dispatch_start(fop); - if (ec_child_select(fop)) - { + if (ec_child_select(fop)) { + ec_sleep(fop); + fop->expected = count = ec->fragments; - fop->first = ec->idx; + fop->first = ec_select_first_by_read_policy(fop->xl->private, fop); idx = fop->first - 1; mask = 0; - while (count-- > 0) - { + while (count-- > 0) { idx = ec_child_next(ec, fop, idx + 1); - mask |= 1ULL << idx; + if (idx < EC_MAX_NODES) + mask |= 1ULL << idx; } ec_dispatch_mask(fop, mask); } } -ec_lock_t * ec_lock_allocate(xlator_t * xl, int32_t kind, loc_t * loc) +void +ec_succeed_all(ec_fop_data_t *fop) { - ec_t * ec = xl->private; - ec_lock_t * lock; + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + fop->expected = gf_bits_count(fop->remaining); + fop->first = 0; + + /* Simulate a successful execution on all bricks */ + ec_trace("SUCCEED", fop, ""); + + fop->good = fop->remaining; + fop->remaining = 0; + } +} + +ec_lock_t * +ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc) +{ + ec_t *ec = fop->xl->private; + ec_lock_t *lock; + int32_t err; if ((loc->inode == NULL) || - (uuid_is_null(loc->gfid) && uuid_is_null(loc->inode->gfid))) - { - gf_log(xl->name, GF_LOG_ERROR, "Trying to lock based on an invalid " - "inode"); + (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))) { + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_INODE, + "Trying to lock based on an invalid " + "inode"); + + __ec_fop_set_error(fop, EINVAL); return NULL; } lock = mem_get0(ec->lock_pool); - if (lock != NULL) - { - lock->kind = kind; - lock->good_mask = -1ULL; + if (lock != NULL) { + lock->good_mask = UINTPTR_MAX; + INIT_LIST_HEAD(&lock->owners); INIT_LIST_HEAD(&lock->waiting); - if (ec_loc_from_loc(xl, &lock->loc, loc) != 0) - { + INIT_LIST_HEAD(&lock->frozen); + err = ec_loc_from_loc(fop->xl, &lock->loc, loc); + if (err != 0) { mem_put(lock); lock = NULL; + + __ec_fop_set_error(fop, -err); } } return lock; } -void ec_lock_destroy(ec_lock_t * lock) +void +ec_lock_destroy(ec_lock_t *lock) { loc_wipe(&lock->loc); + if (lock->fd != NULL) { + fd_unref(lock->fd); + } mem_put(lock); } -int32_t ec_lock_compare(ec_lock_t * lock1, ec_lock_t * lock2) +int32_t +ec_lock_compare(ec_lock_t *lock1, ec_lock_t *lock2) { - return uuid_compare(lock1->loc.gfid, lock2->loc.gfid); + return gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid); } -ec_lock_link_t *ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, - int32_t update) +static void +ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags, loc_t *base, + off_t fl_start, uint64_t fl_size) { - ec_lock_t *new_lock, *tmp; - ec_lock_link_t *link = NULL; - int32_t tmp_update; + ec_lock_link_t *link; - new_lock = lock; + /* This check is only prepared for up to 2 locks per fop. If more locks + * are needed this must be changed. */ if ((fop->lock_count > 0) && - (ec_lock_compare(fop->locks[0].lock, new_lock) > 0)) - { - tmp = fop->locks[0].lock; - fop->locks[0].lock = new_lock; - new_lock = tmp; - - tmp_update = fop->locks_update; - fop->locks_update = update; - update = tmp_update; + (ec_lock_compare(fop->locks[0].lock, lock) < 0)) { + fop->first_lock = fop->lock_count; + } else { + /* When the first lock is added to the current fop, request lock + * counts from locks xlator to be able to determine if there is + * contention and release the lock sooner. */ + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + ec_fop_set_error(fop, ENOMEM); + return; + } + } + if (dict_set_str(fop->xdata, GLUSTERFS_INODELK_DOM_COUNT, + fop->xl->name) != 0) { + ec_fop_set_error(fop, ENOMEM); + return; + } } - fop->locks[fop->lock_count].lock = new_lock; - fop->locks[fop->lock_count].fop = fop; - fop->locks_update |= update << fop->lock_count; + link = &fop->locks[fop->lock_count++]; - fop->lock_count++; + link->lock = lock; + link->fop = fop; + link->update[EC_DATA_TXN] = (flags & EC_UPDATE_DATA) != 0; + link->update[EC_METADATA_TXN] = (flags & EC_UPDATE_META) != 0; + link->base = base; + link->fl_start = fl_start; + link->fl_end = ec_range_end_get(fl_start, fl_size); - if (lock->timer != NULL) { - link = lock->timer->data; - ec_trace("UNLOCK_CANCELLED", link->fop, "lock=%p", lock); - gf_timer_call_cancel(fop->xl->ctx, lock->timer); - lock->timer = NULL; - } else { - lock->refs++; - } - - return link; + lock->refs_pending++; } -void ec_lock_prepare_entry(ec_fop_data_t *fop, loc_t *loc, int32_t update) +static void +ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + loc_t *base, off_t fl_start, uint64_t fl_size) { - ec_lock_t * lock = NULL; - ec_inode_t * ctx = NULL; - ec_lock_link_t *link = NULL; - loc_t tmp; + ec_lock_t *lock = NULL; + ec_inode_t *ctx; - if ((fop->parent != NULL) || (fop->error != 0)) - { + if ((fop->parent != NULL) || (fop->error != 0) || (loc->inode == NULL)) { return; } - /* update is only 0 for 'opendir', which needs to lock the entry pointed - * by loc instead of its parent. - */ - if (update) - { - if (ec_loc_parent(fop->xl, loc, &tmp) != 0) { - ec_fop_set_error(fop, EIO); + LOCK(&loc->inode->lock); - return; - } + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx == NULL) { + __ec_fop_set_error(fop, ENOMEM); + + goto unlock; + } + + if (ctx->inode_lock != NULL) { + lock = ctx->inode_lock; /* If there's another lock, make sure that it's not the same. Otherwise * do not insert it. * * This can only happen on renames where source and target names are * in the same directory. */ - if ((fop->lock_count > 0) && - (fop->locks[0].lock->loc.inode == tmp.inode)) { - goto wipe; - } - } else { - if (ec_loc_from_loc(fop->xl, &tmp, loc) != 0) { - ec_fop_set_error(fop, EIO); + if ((fop->lock_count > 0) && (fop->locks[0].lock == lock)) { + /* Combine data/meta updates */ + fop->locks[0].update[EC_DATA_TXN] |= (flags & EC_UPDATE_DATA) != 0; + fop->locks[0].update[EC_METADATA_TXN] |= (flags & EC_UPDATE_META) != + 0; + + /* Only one base inode is allowed per fop, so there shouldn't be + * overwrites here. */ + if (base != NULL) { + fop->locks[0].base = base; + } - return; + goto update_query; } - } - LOCK(&tmp.inode->lock); - - ctx = __ec_inode_get(tmp.inode, fop->xl); - if (ctx == NULL) - { - __ec_fop_set_error(fop, EIO); - - goto unlock; - } - - if (ctx->entry_lock != NULL) - { - lock = ctx->entry_lock; - ec_trace("LOCK_ENTRYLK", fop, "lock=%p, inode=%p, path=%s" - "Lock already acquired", - lock, tmp.inode, tmp.path); + ec_trace("LOCK_INODELK", fop, + "lock=%p, inode=%p. Lock already " + "acquired", + lock, loc->inode); goto insert; } - lock = ec_lock_allocate(fop->xl, EC_LOCK_ENTRY, &tmp); - if (lock == NULL) - { - __ec_fop_set_error(fop, EIO); - + lock = ec_lock_allocate(fop, loc); + if (lock == NULL) { goto unlock; } ec_trace("LOCK_CREATE", fop, "lock=%p", lock); - lock->type = ENTRYLK_WRLCK; + lock->flock.l_type = F_WRLCK; + lock->flock.l_whence = SEEK_SET; - lock->plock = &ctx->entry_lock; - ctx->entry_lock = lock; + lock->ctx = ctx; + ctx->inode_lock = lock; insert: - link = ec_lock_insert(fop, lock, update); - + ec_lock_insert(fop, lock, flags, base, fl_start, fl_size); +update_query: + lock->query |= (flags & EC_QUERY_INFO) != 0; unlock: - UNLOCK(&tmp.inode->lock); - -wipe: - loc_wipe(&tmp); + UNLOCK(&loc->inode->lock); +} - if (link != NULL) { - ec_resume(link->fop, 0); - } +void +ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + off_t fl_start, uint64_t fl_size) +{ + ec_lock_prepare_inode_internal(fop, loc, flags, NULL, fl_start, fl_size); } -void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, int32_t update) +void +ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base, + uint32_t flags) { - ec_lock_link_t *link = NULL; - ec_lock_t * lock; - ec_inode_t * ctx; + loc_t tmp; + int32_t err; - if ((fop->parent != NULL) || (fop->error != 0) || (loc->inode == NULL)) - { + if (fop->error != 0) { return; } - LOCK(&loc->inode->lock); + err = ec_loc_parent(fop->xl, loc, &tmp); + if (err != 0) { + ec_fop_set_error(fop, -err); - ctx = __ec_inode_get(loc->inode, fop->xl); - if (ctx == NULL) - { - __ec_fop_set_error(fop, EIO); + return; + } - goto unlock; + if ((flags & EC_INODE_SIZE) != 0) { + flags ^= EC_INODE_SIZE; + } else { + base = NULL; } - if (ctx->inode_lock != NULL) - { - lock = ctx->inode_lock; - ec_trace("LOCK_INODELK", fop, "lock=%p, inode=%p. Lock already " - "acquired", lock, loc->inode); + ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, EC_RANGE_FULL); - goto insert; + loc_wipe(&tmp); +} + +void +ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start, + uint64_t fl_size) +{ + loc_t loc; + int32_t err; + + if (fop->error != 0) { + return; } - lock = ec_lock_allocate(fop->xl, EC_LOCK_INODE, loc); - if (lock == NULL) - { - __ec_fop_set_error(fop, EIO); + err = ec_loc_from_fd(fop->xl, &loc, fd); + if (err != 0) { + ec_fop_set_error(fop, -err); - goto unlock; + return; } - ec_trace("LOCK_CREATE", fop, "lock=%p", lock); + ec_lock_prepare_inode_internal(fop, &loc, flags, NULL, fl_start, fl_size); - lock->flock.l_type = F_WRLCK; - lock->flock.l_whence = SEEK_SET; + loc_wipe(&loc); +} - lock->plock = &ctx->inode_lock; - ctx->inode_lock = lock; +gf_boolean_t +ec_config_check(xlator_t *xl, ec_config_t *config) +{ + ec_t *ec; -insert: - link = ec_lock_insert(fop, lock, update); + ec = xl->private; + if ((config->version != EC_CONFIG_VERSION) || + (config->algorithm != EC_CONFIG_ALGORITHM) || + (config->gf_word_size != EC_GF_BITS) || (config->bricks != ec->nodes) || + (config->redundancy != ec->redundancy) || + (config->chunk_size != EC_METHOD_CHUNK_SIZE)) { + uint32_t data_bricks; -unlock: - UNLOCK(&loc->inode->lock); + /* This combination of version/algorithm requires the following + values. Incorrect values for these fields are a sign of + corruption: - if (link != NULL) { - ec_resume(link->fop, 0); + redundancy > 0 + redundancy * 2 < bricks + gf_word_size must be a power of 2 + chunk_size (in bits) must be a multiple of gf_word_size * + (bricks - redundancy) */ + + data_bricks = config->bricks - config->redundancy; + if ((config->redundancy < 1) || + (config->redundancy * 2 >= config->bricks) || + !ec_is_power_of_2(config->gf_word_size) || + ((config->chunk_size * 8) % (config->gf_word_size * data_bricks) != + 0)) { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG, + "Invalid or corrupted config"); + } else { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG, + "Unsupported config " + "(V=%u, A=%u, W=%u, " + "N=%u, R=%u, S=%u)", + config->version, config->algorithm, config->gf_word_size, + config->bricks, config->redundancy, config->chunk_size); + } + + return _gf_false; } + + return _gf_true; } -void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, int32_t update) +gf_boolean_t +ec_set_dirty_flag(ec_lock_link_t *link, ec_inode_t *ctx, uint64_t *dirty) { - loc_t loc; + gf_boolean_t set_dirty = _gf_false; - if ((fop->parent != NULL) || (fop->error != 0)) - { - return; + if (link->update[EC_DATA_TXN] && !ctx->dirty[EC_DATA_TXN]) { + if (!link->optimistic_changelog) + dirty[EC_DATA_TXN] = 1; } - if (ec_loc_from_fd(fop->xl, &loc, fd) == 0) - { - ec_lock_prepare_inode(fop, &loc, update); - - loc_wipe(&loc); + if (link->update[EC_METADATA_TXN] && !ctx->dirty[EC_METADATA_TXN]) { + if (!link->optimistic_changelog) + dirty[EC_METADATA_TXN] = 1; } - else - { - ec_fop_set_error(fop, EIO); + + if (dirty[EC_METADATA_TXN] || dirty[EC_DATA_TXN]) { + set_dirty = _gf_true; } + + return set_dirty; } -int32_t ec_locked(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_prepare_update_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, + dict_t *xdata) { - ec_fop_data_t * fop = cookie; - ec_lock_t * lock = NULL; - - if (op_ret >= 0) - { - lock = fop->data; - lock->mask = fop->good; - lock->acquired = 1; + struct list_head list; + ec_fop_data_t *fop = cookie, *parent, *tmp; + ec_lock_link_t *parent_link = fop->data; + ec_lock_link_t *link = NULL; + ec_lock_t *lock = NULL; + ec_inode_t *ctx; + gf_boolean_t release = _gf_false; + uint64_t provided_flags = 0; + uint64_t dirty[EC_VERSION_SIZE] = {0, 0}; + lock = parent_link->lock; + parent = parent_link->fop; + ctx = lock->ctx; - fop->parent->mask &= fop->good; - fop->parent->locked++; + INIT_LIST_HEAD(&list); + provided_flags = EC_PROVIDED_FLAGS(parent_link->waiting_flags); - ec_trace("LOCKED", fop->parent, "lock=%p", lock); + LOCK(&lock->loc.inode->lock); - ec_lock(fop->parent); - } - else + list_for_each_entry(link, &lock->owners, owner_list) { - gf_log(this->name, GF_LOG_WARNING, "Failed to complete preop lock"); + if ((link->waiting_flags & provided_flags) != 0) { + link->waiting_flags ^= (link->waiting_flags & provided_flags); + if (EC_NEEDED_FLAGS(link->waiting_flags) == 0) + list_add_tail(&link->fop->cbk_list, &list); + } } + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_SIZE_VERS_GET_FAIL, + "Failed to get size and version : %s", ec_msg_str(fop)); - return 0; -} + goto unlock; + } -void ec_lock(ec_fop_data_t * fop) -{ - ec_lock_t * lock; + if (EC_FLAGS_HAVE(provided_flags, EC_FLAG_XATTROP)) { + op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version, + EC_VERSION_SIZE); + if (op_errno != 0) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_VER_XATTR_GET_FAIL, "Unable to get version xattr. %s", + ec_msg_str(fop)); + goto unlock; + } + ctx->post_version[0] += ctx->pre_version[0]; + ctx->post_version[1] += ctx->pre_version[1]; + + ctx->have_version = _gf_true; + + if (lock->loc.inode->ia_type == IA_IFREG || + lock->loc.inode->ia_type == IA_INVAL) { + op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size); + if (op_errno != 0) { + if (lock->loc.inode->ia_type == IA_IFREG) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_SIZE_XATTR_GET_FAIL, + "Unable to get size xattr. %s", ec_msg_str(fop)); + goto unlock; + } + } else { + ctx->post_size = ctx->pre_size; - while (fop->locked < fop->lock_count) - { - lock = fop->locks[fop->locked].lock; + ctx->have_size = _gf_true; + } - LOCK(&lock->loc.inode->lock); + op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config); + if (op_errno != 0) { + if ((lock->loc.inode->ia_type == IA_IFREG) || + (op_errno != ENODATA)) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_CONFIG_XATTR_GET_FAIL, + "Unable to get config xattr. %s", ec_msg_str(fop)); - if (lock->owner != NULL) - { - ec_trace("LOCK_WAIT", fop, "lock=%p", lock); + goto unlock; + } + } else { + if (!ec_config_check(parent->xl, &ctx->config)) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, + EC_MSG_CONFIG_XATTR_INVALID, "Invalid config xattr"); - list_add_tail(&fop->locks[fop->locked].wait_list, &lock->waiting); + op_errno = EINVAL; - ec_sleep(fop); + goto unlock; + } + ctx->have_config = _gf_true; + } + } + ctx->have_info = _gf_true; + } - UNLOCK(&lock->loc.inode->lock); + ec_set_dirty_flag(fop->data, ctx, dirty); + if (dirty[EC_METADATA_TXN] && + (EC_FLAGS_HAVE(provided_flags, EC_FLAG_METADATA_DIRTY))) { + GF_ASSERT(!ctx->dirty[EC_METADATA_TXN]); + ctx->dirty[EC_METADATA_TXN] = 1; + } - break; + if (dirty[EC_DATA_TXN] && + (EC_FLAGS_HAVE(provided_flags, EC_FLAG_DATA_DIRTY))) { + GF_ASSERT(!ctx->dirty[EC_DATA_TXN]); + ctx->dirty[EC_DATA_TXN] = 1; + } + op_errno = 0; +unlock: + + lock->waiting_flags ^= provided_flags; + + if (op_errno == 0) { + /* If the fop fails on any of the good bricks, it is important to mark + * it dirty and update versions right away if dirty was not set before. + */ + if (lock->good_mask & ~(fop->good | fop->remaining)) { + release = _gf_true; } - lock->owner = fop; - UNLOCK(&lock->loc.inode->lock); + if (parent_link->update[0] && !parent_link->dirty[0]) { + lock->release |= release; + } - if (!lock->acquired) - { - ec_owner_set(fop->frame, lock); + if (parent_link->update[1] && !parent_link->dirty[1]) { + lock->release |= release; + } - if (lock->kind == EC_LOCK_ENTRY) - { - ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p, path=%s", - lock, lock->loc.inode, lock->loc.path); + /* We don't allow the main fop to be executed on bricks that have not + * succeeded the initial xattrop. */ + ec_lock_update_good(lock, fop); - ec_entrylk(fop->frame, fop->xl, -1, EC_MINIMUM_ALL, ec_locked, - lock, fop->xl->name, &lock->loc, NULL, - ENTRYLK_LOCK, lock->type, NULL); - } - else - { - ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock, - lock->loc.inode); + /*As of now only data healing marks bricks as healing*/ + lock->healing |= fop->healing; + } - ec_inodelk(fop->frame, fop->xl, -1, EC_MINIMUM_ALL, ec_locked, - lock, fop->xl->name, &lock->loc, F_SETLKW, - &lock->flock, NULL); - } + UNLOCK(&lock->loc.inode->lock); - break; - } + while (!list_empty(&list)) { + tmp = list_entry(list.next, ec_fop_data_t, cbk_list); + list_del_init(&tmp->cbk_list); - ec_trace("LOCK_REUSE", fop, "lock=%p", lock); + if (op_errno == 0) { + tmp->mask &= fop->good; - if (lock->have_size) - { - fop->pre_size = fop->post_size = lock->size; - fop->have_size = 1; + /*As of now only data healing marks bricks as healing*/ + if (ec_is_data_fop(tmp->id)) { + tmp->healing |= fop->healing; + } } - fop->mask &= lock->good_mask; - fop->locked++; + ec_resume(tmp, op_errno); } + + return 0; } -gf_boolean_t -ec_config_check (ec_fop_data_t *fop, dict_t *xdata) +static gf_boolean_t +ec_set_needed_flag(ec_lock_t *lock, ec_lock_link_t *link, uint64_t flag) { - ec_t *ec; + uint64_t current; - if (ec_dict_del_config(xdata, EC_XATTR_CONFIG, &fop->config) < 0) { - gf_log(fop->xl->name, GF_LOG_ERROR, "Failed to get a valid " - "config"); + link->waiting_flags |= EC_FLAG_NEEDS(flag); - ec_fop_set_error(fop, EIO); + current = EC_NEEDED_FLAGS(lock->waiting_flags); + if (!EC_FLAGS_HAVE(current, flag)) { + lock->waiting_flags |= EC_FLAG_NEEDS(flag); + link->waiting_flags |= EC_FLAG_PROVIDES(flag); - return _gf_false; + return _gf_true; } - ec = fop->xl->private; - if ((fop->config.version != EC_CONFIG_VERSION) || - (fop->config.algorithm != EC_CONFIG_ALGORITHM) || - (fop->config.gf_word_size != EC_GF_BITS) || - (fop->config.bricks != ec->nodes) || - (fop->config.redundancy != ec->redundancy) || - (fop->config.chunk_size != EC_METHOD_CHUNK_SIZE)) { - uint32_t data_bricks; + return _gf_false; +} - /* This combination of version/algorithm requires the following - values. Incorrect values for these fields are a sign of - corruption: +static uint64_t +ec_set_xattrop_flags_and_params(ec_lock_t *lock, ec_lock_link_t *link, + uint64_t *dirty) +{ + uint64_t oldflags = 0; + uint64_t newflags = 0; + ec_inode_t *ctx = lock->ctx; - redundancy > 0 - redundancy * 2 < bricks - gf_word_size must be a power of 2 - chunk_size (in bits) must be a multiple of gf_word_size * - (bricks - redundancy) */ + oldflags = EC_NEEDED_FLAGS(lock->waiting_flags); - data_bricks = fop->config.bricks - fop->config.redundancy; - if ((fop->config.redundancy < 1) || - (fop->config.redundancy * 2 >= fop->config.bricks) || - !ec_is_power_of_2(fop->config.gf_word_size) || - ((fop->config.chunk_size * 8) % (fop->config.gf_word_size * - data_bricks) != 0)) { - gf_log(fop->xl->name, GF_LOG_ERROR, "Invalid or corrupted config"); - } else { - gf_log(fop->xl->name, GF_LOG_ERROR, "Unsupported config " - "(V=%u, A=%u, W=%u, " - "N=%u, R=%u, S=%u)", - fop->config.version, fop->config.algorithm, - fop->config.gf_word_size, fop->config.bricks, - fop->config.redundancy, fop->config.chunk_size); - } + if (lock->query && !ctx->have_info) { + ec_set_needed_flag(lock, link, EC_FLAG_XATTROP); + } - ec_fop_set_error(fop, EIO); + if (dirty[EC_DATA_TXN]) { + if (!ec_set_needed_flag(lock, link, EC_FLAG_DATA_DIRTY)) { + dirty[EC_DATA_TXN] = 0; + } + } - return _gf_false; + if (dirty[EC_METADATA_TXN]) { + if (!ec_set_needed_flag(lock, link, EC_FLAG_METADATA_DIRTY)) { + dirty[EC_METADATA_TXN] = 0; + } } + newflags = EC_NEEDED_FLAGS(lock->waiting_flags); - return _gf_true; + return oldflags ^ newflags; } -int32_t ec_get_size_version_set(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, inode_t * inode, - struct iatt * buf, dict_t * xdata, - struct iatt * postparent) +void +ec_get_size_version(ec_lock_link_t *link) { - ec_fop_data_t * fop = cookie; - ec_inode_t * ctx; - ec_lock_t *lock = NULL; + loc_t loc; + ec_lock_t *lock; + ec_inode_t *ctx; + ec_fop_data_t *fop; + dict_t *dict = NULL; + dict_t *xdata = NULL; + ec_t *ec = NULL; + int32_t error = 0; + gf_boolean_t set_dirty = _gf_false; + uint64_t allzero[EC_VERSION_SIZE] = {0, 0}; + uint64_t dirty[EC_VERSION_SIZE] = {0, 0}; + lock = link->lock; + ctx = lock->ctx; + fop = link->fop; + ec = fop->xl->private; + uint64_t changed_flags = 0; - if (op_ret >= 0) - { - if ((buf->ia_type == IA_IFREG) && !ec_config_check(fop, xdata)) { - return 0; + if (ec->optimistic_changelog && !(ec->node_mask & ~link->lock->good_mask) && + !ec_is_data_fop(fop->id)) + link->optimistic_changelog = _gf_true; + + memset(&loc, 0, sizeof(loc)); + + LOCK(&lock->loc.inode->lock); + + set_dirty = ec_set_dirty_flag(link, ctx, dirty); + + /* If ec metadata has already been retrieved, do not try again. */ + if (ctx->have_info) { + if (ec_is_data_fop(fop->id)) { + fop->healing |= lock->healing; } + if (!set_dirty) + goto unlock; + } - LOCK(&inode->lock); + /* Determine if there's something we need to retrieve for the current + * operation. */ + if (!set_dirty && !lock->query && (lock->loc.inode->ia_type != IA_IFREG) && + (lock->loc.inode->ia_type != IA_INVAL)) { + goto unlock; + } - ctx = __ec_inode_get(inode, this); - if (ctx != NULL) { - if (ctx->inode_lock != NULL) { - lock = ctx->inode_lock; - lock->version = fop->answer->version; + changed_flags = ec_set_xattrop_flags_and_params(lock, link, dirty); + if (link->waiting_flags) { + /* This fop needs to wait until all its flags are cleared which + * potentially can be cleared by other xattrops that are already + * wound*/ + ec_sleep(fop); + } else { + GF_ASSERT(!changed_flags); + } - if (buf->ia_type == IA_IFREG) { - lock->have_size = 1; - lock->size = buf->ia_size; - } +unlock: + UNLOCK(&lock->loc.inode->lock); + + if (!changed_flags) + goto out; + + dict = dict_new(); + if (dict == NULL) { + error = -ENOMEM; + goto out; + } + + if (EC_FLAGS_HAVE(changed_flags, EC_FLAG_XATTROP)) { + /* Once we know that an xattrop will be needed, + * we try to get all available information in a + * single call. */ + error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero, + EC_VERSION_SIZE); + if (error != 0) { + goto out; + } + + if (lock->loc.inode->ia_type == IA_IFREG || + lock->loc.inode->ia_type == IA_INVAL) { + error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0); + if (error == 0) { + error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0); } - if (ctx->entry_lock != NULL) { - lock = ctx->entry_lock; - lock->version = fop->answer->version; + if (error != 0) { + goto out; } - } - UNLOCK(&inode->lock); + xdata = dict_new(); + if (xdata == NULL || dict_set_int32(xdata, GF_GET_SIZE, 1)) { + error = -ENOMEM; + goto out; + } + } + } - if (lock != NULL) - { - // Only update parent mask if the lookup has been made with - // inode locked. - fop->parent->mask &= fop->good; + if (memcmp(allzero, dirty, sizeof(allzero))) { + error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (error != 0) { + goto out; } + } + + fop->frame->root->uid = 0; + fop->frame->root->gid = 0; - if (buf->ia_type == IA_IFREG) { - fop->parent->pre_size = fop->parent->post_size = buf->ia_size; - fop->parent->have_size = 1; + /* For normal fops, ec_[f]xattrop() must succeed on at least + * EC_MINIMUM_MIN bricks, however when this is called as part of a + * self-heal operation the mask of target bricks (fop->mask) could + * contain less than EC_MINIMUM_MIN bricks, causing the xattrop to + * always fail. Thus we always use the same minimum used for the main + * fop. + */ + if (lock->fd == NULL) { + error = ec_loc_from_loc(fop->xl, &loc, &lock->loc); + if (error != 0) { + goto out; } + if (gf_uuid_is_null(loc.pargfid)) { + if (loc.parent != NULL) { + inode_unref(loc.parent); + loc.parent = NULL; + } + GF_FREE((char *)loc.path); + loc.path = NULL; + loc.name = NULL; + } + + ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum, + ec_prepare_update_cbk, link, &loc, GF_XATTROP_ADD_ARRAY64, + dict, xdata); + } else { + ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum, + ec_prepare_update_cbk, link, lock->fd, + GF_XATTROP_ADD_ARRAY64, dict, xdata); + } + + error = 0; + +out: + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + loc_wipe(&loc); + + if (dict != NULL) { + dict_unref(dict); + } + + if (xdata != NULL) { + dict_unref(xdata); + } + + if (error != 0) { + ec_fop_set_error(fop, -error); + } +} + +gf_boolean_t +__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size) +{ + ec_inode_t *ctx; + gf_boolean_t found = _gf_false; + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } + + if (ctx->have_size) { + *size = ctx->post_size; + found = _gf_true; } - else + +out: + return found; +} + +gf_boolean_t +ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size) +{ + gf_boolean_t found = _gf_false; + + LOCK(&inode->lock); { - gf_log(this->name, GF_LOG_WARNING, "Failed to get size and version " - "(error %d)", op_errno); - ec_fop_set_error(fop, op_errno); + found = __ec_get_inode_size(fop, inode, size); } + UNLOCK(&inode->lock); - return 0; + return found; } -int32_t ec_prepare_update_cbk(call_frame_t *frame, void *cookie, - xlator_t *this, int32_t op_ret, int32_t op_errno, - dict_t *dict, dict_t *xdata) +gf_boolean_t +__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size) { - ec_fop_data_t *fop = cookie, *parent; - ec_lock_t *lock = NULL; + ec_inode_t *ctx; + gf_boolean_t found = _gf_false; - if (op_ret >= 0) { - parent = fop->parent; - while ((parent != NULL) && (parent->locks[0].lock == NULL)) { - parent = parent->parent; - } - if (parent == NULL) { - return 0; - } + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } - lock = parent->locks[0].lock; - lock->is_dirty = _gf_true; + /* Normal fops always have ctx->have_size set. However self-heal calls this + * to prepare the inode, so ctx->have_size will be false. In this case we + * prepare both pre_size and post_size, and set have_size and have_info to + * true. */ + if (!ctx->have_size) { + ctx->pre_size = size; + ctx->have_size = ctx->have_info = _gf_true; + } + ctx->post_size = size; - if (!ec_config_check(fop, dict)) { - return 0; - } + found = _gf_true; - LOCK(&lock->loc.inode->lock); +out: + return found; +} - if ((ec_dict_del_number(dict, EC_XATTR_VERSION, &lock->version) != 0) || - (ec_dict_del_number(dict, EC_XATTR_SIZE, &lock->size) != 0)) { - UNLOCK(&lock->loc.inode->lock); +gf_boolean_t +ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size) +{ + gf_boolean_t found = _gf_false; - ec_fop_set_error(fop, EIO); + LOCK(&inode->lock); + { + found = __ec_set_inode_size(fop, inode, size); + } + UNLOCK(&inode->lock); - return 0; - } + return found; +} + +static void +ec_release_stripe_cache(ec_inode_t *ctx) +{ + ec_stripe_list_t *stripe_cache = NULL; + ec_stripe_t *stripe = NULL; - lock->have_size = 1; + stripe_cache = &ctx->stripe_cache; + while (!list_empty(&stripe_cache->lru)) { + stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru); + list_del(&stripe->lru); + GF_FREE(stripe); + } + stripe_cache->count = 0; + stripe_cache->max = 0; +} - UNLOCK(&lock->loc.inode->lock); +void +ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode) +{ + ec_inode_t *ctx; + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto unlock; + } + + ec_release_stripe_cache(ctx); + ctx->have_info = _gf_false; + ctx->have_config = _gf_false; + ctx->have_version = _gf_false; + ctx->have_size = _gf_false; - fop->parent->mask &= fop->good; + memset(&ctx->config, 0, sizeof(ctx->config)); + memset(ctx->pre_version, 0, sizeof(ctx->pre_version)); + memset(ctx->post_version, 0, sizeof(ctx->post_version)); + ctx->pre_size = ctx->post_size = 0; + memset(ctx->dirty, 0, sizeof(ctx->dirty)); - fop->parent->pre_size = fop->parent->post_size = lock->size; - fop->parent->have_size = 1; +unlock: + UNLOCK(&inode->lock); +} + +int32_t +ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link; + + if (op_ret >= 0) { + link = fop->data; + link->size = buf->ia_size; } else { - gf_log(this->name, GF_LOG_WARNING, - "Failed to get size and version (error %d: %s)", op_errno, - strerror (op_errno)); - ec_fop_set_error(fop, op_errno); + /* Prevent failure of parent fop. */ + fop->error = 0; } return 0; } -void ec_get_size_version(ec_fop_data_t * fop) +/* This function is used to get the trusted.ec.size xattr from a file when + * no lock is needed on the inode. This is only required to maintain iatt + * structs on fops that manipulate directory entries but do not operate + * directly on the inode, like link, rename, ... + * + * Any error processing this request is ignored. In the worst case, an invalid + * or not up to date value in the iatt could cause some cache invalidation. + */ +void +ec_get_real_size(ec_lock_link_t *link) { - loc_t loc; - dict_t * xdata; - uid_t uid; - gid_t gid; - int32_t error = ENOMEM; + ec_fop_data_t *fop; + dict_t *xdata; - if (fop->have_size) - { + if (link->base == NULL || link->base->inode == NULL) { return; } - if ((fop->parent != NULL) && fop->parent->have_size) - { - fop->pre_size = fop->parent->pre_size; - fop->post_size = fop->parent->post_size; + if (link->base->inode->ia_type != IA_IFREG) { + return; + } - fop->have_size = 1; + fop = link->fop; + if (ec_get_inode_size(fop, link->base->inode, &link->size)) { return; } - memset(&loc, 0, sizeof(loc)); - xdata = dict_new(); - if (xdata == NULL) - { - goto out; + if (xdata == NULL) { + return; } - if ((dict_set_uint64(xdata, EC_XATTR_VERSION, 0) != 0) || - (dict_set_uint64(xdata, EC_XATTR_SIZE, 0) != 0) || - (dict_set_uint64(xdata, EC_XATTR_CONFIG, 0) != 0) || - (dict_set_uint64(xdata, EC_XATTR_DIRTY, 0) != 0)) - { + if (ec_dict_set_number(xdata, EC_XATTR_SIZE, 0) != 0) { goto out; } - uid = fop->frame->root->uid; - gid = fop->frame->root->gid; + /* Send a simple lookup. A single answer is considered ok since this value + * is only used to return an iatt struct related to an inode that is not + * locked and have not suffered any operation. */ + ec_lookup(fop->frame, fop->xl, fop->mask, 1, ec_get_real_size_cbk, link, + link->base, xdata); - fop->frame->root->uid = 0; - fop->frame->root->gid = 0; +out: + if (xdata != NULL) { + dict_unref(xdata); + } +} + +static void +ec_lock_update_fd(ec_lock_t *lock, ec_fop_data_t *fop) +{ + /* If the fop has an fd available, attach it to the lock structure to be + * able to do fxattrop calls instead of xattrop. */ + if (fop->use_fd && (lock->fd == NULL)) { + lock->fd = __fd_ref(fop->fd); + } +} - error = EIO; +static gf_boolean_t +ec_link_has_lock_conflict(ec_lock_link_t *link, gf_boolean_t waitlist_check) +{ + ec_lock_link_t *trav_link = NULL; - if (!fop->use_fd) + list_for_each_entry(trav_link, &link->lock->owners, owner_list) { - if (ec_loc_from_loc(fop->xl, &loc, &fop->loc[0]) != 0) - { - goto out; + if (ec_lock_conflict(trav_link, link)) + return _gf_true; + } + + if (!waitlist_check) + return _gf_false; + + list_for_each_entry(trav_link, &link->lock->waiting, wait_list) + { + if (ec_lock_conflict(trav_link, link)) + return _gf_true; + } + + return _gf_false; +} + +static void +ec_lock_wake_shared(ec_lock_t *lock, struct list_head *list) +{ + ec_fop_data_t *fop; + ec_lock_link_t *link; + gf_boolean_t conflict = _gf_false; + + while (!conflict && !list_empty(&lock->waiting)) { + link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list); + fop = link->fop; + + /* If lock is not acquired, at most one fop can be assigned as owner. + * The following fops will need to wait in the lock->waiting queue + * until the lock has been fully acquired. */ + conflict = !lock->acquired; + + /* If the fop is not shareable, only this fop can be assigned as owner. + * Other fops will need to wait until this one finishes. */ + if (ec_link_has_lock_conflict(link, _gf_false)) { + conflict = _gf_true; } - if (uuid_is_null(loc.pargfid)) - { - if (loc.parent != NULL) - { - inode_unref(loc.parent); - loc.parent = NULL; - } - GF_FREE((char *)loc.path); - loc.path = NULL; - loc.name = NULL; + + /* If only one fop is allowed, it can be assigned as the owner of the + * lock only if there weren't any other owner. */ + if (conflict && !list_empty(&lock->owners)) { + break; } - } else if (ec_loc_from_fd(fop->xl, &loc, fop->fd) != 0) { - goto out; + + list_move_tail(&link->wait_list, list); + + list_add_tail(&link->owner_list, &lock->owners); + lock->refs_owners++; + + ec_lock_update_fd(lock, fop); } +} - /* For normal fops, ec_lookup() must succeed on at least EC_MINIMUM_MIN - * bricks, however when this is called as part of a self-heal operation - * the mask of target bricks (fop->mask) could contain less than - * EC_MINIMUM_MIN bricks, causing the lookup to always fail. Thus we - * always use the same minimum used for the main fop. - */ - ec_lookup(fop->frame, fop->xl, fop->mask, fop->minimum, - ec_get_size_version_set, NULL, &loc, xdata); +static void +ec_lock_apply(ec_lock_link_t *link) +{ + ec_fop_data_t *fop = link->fop; - fop->frame->root->uid = uid; - fop->frame->root->gid = gid; + fop->mask &= link->lock->good_mask; + fop->locked++; - error = 0; + ec_get_size_version(link); + ec_get_real_size(link); +} -out: - loc_wipe(&loc); +gf_boolean_t +ec_lock_acquire(ec_lock_link_t *link); - if (xdata != NULL) - { - dict_unref(xdata); - } +static void +ec_lock_resume_shared(struct list_head *list) +{ + ec_lock_link_t *link; - if (error != 0) { - ec_fop_set_error(fop, error); + while (!list_empty(list)) { + link = list_entry(list->next, ec_lock_link_t, wait_list); + list_del_init(&link->wait_list); + + if (link->lock->acquired) { + ec_lock_apply(link); + ec_lock(link->fop); + } else { + GF_ASSERT(list_empty(list)); + + ec_lock_acquire(link); + } + + ec_resume(link->fop, 0); } } -void ec_prepare_update(ec_fop_data_t *fop) +void +ec_lock_acquired(ec_lock_link_t *link) { - loc_t loc; - dict_t *xdata; - ec_fop_data_t *tmp; + struct list_head list; ec_lock_t *lock; - uid_t uid; - gid_t gid; - int32_t error = ENOMEM; + ec_fop_data_t *fop; + + lock = link->lock; + fop = link->fop; + + ec_trace("LOCKED", fop, "lock=%p", lock); - tmp = fop; - while ((tmp != NULL) && (tmp->locks[0].lock == NULL)) { - tmp = tmp->parent; + INIT_LIST_HEAD(&list); + + LOCK(&lock->loc.inode->lock); + + lock->acquired = _gf_true; + if (lock->contention) { + lock->release = _gf_true; + lock->contention = _gf_false; } - if ((tmp != NULL) && tmp->locks[0].lock->is_dirty) { - lock = tmp->locks[0].lock; - fop->pre_size = fop->post_size = lock->size; - fop->have_size = 1; + ec_lock_update_fd(lock, fop); + ec_lock_wake_shared(lock, &list); - return; + UNLOCK(&lock->loc.inode->lock); + + ec_lock_apply(link); + + if (fop->use_fd && + (link->update[EC_DATA_TXN] || link->update[EC_METADATA_TXN])) { + /* Try to reopen closed fd's only if lock has succeeded. */ + ec_fix_open(fop, lock->mask); } - memset(&loc, 0, sizeof(loc)); + ec_lock_resume_shared(&list); +} - xdata = dict_new(); - if (xdata == NULL) { - goto out; +int32_t +ec_locked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link = NULL; + ec_lock_t *lock = NULL; + + link = fop->data; + lock = link->lock; + if (op_ret >= 0) { + lock->mask = lock->good_mask = fop->good; + lock->healing = 0; + + ec_lock_acquired(link); + ec_lock(fop->parent); + } else { + LOCK(&lock->loc.inode->lock); + { + lock->contention = _gf_false; + } + UNLOCK(&lock->loc.inode->lock); + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_PREOP_LOCK_FAILED, + "Failed to complete preop lock"); } - if ((ec_dict_set_number(xdata, EC_XATTR_VERSION, 0) != 0) || - (ec_dict_set_number(xdata, EC_XATTR_SIZE, 0) != 0) || - (ec_dict_set_number(xdata, EC_XATTR_CONFIG, 0) != 0) || - (ec_dict_set_number(xdata, EC_XATTR_DIRTY, 1) != 0)) { - goto out; + + return 0; +} + +gf_boolean_t +ec_lock_acquire(ec_lock_link_t *link) +{ + ec_lock_t *lock; + ec_fop_data_t *fop; + gf_lkowner_t lk_owner; + + lock = link->lock; + fop = link->fop; + + if (!lock->acquired) { + set_lk_owner_from_ptr(&lk_owner, lock); + + ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock, + lock->loc.inode); + + lock->flock.l_type = F_WRLCK; + ec_inodelk(fop->frame, fop->xl, &lk_owner, -1, EC_MINIMUM_ALL, + ec_locked, link, fop->xl->name, &lock->loc, F_SETLKW, + &lock->flock, NULL); + + return _gf_false; } - uid = fop->frame->root->uid; - gid = fop->frame->root->gid; + ec_trace("LOCK_REUSE", fop, "lock=%p", lock); - fop->frame->root->uid = 0; - fop->frame->root->gid = 0; + ec_lock_acquired(link); - error = EIO; + return _gf_true; +} - if (!fop->use_fd) { - if (ec_loc_from_loc(fop->xl, &loc, &fop->loc[0]) != 0) { - goto out; - } +static ec_lock_link_t * +ec_lock_timer_cancel(xlator_t *xl, ec_lock_t *lock) +{ + ec_lock_link_t *timer_link; - ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum, - ec_prepare_update_cbk, NULL, &loc, GF_XATTROP_ADD_ARRAY64, - xdata, NULL); + /* If we don't have any timer, there's nothing to cancel. */ + if (lock->timer == NULL) { + return NULL; + } + + /* We are trying to access a lock that has an unlock timer active. + * This means that the lock must be idle, i.e. no fop can be in the + * owner, waiting or frozen lists. It also means that the lock cannot + * have been marked as being released (this is done without timers). + * There should only be one owner reference, but it's possible that + * some fops are being prepared to use this lock. */ + GF_ASSERT((lock->refs_owners == 1) && list_empty(&lock->owners) && + list_empty(&lock->waiting)); + + /* We take the timer_link before cancelling the timer, since a + * successful cancellation will destroy it. It must not be NULL + * because it references the fop responsible for the delayed unlock + * that we are currently trying to cancel. */ + timer_link = lock->timer->data; + GF_ASSERT(timer_link != NULL); + + if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) { + /* It's too late to avoid the execution of the timer callback. + * Since we need to be sure that the callback has access to all + * needed resources, we cannot resume the execution of the + * timer fop now. This will be done in the callback. */ + timer_link = NULL; } else { - ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum, - ec_prepare_update_cbk, NULL, fop->fd, - GF_XATTROP_ADD_ARRAY64, xdata, NULL); + /* The timer has been cancelled. The fop referenced by + * timer_link holds the last reference. The caller is + * responsible to release it when not needed anymore. */ + ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock); + } + + /* We have two options here: + * + * 1. The timer has been successfully cancelled. + * + * This is the easiest case and we can continue with the currently + * acquired lock. + * + * 2. The timer callback has already been fired. + * + * In this case we have not been able to cancel the timer before + * the timer callback has been fired, but we also know that + * lock->timer != NULL. This means that the timer callback is still + * trying to acquire the inode mutex that we currently own. We are + * safe until we release it. In this case we can safely clear + * lock->timer. This will cause that the timer callback does nothing + * once it acquires the mutex. + */ + lock->timer = NULL; + + return timer_link; +} + +static gf_boolean_t +ec_lock_assign_owner(ec_lock_link_t *link) +{ + ec_fop_data_t *fop; + ec_lock_t *lock; + ec_lock_link_t *timer_link = NULL; + gf_boolean_t assigned = _gf_false; + + /* The link cannot be in any list because we have just finished preparing + * it. */ + GF_ASSERT(list_empty(&link->wait_list)); + + fop = link->fop; + lock = link->lock; + + LOCK(&lock->loc.inode->lock); + + /* Since the link has just been prepared but it's not active yet, the + * refs_pending must be one at least (the ref owned by this link). */ + GF_ASSERT(lock->refs_pending > 0); + /* The link is not pending any more. It will be assigned to the owner, + * waiting or frozen list. */ + lock->refs_pending--; + + if (lock->release) { + ec_trace("LOCK_QUEUE_FREEZE", fop, "lock=%p", lock); + + /* When lock->release is set, we'll unlock the lock as soon as + * possible, meaning that we won't use a timer. */ + GF_ASSERT(lock->timer == NULL); + + /* The lock is marked to be released. We can still have owners and fops + * in the waiting ilist f they have been added before the lock has been + * marked to be released. However new fops are put into the frozen list + * to wait for the next unlock/lock cycle. */ + list_add_tail(&link->wait_list, &lock->frozen); + + goto unlock; } - fop->frame->root->uid = uid; - fop->frame->root->gid = gid; + /* The lock is not marked to be released, so the frozen list should be + * empty. */ + GF_ASSERT(list_empty(&lock->frozen)); - error = 0; + timer_link = ec_lock_timer_cancel(fop->xl, lock); -out: - loc_wipe(&loc); + if (!list_empty(&lock->owners)) { + /* There are other owners of this lock. We can only take ownership if + * the lock is already acquired and doesn't have conflict with existing + * owners, or waiters(to prevent starvation). + * Otherwise we need to wait. + */ + if (!lock->acquired || ec_link_has_lock_conflict(link, _gf_true)) { + ec_trace("LOCK_QUEUE_WAIT", fop, "lock=%p", lock); - if (xdata != NULL) { - dict_unref(xdata); + list_add_tail(&link->wait_list, &lock->waiting); + + goto unlock; + } } - if (error != 0) { - ec_fop_set_error(fop, error); + list_add_tail(&link->owner_list, &lock->owners); + + /* If timer_link is not NULL, it means that we have inherited the owner + * reference assigned to the timer fop. In this case we simply reuse it. + * Otherwise we need to increase the number of owners. */ + if (timer_link == NULL) { + lock->refs_owners++; } + + assigned = _gf_true; + +unlock: + if (!assigned) { + /* We have not been able to take ownership of this lock. The fop must + * be put to sleep. */ + ec_sleep(fop); + } + + UNLOCK(&lock->loc.inode->lock); + + /* If we have cancelled the timer, we need to resume the fop that was + * waiting for it. */ + if (timer_link != NULL) { + ec_resume(timer_link->fop, 0); + } + + return assigned; } -int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno, dict_t *xdata) +static void +ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk, + gf_boolean_t release) { - ec_fop_data_t *fop = cookie; + struct list_head list; + ec_lock_t *lock = link->lock; + ec_fop_data_t *fop = link->fop; + ec_inode_t *ctx = lock->ctx; - if (op_ret < 0) { - gf_log(this->name, GF_LOG_WARNING, "entry/inode unlocking failed (%s)", - ec_fop_name(fop->parent->id)); - } else { - ec_trace("UNLOCKED", fop->parent, "lock=%p", fop->data); + INIT_LIST_HEAD(&list); + + LOCK(&lock->loc.inode->lock); + + ec_trace("LOCK_DONE", fop, "lock=%p", lock); + + /* Current link must belong to the owner list of the lock. We don't + * decrement lock->refs_owners here because the inode mutex is released + * before ec_unlock() is called and we need to know when the last owner + * unlocks the lock to do proper cleanup. lock->refs_owners is used for + * this task. */ + GF_ASSERT((lock->refs_owners > 0) && !list_empty(&link->owner_list)); + list_del_init(&link->owner_list); + + lock->release |= release; + + if ((fop->error == 0) && (cbk != NULL) && (cbk->op_ret >= 0)) { + if (link->update[0]) { + ctx->post_version[0]++; + } + if (link->update[1]) { + ctx->post_version[1]++; + } + /* If the fop fails on any of the good bricks, it is important to mark + * it dirty and update versions right away. */ + if (link->update[0] || link->update[1]) { + if (lock->good_mask & ~(fop->good | fop->remaining)) { + lock->release = _gf_true; + } + } } - return 0; + if (fop->healing) { + lock->healing = fop->healing & (fop->good | fop->remaining); + } + ec_lock_update_good(lock, fop); + + ec_lock_wake_shared(lock, &list); + + UNLOCK(&lock->loc.inode->lock); + + ec_lock_resume_shared(&list); } -void ec_unlock_lock(ec_fop_data_t *fop, ec_lock_t *lock) +void +ec_lock(ec_fop_data_t *fop) { - if ((lock->mask != 0) && lock->acquired) { - ec_owner_set(fop->frame, lock); + ec_lock_link_t *link; - switch (lock->kind) { - case EC_LOCK_ENTRY: - ec_trace("UNLOCK_ENTRYLK", fop, "lock=%p, inode=%p, path=%s", lock, - lock->loc.inode, lock->loc.path); + /* There is a chance that ec_resume is called on fop even before ec_sleep. + * Which can result in refs == 0 for fop leading to use after free in this + * function when it calls ec_sleep so do ec_sleep at start and ec_resume at + * the end of this function.*/ + ec_sleep(fop); - ec_entrylk(fop->frame, fop->xl, lock->mask, EC_MINIMUM_ALL, - ec_unlocked, lock, fop->xl->name, &lock->loc, NULL, - ENTRYLK_UNLOCK, lock->type, NULL); + while (fop->locked < fop->lock_count) { + /* Since there are only up to 2 locks per fop, this xor will change + * the order of the locks if fop->first_lock is 1. */ + link = &fop->locks[fop->locked ^ fop->first_lock]; + if (!ec_lock_assign_owner(link) || !ec_lock_acquire(link)) { break; + } + } + + ec_resume(fop, 0); +} - case EC_LOCK_INODE: - lock->flock.l_type = F_UNLCK; - ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock, - lock->loc.inode); +void +ec_lock_unfreeze(ec_lock_link_t *link) +{ + struct list_head list; + ec_lock_t *lock; + gf_boolean_t destroy = _gf_false; - ec_inodelk(fop->frame, fop->xl, lock->mask, EC_MINIMUM_ALL, - ec_unlocked, lock, fop->xl->name, &lock->loc, F_SETLK, - &lock->flock, NULL); + lock = link->lock; - break; + INIT_LIST_HEAD(&list); - default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Invalid lock type"); - } + LOCK(&lock->loc.inode->lock); + + /* The lock must be marked to be released here, since we have just released + * it and any attempt to assign it to more fops must have added them to the + * frozen list. We can only have one active reference here: the one that + * is processing this unfreeze. */ + GF_ASSERT(lock->release && (lock->refs_owners == 1)); + lock->release = _gf_false; + lock->refs_owners = 0; + + lock->acquired = _gf_false; + + /* We are unfreezing a lock. This means that the lock has already been + * released. In this state it shouldn't have a pending timer nor have any + * owner, and the waiting list should be empty. Only the frozen list can + * contain some fop. */ + GF_ASSERT((lock->timer == NULL) && list_empty(&lock->waiting) && + list_empty(&lock->owners)); + + /* We move all frozen fops to the waiting list. */ + list_splice_init(&lock->frozen, &lock->waiting); + + /* If we don't have any fop waiting nor there are any prepared fops using + * this lock, we can finally dispose it. */ + destroy = list_empty(&lock->waiting) && (lock->refs_pending == 0); + if (destroy) { + ec_trace("LOCK_DESTROY", link->fop, "lock=%p", lock); + + lock->ctx->inode_lock = NULL; + } else { + ec_trace("LOCK_UNFREEZE", link->fop, "lock=%p", lock); + + ec_lock_wake_shared(lock, &list); } - ec_trace("LOCK_DESTROY", fop, "lock=%p", lock); + UNLOCK(&lock->loc.inode->lock); + + ec_lock_resume_shared(&list); - ec_lock_destroy(lock); + if (destroy) { + ec_lock_destroy(lock); + } } -int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, dict_t * xattr, - dict_t * xdata) +int32_t +ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = cookie; + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link = fop->data; - if (op_ret < 0) - { - gf_log(fop->xl->name, GF_LOG_ERROR, "Failed to update version and " - "size (error %d)", op_errno); - } - else - { - fop->parent->mask &= fop->good; + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED, + "entry/inode unlocking failed :(%s)", ec_msg_str(link->fop)); + } else { + ec_trace("UNLOCKED", link->fop, "lock=%p", link->lock); } - if (fop->data != NULL) { - ec_unlock_lock(fop->parent, fop->data); - } + ec_lock_unfreeze(link); return 0; } -void ec_update_size_version(ec_fop_data_t *fop, loc_t *loc, uint64_t version, - uint64_t size, gf_boolean_t dirty, ec_lock_t *lock) +void +ec_unlock_lock(ec_lock_link_t *link) { - dict_t * dict; - uid_t uid; - gid_t gid; + ec_lock_t *lock; + ec_fop_data_t *fop; + gf_lkowner_t lk_owner; - if (fop->parent != NULL) + lock = link->lock; + fop = link->fop; + + lock->unlock_now = _gf_false; + ec_clear_inode_info(fop, lock->loc.inode); + + if ((lock->mask != 0) && lock->acquired) { + set_lk_owner_from_ptr(&lk_owner, lock); + lock->flock.l_type = F_UNLCK; + ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock, + lock->loc.inode); + + ec_inodelk(fop->frame, fop->xl, &lk_owner, lock->mask, EC_MINIMUM_ONE, + ec_unlocked, link, fop->xl->name, &lock->loc, F_SETLK, + &lock->flock, NULL); + } else { + ec_lock_unfreeze(link); + } +} + +void +ec_inode_bad_inc(inode_t *inode, xlator_t *xl) +{ + ec_inode_t *ctx = NULL; + + LOCK(&inode->lock); { - fop->parent->post_size = fop->post_size; + ctx = __ec_inode_get(inode, xl); + if (ctx == NULL) { + goto unlock; + } + ctx->bad_version++; + } +unlock: + UNLOCK(&inode->lock); +} - return; +int32_t +ec_update_size_version_done(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xattr, + dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link; + ec_lock_t *lock; + ec_inode_t *ctx; + + link = fop->data; + lock = link->lock; + ctx = lock->ctx; + + if (op_ret < 0) { + if (link->lock->fd == NULL) { + ec_inode_bad_inc(link->lock->loc.inode, this); + } else { + ec_inode_bad_inc(link->lock->fd->inode, this); + } + + gf_msg(fop->xl->name, fop_log_level(fop->id, op_errno), op_errno, + EC_MSG_SIZE_VERS_UPDATE_FAIL, + "Failed to update version and size. %s", ec_msg_str(fop)); + } else { + fop->parent->good &= fop->good; + + ec_lock_update_good(lock, fop); + + if (ec_dict_del_array(xattr, EC_XATTR_VERSION, ctx->post_version, + EC_VERSION_SIZE) == 0) { + ctx->pre_version[0] = ctx->post_version[0]; + ctx->pre_version[1] = ctx->post_version[1]; + + ctx->have_version = _gf_true; + } + if (ec_dict_del_number(xattr, EC_XATTR_SIZE, &ctx->post_size) == 0) { + ctx->pre_size = ctx->post_size; + + ctx->have_size = _gf_true; + } + if ((ec_dict_del_config(xdata, EC_XATTR_CONFIG, &ctx->config) == 0) && + ec_config_check(fop->xl, &ctx->config)) { + ctx->have_config = _gf_true; + } + + ctx->have_info = _gf_true; + } + /* If we are here because of fop's and other than unlock request, + * that means we are still holding a lock. That make sure + * lock->unlock_now can not be modified. + */ + if (lock->unlock_now) { + ec_unlock_lock(fop->data); } - ec_trace("UPDATE", fop, "version=%ld, size=%ld, dirty=%u", version, size, - dirty); + return 0; +} + +void +ec_update_size_version(ec_lock_link_t *link, uint64_t *version, uint64_t size, + uint64_t *dirty) +{ + ec_fop_data_t *fop; + ec_lock_t *lock; + ec_inode_t *ctx; + dict_t *dict = NULL; + uintptr_t update_on = 0; + int32_t err = -ENOMEM; + + fop = link->fop; + lock = link->lock; + ctx = lock->ctx; + + ec_trace("UPDATE", fop, "version=%ld/%ld, size=%ld, dirty=%ld/%ld", + version[0], version[1], size, dirty[0], dirty[1]); dict = dict_new(); - if (dict == NULL) - { + if (dict == NULL) { goto out; } - if (version != 0) { - if (ec_dict_set_number(dict, EC_XATTR_VERSION, version) != 0) { + /* If we don't have version information or it has been modified, we + * update it. */ + if (!ctx->have_version || (version[0] != 0) || (version[1] != 0)) { + err = ec_dict_set_array(dict, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { goto out; } } + if (size != 0) { - if (ec_dict_set_number(dict, EC_XATTR_SIZE, size) != 0) { + /* If size has been changed, we should already + * know the previous size of the file. */ + GF_ASSERT(ctx->have_size); + + err = ec_dict_set_number(dict, EC_XATTR_SIZE, size); + if (err != 0) { goto out; } } - if (dirty) { - if (ec_dict_set_number(dict, EC_XATTR_DIRTY, -1) != 0) { + + if (dirty[0] || dirty[1]) { + err = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (err != 0) { goto out; } } - uid = fop->frame->root->uid; - gid = fop->frame->root->gid; + /* If config information is not known, we request it now. */ + if ((lock->loc.inode->ia_type == IA_IFREG) && !ctx->have_config) { + /* A failure requesting this xattr is ignored because it's not + * absolutely required right now. */ + (void)ec_dict_set_number(dict, EC_XATTR_CONFIG, 0); + } fop->frame->root->uid = 0; fop->frame->root->gid = 0; - ec_xattrop(fop->frame, fop->xl, fop->mask, EC_MINIMUM_MIN, - ec_update_size_version_done, lock, loc, - GF_XATTROP_ADD_ARRAY64, dict, NULL); + update_on = lock->good_mask | lock->healing; - fop->frame->root->uid = uid; - fop->frame->root->gid = gid; + if (link->lock->fd == NULL) { + ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN, + ec_update_size_version_done, link, &link->lock->loc, + GF_XATTROP_ADD_ARRAY64, dict, NULL); + } else { + ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN, + ec_update_size_version_done, link, link->lock->fd, + GF_XATTROP_ADD_ARRAY64, dict, NULL); + } + + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; dict_unref(dict); return; out: - if (dict != NULL) - { + if (dict != NULL) { dict_unref(dict); } - ec_fop_set_error(fop, EIO); + ec_fop_set_error(fop, -err); - gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to update version and size"); + gf_msg(fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL, + "Unable to update version and size. %s", ec_msg_str(fop)); + + if (lock->unlock_now) { + ec_unlock_lock(fop->data); + } } -void ec_unlock_now(ec_fop_data_t *fop, ec_lock_t *lock) +gf_boolean_t +ec_update_info(ec_lock_link_t *link) { - ec_trace("UNLOCK_NOW", fop, "lock=%p", lock); + ec_lock_t *lock; + ec_inode_t *ctx; + uint64_t version[2] = {0, 0}; + uint64_t dirty[2] = {0, 0}; + uint64_t size; + ec_t *ec = NULL; + uintptr_t mask; - if ((lock->version_delta != 0) || lock->is_dirty) { - ec_update_size_version(fop, &lock->loc, lock->version_delta, - lock->size_delta, lock->is_dirty, lock); - } else { - ec_unlock_lock(fop, lock); + lock = link->lock; + ctx = lock->ctx; + ec = link->fop->xl->private; + + /* pre_version[*] will be 0 if have_version is false */ + version[EC_DATA_TXN] = ctx->post_version[EC_DATA_TXN] - + ctx->pre_version[EC_DATA_TXN]; + version[EC_METADATA_TXN] = ctx->post_version[EC_METADATA_TXN] - + ctx->pre_version[EC_METADATA_TXN]; + + size = ctx->post_size - ctx->pre_size; + /* If we set the dirty flag for update fop, we have to unset it. + * If fop has failed on some bricks, leave the dirty as marked. */ + + if (lock->unlock_now) { + if (version[EC_DATA_TXN]) { + /*A data fop will have difference in post and pre version + *and for data fop we send writes on healing bricks also */ + mask = lock->good_mask | lock->healing; + } else { + mask = lock->good_mask; + } + /* Ensure that nodes are up while doing final + * metadata update.*/ + if (!(ec->node_mask & ~(mask)) && !(ec->node_mask & ~ec->xl_up)) { + if (ctx->dirty[EC_DATA_TXN] != 0) { + dirty[EC_DATA_TXN] = -1; + } + if (ctx->dirty[EC_METADATA_TXN] != 0) { + dirty[EC_METADATA_TXN] = -1; + } + /*If everything is fine and we already + *have version xattr set on entry, there + *is no need to update version again*/ + if (ctx->pre_version[EC_DATA_TXN]) { + version[EC_DATA_TXN] = 0; + } + if (ctx->pre_version[EC_METADATA_TXN]) { + version[EC_METADATA_TXN] = 0; + } + } else { + link->optimistic_changelog = _gf_false; + ec_set_dirty_flag(link, ctx, dirty); + } + memset(ctx->dirty, 0, sizeof(ctx->dirty)); } - ec_resume(fop, 0); + if ((version[EC_DATA_TXN] != 0) || (version[EC_METADATA_TXN] != 0) || + (dirty[EC_DATA_TXN] != 0) || (dirty[EC_METADATA_TXN] != 0)) { + ec_update_size_version(link, version, size, dirty); + return _gf_true; + } + + return _gf_false; } -void ec_unlock_timer_cbk(void *data) +void +ec_unlock_now(ec_lock_link_t *link) { - ec_lock_link_t *link = data; - ec_lock_t *lock = link->lock; - ec_fop_data_t *fop = NULL; + ec_lock_t *lock; + lock = link->lock; - LOCK(&lock->loc.inode->lock); + ec_trace("UNLOCK_NOW", link->fop, "lock=%p", link->lock); + /*At this point, lock is not being used by any fop and + *can not be reused by any fop as it is going to be released. + *lock->unlock_now can not be modified at any other place. + */ + lock->unlock_now = _gf_true; - if (lock->timer != NULL) { - fop = link->fop; + if (!ec_update_info(link)) { + ec_unlock_lock(link); + } + + ec_resume(link->fop, 0); +} + +void +ec_lock_release(ec_t *ec, inode_t *inode) +{ + ec_lock_t *lock; + ec_inode_t *ctx; + ec_lock_link_t *timer_link = NULL; + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, ec->xl); + if (ctx == NULL) { + goto done; + } + lock = ctx->inode_lock; + if ((lock == NULL) || lock->release) { + goto done; + } + + gf_msg_debug(ec->xl->name, 0, "Releasing inode %p due to lock contention", + inode); + + if (!lock->acquired) { + /* This happens if some bricks already got the lock while inodelk is in + * progress. Set release to true after lock is acquired*/ + lock->contention = _gf_true; + goto done; + } + + /* The lock is not marked to be released, so the frozen list should be + * empty. */ + GF_ASSERT(list_empty(&lock->frozen)); + + timer_link = ec_lock_timer_cancel(ec->xl, lock); - ec_trace("UNLOCK_DELAYED", fop, "lock=%p", lock); + /* We mark the lock to be released as soon as possible. */ + lock->release = _gf_true; - GF_ASSERT(lock->refs == 1); +done: + UNLOCK(&inode->lock); - gf_timer_call_cancel(fop->xl->ctx, lock->timer); + /* If we have cancelled the timer, we need to start the unlock of the + * inode. If there was a timer but we have been unable to cancel it + * because it was just triggered, the timer callback will take care + * of releasing the inode. */ + if (timer_link != NULL) { + ec_unlock_now(timer_link); + } +} + +void +ec_unlock_timer_add(ec_lock_link_t *link); + +void +ec_unlock_timer_del(ec_lock_link_t *link) +{ + ec_lock_t *lock; + inode_t *inode; + gf_boolean_t now = _gf_false; + + /* If we are here, it means that the timer has expired before having + * been cancelled. This guarantees that 'link' is still valid because + * the fop that contains it must be pending (if timer cancellation in + * ec_lock_assign_owner() fails, the fop is left sleeping). + * + * At the same time, the fop still has a reference to the lock, so + * it must also be valid. + */ + lock = link->lock; + + /* 'lock' must have a valid inode since it can only be destroyed + * when the lock itself is destroyed, but we have a reference to the + * lock to avoid this. + */ + inode = lock->loc.inode; + + LOCK(&inode->lock); + + if (lock->timer != NULL) { + ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock); + + /* The unlock timer has expired without anyone cancelling it. + * This means that it shouldn't have any owner, and the waiting + * and frozen lists should be empty. It must have only one + * owner reference, but there can be fops being prepared + * though. + * */ + GF_ASSERT(!lock->release && (lock->refs_owners == 1) && + list_empty(&lock->owners) && list_empty(&lock->waiting) && + list_empty(&lock->frozen)); + + gf_timer_call_cancel(link->fop->xl->ctx, lock->timer); lock->timer = NULL; - *lock->plock = NULL; + + /* Any fop being processed from now on, will need to wait + * until the next unlock/lock cycle. */ + lock->release = now = _gf_true; } - UNLOCK(&lock->loc.inode->lock); + UNLOCK(&inode->lock); - if (fop != NULL) { - ec_unlock_now(fop, lock); + if (now) { + ec_unlock_now(link); + } else { + /* The timer has been cancelled just after firing it but before + * getting here. This means that another fop has used the lock + * and everything should be handled as if this callback were + * have not been executed. However we still have an owner + * reference. + * + * We need to release our reference. If this is not the last + * reference (the most common case because another fop has + * taken another ref) we only need to decrement the counter. + * Otherwise we have been delayed enough so that the other fop + * has had time to acquire the reference, do its operation and + * release it. At the time of releasing it, the fop did found + * that the ref counter was > 1 (our reference), so the delayed + * unlock timer wasn't started. We need to start it again if we + * are the last reference. + * + * ec_unlock_timer_add() handles both cases. + */ + ec_unlock_timer_add(link); + + /* We need to resume the fop that was waiting for the delayed + * unlock. + */ + ec_resume(link->fop, 0); + } +} + +void +ec_unlock_timer_cbk(void *data) +{ + ec_unlock_timer_del(data); +} + +static gf_boolean_t +ec_eager_lock_used(ec_t *ec, ec_fop_data_t *fop) +{ + /* Fops with no locks at this point mean that they are sent as sub-fops + * of other higher level fops. In this case we simply assume that the + * parent fop will take correct care of the eager lock. */ + if (fop->lock_count == 0) { + return _gf_true; + } + + /* We may have more than one lock, but this only happens in the rename + * fop, and both locks will reference an inode of the same type (a + * directory in this case), so we only need to check the first lock. */ + if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) { + return ec->eager_lock; + } + + return ec->other_eager_lock; +} + +static uint32_t +ec_eager_lock_timeout(ec_t *ec, ec_lock_t *lock) +{ + if (lock->loc.inode->ia_type == IA_IFREG) { + return ec->eager_lock_timeout; } + + return ec->other_eager_lock_timeout; } -void ec_unlock_timer_add(ec_lock_link_t *link) +static gf_boolean_t +ec_lock_delay_create(ec_lock_link_t *link) { struct timespec delay; ec_fop_data_t *fop = link->fop; ec_lock_t *lock = link->lock; - int32_t refs = 1; + + delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock); + delay.tv_nsec = 0; + lock->timer = gf_timer_call_after(fop->xl->ctx, delay, ec_unlock_timer_cbk, + link); + if (lock->timer == NULL) { + gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM, + EC_MSG_UNLOCK_DELAY_FAILED, "Unable to delay an unlock"); + + return _gf_false; + } + + return _gf_true; +} + +void +ec_unlock_timer_add(ec_lock_link_t *link) +{ + ec_fop_data_t *fop = link->fop; + ec_lock_t *lock = link->lock; + gf_boolean_t now = _gf_false; LOCK(&lock->loc.inode->lock); - GF_ASSERT(lock->timer == NULL); + /* We are trying to unlock the lock. We can have multiple scenarios here, + * but all of them need to have lock->timer == NULL: + * + * 1. There are other owners currently running that can call ec_unlock(). + * + * None of them can have started the timer until the last one. But this + * call should be the consequence of this lastest one. + * + * 2. There are fops in the waiting or frozen lists. + * + * These fops cannot call ec_unlock(). So we should be here. + * + * We must reach here with at least one owner reference. + */ + GF_ASSERT((lock->timer == NULL) && (lock->refs_owners > 0)); + + /* If the fop detects that a heal is needed, we mark the lock to be + * released as soon as possible. */ + lock->release |= ec_fop_needs_heal(fop); - if (lock->refs != 1) { + if (lock->refs_owners > 1) { ec_trace("UNLOCK_SKIP", fop, "lock=%p", lock); - lock->refs--; + /* If there are other owners we cannot do anything else with the lock. + * Note that the current fop has already been removed from the owners + * list in ec_lock_reuse(). */ + lock->refs_owners--; UNLOCK(&lock->loc.inode->lock); } else if (lock->acquired) { - delay.tv_sec = 1; - delay.tv_nsec = 0; + /* There are no other owners and the lock is acquired. If there were + * fops waiting, at least one of them should have been promoted to an + * owner, so the waiting list should be empty. */ + GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting)); + ec_t *ec = fop->xl->private; + + /* If everything goes as expected this fop will be put to sleep until + * the timer callback is executed. */ ec_sleep(fop); - /* If healing is needed, do not delay lock release to let self-heal - * start working as soon as possible. */ - if (!ec_fop_needs_heal(fop)) { - ec_trace("UNLOCK_DELAY", fop, "lock=%p", lock); - - delay.tv_sec = 1; - delay.tv_nsec = 0; - lock->timer = gf_timer_call_after(fop->xl->ctx, delay, - ec_unlock_timer_cbk, link); - if (lock->timer == NULL) { - gf_log(fop->xl->name, GF_LOG_WARNING, "Unable to delay an " - "unlock"); - - *lock->plock = NULL; - refs = 0; + /* If the lock needs to be released, or ec is shutting down, do not + * delay lock release. */ + if (!lock->release && !ec->shutdown) { + ec_trace("UNLOCK_DELAY", fop, "lock=%p, release=%d", lock, + lock->release); + + if (!ec_lock_delay_create(link)) { + /* We are unable to create a new timer. We immediately release + * the lock. */ + lock->release = now = _gf_true; } + } else { - ec_trace("UNLOCK_FORCE", fop, "lock=%p", lock); - *lock->plock = NULL; - refs = 0; + ec_trace("UNLOCK_FORCE", fop, "lock=%p, release=%d", lock, + lock->release); + lock->release = now = _gf_true; } UNLOCK(&lock->loc.inode->lock); - if (refs == 0) { - ec_unlock_now(fop, lock); + if (now) { + ec_unlock_now(link); } } else { - *lock->plock = NULL; + /* There are no owners and the lock is not acquired. This can only + * happen if a lock attempt has failed and we get to the unlock step + * of the fop. As in the previous case, the waiting list must be + * empty. */ + GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting)); + + /* We need to mark the lock to be released to correctly handle fops + * that may get in after we release the inode mutex but before + * ec_lock_unfreeze() is processed. */ + lock->release = _gf_true; UNLOCK(&lock->loc.inode->lock); - ec_lock_destroy(lock); + ec_lock_unfreeze(link); } } -void ec_unlock(ec_fop_data_t *fop) +void +ec_unlock(ec_fop_data_t *fop) { int32_t i; @@ -1525,103 +2839,155 @@ void ec_unlock(ec_fop_data_t *fop) } } -void ec_flush_size_version(ec_fop_data_t * fop) +void +ec_flush_size_version(ec_fop_data_t *fop) { - ec_lock_t * lock; - uint64_t version, delta; - GF_ASSERT(fop->lock_count == 1); + ec_update_info(&fop->locks[0]); +} - lock = fop->locks[0].lock; - - LOCK(&lock->loc.inode->lock); - - GF_ASSERT(lock->owner == fop); - - version = lock->version_delta; - delta = lock->size_delta; - lock->version_delta = 0; - lock->size_delta = 0; - - UNLOCK(&lock->loc.inode->lock); +static void +ec_update_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache, ec_stripe_t *stripe, + ec_fop_data_t *fop) +{ + off_t base; + + /* On write fops, we only update existing fragments if the write has + * succeeded. Otherwise, we remove them from the cache. */ + if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) && + (fop->answer->op_ret >= 0)) { + base = stripe->frag_offset - fop->frag_range.first; + base *= ec->fragments; + + /* We check if the stripe offset falls inside the real region + * modified by the write fop (a write request is allowed, + * though uncommon, to write less bytes than requested). The + * current write fop implementation doesn't allow partial + * writes of fragments, so if there's no error, we are sure + * that a full stripe has been completely modified or not + * touched at all. The value of op_ret may not be a multiple + * of the stripe size because it depends on the requested + * size by the user, so we update the stripe if the write has + * modified at least one byte (meaning ec has written the full + * stripe). */ + if (base < fop->answer->op_ret + fop->head) { + memcpy(stripe->data, fop->vector[0].iov_base + base, + ec->stripe_size); + list_move_tail(&stripe->lru, &stripe_cache->lru); + + GF_ATOMIC_INC(ec->stats.stripe_cache.updates); + } + } else { + stripe->frag_offset = -1; + list_move(&stripe->lru, &stripe_cache->lru); - if (version > 0) - { - ec_update_size_version(fop, &lock->loc, version, delta, _gf_false, - NULL); + GF_ATOMIC_INC(ec->stats.stripe_cache.invals); } } -void ec_lock_reuse(ec_fop_data_t *fop) +static void +ec_update_cached_stripes(ec_fop_data_t *fop) { - ec_fop_data_t * wait_fop; - ec_lock_t * lock; - ec_lock_link_t * link; - int32_t i; - - for (i = 0; i < fop->lock_count; i++) - { - wait_fop = NULL; - - lock = fop->locks[i].lock; - - LOCK(&lock->loc.inode->lock); + uint64_t first; + uint64_t last; + ec_stripe_t *stripe = NULL; + ec_inode_t *ctx = NULL; + ec_stripe_list_t *stripe_cache = NULL; + inode_t *inode = NULL; + struct list_head *temp; + struct list_head sentinel; + + first = fop->frag_range.first; + /* 'last' represents the first stripe not touched by the operation */ + last = fop->frag_range.last; + + /* If there are no modified stripes, we don't need to do anything + * else. */ + if (last <= first) { + return; + } - ec_trace("LOCK_DONE", fop, "lock=%p", lock); + if (!fop->use_fd) { + inode = fop->loc[0].inode; + } else { + inode = fop->fd->inode; + } - GF_ASSERT(lock->owner == fop); - lock->owner = NULL; + LOCK(&inode->lock); - if (((fop->locks_update >> i) & 1) != 0) { - if (fop->error == 0) - { - lock->version_delta++; - lock->size_delta += fop->post_size - fop->pre_size; - if (fop->have_size) { - lock->size = fop->post_size; - lock->have_size = 1; - } - } + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } + stripe_cache = &ctx->stripe_cache; + + /* Since we'll be moving elements of the list to the tail, we might + * end in an infinite loop. To avoid it, we insert a sentinel element + * into the list, so that it will be used to detect when we have + * traversed all existing elements once. */ + list_add_tail(&sentinel, &stripe_cache->lru); + temp = stripe_cache->lru.next; + while (temp != &sentinel) { + stripe = list_entry(temp, ec_stripe_t, lru); + temp = temp->next; + if ((first <= stripe->frag_offset) && (stripe->frag_offset < last)) { + ec_update_stripe(fop->xl->private, stripe_cache, stripe, fop); } + } + list_del(&sentinel); - lock->good_mask &= fop->mask; - - if (!list_empty(&lock->waiting)) - { - link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list); - list_del_init(&link->wait_list); - - wait_fop = link->fop; +out: + UNLOCK(&inode->lock); +} - if (lock->kind == EC_LOCK_INODE) - { - wait_fop->pre_size = wait_fop->post_size = fop->post_size; - wait_fop->have_size = fop->have_size; +void +ec_lock_reuse(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk; + ec_t *ec = NULL; + int32_t i, count; + gf_boolean_t release = _gf_false; + ec = fop->xl->private; + cbk = fop->answer; + + if (ec_eager_lock_used(ec, fop) && cbk != NULL) { + if (cbk->xdata != NULL) { + if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT, &count) == + 0) && + (count > 1)) { + release = _gf_true; + } + if (release) { + gf_msg_debug(fop->xl->name, 0, "Lock contention detected"); } - wait_fop->mask &= fop->mask; } + } else { + /* If eager lock is disabled or if we haven't get + * an answer with enough quorum, we always release + * the lock. */ + release = _gf_true; + } + ec_update_cached_stripes(fop); - UNLOCK(&lock->loc.inode->lock); - - if (wait_fop != NULL) - { - ec_lock(wait_fop); - - ec_resume(wait_fop, 0); - } + for (i = 0; i < fop->lock_count; i++) { + ec_lock_next_owner(&fop->locks[i], cbk, release); } } -void __ec_manager(ec_fop_data_t * fop, int32_t error) +void +__ec_manager(ec_fop_data_t *fop, int32_t error) { ec_t *ec = fop->xl->private; do { ec_trace("MANAGER", fop, "error=%d", error); - if (ec->xl_up_count < ec->fragments) { - error = ENOTCONN; + if (!ec_must_wind(fop)) { + if (ec->xl_up_count < ec->fragments) { + error = ENOTCONN; + } } + if (error != 0) { fop->error = error; fop->state = -fop->state; @@ -1633,22 +2999,44 @@ void __ec_manager(ec_fop_data_t * fop, int32_t error) break; } + /* At each state, fop must not be used anywhere else and there + * shouldn't be any pending subfop going on. */ + GF_ASSERT(fop->jobs == 0); + + /* While the manager is running we need to avoid that subfops launched + * from it could finish and call ec_resume() before the fop->handler + * has completed. This could lead to the same manager being executed + * by two threads concurrently. ec_check_complete() will take care of + * this reference. */ + fop->jobs = 1; + fop->state = fop->handler(fop, fop->state); + GF_ASSERT(fop->state >= 0); error = ec_check_complete(fop, __ec_manager); } while (error >= 0); } -void ec_manager(ec_fop_data_t * fop, int32_t error) +void +ec_manager(ec_fop_data_t *fop, int32_t error) { GF_ASSERT(fop->jobs == 0); GF_ASSERT(fop->winds == 0); GF_ASSERT(fop->error == 0); - if (fop->state == EC_STATE_START) - { + if (fop->state == EC_STATE_START) { fop->state = EC_STATE_INIT; } __ec_manager(fop, error); } + +gf_boolean_t +__ec_is_last_fop(ec_t *ec) +{ + if ((list_empty(&ec->pending_fops)) && + (GF_ATOMIC_GET(ec->async_fop_count) == 0)) { + return _gf_true; + } + return _gf_false; +} diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h index aaae16e71c3..51493612ac6 100644 --- a/xlators/cluster/ec/src/ec-common.h +++ b/xlators/cluster/ec/src/ec-common.h @@ -11,93 +11,224 @@ #ifndef __EC_COMMON_H__ #define __EC_COMMON_H__ -#include "xlator.h" - +#include "glusterfs/compat-errno.h" // for ENODATA on BSD #include "ec-data.h" +typedef enum { EC_DATA_TXN, EC_METADATA_TXN } ec_txn_t; + +#define EC_FOP_HEAL -1 +#define EC_FOP_FHEAL -2 + #define EC_CONFIG_VERSION 0 #define EC_CONFIG_ALGORITHM 0 -#define EC_FLAG_UPDATE_LOC_PARENT 0x0001 -#define EC_FLAG_UPDATE_LOC_INODE 0x0002 -#define EC_FLAG_UPDATE_FD 0x0004 -#define EC_FLAG_UPDATE_FD_INODE 0x0008 - -#define EC_FLAG_WAITING_WINDS 0x0010 - -#define EC_MINIMUM_ONE -1 -#define EC_MINIMUM_MIN -2 -#define EC_MINIMUM_ALL -3 - -#define EC_LOCK_ENTRY 0 -#define EC_LOCK_INODE 1 - -#define EC_STATE_START 0 -#define EC_STATE_END 0 -#define EC_STATE_INIT 1 -#define EC_STATE_LOCK 2 -#define EC_STATE_GET_SIZE_AND_VERSION 3 -#define EC_STATE_DISPATCH 4 -#define EC_STATE_PREPARE_ANSWER 5 -#define EC_STATE_REPORT 6 -#define EC_STATE_LOCK_REUSE 7 -#define EC_STATE_UNLOCK 8 - -#define EC_STATE_DELAYED_START 100 - -#define EC_STATE_HEAL_ENTRY_LOOKUP 200 -#define EC_STATE_HEAL_ENTRY_PREPARE 201 -#define EC_STATE_HEAL_PRE_INODELK_LOCK 202 -#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203 -#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204 -#define EC_STATE_HEAL_XATTRIBUTES_SET 205 -#define EC_STATE_HEAL_ATTRIBUTES 206 -#define EC_STATE_HEAL_OPEN 207 -#define EC_STATE_HEAL_REOPEN_FD 208 -#define EC_STATE_HEAL_UNLOCK 209 -#define EC_STATE_HEAL_UNLOCK_ENTRY 210 -#define EC_STATE_HEAL_DATA_LOCK 211 -#define EC_STATE_HEAL_DATA_COPY 212 -#define EC_STATE_HEAL_DATA_UNLOCK 213 -#define EC_STATE_HEAL_POST_INODELK_LOCK 214 -#define EC_STATE_HEAL_POST_INODE_LOOKUP 215 -#define EC_STATE_HEAL_SETATTR 216 -#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217 -#define EC_STATE_HEAL_DISPATCH 218 - -int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret, - int32_t op_errno); -int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx); - -void ec_complete(ec_fop_data_t * fop); - -void ec_update_bad(ec_fop_data_t * fop, uintptr_t good); - -void ec_fop_set_error(ec_fop_data_t * fop, int32_t error); - -void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, int32_t update); -void ec_lock_prepare_entry(ec_fop_data_t *fop, loc_t *loc, int32_t update); -void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, int32_t update); -void ec_lock(ec_fop_data_t * fop); -void ec_lock_reuse(ec_fop_data_t *fop); -void ec_unlock(ec_fop_data_t * fop); - -void ec_get_size_version(ec_fop_data_t * fop); -void ec_prepare_update(ec_fop_data_t *fop); -void ec_flush_size_version(ec_fop_data_t * fop); - -void ec_dispatch_all(ec_fop_data_t * fop); -void ec_dispatch_inc(ec_fop_data_t * fop); -void ec_dispatch_min(ec_fop_data_t * fop); -void ec_dispatch_one(ec_fop_data_t * fop); - -void ec_wait_winds(ec_fop_data_t * fop); - -void ec_sleep(ec_fop_data_t *fop); -void ec_resume(ec_fop_data_t * fop, int32_t error); -void ec_resume_parent(ec_fop_data_t * fop, int32_t error); - -void ec_manager(ec_fop_data_t * fop, int32_t error); - +#define EC_FLAG_LOCK_SHARED 0x0001 + +#define QUORUM_CBK(fn, fop, frame, cookie, this, op_ret, op_errno, params...) \ + do { \ + ec_t *__ec = fop->xl->private; \ + int32_t __op_ret = 0; \ + int32_t __op_errno = 0; \ + int32_t __success_count = gf_bits_count(fop->good); \ + \ + __op_ret = op_ret; \ + __op_errno = op_errno; \ + if (!fop->parent && frame && \ + (GF_CLIENT_PID_SELF_HEALD != frame->root->pid) && \ + __ec->quorum_count && (__success_count < __ec->quorum_count) && \ + op_ret >= 0) { \ + __op_ret = -1; \ + __op_errno = EIO; \ + gf_msg(__ec->xl->name, GF_LOG_ERROR, 0, \ + EC_MSG_CHILDS_INSUFFICIENT, \ + "Insufficient available children for this request " \ + "(have %d, need %d). %s", \ + __success_count, __ec->quorum_count, ec_msg_str(fop)); \ + } \ + fn(frame, cookie, this, __op_ret, __op_errno, params); \ + } while (0) + +enum _ec_xattrop_flags { + EC_FLAG_XATTROP, + EC_FLAG_DATA_DIRTY, + EC_FLAG_METADATA_DIRTY, + + /* Add any new flag here, before EC_FLAG_MAX. The maximum number of + * flags that can be defined is 16. */ + + EC_FLAG_MAX +}; + +/* We keep two sets of flags. One to determine what's really providing the + * current xattrop and the other to know what the parent fop of the xattrop + * needs to proceed. It might happen that a fop needs some information that + * is being already requested by a previous fop. The two sets are stored + * contiguously. */ + +#define EC_FLAG_NEEDS(_flag) (1 << (_flag)) +#define EC_FLAG_PROVIDES(_flag) (1 << ((_flag) + EC_FLAG_MAX)) + +#define EC_NEEDED_FLAGS(_flags) ((_flags) & ((1 << EC_FLAG_MAX) - 1)) + +#define EC_PROVIDED_FLAGS(_flags) EC_NEEDED_FLAGS((_flags) >> EC_FLAG_MAX) + +#define EC_FLAGS_HAVE(_flags, _flag) (((_flags) & (1 << (_flag))) != 0) + +#define EC_SELFHEAL_BIT 62 + +#define EC_MINIMUM_ONE (1 << 6) +#define EC_MINIMUM_MIN (2 << 6) +#define EC_MINIMUM_ALL (3 << 6) +#define EC_FOP_NO_PROPAGATE_ERROR (1 << 8) +#define EC_FOP_MINIMUM(_flags) ((_flags)&255) +#define EC_FOP_FLAGS(_flags) ((_flags) & ~255) + +#define EC_UPDATE_DATA 1 +#define EC_UPDATE_META 2 +#define EC_QUERY_INFO 4 +#define EC_INODE_SIZE 8 + +#define EC_STATE_START 0 +#define EC_STATE_END 0 +#define EC_STATE_INIT 1 +#define EC_STATE_LOCK 2 +#define EC_STATE_DISPATCH 3 +#define EC_STATE_PREPARE_ANSWER 4 +#define EC_STATE_REPORT 5 +#define EC_STATE_LOCK_REUSE 6 +#define EC_STATE_UNLOCK 7 + +#define EC_STATE_DELAYED_START 100 + +#define EC_STATE_HEAL_ENTRY_LOOKUP 200 +#define EC_STATE_HEAL_ENTRY_PREPARE 201 +#define EC_STATE_HEAL_PRE_INODELK_LOCK 202 +#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203 +#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204 +#define EC_STATE_HEAL_XATTRIBUTES_SET 205 +#define EC_STATE_HEAL_ATTRIBUTES 206 +#define EC_STATE_HEAL_OPEN 207 +#define EC_STATE_HEAL_REOPEN_FD 208 +#define EC_STATE_HEAL_UNLOCK 209 +#define EC_STATE_HEAL_UNLOCK_ENTRY 210 +#define EC_STATE_HEAL_DATA_LOCK 211 +#define EC_STATE_HEAL_DATA_COPY 212 +#define EC_STATE_HEAL_DATA_UNLOCK 213 +#define EC_STATE_HEAL_POST_INODELK_LOCK 214 +#define EC_STATE_HEAL_POST_INODE_LOOKUP 215 +#define EC_STATE_HEAL_SETATTR 216 +#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217 +#define EC_STATE_HEAL_DISPATCH 218 + +/* Value to cover the full range of a file */ +#define EC_RANGE_FULL ((uint64_t)LLONG_MAX + 1) + +gf_boolean_t +ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk); +void +ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx); + +void +ec_complete(ec_fop_data_t *fop); + +void +ec_update_good(ec_fop_data_t *fop, uintptr_t good); + +void +ec_fop_set_error(ec_fop_data_t *fop, int32_t error); + +void +__ec_fop_set_error(ec_fop_data_t *fop, int32_t error); + +ec_cbk_data_t * +ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro); + +gf_boolean_t +ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro); + +void +ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + off_t fl_start, uint64_t fl_size); +void +ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base, + uint32_t flags); +void +ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start, + uint64_t fl_size); +void +ec_lock(ec_fop_data_t *fop); +void +ec_lock_reuse(ec_fop_data_t *fop); +void +ec_unlock(ec_fop_data_t *fop); +void +ec_lock_release(ec_t *ec, inode_t *inode); + +gf_boolean_t +ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size); +gf_boolean_t +__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size); +gf_boolean_t +ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size); +gf_boolean_t +__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size); +void +ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode); + +void +ec_flush_size_version(ec_fop_data_t *fop); + +void +ec_dispatch_all(ec_fop_data_t *fop); +void +ec_dispatch_inc(ec_fop_data_t *fop); +void +ec_dispatch_min(ec_fop_data_t *fop); +void +ec_dispatch_one(ec_fop_data_t *fop); + +void +ec_succeed_all(ec_fop_data_t *fop); + +void +ec_sleep(ec_fop_data_t *fop); +void +ec_resume(ec_fop_data_t *fop, int32_t error); +void +ec_resume_parent(ec_fop_data_t *fop); + +void +ec_manager(ec_fop_data_t *fop, int32_t error); +gf_boolean_t +ec_is_recoverable_error(int32_t op_errno); +void +ec_handle_healers_done(ec_fop_data_t *fop); + +int32_t +ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal); +int32_t +ec_get_heal_info(xlator_t *this, loc_t *loc, dict_t **dict); + +int32_t +ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata); + +void +ec_update_fd_status(fd_t *fd, xlator_t *xl, int child_index, + int32_t ret_status); +gf_boolean_t +ec_is_entry_healing(ec_fop_data_t *fop); +void +ec_set_entry_healing(ec_fop_data_t *fop); +void +ec_reset_entry_healing(ec_fop_data_t *fop); +char * +ec_msg_str(ec_fop_data_t *fop); +gf_boolean_t +__ec_is_last_fop(ec_t *ec); +void +ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop); #endif /* __EC_COMMON_H__ */ diff --git a/xlators/cluster/ec/src/ec-data.c b/xlators/cluster/ec/src/ec-data.c index fb47aea90a8..06388833546 100644 --- a/xlators/cluster/ec/src/ec-data.c +++ b/xlators/cluster/ec/src/ec-data.c @@ -8,49 +8,49 @@ cases as published by the Free Software Foundation. */ -#include "ec-mem-types.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-data.h" +#include "ec-messages.h" -ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this, - ec_fop_data_t * fop, int32_t id, - int32_t idx, int32_t op_ret, - int32_t op_errno) +ec_cbk_data_t * +ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop, + int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno) { - ec_cbk_data_t * cbk; - ec_t * ec = this->private; + ec_cbk_data_t *cbk; + ec_t *ec = this->private; - if (fop->xl != this) - { - gf_log(this->name, GF_LOG_ERROR, "Mismatching xlators between request " - "and answer (req=%s, ans=%s).", - fop->xl->name, this->name); + if (fop->xl != this) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_MISMATCH, + "Mismatching xlators between request " + "and answer (req=%s, ans=%s).", + fop->xl->name, this->name); return NULL; } - if (fop->frame != frame) - { - gf_log(this->name, GF_LOG_ERROR, "Mismatching frames between request " - "and answer (req=%p, ans=%p).", - fop->frame, frame); + if (fop->frame != frame) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FRAME_MISMATCH, + "Mismatching frames between request " + "and answer (req=%p, ans=%p).", + fop->frame, frame); return NULL; } - if (fop->id != id) - { - gf_log(this->name, GF_LOG_ERROR, "Mismatching fops between request " - "and answer (req=%d, ans=%d).", - fop->id, id); + if (fop->id != id) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FOP_MISMATCH, + "Mismatching fops between request " + "and answer (req=%d, ans=%d).", + fop->id, id); return NULL; } cbk = mem_get0(ec->cbk_pool); - if (cbk == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to allocate memory for an " - "answer."); + if (cbk == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate memory for an " + "answer."); + return NULL; } cbk->fop = fop; @@ -59,6 +59,7 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this, cbk->count = 1; cbk->op_ret = op_ret; cbk->op_errno = op_errno; + INIT_LIST_HEAD(&cbk->entries.list); LOCK(&fop->lock); @@ -69,51 +70,58 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this, return cbk; } -void ec_cbk_data_destroy(ec_cbk_data_t * cbk) +void +ec_cbk_data_destroy(ec_cbk_data_t *cbk) { - if (cbk->xdata != NULL) - { + if (cbk->xdata != NULL) { dict_unref(cbk->xdata); } - if (cbk->dict != NULL) - { + if (cbk->dict != NULL) { dict_unref(cbk->dict); } - if (cbk->inode != NULL) - { + if (cbk->inode != NULL) { inode_unref(cbk->inode); } - if (cbk->fd != NULL) - { + if (cbk->fd != NULL) { fd_unref(cbk->fd); } - if (cbk->buffers != NULL) - { + if (cbk->buffers != NULL) { iobref_unref(cbk->buffers); } GF_FREE(cbk->vector); + gf_dirent_free(&cbk->entries); + GF_FREE(cbk->str); mem_put(cbk); } -ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this, - int32_t id, uint32_t flags, - uintptr_t target, int32_t minimum, - ec_wind_f wind, ec_handler_f handler, - ec_cbk_t cbks, void * data) +ec_fop_data_t * +ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id, + uint32_t flags, uintptr_t target, uint32_t fop_flags, + ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks, + void *data) { - ec_fop_data_t * fop, * parent; - ec_t * ec = this->private; + ec_fop_data_t *fop, *parent; + ec_t *ec = this->private; fop = mem_get0(ec->fop_pool); - if (fop == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to allocate memory for a " - "request."); + if (fop == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate memory for a " + "request."); return NULL; } + INIT_LIST_HEAD(&fop->cbk_list); + INIT_LIST_HEAD(&fop->healer); + INIT_LIST_HEAD(&fop->answer_list); + INIT_LIST_HEAD(&fop->pending_list); + INIT_LIST_HEAD(&fop->locks[0].owner_list); + INIT_LIST_HEAD(&fop->locks[0].wait_list); + INIT_LIST_HEAD(&fop->locks[1].owner_list); + INIT_LIST_HEAD(&fop->locks[1].wait_list); + fop->xl = this; fop->req_frame = frame; @@ -124,18 +132,15 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this, * TODO: minimize usage of private frames. Reuse req_frame as much as * possible. */ - if (frame != NULL) - { + if (frame != NULL) { fop->frame = copy_frame(frame); - } - else - { + } else { fop->frame = create_frame(this, this->ctx->pool); } - if (fop->frame == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to create a private frame " - "for a request"); + if (fop->frame == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to create a private frame " + "for a request"); mem_put(fop); @@ -145,41 +150,42 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this, fop->refs = 1; fop->flags = flags; - fop->minimum = minimum; + fop->minimum = EC_FOP_MINIMUM(fop_flags); + fop->fop_flags = EC_FOP_FLAGS(fop_flags); fop->mask = target; - INIT_LIST_HEAD(&fop->cbk_list); - INIT_LIST_HEAD(&fop->answer_list); - fop->wind = wind; fop->handler = handler; fop->cbks = cbks; fop->data = data; + fop->uid = fop->frame->root->uid; + fop->gid = fop->frame->root->gid; + LOCK_INIT(&fop->lock); fop->frame->local = fop; - if (frame != NULL) - { + if (frame != NULL) { parent = frame->local; - if (parent != NULL) - { - LOCK(&parent->lock); - - parent->jobs++; - parent->refs++; - - UNLOCK(&parent->lock); + if (parent != NULL) { + ec_sleep(parent); } fop->parent = parent; } + LOCK(&ec->lock); + + list_add_tail(&fop->pending_list, &ec->pending_fops); + + UNLOCK(&ec->lock); + return fop; } -void ec_fop_data_acquire(ec_fop_data_t * fop) +void +ec_fop_data_acquire(ec_fop_data_t *fop) { LOCK(&fop->lock); @@ -190,44 +196,74 @@ void ec_fop_data_acquire(ec_fop_data_t * fop) UNLOCK(&fop->lock); } -void ec_fop_data_release(ec_fop_data_t * fop) +static void +ec_handle_last_pending_fop_completion(ec_fop_data_t *fop, gf_boolean_t *notify) +{ + ec_t *ec = fop->xl->private; + + *notify = _gf_false; + + if (!list_empty(&fop->pending_list)) { + LOCK(&ec->lock); + { + list_del_init(&fop->pending_list); + *notify = __ec_is_last_fop(ec); + } + UNLOCK(&ec->lock); + } +} + +void +ec_fop_cleanup(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk, *tmp; + + list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list) + { + list_del_init(&cbk->answer_list); + + ec_cbk_data_destroy(cbk); + } + INIT_LIST_HEAD(&fop->cbk_list); + + fop->answer = NULL; +} + +void +ec_fop_data_release(ec_fop_data_t *fop) { - ec_cbk_data_t * cbk, * tmp; + ec_t *ec = NULL; int32_t refs; + gf_boolean_t notify = _gf_false; LOCK(&fop->lock); ec_trace("RELEASE", fop, ""); + GF_ASSERT(fop->refs > 0); refs = --fop->refs; UNLOCK(&fop->lock); - if (refs == 0) - { + if (refs == 0) { fop->frame->local = NULL; STACK_DESTROY(fop->frame->root); LOCK_DESTROY(&fop->lock); - if (fop->xdata != NULL) - { + if (fop->xdata != NULL) { dict_unref(fop->xdata); } - if (fop->dict != NULL) - { + if (fop->dict != NULL) { dict_unref(fop->dict); } - if (fop->inode != NULL) - { + if (fop->inode != NULL) { inode_unref(fop->inode); } - if (fop->fd != NULL) - { + if (fop->fd != NULL) { fd_unref(fop->fd); } - if (fop->buffers != NULL) - { + if (fop->buffers != NULL) { iobref_unref(fop->buffers); } GF_FREE(fop->vector); @@ -235,16 +271,18 @@ void ec_fop_data_release(ec_fop_data_t * fop) GF_FREE(fop->str[1]); loc_wipe(&fop->loc[0]); loc_wipe(&fop->loc[1]); + GF_FREE(fop->errstr); - ec_resume_parent(fop, fop->error); + ec_resume_parent(fop); - list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list) - { - list_del_init(&cbk->answer_list); - - ec_cbk_data_destroy(cbk); - } + ec_fop_cleanup(fop); + ec = fop->xl->private; + ec_handle_last_pending_fop_completion(fop, ¬ify); + ec_handle_healers_done(fop); mem_put(fop); + if (notify) { + ec_pending_fops_completed(ec); + } } } diff --git a/xlators/cluster/ec/src/ec-data.h b/xlators/cluster/ec/src/ec-data.h index b17f197837b..c8a74ffe1ed 100644 --- a/xlators/cluster/ec/src/ec-data.h +++ b/xlators/cluster/ec/src/ec-data.h @@ -11,287 +11,25 @@ #ifndef __EC_DATA_H__ #define __EC_DATA_H__ -#include "xlator.h" - -#include "ec.h" - -struct _ec_config; -typedef struct _ec_config ec_config_t; - -struct _ec_fd; -typedef struct _ec_fd ec_fd_t; - -struct _ec_inode; -typedef struct _ec_inode ec_inode_t; - -union _ec_cbk; -typedef union _ec_cbk ec_cbk_t; - -struct _ec_lock; -typedef struct _ec_lock ec_lock_t; - -struct _ec_lock_link; -typedef struct _ec_lock_link ec_lock_link_t; - -struct _ec_fop_data; -typedef struct _ec_fop_data ec_fop_data_t; - -struct _ec_cbk_data; -typedef struct _ec_cbk_data ec_cbk_data_t; - -struct _ec_heal; -typedef struct _ec_heal ec_heal_t; - -typedef void (* ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t); -typedef int32_t (* ec_handler_f)(ec_fop_data_t *, int32_t); -typedef void (* ec_resume_f)(ec_fop_data_t *, int32_t); - -struct _ec_config -{ - uint32_t version; - uint8_t algorithm; - uint8_t gf_word_size; - uint8_t bricks; - uint8_t redundancy; - uint32_t chunk_size; -}; - -struct _ec_fd -{ - uintptr_t bad; - loc_t loc; - uintptr_t open; - int32_t flags; -}; - -struct _ec_inode -{ - uintptr_t bad; - ec_lock_t *entry_lock; - ec_lock_t *inode_lock; - struct list_head heal; - -}; - -typedef int32_t (* fop_heal_cbk_t)(call_frame_t *, void * cookie, xlator_t *, - int32_t, int32_t, uintptr_t, uintptr_t, - uintptr_t, dict_t *); -typedef int32_t (* fop_fheal_cbk_t)(call_frame_t *, void * cookie, xlator_t *, - int32_t, int32_t, uintptr_t, uintptr_t, - uintptr_t, dict_t *); - - -union _ec_cbk -{ - fop_access_cbk_t access; - fop_create_cbk_t create; - fop_discard_cbk_t discard; - fop_entrylk_cbk_t entrylk; - fop_fentrylk_cbk_t fentrylk; - fop_fallocate_cbk_t fallocate; - fop_flush_cbk_t flush; - fop_fsync_cbk_t fsync; - fop_fsyncdir_cbk_t fsyncdir; - fop_getxattr_cbk_t getxattr; - fop_fgetxattr_cbk_t fgetxattr; - fop_heal_cbk_t heal; - fop_fheal_cbk_t fheal; - fop_inodelk_cbk_t inodelk; - fop_finodelk_cbk_t finodelk; - fop_link_cbk_t link; - fop_lk_cbk_t lk; - fop_lookup_cbk_t lookup; - fop_mkdir_cbk_t mkdir; - fop_mknod_cbk_t mknod; - fop_open_cbk_t open; - fop_opendir_cbk_t opendir; - fop_readdir_cbk_t readdir; - fop_readdirp_cbk_t readdirp; - fop_readlink_cbk_t readlink; - fop_readv_cbk_t readv; - fop_removexattr_cbk_t removexattr; - fop_fremovexattr_cbk_t fremovexattr; - fop_rename_cbk_t rename; - fop_rmdir_cbk_t rmdir; - fop_setattr_cbk_t setattr; - fop_fsetattr_cbk_t fsetattr; - fop_setxattr_cbk_t setxattr; - fop_fsetxattr_cbk_t fsetxattr; - fop_stat_cbk_t stat; - fop_fstat_cbk_t fstat; - fop_statfs_cbk_t statfs; - fop_symlink_cbk_t symlink; - fop_truncate_cbk_t truncate; - fop_ftruncate_cbk_t ftruncate; - fop_unlink_cbk_t unlink; - fop_writev_cbk_t writev; - fop_xattrop_cbk_t xattrop; - fop_fxattrop_cbk_t fxattrop; - fop_zerofill_cbk_t zerofill; -}; - -struct _ec_lock -{ - ec_lock_t **plock; - gf_timer_t *timer; - struct list_head waiting; - uintptr_t mask; - uintptr_t good_mask; - int32_t kind; - int32_t refs; - int32_t acquired; - int32_t have_size; - uint64_t size; - uint64_t size_delta; - uint64_t version; - uint64_t version_delta; - ec_fop_data_t *owner; - loc_t loc; - union - { - entrylk_type type; - struct gf_flock flock; - }; - gf_boolean_t is_dirty; -}; - -struct _ec_lock_link -{ - ec_lock_t * lock; - ec_fop_data_t * fop; - struct list_head wait_list; -}; - -struct _ec_fop_data -{ - int32_t id; - int32_t refs; - int32_t state; - int32_t minimum; - int32_t expected; - int32_t winds; - int32_t jobs; - int32_t error; - ec_fop_data_t * parent; - xlator_t * xl; - call_frame_t * req_frame; // frame of the calling xlator - call_frame_t * frame; // frame used by this fop - struct list_head cbk_list; // sorted list of groups of answers - struct list_head answer_list; // list of answers - ec_cbk_data_t * answer; // accepted answer - int32_t lock_count; - int32_t locked; - ec_lock_link_t locks[2]; - int32_t locks_update; - int32_t have_size; - uint64_t pre_size; - uint64_t post_size; - gf_lock_t lock; - ec_config_t config; - - uint32_t flags; - uint32_t first; - uintptr_t mask; - uintptr_t remaining; - uintptr_t good; - uintptr_t bad; - - ec_wind_f wind; - ec_handler_f handler; - ec_resume_f resume; - ec_cbk_t cbks; - void * data; - ec_heal_t *heal; - - uint64_t user_size; - uint32_t head; - - int32_t use_fd; - - dict_t * xdata; - dict_t * dict; - int32_t int32; - uint32_t uint32; - uint64_t size; - off_t offset; - mode_t mode[2]; - entrylk_cmd entrylk_cmd; - entrylk_type entrylk_type; - gf_xattrop_flags_t xattrop_flags; - dev_t dev; - inode_t * inode; - fd_t * fd; - struct iatt iatt; - char * str[2]; - loc_t loc[2]; - struct gf_flock flock; - struct iovec * vector; - struct iobref * buffers; -}; - -struct _ec_cbk_data -{ - struct list_head list; // item in the sorted list of groups - struct list_head answer_list; // item in the list of answers - ec_fop_data_t * fop; - ec_cbk_data_t * next; // next answer in the same group - int32_t idx; - int32_t op_ret; - int32_t op_errno; - int32_t count; - uintptr_t mask; - - dict_t * xdata; - dict_t * dict; - int32_t int32; - uintptr_t uintptr[3]; - uint64_t size; - uint64_t version; - inode_t * inode; - fd_t * fd; - struct statvfs statvfs; - struct iatt iatt[5]; - struct gf_flock flock; - struct iovec * vector; - struct iobref * buffers; - gf_boolean_t dirty; -}; - -struct _ec_heal -{ - struct list_head list; - gf_lock_t lock; - xlator_t *xl; - ec_fop_data_t *fop; - void *data; - ec_fop_data_t *lookup; - loc_t loc; - struct iatt iatt; - char *symlink; - fd_t *fd; - int32_t partial; - int32_t done; - uintptr_t available; - uintptr_t good; - uintptr_t bad; - uintptr_t open; - uintptr_t fixed; - uint64_t offset; - uint64_t size; - uint64_t version; - uint64_t raw_size; -}; - -ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this, - ec_fop_data_t * fop, int32_t id, - int32_t idx, int32_t op_ret, - int32_t op_errno); -ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this, - int32_t id, uint32_t flags, - uintptr_t target, int32_t minimum, - ec_wind_f wind, ec_handler_f handler, - ec_cbk_t cbks, void * data); -void ec_fop_data_acquire(ec_fop_data_t * fop); -void ec_fop_data_release(ec_fop_data_t * fop); +#include "ec-types.h" + +ec_cbk_data_t * +ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop, + int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno); +ec_fop_data_t * +ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id, + uint32_t flags, uintptr_t target, uint32_t fop_flags, + ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks, + void *data); +void +ec_fop_data_acquire(ec_fop_data_t *fop); +void +ec_fop_data_release(ec_fop_data_t *fop); + +void +ec_fop_cleanup(ec_fop_data_t *fop); + +void +ec_pending_fops_completed(ec_t *ec); #endif /* __EC_DATA_H__ */ diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c index c705b80fe82..f71dcfac293 100644 --- a/xlators/cluster/ec/src/ec-dir-read.c +++ b/xlators/cluster/ec/src/ec-dir-read.c @@ -8,24 +8,26 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" - +#include "ec.h" +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" -#include "ec-method.h" #include "ec-fops.h" -/* FOP: opendir */ +/**************************************************************** + * + * File Operation: opendir + * + ***************************************************************/ -int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_opendir(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (dst->fd != src->fd) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching fd in answers " - "of 'GF_FOP_OPENDIR': %p <-> %p", + if (dst->fd != src->fd) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH, + "Mismatching fd in answers " + "of 'GF_FOP_OPENDIR': %p <-> %p", dst->fd, src->fd); return 0; @@ -34,12 +36,12 @@ int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, fd_t * fd, - dict_t * xdata) +int32_t +ec_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -49,52 +51,51 @@ int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPENDIR, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (fd != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (fd != NULL) { cbk->fd = fd_ref(fd); - if (cbk->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (cbk->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, + EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } ec_combine(cbk, ec_combine_opendir); + + ec_update_fd_status(fd, this, idx, op_ret); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_opendir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -103,39 +104,45 @@ void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->fd, fop->xdata); } -int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_opendir(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; ec_fd_t *ctx; + int32_t err; - switch (state) - { + switch (state) { case EC_STATE_INIT: LOCK(&fop->fd->lock); ctx = __ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || - (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) { + if (ctx == NULL) { UNLOCK(&fop->fd->lock); - fop->error = EIO; + fop->error = ENOMEM; return EC_STATE_REPORT; } + if (!ctx->loc.inode) { + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { + UNLOCK(&fop->fd->lock); + + fop->error = -err; + + return EC_STATE_REPORT; + } + } UNLOCK(&fop->fd->lock); /* Fall through */ case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 0); + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -144,38 +151,20 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + /* Save which subvolumes successfully opened the directory. + * If ctx->open is 0, it means that readdir cannot be + * processed in this directory. + */ + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; } - if (cbk->op_ret >= 0) { - /* Save which subvolumes successfully opened the directory. - * If ctx->open is 0, it means that readdir cannot be - * processed in this directory. - */ - LOCK(&fop->fd->lock); - - ctx = __ec_fd_get(fop->fd, fop->xl); - if (ctx != NULL) { - ctx->open |= cbk->mask; - } - UNLOCK(&fop->fd->lock); - } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + UNLOCK(&fop->fd->lock); } return EC_STATE_REPORT; @@ -185,8 +174,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.opendir != NULL) - { + if (fop->cbks.opendir != NULL) { fop->cbks.opendir(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->fd, cbk->xdata); } @@ -195,14 +183,12 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.opendir != NULL) - { + if (fop->cbks.opendir != NULL) { fop->cbks.opendir(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } @@ -222,62 +208,59 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_opendir_cbk_t func, void * data, - loc_t * loc, fd_t * fd, dict_t * xdata) +void +ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc, + fd_t *fd, dict_t *xdata) { - ec_cbk_t callback = { .opendir = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.opendir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(OPENDIR) %p", frame); + gf_msg_trace("ec", 0, "EC(OPENDIR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_opendir, + fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_opendir, ec_manager_opendir, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -286,48 +269,72 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); +} + +/* Returns -1 if client_id is invalid else index of child subvol in xl_list */ +int +ec_deitransform(xlator_t *this, off_t offset) +{ + int idx = -1; + int client_id = -1; + ec_t *ec = this->private; + char id[32] = {0}; + int err; + + client_id = gf_deitransform(this, offset); + sprintf(id, "%d", client_id); + err = dict_get_int32(ec->leaf_to_subvolid, id, &idx); + if (err < 0) { + idx = err; + goto out; } + +out: + if (idx < 0) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REQUEST, + "Invalid index %d in readdirp request", client_id); + idx = -EINVAL; + } + return idx; } /* FOP: readdir */ -void ec_adjust_readdir(ec_t * ec, int32_t idx, gf_dirent_t * entries) +void +ec_adjust_readdirp(ec_t *ec, int32_t idx, gf_dirent_t *entries) { - gf_dirent_t * entry; + gf_dirent_t *entry; list_for_each_entry(entry, &entries->list, list) { - entry->d_off = ec_itransform(ec, idx, entry->d_off); + if (!entry->inode) + continue; - if (entry->d_stat.ia_type == IA_IFREG) - { + if (entry->d_stat.ia_type == IA_IFREG) { if ((entry->dict == NULL) || (ec_dict_del_number(entry->dict, EC_XATTR_SIZE, - &entry->d_stat.ia_size) != 0)) - { - gf_log(ec->xl->name, GF_LOG_WARNING, "Unable to get exact " - "file size."); - - entry->d_stat.ia_size *= ec->fragments; + &entry->d_stat.ia_size) != 0)) { + inode_unref(entry->inode); + entry->inode = NULL; + } else { + ec_iatt_rebuild(ec, &entry->d_stat, 1, 1); } - - ec_iatt_rebuild(ec, &entry->d_stat, 1, 1); } } } -int32_t ec_readdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, gf_dirent_t * entries, - dict_t * xdata) +int32_t +ec_common_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, + dict_t *xdata) { - ec_fop_data_t * fop = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -337,145 +344,199 @@ int32_t ec_readdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - if (op_ret > 0) - { - ec_adjust_readdir(fop->xl->private, idx, entries); - } + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno)) - { - if (fop->cbks.readdir != NULL) - { - fop->cbks.readdir(fop->req_frame, fop, this, op_ret, op_errno, - entries, xdata); - } + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); + if (cbk->op_ret >= 0) + list_splice_init(&entries->list, &cbk->entries.list); + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_readdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); - STACK_WIND_COOKIE(fop->frame, ec_readdir_cbk, (void *)(uintptr_t)idx, + STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx, ec->xl_list[idx], ec->xl_list[idx]->fops->readdir, fop->fd, fop->size, fop->offset, fop->xdata); } -int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_readdir(ec_fop_data_t *fop, int32_t state) { - ec_fd_t *ctx; + ec_fd_t *ctx = NULL; + ec_cbk_data_t *cbk = NULL; - switch (state) - { + switch (state) { case EC_STATE_INIT: /* Return error if opendir has not been successfully called on * any subvolume. */ ctx = ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || (ctx->open == 0)) { - fop->error = EINVAL; + if (ctx == NULL) { + fop->error = ENOMEM; + } else if (ctx->open == 0) { + fop->error = EBADFD; + } + if (fop->error) { + gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error, + EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s", + ec_msg_str(fop)); return EC_STATE_REPORT; } - if (fop->xdata == NULL) - { - fop->xdata = dict_new(); - if (fop->xdata == NULL) - { - gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare " - "readdirp request"); + if (fop->id == GF_FOP_READDIRP) { + int32_t err; - fop->error = EIO; + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = ENOMEM; - return EC_STATE_REPORT; + return EC_STATE_REPORT; + } } - } - if (dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0) != 0) - { - gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare " - "readdirp request"); - fop->error = EIO; + err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; - return EC_STATE_REPORT; + return EC_STATE_REPORT; + } } - if (fop->offset != 0) - { - int32_t idx; + if (fop->offset != 0) { + /* Non-zero offset is irrecoverable error as the offset may not + * be valid on other bricks*/ + int32_t idx = -1; - fop->offset = ec_deitransform(fop->xl->private, &idx, - fop->offset); + idx = ec_deitransform(fop->xl, fop->offset); + + if (idx < 0) { + fop->error = -idx; + return EC_STATE_REPORT; + } fop->mask &= 1ULL << idx; + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); } - /* Fall through */ + return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: ec_dispatch_one(fop); + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + + if ((cbk != NULL) && (cbk->op_ret > 0) && + (fop->id == GF_FOP_READDIRP)) { + ec_adjust_readdirp(fop->xl->private, cbk->idx, &cbk->entries); + } + return EC_STATE_REPORT; + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->id == GF_FOP_READDIR) { + if (fop->cbks.readdir != NULL) { + fop->cbks.readdir(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->entries, cbk->xdata); + } + } else { + if (fop->cbks.readdirp != NULL) { + fop->cbks.readdirp(fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, + &cbk->entries, cbk->xdata); + } + } + if (fop->offset == 0) + return EC_STATE_LOCK_REUSE; + else + return EC_STATE_END; + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: - if (fop->id == GF_FOP_READDIR) - { - if (fop->cbks.readdir != NULL) - { + if (fop->id == GF_FOP_READDIR) { + if (fop->cbks.readdir != NULL) { fop->cbks.readdir(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } - } - else - { - if (fop->cbks.readdirp != NULL) - { + } else { + if (fop->cbks.readdirp != NULL) { fop->cbks.readdirp(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } } + if (fop->offset == 0) + return EC_STATE_LOCK_REUSE; + else + return EC_STATE_END; - case EC_STATE_REPORT: - return EC_STATE_END; + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + GF_ASSERT(fop->offset == 0); + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + GF_ASSERT(fop->offset == 0); + ec_unlock(fop); + return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readdir_cbk_t func, void * data, - fd_t * fd, size_t size, off_t offset, dict_t * xdata) +void +ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata) { - ec_cbk_t callback = { .readdir = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.readdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(READDIR) %p", frame); + gf_msg_trace("ec", 0, "EC(READDIR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, 0, target, minimum, - ec_wind_readdir, ec_manager_readdir, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_readdir, + ec_manager_readdir, callback, data); + if (fop == NULL) { goto out; } @@ -484,24 +545,22 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->size = size; fop->offset = offset; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -510,86 +569,44 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: readdirp */ -int32_t ec_readdirp_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - gf_dirent_t * entries, dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - if (op_ret > 0) - { - ec_adjust_readdir(fop->xl->private, idx, entries); - } - - if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno)) - { - if (fop->cbks.readdirp != NULL) - { - fop->cbks.readdirp(fop->req_frame, fop, this, op_ret, op_errno, - entries, xdata); - } - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; -} - -void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_readdirp(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); - STACK_WIND_COOKIE(fop->frame, ec_readdirp_cbk, (void *)(uintptr_t)idx, + STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx, ec->xl_list[idx], ec->xl_list[idx]->fops->readdirp, fop->fd, fop->size, fop->offset, fop->xdata); } -void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readdirp_cbk_t func, void * data, - fd_t * fd, size_t size, off_t offset, dict_t * xdata) +void +ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata) { - ec_cbk_t callback = { .readdirp = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.readdirp = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(READDIRP) %p", frame); + gf_msg_trace("ec", 0, "EC(READDIRP) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIRP, 0, target, - minimum, ec_wind_readdirp, ec_manager_readdir, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate( + frame, this, GF_FOP_READDIRP, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_readdirp, ec_manager_readdir, callback, data); + if (fop == NULL) { goto out; } @@ -598,24 +615,22 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->size = size; fop->offset = offset; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -624,12 +639,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c index d48be20470d..53d27d895c3 100644 --- a/xlators/cluster/ec/src/ec-dir-write.c +++ b/xlators/cluster/ec/src/ec-dir-write.c @@ -8,49 +8,24 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" - +#include "ec.h" +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" #include "ec-method.h" #include "ec-fops.h" -/* FOP: create */ - -int32_t ec_combine_create(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) -{ - if (dst->fd != src->fd) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching fd in answers " - "of 'GF_FOP_CREATE': %p <-> %p", - dst->fd, src->fd); - - return 0; - } - - if (!ec_iatt_combine(dst->iatt, src->iatt, 3)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_CREATE'"); - - return 0; - } - - return 1; -} - -int32_t ec_create_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, fd_t * fd, - inode_t * inode, struct iatt * buf, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) +int +ec_dir_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, int op_ret, + int op_errno, struct iatt *poststat, struct iatt *preparent, + struct iatt *postparent, struct iatt *preparent2, + struct iatt *postparent2, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int i = 0; + int idx = 0; VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); @@ -58,76 +33,59 @@ int32_t ec_create_cbk(call_frame_t * frame, void * cookie, xlator_t * this, GF_VALIDATE_OR_GOTO(this->name, this->private, out); fop = frame->local; + idx = (long)cookie; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_CREATE, idx, op_ret, + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (fd != NULL) - { - cbk->fd = fd_ref(fd); - if (cbk->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); - - goto out; - } - } - if (inode != NULL) - { - cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, - "Failed to reference an inode."); - - goto out; - } - } - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preparent != NULL) - { - cbk->iatt[1] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[2] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } + if (!cbk) + goto out; - ec_combine(cbk, ec_combine_create); - } + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (op_ret < 0) + goto out; + + if (poststat) + cbk->iatt[i++] = *poststat; + + if (preparent) + cbk->iatt[i++] = *preparent; + + if (postparent) + cbk->iatt[i++] = *postparent; + + if (preparent2) + cbk->iatt[i++] = *preparent2; + + if (postparent2) + cbk->iatt[i++] = *postparent2; out: - if (fop != NULL) - { - ec_complete(fop); - } + if (cbk) + ec_combine(cbk, ec_combine_write); + if (fop) + ec_complete(fop); return 0; } -void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +/* FOP: create */ + +int32_t +ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_create(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -137,23 +95,33 @@ void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->fd, fop->xdata); } -int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_create(ec_fop_data_t *fop, int32_t state) { - ec_t * ec; - ec_cbk_data_t * cbk; - ec_fd_t * ctx; + ec_config_t config; + ec_t *ec; + ec_cbk_data_t *cbk; + ec_fd_t *ctx; + uint64_t version[2] = {0, 0}; + int32_t err; - switch (state) - { + switch (state) { case EC_STATE_INIT: LOCK(&fop->fd->lock); ctx = __ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || - (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) { + if (ctx == NULL) { + UNLOCK(&fop->fd->lock); + + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { UNLOCK(&fop->fd->lock); - fop->error = EIO; + fop->error = -err; return EC_STATE_REPORT; } @@ -162,12 +130,10 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) UNLOCK(&fop->fd->lock); - if (fop->xdata == NULL) - { + if (fop->xdata == NULL) { fop->xdata = dict_new(); - if (fop->xdata == NULL) - { - fop->error = EIO; + if (fop->xdata == NULL) { + fop->error = ENOMEM; return EC_STATE_REPORT; } @@ -175,29 +141,29 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) ec = fop->xl->private; - fop->config.version = EC_CONFIG_VERSION; - fop->config.algorithm = EC_CONFIG_ALGORITHM; - fop->config.gf_word_size = EC_GF_BITS; - fop->config.bricks = ec->nodes; - fop->config.redundancy = ec->redundancy; - fop->config.chunk_size = EC_METHOD_CHUNK_SIZE; + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; - if (ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, - &fop->config) < 0) - { - fop->error = EIO; + err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } - - if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { - fop->error = EIO; + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } - - if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) { - fop->error = EIO; + err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } @@ -206,10 +172,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) * need to remove O_APPEND from flags (if present) */ fop->int32 &= ~O_APPEND; - /* Fall through */ + /* Fall through */ case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); return EC_STATE_DISPATCH; @@ -220,43 +187,24 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret >= 0) { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, - cbk->count); - - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } else { - LOCK(&fop->fd->lock); - - ctx = __ec_fd_get(fop->fd, fop->xl); - if (ctx != NULL) { - ctx->open |= cbk->mask; - } - - UNLOCK(&fop->fd->lock); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + if (!ec_cbk_set_error(cbk, -err, _gf_false)) { + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; } + + UNLOCK(&fop->fd->lock); } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); } return EC_STATE_REPORT; @@ -266,12 +214,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.create != NULL) - { - fop->cbks.create(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->fd, cbk->inode, - &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], - cbk->xdata); + if (fop->cbks.create != NULL) { + QUORUM_CBK(fop->cbks.create, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->fd, + fop->loc[0].inode, &cbk->iatt[0], &cbk->iatt[1], + &cbk->iatt[2], cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -283,8 +230,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.create != NULL) - { + if (fop->cbks.create != NULL) { fop->cbks.create(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL, NULL, NULL); } @@ -304,35 +250,32 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_create_cbk_t func, void * data, - loc_t * loc, int32_t flags, mode_t mode, mode_t umask, - fd_t * fd, dict_t * xdata) +void +ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) { - ec_cbk_t callback = { .create = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.create = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(CREATE) %p", frame); + gf_msg_trace("ec", 0, "EC(CREATE) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, - EC_FLAG_UPDATE_LOC_PARENT | - EC_FLAG_UPDATE_FD_INODE, target, minimum, + fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, fop_flags, ec_wind_create, ec_manager_create, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } @@ -340,33 +283,30 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->mode[0] = mode; fop->mode[1] = umask; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -375,106 +315,26 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL); } } /* FOP: link */ -int32_t ec_combine_link(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 3)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_LINK'"); - - return 0; - } - - return 1; -} - -int32_t ec_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LINK, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (inode != NULL) - { - cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, - "Failed to reference an inode."); - - goto out; - } - } - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preparent != NULL) - { - cbk->iatt[1] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[2] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_link); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); } -void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_link(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -483,25 +343,19 @@ void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], &fop->loc[1], fop->xdata); } -int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_link(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - // Parent entry of fop->loc[0] should be locked, but I don't - // receive enough information to do it (fop->loc[0].parent is - // NULL). - ec_lock_prepare_entry(fop, &fop->loc[1], 1); + ec_lock_prepare_parent_inode( + fop, &fop->loc[1], &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -510,37 +364,19 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret >= 0) { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, - cbk->count); - if (cbk->iatt[0].ia_type == IA_IFREG) { - cbk->iatt[0].ia_size = fop->pre_size; - } + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + if (cbk->iatt[0].ia_type == IA_IFREG) { + cbk->iatt[0].ia_size = fop->locks[0].size; } - } - else - { - ec_fop_set_error(fop, EIO); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); } return EC_STATE_REPORT; @@ -550,25 +386,23 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.link != NULL) - { - fop->cbks.link(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->inode, &cbk->iatt[0], - &cbk->iatt[1], &cbk->iatt[2], cbk->xdata); + if (fop->cbks.link != NULL) { + QUORUM_CBK(fop->cbks.link, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); } return EC_STATE_LOCK_REUSE; case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.link != NULL) - { + if (fop->cbks.link != NULL) { fop->cbks.link(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL, NULL); } @@ -588,59 +422,56 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_link_cbk_t func, void * data, loc_t * oldloc, - loc_t * newloc, dict_t * xdata) +void +ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) { - ec_cbk_t callback = { .link = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.link = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(LINK) %p", frame); + gf_msg_trace("ec", 0, "EC(LINK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, minimum, + fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, fop_flags, ec_wind_link, ec_manager_link, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } - if (oldloc != NULL) - { - if (loc_copy(&fop->loc[0], oldloc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (oldloc != NULL) { + if (loc_copy(&fop->loc[0], oldloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (newloc != NULL) - { - if (loc_copy(&fop->loc[1], newloc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (newloc != NULL) { + if (loc_copy(&fop->loc[1], newloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -649,106 +480,26 @@ void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); } } /* FOP: mkdir */ -int32_t ec_combine_mkdir(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 3)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_MKDIR'"); - - return 0; - } - - return 1; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); } -int32_t ec_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_MKDIR, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (inode != NULL) - { - cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, - "Failed to reference an inode."); - - goto out; - } - } - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preparent != NULL) - { - cbk->iatt[1] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[2] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_mkdir); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; -} - -void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_mkdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -757,32 +508,36 @@ void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->mode[0], fop->mode[1], fop->xdata); } -int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_mkdir(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + uint64_t version[2] = {0, 0}; + int32_t err; - switch (state) - { + switch (state) { case EC_STATE_INIT: if (fop->xdata == NULL) { fop->xdata = dict_new(); if (fop->xdata == NULL) { - fop->error = EIO; + fop->error = ENOMEM; return EC_STATE_REPORT; } } - if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { - fop->error = EIO; - + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } - /* Fall through */ + /* Fall through */ case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); return EC_STATE_DISPATCH; @@ -793,34 +548,15 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret >= 0) { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, - cbk->count); - - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); } return EC_STATE_REPORT; @@ -830,11 +566,11 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.mkdir != NULL) - { - fop->cbks.mkdir(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->inode, &cbk->iatt[0], - &cbk->iatt[1], &cbk->iatt[2], cbk->xdata); + if (fop->cbks.mkdir != NULL) { + QUORUM_CBK(fop->cbks.mkdir, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -844,12 +580,13 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: + cbk = fop->answer; GF_ASSERT(fop->error != 0); - if (fop->cbks.mkdir != NULL) - { + if (fop->cbks.mkdir != NULL) { fop->cbks.mkdir(fop->req_frame, fop, fop->xl, -1, fop->error, - NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, + ((cbk) ? cbk->xdata : NULL)); } return EC_STATE_LOCK_REUSE; @@ -867,55 +604,51 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_mkdir_cbk_t func, void * data, loc_t * loc, - mode_t mode, mode_t umask, dict_t * xdata) +void +ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc, + mode_t mode, mode_t umask, dict_t *xdata) { - ec_cbk_t callback = { .mkdir = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.mkdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(MKDIR) %p", frame); + gf_msg_trace("ec", 0, "EC(MKDIR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, - ec_wind_mkdir, ec_manager_mkdir, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, fop_flags, + ec_wind_mkdir, ec_manager_mkdir, callback, data); + if (fop == NULL) { goto out; } fop->mode[0] = mode; fop->mode[1] = umask; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -924,106 +657,26 @@ void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); } } /* FOP: mknod */ -int32_t ec_combine_mknod(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 3)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_MKNOD'"); - - return 0; - } - - return 1; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); } -int32_t ec_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_MKNOD, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (inode != NULL) - { - cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, - "Failed to reference an inode."); - - goto out; - } - } - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preparent != NULL) - { - cbk->iatt[1] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[2] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_mknod); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; -} - -void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_mknod(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1033,19 +686,23 @@ void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->xdata); } -int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_mknod(ec_fop_data_t *fop, int32_t state) { + ec_config_t config; ec_t *ec; - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + uint64_t version[2] = {0, 0}; - switch (state) - { + switch (state) { case EC_STATE_INIT: if (S_ISREG(fop->mode[0])) { + int32_t err; + if (fop->xdata == NULL) { fop->xdata = dict_new(); if (fop->xdata == NULL) { - fop->error = EIO; + fop->error = ENOMEM; return EC_STATE_REPORT; } @@ -1053,37 +710,39 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) ec = fop->xl->private; - fop->config.version = EC_CONFIG_VERSION; - fop->config.algorithm = EC_CONFIG_ALGORITHM; - fop->config.gf_word_size = EC_GF_BITS; - fop->config.bricks = ec->nodes; - fop->config.redundancy = ec->redundancy; - fop->config.chunk_size = EC_METHOD_CHUNK_SIZE; + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; - if (ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, - &fop->config) < 0) { - fop->error = EIO; + err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } - - if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) { - fop->error = EIO; + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } - - if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) { - fop->error = EIO; + err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; return EC_STATE_REPORT; } } - /* Fall through */ + /* Fall through */ case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); return EC_STATE_DISPATCH; @@ -1094,34 +753,15 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret >= 0) { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, - cbk->count); - - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); } return EC_STATE_REPORT; @@ -1131,11 +771,11 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.mknod != NULL) - { - fop->cbks.mknod(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->inode, &cbk->iatt[0], - &cbk->iatt[1], &cbk->iatt[2], cbk->xdata); + if (fop->cbks.mknod != NULL) { + QUORUM_CBK(fop->cbks.mknod, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -1147,8 +787,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.mknod != NULL) - { + if (fop->cbks.mknod != NULL) { fop->cbks.mknod(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL, NULL); } @@ -1168,33 +807,31 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_mknod_cbk_t func, void * data, loc_t * loc, - mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata) +void +ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc, + mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata) { - ec_cbk_t callback = { .mknod = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.mknod = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(MKNOD) %p", frame); + gf_msg_trace("ec", 0, "EC(MKNOD) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, - ec_wind_mknod, ec_manager_mknod, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, fop_flags, + ec_wind_mknod, ec_manager_mknod, callback, data); + if (fop == NULL) { goto out; } @@ -1202,22 +839,20 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->dev = rdev; fop->mode[1] = umask; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1226,104 +861,28 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); } } /* FOP: rename */ -int32_t ec_combine_rename(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, struct iatt *preoldparent, + struct iatt *postoldparent, struct iatt *prenewparent, + struct iatt *postnewparent, dict_t *xdata) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 5)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_RENAME'"); - - return 0; - } - - return 1; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preoldparent, postoldparent, prenewparent, + postnewparent, xdata); } -int32_t ec_rename_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * buf, - struct iatt * preoldparent, struct iatt * postoldparent, - struct iatt * prenewparent, struct iatt * postnewparent, - dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_RENAME, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preoldparent != NULL) - { - cbk->iatt[1] = *preoldparent; - } - if (postoldparent != NULL) - { - cbk->iatt[2] = *postoldparent; - } - if (prenewparent != NULL) - { - cbk->iatt[3] = *prenewparent; - } - if (postnewparent != NULL) - { - cbk->iatt[4] = *postnewparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_rename); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; -} - -void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_rename(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1332,23 +891,21 @@ void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], &fop->loc[1], fop->xdata); } -int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_rename(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); - ec_lock_prepare_entry(fop, &fop->loc[1], 1); + ec_lock_prepare_parent_inode( + fop, &fop->loc[0], &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE); + ec_lock_prepare_parent_inode(fop, &fop->loc[1], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -1357,36 +914,14 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 5, - cbk->count); - - if (cbk->iatt[0].ia_type == IA_IFREG) - { - cbk->iatt[0].ia_size = fop->pre_size; - } + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 5, cbk->count); + + if (cbk->iatt[0].ia_type == IA_IFREG) { + cbk->iatt[0].ia_size = fop->locks[0].size; } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -1395,26 +930,23 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.rename != NULL) - { - fop->cbks.rename(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], - &cbk->iatt[2], &cbk->iatt[3], &cbk->iatt[4], - cbk->xdata); + if (fop->cbks.rename != NULL) { + QUORUM_CBK(fop->cbks.rename, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], &cbk->iatt[2], &cbk->iatt[3], + &cbk->iatt[4], cbk->xdata); } return EC_STATE_LOCK_REUSE; case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.rename != NULL) - { + if (fop->cbks.rename != NULL) { fop->cbks.rename(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL, NULL, NULL); } @@ -1434,61 +966,57 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_rename_cbk_t func, void * data, - loc_t * oldloc, loc_t * newloc, dict_t * xdata) +void +ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) { - ec_cbk_t callback = { .rename = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.rename = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(RENAME) %p", frame); + gf_msg_trace("ec", 0, "EC(RENAME) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, + fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, fop_flags, ec_wind_rename, ec_manager_rename, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } - if (oldloc != NULL) - { - if (loc_copy(&fop->loc[0], oldloc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (oldloc != NULL) { + if (loc_copy(&fop->loc[0], oldloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (newloc != NULL) - { - if (loc_copy(&fop->loc[1], newloc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (newloc != NULL) { + if (loc_copy(&fop->loc[1], newloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1497,90 +1025,26 @@ void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL); } } /* FOP: rmdir */ -int32_t ec_combine_rmdir(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *preparent, struct iatt *postparent, + dict_t *xdata) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_RMDIR'"); - - return 0; - } - - return 1; -} - -int32_t ec_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) -{ - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_RMDIR, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (preparent != NULL) - { - cbk->iatt[0] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[1] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_rmdir); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, + preparent, postparent, NULL, NULL, xdata); } -void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_rmdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1589,15 +1053,16 @@ void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->int32, fop->xdata); } -int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_rmdir(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); return EC_STATE_DISPATCH; @@ -1608,26 +1073,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + ec_fop_prepare_answer(fop, _gf_false); return EC_STATE_REPORT; @@ -1636,11 +1082,10 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.rmdir != NULL) - { - fop->cbks.rmdir(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + if (fop->cbks.rmdir != NULL) { + QUORUM_CBK(fop->cbks.rmdir, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -1652,8 +1097,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.rmdir != NULL) - { + if (fop->cbks.rmdir != NULL) { fop->cbks.rmdir(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -1673,54 +1117,50 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_rmdir_cbk_t func, void * data, loc_t * loc, - int xflags, dict_t * xdata) +void +ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata) { - ec_cbk_t callback = { .rmdir = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.rmdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(RMDIR) %p", frame); + gf_msg_trace("ec", 0, "EC(RMDIR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, - ec_wind_rmdir, ec_manager_rmdir, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, fop_flags, + ec_wind_rmdir, ec_manager_rmdir, callback, data); + if (fop == NULL) { goto out; } fop->int32 = xflags; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1729,106 +1169,27 @@ void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: symlink */ -int32_t ec_combine_symlink(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) -{ - if (!ec_iatt_combine(dst->iatt, src->iatt, 3)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_SYMLINK'"); - - return 0; - } - - return 1; -} - -int32_t ec_symlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) +int32_t +ec_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SYMLINK, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (inode != NULL) - { - cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, - "Failed to reference an inode."); - - goto out; - } - } - if (buf != NULL) - { - cbk->iatt[0] = *buf; - } - if (preparent != NULL) - { - cbk->iatt[1] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[2] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_symlink); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); } -void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_symlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1837,15 +1198,16 @@ void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->str[0], &fop->loc[0], fop->mode[0], fop->xdata); } -int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_symlink(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); return EC_STATE_DISPATCH; @@ -1856,34 +1218,15 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret >= 0) { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, - cbk->count); - - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, - &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); } return EC_STATE_REPORT; @@ -1893,11 +1236,11 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.symlink != NULL) - { - fop->cbks.symlink(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->inode, &cbk->iatt[0], - &cbk->iatt[1], &cbk->iatt[2], cbk->xdata); + if (fop->cbks.symlink != NULL) { + QUORUM_CBK(fop->cbks.symlink, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -1909,8 +1252,7 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.symlink != NULL) - { + if (fop->cbks.symlink != NULL) { fop->cbks.symlink(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL, NULL); } @@ -1930,65 +1272,60 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_symlink_cbk_t func, void * data, - const char * linkname, loc_t * loc, mode_t umask, - dict_t * xdata) +void +ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_symlink_cbk_t func, void *data, + const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata) { - ec_cbk_t callback = { .symlink = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.symlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(SYMLINK) %p", frame); + gf_msg_trace("ec", 0, "EC(SYMLINK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, - ec_wind_symlink, ec_manager_symlink, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target, + fop_flags, ec_wind_symlink, ec_manager_symlink, + callback, data); + if (fop == NULL) { goto out; } fop->mode[0] = umask; - if (linkname != NULL) - { + if (linkname != NULL) { fop->str[0] = gf_strdup(linkname); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1997,91 +1334,26 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); } } /* FOP: unlink */ -int32_t ec_combine_unlink(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) -{ - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_UNLINK'"); - - return 0; - } - - return 1; -} - -int32_t ec_unlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) +int32_t +ec_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *preparent, struct iatt *postparent, + dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_UNLINK, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (preparent != NULL) - { - cbk->iatt[0] = *preparent; - } - if (postparent != NULL) - { - cbk->iatt[1] = *postparent; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_unlink); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, + preparent, postparent, NULL, NULL, xdata); } -void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_unlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -2090,22 +1362,18 @@ void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->int32, fop->xdata); } -int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_unlink(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_entry(fop, &fop->loc[0], 1); + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -2114,26 +1382,7 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + ec_fop_prepare_answer(fop, _gf_false); return EC_STATE_REPORT; @@ -2142,25 +1391,22 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.unlink != NULL) - { - fop->cbks.unlink(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + if (fop->cbks.unlink != NULL) { + QUORUM_CBK(fop->cbks.unlink, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); } return EC_STATE_LOCK_REUSE; case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.unlink != NULL) - { + if (fop->cbks.unlink != NULL) { fop->cbks.unlink(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -2180,54 +1426,51 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_unlink_cbk_t func, void * data, - loc_t * loc, int xflags, dict_t * xdata) +void +ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata) { - ec_cbk_t callback = { .unlink = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.unlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(UNLINK) %p", frame); + gf_msg_trace("ec", 0, "EC(UNLINK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, - EC_FLAG_UPDATE_LOC_PARENT, target, minimum, + fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, fop_flags, ec_wind_unlink, ec_manager_unlink, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } fop->int32 = xflags; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -2236,12 +1479,9 @@ void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } diff --git a/xlators/cluster/ec/src/ec-fops.h b/xlators/cluster/ec/src/ec-fops.h index d6b9770f720..07edf8a7fec 100644 --- a/xlators/cluster/ec/src/ec-fops.h +++ b/xlators/cluster/ec/src/ec-fops.h @@ -11,191 +11,244 @@ #ifndef __EC_FOPS_H__ #define __EC_FOPS_H__ -#include "xlator.h" +#include <glusterfs/xlator.h> -#include "ec-data.h" +#include "ec-types.h" #include "ec-common.h" -#define EC_FOP_HEAL -1 -#define EC_FOP_FHEAL -2 - -void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_access_cbk_t func, void *data, loc_t * loc, - int32_t mask, dict_t * xdata); - -void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_create_cbk_t func, void *data, loc_t * loc, - int32_t flags, mode_t mode, mode_t umask, fd_t * fd, - dict_t * xdata); - -void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_entrylk_cbk_t func, void *data, - const char * volume, loc_t * loc, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata); - -void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fentrylk_cbk_t func, void *data, - const char * volume, fd_t * fd, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata); - -void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_flush_cbk_t func, void *data, fd_t * fd, - dict_t * xdata); - -void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsync_cbk_t func, void *data, fd_t * fd, - int32_t datasync, dict_t * xdata); - -void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsyncdir_cbk_t func, void *data, - fd_t * fd, int32_t datasync, dict_t * xdata); - -void ec_getxattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_getxattr_cbk_t func, void *data, - loc_t * loc, const char * name, dict_t * xdata); - -void ec_fgetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fgetxattr_cbk_t func, void *data, - fd_t * fd, const char * name, dict_t * xdata); - -void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_heal_cbk_t func, void *data, loc_t * loc, - int32_t partial, dict_t *xdata); - -void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fheal_cbk_t func, void *data, fd_t * fd, - int32_t partial, dict_t *xdata); - -void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_inodelk_cbk_t func, void *data, - const char * volume, loc_t * loc, int32_t cmd, - struct gf_flock * flock, dict_t * xdata); - -void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_finodelk_cbk_t func, void *data, - const char * volume, fd_t * fd, int32_t cmd, - struct gf_flock * flock, dict_t * xdata); - -void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_link_cbk_t func, void *data, loc_t * oldloc, - loc_t * newloc, dict_t * xdata); - -void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_lk_cbk_t func, void *data, fd_t * fd, - int32_t cmd, struct gf_flock * flock, dict_t * xdata); - -void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_lookup_cbk_t func, void *data, loc_t * loc, - dict_t * xdata); - -void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_mkdir_cbk_t func, void *data, loc_t * loc, - mode_t mode, mode_t umask, dict_t * xdata); - -void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_mknod_cbk_t func, void *data, loc_t * loc, - mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata); - -void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_open_cbk_t func, void *data, loc_t * loc, - int32_t flags, fd_t * fd, dict_t * xdata); - -void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_opendir_cbk_t func, void *data, - loc_t * loc, fd_t * fd, dict_t * xdata); - -void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readdir_cbk_t func, void *data, fd_t * fd, - size_t size, off_t offset, dict_t * xdata); - -void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readdirp_cbk_t func, void *data, - fd_t * fd, size_t size, off_t offset, dict_t * xdata); - -void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readlink_cbk_t func, void *data, - loc_t * loc, size_t size, dict_t * xdata); - -void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readv_cbk_t func, void *data, fd_t * fd, - size_t size, off_t offset, uint32_t flags, dict_t * xdata); - -void ec_removexattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_removexattr_cbk_t func, void *data, - loc_t * loc, const char * name, dict_t * xdata); - -void ec_fremovexattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fremovexattr_cbk_t func, void *data, - fd_t * fd, const char * name, dict_t * xdata); - -void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_rename_cbk_t func, void *data, - loc_t * oldloc, loc_t * newloc, dict_t * xdata); - -void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_rmdir_cbk_t func, void *data, loc_t * loc, - int xflags, dict_t * xdata); - -void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_setattr_cbk_t func, void *data, - loc_t * loc, struct iatt * stbuf, int32_t valid, - dict_t * xdata); - -void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsetattr_cbk_t func, void *data, - fd_t * fd, struct iatt * stbuf, int32_t valid, - dict_t * xdata); - -void ec_setxattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_setxattr_cbk_t func, void *data, - loc_t * loc, dict_t * dict, int32_t flags, dict_t * xdata); - -void ec_fsetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsetxattr_cbk_t func, void *data, - fd_t * fd, dict_t * dict, int32_t flags, dict_t * xdata); - -void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_stat_cbk_t func, void *data, loc_t * loc, - dict_t * xdata); - -void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fstat_cbk_t func, void *data, fd_t * fd, - dict_t * xdata); - -void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_statfs_cbk_t func, void *data, loc_t * loc, - dict_t * xdata); - -void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_symlink_cbk_t func, void *data, - const char * linkname, loc_t * loc, mode_t umask, - dict_t * xdata); - -void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_truncate_cbk_t func, void *data, - loc_t * loc, off_t offset, dict_t * xdata); - -void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_ftruncate_cbk_t func, void *data, - fd_t * fd, off_t offset, dict_t * xdata); - -void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_unlink_cbk_t func, void *data, loc_t * loc, - int xflags, dict_t * xdata); - -void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_writev_cbk_t func, void *data, fd_t * fd, - struct iovec * vector, int32_t count, off_t offset, - uint32_t flags, struct iobref * iobref, dict_t * xdata); - -void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_xattrop_cbk_t func, void *data, - loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata); - -void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fxattrop_cbk_t func, void *data, - fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata); +void +ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc, + int32_t mask, dict_t *xdata); + +void +ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata); + +void +ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_entrylk_cbk_t func, void *data, + const char *volume, loc_t *loc, const char *basename, + entrylk_cmd cmd, entrylk_type type, dict_t *xdata); + +void +ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data, + const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata); + +void +ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd, + dict_t *xdata); + +void +ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata); + +void +ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata); + +void +ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc, + const char *name, dict_t *xdata); + +void +ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd, + const char *name, dict_t *xdata); + +void +ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc, + int32_t partial, dict_t *xdata); + +void +ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd, + int32_t partial, dict_t *xdata); + +void +ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func, + void *data, const char *volume, loc_t *loc, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func, + void *data, const char *volume, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata); + +void +ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags, + fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc, + mode_t mode, mode_t umask, dict_t *xdata); + +void +ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc, + mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata); + +void +ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc, + int32_t flags, fd_t *fd, dict_t *xdata); + +void +ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc, + fd_t *fd, dict_t *xdata); + +void +ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata); + +void +ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata); + +void +ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc, + size_t size, dict_t *xdata); + +void +ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, uint32_t flags, dict_t *xdata); + +void +ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_removexattr_cbk_t func, void *data, + loc_t *loc, const char *name, dict_t *xdata); + +void +ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data, + fd_t *fd, const char *name, dict_t *xdata); + +void +ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata); + +void +ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata); + +void +ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata); + +void +ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata); + +void +ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc, + dict_t *dict, int32_t flags, dict_t *xdata); + +void +ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd, + dict_t *dict, int32_t flags, dict_t *xdata); + +void +ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd, + dict_t *xdata); + +void +ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_symlink_cbk_t func, void *data, + const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata); + +void +ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd, + int32_t mode, off_t offset, size_t len, dict_t *xdata); + +void +ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd, + off_t offset, size_t len, dict_t *xdata); + +void +ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc, + off_t offset, dict_t *xdata); + +void +ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd, + off_t offset, dict_t *xdata); + +void +ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata); + +void +ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata); + +void +ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata); + +void +ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata); + +void +ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd, + off_t offset, gf_seek_what_t what, dict_t *xdata); + +void +ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op, + dict_t *xdata); #endif /* __EC_FOPS_H__ */ diff --git a/xlators/cluster/ec/src/ec-galois.c b/xlators/cluster/ec/src/ec-galois.c new file mode 100644 index 00000000000..6e4990c71f5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-galois.c @@ -0,0 +1,183 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> + +#include "ec-mem-types.h" +#include "ec-gf8.h" +#include "ec-helpers.h" + +static ec_gf_t * +ec_gf_alloc(uint32_t bits, uint32_t mod) +{ + ec_gf_t *gf; + + gf = GF_MALLOC(sizeof(ec_gf_t), ec_mt_ec_gf_t); + if (gf == NULL) { + goto failed; + } + + gf->bits = bits; + gf->size = 1 << bits; + gf->mod = mod; + + gf->log = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1), + gf_common_mt_int); + if (gf->log == NULL) { + goto failed_gf; + } + gf->pow = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1), + gf_common_mt_int); + if (gf->pow == NULL) { + goto failed_log; + } + + return gf; + +failed_log: + GF_FREE(gf->log); +failed_gf: + GF_FREE(gf); +failed: + return EC_ERR(ENOMEM); +} + +static void +ec_gf_init_tables(ec_gf_t *gf) +{ + uint32_t i, tmp; + + memset(gf->log, -1, sizeof(uint32_t) * gf->size); + + gf->pow[0] = 1; + gf->log[0] = gf->size; + gf->log[1] = 0; + for (i = 1; i < gf->size; i++) { + tmp = gf->pow[i - 1] << 1; + if (tmp >= gf->size) { + tmp ^= gf->mod; + } + gf->pow[i + gf->size - 1] = gf->pow[i] = tmp; + gf->log[tmp + gf->size - 1] = gf->log[tmp] = i; + } +} + +ec_gf_t * +ec_gf_prepare(uint32_t bits, uint32_t mod) +{ + ec_gf_mul_t **tbl; + ec_gf_t *gf; + uint32_t i, j; + + if (bits != 8) { + return EC_ERR(EINVAL); + } + + tbl = ec_gf8_mul; + if (mod == 0) { + mod = 0x11d; + } + + gf = ec_gf_alloc(bits, mod); + if (EC_IS_ERR(gf)) { + return gf; + } + ec_gf_init_tables(gf); + + gf->table = tbl; + gf->min_ops = bits * bits; + gf->max_ops = 0; + gf->avg_ops = 0; + for (i = 1; i < gf->size; i++) { + for (j = 0; tbl[i]->ops[j].op != EC_GF_OP_END; j++) { + } + if (gf->max_ops < j) { + gf->max_ops = j; + } + if (gf->min_ops > j) { + gf->min_ops = j; + } + gf->avg_ops += j; + } + gf->avg_ops /= gf->size; + + return gf; +} + +void +ec_gf_destroy(ec_gf_t *gf) +{ + GF_FREE(gf->pow); + GF_FREE(gf->log); + GF_FREE(gf); +} + +uint32_t +ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + return a ^ b; +} + +uint32_t +ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + if ((a != 0) && (b != 0)) { + return gf->pow[gf->log[a] + gf->log[b]]; + } + + return 0; +} + +uint32_t +ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + if (b != 0) { + if (a != 0) { + return gf->pow[gf->size - 1 + gf->log[a] - gf->log[b]]; + } + + return 0; + } + + return gf->size; +} + +uint32_t +ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + uint32_t r; + + if ((a >= gf->size) || ((a == 0) && (b == 0))) { + return gf->size; + } + + r = 1; + while (b != 0) { + if ((b & 1) != 0) { + r = ec_gf_mul(gf, r, a); + } + a = ec_gf_mul(gf, a, a); + b >>= 1; + } + + return r; +} diff --git a/xlators/cluster/ec/src/ec-galois.h b/xlators/cluster/ec/src/ec-galois.h new file mode 100644 index 00000000000..ed55d53e419 --- /dev/null +++ b/xlators/cluster/ec/src/ec-galois.h @@ -0,0 +1,32 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_GALOIS_H__ +#define __EC_GALOIS_H__ + +#include <inttypes.h> + +#include "ec-types.h" + +ec_gf_t * +ec_gf_prepare(uint32_t bits, uint32_t mod); +void +ec_gf_destroy(ec_gf_t *gf); + +uint32_t +ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b); + +#endif /* __EC_GALOIS_H__ */ diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c index 73df0d89db8..884deb93669 100644 --- a/xlators/cluster/ec/src/ec-generic.c +++ b/xlators/cluster/ec/src/ec-generic.c @@ -8,22 +8,23 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" +#include <glusterfs/byte-order.h> +#include "ec.h" +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" -#include "ec-method.h" #include "ec-fops.h" /* FOP: flush */ -int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -33,20 +34,18 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FLUSH, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -56,15 +55,15 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_flush(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -73,15 +72,15 @@ void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->xdata); } -int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_flush(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_fd(fop, fop->fd, 0); + ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL); ec_lock(fop); return EC_STATE_DISPATCH; @@ -97,26 +96,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + ec_fop_prepare_answer(fop, _gf_false); return EC_STATE_REPORT; @@ -125,8 +105,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.flush != NULL) - { + if (fop->cbks.flush != NULL) { fop->cbks.flush(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->xdata); } @@ -135,13 +114,13 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: + case -EC_STATE_DELAYED_START: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.flush != NULL) - { + if (fop->cbks.flush != NULL) { fop->cbks.flush(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } @@ -161,55 +140,93 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_flush_cbk_t func, void * data, fd_t * fd, - dict_t * xdata) +static int32_t +ec_validate_fd(fd_t *fd, xlator_t *xl) +{ + uint64_t iversion = 0; + uint64_t fversion = 0; + ec_inode_t *inode_ctx = NULL; + ec_fd_t *fd_ctx = NULL; + + LOCK(&fd->lock); + { + fd_ctx = __ec_fd_get(fd, xl); + if (fd_ctx) { + fversion = fd_ctx->bad_version; + } + } + UNLOCK(&fd->lock); + + LOCK(&fd->inode->lock); + { + inode_ctx = __ec_inode_get(fd->inode, xl); + if (inode_ctx) { + iversion = inode_ctx->bad_version; + } + } + UNLOCK(&fd->inode->lock); + if (fversion < iversion) { + return EBADF; + } + return 0; +} + +void +ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd, + dict_t *xdata) { - ec_cbk_t callback = { .flush = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.flush = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FLUSH) %p", frame); + gf_msg_trace("ec", 0, "EC(FLUSH) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_flush, - ec_manager_flush, callback, data); - if (fop == NULL) - { + if (fd) { + error = ec_validate_fd(fd, this); + if (error) { + gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD, + "Failing %s on %s", gf_fop_list[GF_FOP_FLUSH], + fd->inode ? uuid_utoa(fd->inode->gfid) : ""); + goto out; + } + } + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, fop_flags, + ec_wind_flush, ec_manager_flush, callback, data); + if (fop == NULL) { goto out; } fop->use_fd = 1; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -218,25 +235,22 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: fsync */ -int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_fsync(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_FSYNC'"); + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_FSYNC'"); return 0; } @@ -244,12 +258,13 @@ int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * prebuf, - struct iatt * postbuf, dict_t * xdata) +int32_t +ec_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf, + dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -259,31 +274,26 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNC, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (prebuf != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (prebuf != NULL) { cbk->iatt[0] = *prebuf; } - if (postbuf != NULL) - { + if (postbuf != NULL) { cbk->iatt[1] = *postbuf; } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -293,15 +303,15 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fsync(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -310,22 +320,17 @@ void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->int32, fop->xdata); } -int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_fsync(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_fd(fop, fop->fd, 0); + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, EC_RANGE_FULL); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -339,33 +344,14 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, - cbk->count); - - cbk->iatt[0].ia_size = fop->pre_size; - cbk->iatt[1].ia_size = fop->post_size; - } - } - else - { - ec_fop_set_error(fop, EIO); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; } return EC_STATE_REPORT; @@ -375,8 +361,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.fsync != NULL) - { + if (fop->cbks.fsync != NULL) { fop->cbks.fsync(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); @@ -386,14 +371,13 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: + case -EC_STATE_DELAYED_START: GF_ASSERT(fop->error != 0); - if (fop->cbks.fsync != NULL) - { + if (fop->cbks.fsync != NULL) { fop->cbks.fsync(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -413,32 +397,41 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsync_cbk_t func, void * data, fd_t * fd, - int32_t datasync, dict_t * xdata) +void +ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata) { - ec_cbk_t callback = { .fsync = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fsync = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FSYNC) %p", frame); + gf_msg_trace("ec", 0, "EC(FSYNC) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_fsync, - ec_manager_fsync, callback, data); - if (fop == NULL) - { + if (fd) { + error = ec_validate_fd(fd, this); + if (error) { + gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD, + "Failing %s on %s", gf_fop_list[GF_FOP_FSYNC], + fd->inode ? uuid_utoa(fd->inode->gfid) : ""); + goto out; + } + } + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, fop_flags, + ec_wind_fsync, ec_manager_fsync, callback, data); + if (fop == NULL) { goto out; } @@ -446,24 +439,22 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->int32 = datasync; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -472,23 +463,21 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: fsyncdir */ -int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -498,20 +487,18 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNCDIR, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -521,15 +508,15 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fsyncdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -538,15 +525,15 @@ void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->fd, fop->int32, fop->xdata); } -int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_fsyncdir(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_fd(fop, fop->fd, 0); + ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL); ec_lock(fop); return EC_STATE_DISPATCH; @@ -562,26 +549,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + ec_fop_prepare_answer(fop, _gf_false); return EC_STATE_REPORT; @@ -590,8 +558,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.fsyncdir != NULL) - { + if (fop->cbks.fsyncdir != NULL) { fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->xdata); } @@ -603,12 +570,12 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state) case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: + case -EC_STATE_DELAYED_START: GF_ASSERT(fop->error != 0); - if (fop->cbks.fsyncdir != NULL) - { - fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL); + if (fop->cbks.fsyncdir != NULL) { + fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); } return EC_STATE_LOCK_REUSE; @@ -626,32 +593,32 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsyncdir_cbk_t func, void * data, - fd_t * fd, int32_t datasync, dict_t * xdata) +void +ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata) { - ec_cbk_t callback = { .fsyncdir = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fsyncdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FSYNCDIR) %p", frame); + gf_msg_trace("ec", 0, "EC(FSYNCDIR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_fsyncdir, - ec_manager_fsyncdir, callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, 0, target, + fop_flags, ec_wind_fsyncdir, ec_manager_fsyncdir, + callback, data); + if (fop == NULL) { goto out; } @@ -659,24 +626,22 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->int32 = datasync; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -685,139 +650,66 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: lookup */ -void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk) +void +ec_lookup_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk) { - ec_cbk_data_t * ans = NULL; - ec_inode_t * ctx = NULL; - ec_lock_t * lock = NULL; - data_t * data = NULL; - uint8_t * buff = NULL; + ec_inode_t *ctx = NULL; uint64_t size = 0; - int32_t i = 0, have_size = 0; + int32_t have_size = 0, err; - if (cbk->op_ret < 0) - { + if (cbk->op_ret < 0) { return; } - ec_dict_del_number(cbk->xdata, EC_XATTR_VERSION, &cbk->version); - - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, &cbk->iatt[0]) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; + ec_dict_del_array(cbk->xdata, EC_XATTR_VERSION, cbk->version, + EC_VERSION_SIZE); + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, &cbk->iatt[0]); + if (ec_cbk_set_error(cbk, -err, _gf_true)) { return; } LOCK(&cbk->inode->lock); ctx = __ec_inode_get(cbk->inode, fop->xl); - if ((ctx != NULL) && (ctx->inode_lock != NULL)) - { - lock = ctx->inode_lock; - cbk->version = lock->version; - if (lock->have_size) - { - size = lock->size; + if (ctx != NULL) { + if (ctx->have_version) { + cbk->version[0] = ctx->post_version[0]; + cbk->version[1] = ctx->post_version[1]; + } + if (ctx->have_size) { + size = ctx->post_size; have_size = 1; } } UNLOCK(&cbk->inode->lock); - if (cbk->iatt[0].ia_type == IA_IFREG) - { - uint8_t * blocks[cbk->count]; - uint32_t values[cbk->count]; - + if (cbk->iatt[0].ia_type == IA_IFREG) { cbk->size = cbk->iatt[0].ia_size; ec_dict_del_number(cbk->xdata, EC_XATTR_SIZE, &cbk->iatt[0].ia_size); - if (have_size) - { + if (have_size) { cbk->iatt[0].ia_size = size; } - - size = UINT64_MAX; - for (i = 0, ans = cbk; (ans != NULL) && (i < ec->fragments); - ans = ans->next) - { - if (!ans->dirty) { - data = dict_get(ans->xdata, GF_CONTENT_KEY); - if (data != NULL) - { - values[i] = ans->idx; - blocks[i] = (uint8_t *)data->data; - if (size > data->len) { - size = data->len; - } - i++; - } - } - } - - if (i >= ec->fragments) - { - size -= size % ec->fragment_size; - if (size > 0) - { - buff = GF_MALLOC(size * ec->fragments, gf_common_mt_char); - if (buff != NULL) - { - size = ec_method_decode(size, ec->fragments, values, - blocks, buff); - if (size > fop->size) - { - size = fop->size; - } - if (size > cbk->iatt[0].ia_size) - { - size = cbk->iatt[0].ia_size; - } - - if (dict_set_bin(cbk->xdata, GF_CONTENT_KEY, buff, - size) != 0) - { - GF_FREE(buff); - buff = NULL; - gf_log(fop->xl->name, GF_LOG_WARNING, "Lookup " - "read-ahead " - "failed"); - } - } - else - { - gf_log(fop->xl->name, GF_LOG_WARNING, "Lookup read-ahead " - "failed"); - } - } - } - - if (buff == NULL) - { - dict_del(cbk->xdata, GF_CONTENT_KEY); - } } } -int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_lookup(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_LOOKUP'"); + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_LOOKUP'"); return 0; } @@ -825,14 +717,15 @@ int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, dict_t * xdata, - struct iatt * postparent) +int32_t +ec_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata, + struct iatt *postparent) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; + uint64_t dirty[2] = {0}; VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); @@ -841,65 +734,54 @@ int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LOOKUP, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (inode != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (inode != NULL) { cbk->inode = inode_ref(inode); - if (cbk->inode == NULL) - { - gf_log(this->name, GF_LOG_ERROR, + if (cbk->inode == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_INODE_REF_FAIL, "Failed to reference an inode."); goto out; } } - if (buf != NULL) - { + if (buf != NULL) { cbk->iatt[0] = *buf; } - if (postparent != NULL) - { + if (postparent != NULL) { cbk->iatt[1] = *postparent; } } - if (xdata != NULL) - { - uint64_t dirty; - + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } - if (ec_dict_del_number(cbk->xdata, EC_XATTR_DIRTY, &dirty) == 0) { - cbk->dirty = dirty != 0; - } + ec_dict_del_array(xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); } ec_combine(cbk, ec_combine_lookup); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_lookup(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -908,54 +790,49 @@ void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->xdata); } -int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_lookup(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + int32_t err; - switch (state) - { + switch (state) { case EC_STATE_INIT: - if (fop->xdata == NULL) - { + if (fop->xdata == NULL) { fop->xdata = dict_new(); - if (fop->xdata == NULL) - { - gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare " - "lookup request"); + if (fop->xdata == NULL) { + gf_msg(fop->xl->name, GF_LOG_ERROR, ENOMEM, + EC_MSG_LOOKUP_REQ_PREP_FAIL, + "Unable to prepare " + "lookup request"); - fop->error = EIO; + fop->error = ENOMEM; return EC_STATE_REPORT; } + } else { + /*TODO: To be handled once we have 'syndromes' */ + dict_del(fop->xdata, GF_CONTENT_KEY); } - else - { - uint64_t size; - - if (dict_get_uint64(fop->xdata, GF_CONTENT_KEY, &size) == 0) - { - fop->size = size; - size = ec_adjust_size(fop->xl->private, size, 1); - if (dict_set_uint64(fop->xdata, GF_CONTENT_KEY, size) != 0) - { - gf_log("ec", GF_LOG_DEBUG, "Unable to update lookup " - "content size"); - } - } + err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0); + if (err == 0) { + err = dict_set_uint64(fop->xdata, EC_XATTR_VERSION, 0); } - if ((dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0) != 0) || - (dict_set_uint64(fop->xdata, EC_XATTR_VERSION, 0) != 0) || - (dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0) != 0)) - { - gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare lookup " - "request"); + if (err == 0) { + err = dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0); + } + if (err != 0) { + gf_msg(fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_LOOKUP_REQ_PREP_FAIL, + "Unable to prepare lookup " + "request"); - fop->error = EIO; + fop->error = -err; return EC_STATE_REPORT; } - /* Fall through */ + /* Fall through */ case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -963,32 +840,23 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, - cbk->count); - - ec_lookup_rebuild(fop->xl->private, fop, cbk); - } + /* + * Lookup happens without any lock, so there is a chance that it + * will have answers before modification happened and after + * modification happened in the same response. So choose the next + * best answer when the answers don't match for EC_MINIMUM_MIN + */ + + if (!fop->answer && !list_empty(&fop->cbk_list)) { + fop->answer = list_entry(fop->cbk_list.next, ec_cbk_data_t, + list); } - else - { - ec_fop_set_error(fop, EIO); + + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + ec_lookup_rebuild(fop->xl->private, fop, cbk); } return EC_STATE_REPORT; @@ -998,8 +866,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.lookup != NULL) - { + if (fop->cbks.lookup != NULL) { fop->cbks.lookup(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->inode, &cbk->iatt[0], cbk->xdata, &cbk->iatt[1]); @@ -1013,8 +880,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.lookup != NULL) - { + if (fop->cbks.lookup != NULL) { fop->cbks.lookup(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL, NULL); } @@ -1022,85 +888,77 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_lookup_cbk_t func, void * data, - loc_t * loc, dict_t * xdata) +void +ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) { - ec_cbk_t callback = { .lookup = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.lookup = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(LOOKUP) %p", frame); + gf_msg_trace("ec", 0, "EC(LOOKUP) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, 0, target, minimum, - ec_wind_lookup, ec_manager_lookup, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_lookup, + ec_manager_lookup, callback, data); + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + /* Do not log failures here as a memory problem would have already + * been logged by the corresponding alloc functions */ if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - goto out; - } } error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL); } } /* FOP: statfs */ -int32_t ec_combine_statfs(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_statfs(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { ec_statvfs_combine(&dst->statvfs, &src->statvfs); return 1; } -int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct statvfs * buf, - dict_t * xdata) +int32_t +ec_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct statvfs *buf, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1110,27 +968,23 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STATFS, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (buf != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { cbk->statvfs = *buf; } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1140,15 +994,15 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_statfs(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1157,12 +1011,14 @@ void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->xdata); } -int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_statfs(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk = NULL; + gf_boolean_t deem_statfs_enabled = _gf_false; + int32_t err = 0; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -1170,34 +1026,24 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + ec_t *ec = fop->xl->private; + + if (cbk->xdata) { + err = dict_get_int8(cbk->xdata, "quota-deem-statfs", + (int8_t *)&deem_statfs_enabled); + if (err != -ENOENT) { + ec_cbk_set_error(cbk, -err, _gf_true); } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_t * ec = fop->xl->private; + if (err != 0 || deem_statfs_enabled == _gf_false) { cbk->statvfs.f_blocks *= ec->fragments; cbk->statvfs.f_bfree *= ec->fragments; cbk->statvfs.f_bavail *= ec->fragments; } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -1206,8 +1052,7 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.statfs != NULL) - { + if (fop->cbks.statfs != NULL) { fop->cbks.statfs(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, &cbk->statvfs, cbk->xdata); } @@ -1220,8 +1065,7 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.statfs != NULL) - { + if (fop->cbks.statfs != NULL) { fop->cbks.statfs(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } @@ -1229,51 +1073,49 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_statfs_cbk_t func, void * data, - loc_t * loc, dict_t * xdata) +void +ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) { - ec_cbk_t callback = { .statfs = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.statfs = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(STATFS) %p", frame); + gf_msg_trace("ec", 0, "EC(STATFS) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, 0, target, minimum, - ec_wind_statfs, ec_manager_statfs, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_statfs, + ec_manager_statfs, callback, data); + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1282,25 +1124,22 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: xattrop */ -int32_t ec_combine_xattrop(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_dict_compare(dst->dict, src->dict)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching dictionary in " - "answers of 'GF_FOP_XATTROP'"); + if (!ec_dict_compare(dst->dict, src->dict)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_DICT_MISMATCH, + "Mismatching dictionary in " + "answers of 'GF_FOP_XATTROP'"); return 0; } @@ -1308,12 +1147,16 @@ int32_t ec_combine_xattrop(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_xattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xattr, - dict_t * xdata) +int32_t +ec_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_lock_link_t *link = NULL; + ec_cbk_data_t *cbk = NULL; + uint64_t dirty[2] = {0}; + data_t *data; + uint64_t *version; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1323,57 +1166,53 @@ int32_t ec_xattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_XATTROP, idx, op_ret, + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (xattr != NULL) - { - cbk->dict = dict_ref(xattr); - if (cbk->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (!cbk) + goto out; - goto out; - } - } - } - if (xdata != NULL) - { - uint64_t dirty; + if (op_ret >= 0) { + cbk->dict = dict_ref(xattr); - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + data = dict_get(cbk->dict, EC_XATTR_VERSION); + if ((data != NULL) && (data->len >= sizeof(uint64_t))) { + version = (uint64_t *)data->data; - goto out; - } - if (ec_dict_del_number(cbk->xdata, EC_XATTR_DIRTY, &dirty) == 0) { - cbk->dirty = dirty != 0; + if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) { + LOCK(&fop->lock); + + fop->healing |= 1ULL << idx; + + UNLOCK(&fop->lock); } } - ec_combine(cbk, ec_combine_xattrop); + ec_dict_del_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + link = fop->data; + if (link) { + /*Keep a note of if the dirty is already set or not*/ + link->dirty[0] |= (dirty[0] != 0); + link->dirty[1] |= (dirty[1] != 0); + } } + if (xdata) + cbk->xdata = dict_ref(xdata); + + ec_combine(cbk, ec_combine_xattrop); + out: - if (fop != NULL) - { + if (fop) ec_complete(fop); - } return 0; } -void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_xattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1382,21 +1221,20 @@ void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->xattrop_flags, fop->dict, fop->xdata); } -int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_xattrop(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - if (fop->fd == NULL) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 1); - } - else - { - ec_lock_prepare_fd(fop, fop->fd, 1); + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META, 0, + EC_RANGE_FULL); } ec_lock(fop); @@ -1408,27 +1246,12 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA) || - ((cbk->op_ret >= 0) && !ec_dict_combine(cbk, - EC_COMBINE_DICT))) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + err = ec_dict_combine(cbk, EC_COMBINE_DICT); + ec_cbk_set_error(cbk, -err, _gf_false); } return EC_STATE_REPORT; @@ -1438,19 +1261,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_XATTROP) - { - if (fop->cbks.xattrop != NULL) - { - fop->cbks.xattrop(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, cbk->dict, - cbk->xdata); + if (fop->id == GF_FOP_XATTROP) { + if (fop->cbks.xattrop != NULL) { + fop->cbks.xattrop(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->dict, cbk->xdata); } - } - else - { - if (fop->cbks.fxattrop != NULL) - { + } else { + if (fop->cbks.fxattrop != NULL) { fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->dict, cbk->xdata); @@ -1466,18 +1283,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_XATTROP) - { - if (fop->cbks.xattrop != NULL) - { + if (fop->id == GF_FOP_XATTROP) { + if (fop->cbks.xattrop != NULL) { fop->cbks.xattrop(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } - } - else - { - if (fop->cbks.fxattrop != NULL) - { + } else { + if (fop->cbks.fxattrop != NULL) { fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } @@ -1498,66 +1310,61 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_xattrop_cbk_t func, void * data, - loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata) +void +ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) { - ec_cbk_t callback = { .xattrop = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.xattrop = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(XATTROP) %p", frame); + gf_msg_trace("ec", 0, "EC(XATTROP) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_xattrop, ec_manager_xattrop, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target, + fop_flags, ec_wind_xattrop, ec_manager_xattrop, + callback, data); + if (fop == NULL) { goto out; } fop->xattrop_flags = optype; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xattr != NULL) - { + if (xattr != NULL) { fop->dict = dict_ref(xattr); - if (fop->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1566,24 +1373,98 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); +} + +void +ec_wind_fxattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_xattrop_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fxattrop, + fop->fd, fop->xattrop_flags, fop->dict, fop->xdata); +} + +void +ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) +{ + ec_cbk_t callback = {.fxattrop = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FXATTROP) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, 0, target, + fop_flags, ec_wind_fxattrop, ec_manager_xattrop, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->xattrop_flags = optype; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xattr != NULL) { + fop->dict = dict_ref(xattr); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } -/* FOP: fxattrop */ +/* FOP: IPC */ -int32_t ec_fxattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xattr, - dict_t * xdata) +int32_t +ec_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1593,131 +1474,118 @@ int32_t ec_fxattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FXATTROP, idx, op_ret, + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_IPC, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (xattr != NULL) - { - cbk->dict = dict_ref(xattr); - if (cbk->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - goto out; - } - } - } - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } } - ec_combine(cbk, ec_combine_xattrop); + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_ipc(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); - STACK_WIND_COOKIE(fop->frame, ec_fxattrop_cbk, (void *)(uintptr_t)idx, - ec->xl_list[idx], ec->xl_list[idx]->fops->fxattrop, - fop->fd, fop->xattrop_flags, fop->dict, fop->xdata); + STACK_WIND_COOKIE(fop->frame, ec_ipc_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->ipc, fop->int32, + fop->xdata); } -void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fxattrop_cbk_t func, void * data, - fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata) +int32_t +ec_manager_ipc(ec_fop_data_t *fop, int32_t state) { - ec_cbk_t callback = { .fxattrop = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_data_t *cbk; - gf_log("ec", GF_LOG_TRACE, "EC(FXATTROP) %p", frame); + switch (state) { + case EC_STATE_INIT: + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); + return EC_STATE_PREPARE_ANSWER; - fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fxattrop, ec_manager_xattrop, callback, - data); - if (fop == NULL) - { - goto out; - } + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_true); - fop->use_fd = 1; + return EC_STATE_REPORT; - fop->xattrop_flags = optype; + case EC_STATE_REPORT: + cbk = fop->answer; - if (fd != NULL) - { - fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + GF_ASSERT(cbk != NULL); + if (fop->cbks.ipc != NULL) { + fop->cbks.ipc(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } - goto out; - } + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.ipc != NULL) { + fop->cbks.ipc(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; } - if (xattr != NULL) - { - fop->dict = dict_ref(xattr); - if (fop->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); +} - goto out; - } +void +ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op, + dict_t *xdata) +{ + ec_cbk_t callback = {.ipc = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(IPC) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, fop_flags, + ec_wind_ipc, ec_manager_ipc, callback, data); + if (fop == NULL) { + goto out; } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } } + fop->int32 = op; error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } diff --git a/xlators/cluster/ec/src/ec-gf.c b/xlators/cluster/ec/src/ec-gf.c deleted file mode 100644 index 1ae8928f20b..00000000000 --- a/xlators/cluster/ec/src/ec-gf.c +++ /dev/null @@ -1,11635 +0,0 @@ -/* - Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> - This file is part of GlusterFS. - - This file is licensed to you under your choice of the GNU Lesser - General Public License, version 3 or any later version (LGPLv3 or - later), or the GNU General Public License, version 2 (GPLv2), in all - cases as published by the Free Software Foundation. -*/ - -#include <inttypes.h> -#include <string.h> - -#include "ec-gf.h" - -static void gf8_muladd_00(uint8_t * out, uint8_t * in, unsigned int width) -{ - memcpy(out, in, sizeof(uint64_t) * 8 * width); -} - -static void gf8_muladd_01(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - out_ptr[0] ^= in_ptr[0]; - out_ptr[width] ^= in_ptr[width]; - out_ptr[width * 2] ^= in_ptr[width * 2]; - out_ptr[width * 3] ^= in_ptr[width * 3]; - out_ptr[width * 4] ^= in_ptr[width * 4]; - out_ptr[width * 5] ^= in_ptr[width * 5]; - out_ptr[width * 6] ^= in_ptr[width * 6]; - out_ptr[width * 7] ^= in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_02(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in7; - out1 = in0; - out7 = in6; - out5 = in4; - out6 = in5; - out3 = in2 ^ in7; - out4 = in3 ^ in7; - out2 = in1 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_03(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in0 ^ in7; - tmp0 = in2 ^ in7; - out1 = in0 ^ in1; - out7 = in6 ^ in7; - out5 = in4 ^ in5; - out6 = in5 ^ in6; - out4 = in3 ^ in4 ^ in7; - out2 = tmp0 ^ in1; - out3 = tmp0 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_04(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in6; - out1 = in7; - out7 = in5; - out6 = in4; - tmp0 = in6 ^ in7; - out2 = in0 ^ in6; - out5 = in3 ^ in7; - out3 = tmp0 ^ in1; - out4 = tmp0 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_05(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in0 ^ in6; - out1 = in1 ^ in7; - out7 = in5 ^ in7; - out6 = in4 ^ in6; - out2 = out0 ^ in2; - out3 = out1 ^ in3 ^ in6; - out5 = out7 ^ in3; - out4 = out6 ^ in2 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_06(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in6 ^ in7; - tmp0 = in1 ^ in6; - out1 = in0 ^ in7; - out7 = in5 ^ in6; - out6 = in4 ^ in5; - out4 = in2 ^ in3 ^ in6; - out5 = in3 ^ in4 ^ in7; - out3 = tmp0 ^ in2; - out2 = tmp0 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_07(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in6; - tmp1 = in5 ^ in6; - tmp2 = in0 ^ in7; - tmp3 = tmp0 ^ in3; - out6 = tmp1 ^ in4; - out7 = tmp1 ^ in7; - out0 = tmp2 ^ in6; - out1 = tmp2 ^ in1; - out3 = tmp3 ^ in1; - out4 = tmp3 ^ in4; - out5 = out4 ^ out7 ^ in2; - out2 = tmp0 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_08(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in5; - out1 = in6; - out7 = in4; - out6 = in3 ^ in7; - out3 = in0 ^ in5 ^ in6; - out5 = in2 ^ in6 ^ in7; - out2 = in5 ^ in7; - out4 = out2 ^ in1 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_09(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in0 ^ in5; - tmp0 = in3 ^ in6; - out1 = in1 ^ in6; - out7 = in4 ^ in7; - out2 = in2 ^ in5 ^ in7; - out3 = tmp0 ^ out0; - out6 = tmp0 ^ in7; - out4 = out1 ^ out7 ^ in5; - out5 = out2 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in5 ^ in7; - out1 = in0 ^ in6; - out7 = in4 ^ in6; - out2 = in1 ^ in5; - out6 = out0 ^ in3; - out3 = out0 ^ out1 ^ in2; - out5 = out7 ^ in2 ^ in7; - out4 = out2 ^ in3 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = in0 ^ in6; - tmp2 = in4 ^ in7; - out0 = in0 ^ in5 ^ in7; - out2 = tmp0 ^ in1; - out1 = tmp1 ^ in1; - out6 = tmp1 ^ out0 ^ in3; - out7 = tmp2 ^ in6; - out4 = tmp2 ^ out6 ^ in1; - out3 = out6 ^ in0 ^ in2; - out5 = tmp0 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in5 ^ in6; - out1 = in6 ^ in7; - out7 = in4 ^ in5; - tmp0 = in1 ^ in5; - tmp1 = in0 ^ in7; - out5 = in2 ^ in3 ^ in6; - out6 = in3 ^ in4 ^ in7; - out2 = tmp1 ^ out0; - out4 = tmp0 ^ in2; - out3 = tmp0 ^ tmp1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in5; - tmp1 = in5 ^ in6; - out1 = in1 ^ in6 ^ in7; - out7 = tmp0 ^ in7; - out4 = tmp0 ^ in1 ^ in2; - out0 = tmp1 ^ in0; - tmp2 = tmp1 ^ in3; - out6 = tmp2 ^ out7; - out2 = out0 ^ in2 ^ in7; - out3 = out0 ^ out1 ^ in3; - out5 = tmp2 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in2 ^ in5; - tmp2 = in5 ^ in6; - out1 = in0 ^ in6 ^ in7; - out3 = tmp0 ^ tmp1; - out2 = tmp0 ^ tmp2; - tmp3 = tmp1 ^ in3; - out7 = tmp2 ^ in4; - out0 = tmp2 ^ in7; - out4 = tmp3 ^ in1 ^ in7; - out5 = tmp3 ^ out7; - out6 = out0 ^ out5 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_0F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in6 ^ in7; - tmp1 = tmp0 ^ in1; - tmp2 = tmp0 ^ in5; - out1 = tmp1 ^ in0; - out7 = tmp2 ^ in4; - out0 = tmp2 ^ in0; - out6 = out7 ^ in3; - out5 = out6 ^ in2 ^ in7; - tmp3 = tmp1 ^ out0 ^ in2; - out4 = tmp1 ^ out5; - out2 = tmp3 ^ in6; - out3 = tmp3 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_10(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in4; - out1 = in5; - out7 = in3 ^ in7; - tmp0 = in6 ^ in7; - out2 = in4 ^ in6; - tmp1 = out2 ^ in5; - out6 = tmp0 ^ in2; - out3 = tmp0 ^ tmp1; - out5 = out2 ^ out3 ^ in1; - out4 = tmp1 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_11(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out7 = in3; - out0 = in0 ^ in4; - out1 = in1 ^ in5; - out6 = in2 ^ in7; - out4 = in0 ^ in5 ^ in6; - out5 = in1 ^ in6 ^ in7; - out2 = in2 ^ in4 ^ in6; - out3 = in3 ^ in4 ^ in5 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_12(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in4 ^ in7; - out1 = in0 ^ in5; - out3 = in2 ^ in4 ^ in5; - tmp0 = out0 ^ in6; - out2 = tmp0 ^ in1; - tmp1 = tmp0 ^ in3; - out6 = tmp0 ^ out3; - out5 = out2 ^ in5; - out7 = tmp1 ^ in4; - out4 = tmp1 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_13(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out7 = in3 ^ in6; - tmp0 = in0 ^ in5; - tmp1 = in4 ^ in7; - out6 = in2 ^ in5 ^ in7; - out4 = tmp0 ^ out7 ^ in7; - out1 = tmp0 ^ in1; - out0 = tmp1 ^ in0; - out5 = tmp1 ^ in1 ^ in6; - out3 = tmp1 ^ out6 ^ in3; - out2 = out5 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_14(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in4 ^ in6; - out1 = in5 ^ in7; - out2 = in0 ^ in4; - tmp0 = out0 ^ in5; - out7 = out1 ^ in3; - tmp1 = out1 ^ in2; - out3 = tmp0 ^ in1; - out6 = tmp0 ^ tmp1; - out4 = tmp1 ^ out2; - out5 = out3 ^ in3 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_15(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out7 = in3 ^ in5; - tmp0 = in0 ^ in4; - out1 = in1 ^ in5 ^ in7; - out5 = in1 ^ in3 ^ in6; - out0 = tmp0 ^ in6; - out2 = tmp0 ^ in2; - out3 = out5 ^ in4 ^ in5; - out6 = out2 ^ in0 ^ in7; - out4 = tmp0 ^ out6 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_16(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in5; - tmp1 = in4 ^ in7; - tmp2 = in2 ^ in3 ^ in4; - out1 = tmp0 ^ in7; - out4 = tmp0 ^ tmp2; - out0 = tmp1 ^ in6; - tmp3 = tmp1 ^ in1; - out6 = out0 ^ in2 ^ in5; - out2 = tmp3 ^ in0; - out3 = out6 ^ in1; - out7 = tmp2 ^ out6; - out5 = tmp3 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_17(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = in3 ^ in6; - tmp2 = tmp0 ^ in4; - out4 = tmp0 ^ in0 ^ in3; - out7 = tmp1 ^ in5; - tmp3 = tmp1 ^ in1; - out6 = tmp2 ^ in7; - out5 = tmp3 ^ in4; - out3 = tmp3 ^ out6; - out0 = out3 ^ out4 ^ in1; - out2 = out3 ^ out7 ^ in0; - out1 = tmp2 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_18(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in4 ^ in5; - out1 = in5 ^ in6; - tmp0 = in4 ^ in7; - out5 = in1 ^ in2 ^ in5; - out6 = in2 ^ in3 ^ in6; - out2 = tmp0 ^ out1; - out7 = tmp0 ^ in3; - tmp1 = tmp0 ^ in0; - out3 = tmp1 ^ in6; - out4 = tmp1 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_19(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in1 ^ in2; - out7 = in3 ^ in4; - tmp0 = in0 ^ in7; - out6 = in2 ^ in3; - out1 = in1 ^ in5 ^ in6; - out0 = in0 ^ in4 ^ in5; - out4 = tmp0 ^ in1; - tmp1 = tmp0 ^ in6; - out2 = tmp1 ^ out0 ^ in2; - out3 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in5; - tmp1 = in5 ^ in6; - tmp2 = tmp0 ^ in1; - out0 = tmp0 ^ in7; - out1 = tmp1 ^ in0; - tmp3 = tmp1 ^ in3; - out5 = tmp2 ^ in2; - out2 = tmp2 ^ in6; - out7 = tmp3 ^ out0; - out6 = tmp3 ^ in2; - out4 = tmp3 ^ out2 ^ in0; - out3 = tmp0 ^ out1 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in4; - tmp1 = in2 ^ in5; - tmp2 = in3 ^ in6; - out5 = tmp0 ^ in1; - tmp3 = tmp0 ^ in0; - out6 = tmp1 ^ in3; - out0 = tmp1 ^ tmp3 ^ in7; - out7 = tmp2 ^ in4; - tmp4 = out5 ^ in6; - out3 = tmp2 ^ tmp3; - out2 = tmp4 ^ in5; - out4 = tmp4 ^ out3; - out1 = tmp3 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - tmp1 = in4 ^ in6; - tmp2 = in5 ^ in7; - out6 = tmp0 ^ tmp1; - out0 = tmp1 ^ in5; - out1 = tmp2 ^ in6; - tmp3 = tmp2 ^ in1; - tmp4 = tmp2 ^ in4; - out2 = tmp4 ^ in0; - out7 = tmp4 ^ in3; - out5 = tmp0 ^ tmp3; - out3 = tmp3 ^ out2; - out4 = out3 ^ in2 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in3; - tmp1 = in0 ^ in4; - tmp2 = in3 ^ in4; - tmp3 = in2 ^ in7; - out3 = tmp0 ^ tmp1; - out5 = tmp0 ^ tmp3; - tmp4 = tmp1 ^ in5; - out6 = tmp2 ^ in2; - out7 = tmp2 ^ in5; - out2 = tmp3 ^ tmp4; - out4 = out3 ^ out6 ^ in6; - out0 = tmp4 ^ in6; - out1 = out2 ^ out4 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in4; - tmp1 = in2 ^ in7; - tmp2 = tmp0 ^ in1; - out3 = tmp1 ^ tmp2; - out2 = tmp2 ^ in5; - out4 = out3 ^ in3 ^ in6; - tmp3 = out4 ^ in7; - out6 = tmp3 ^ out2 ^ in4; - out7 = tmp1 ^ out6; - out0 = out7 ^ in3; - out1 = tmp0 ^ out0; - out5 = tmp3 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_1F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in6; - tmp1 = tmp0 ^ in5; - out7 = tmp1 ^ in3; - out0 = tmp1 ^ in0 ^ in7; - out6 = out7 ^ in2 ^ in6; - out1 = out0 ^ in1 ^ in4; - out4 = out0 ^ out6 ^ in1; - out3 = tmp0 ^ out4; - out2 = out4 ^ out7 ^ in7; - out5 = out3 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_20(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in4; - out0 = in3 ^ in7; - tmp0 = in3 ^ in4; - tmp1 = in6 ^ in7; - out2 = out0 ^ in5; - out4 = tmp0 ^ in5; - out3 = tmp0 ^ tmp1; - out7 = tmp1 ^ in2; - out6 = tmp1 ^ in1 ^ in5; - out5 = out2 ^ out3 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_21(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in1 ^ in4; - tmp0 = in4 ^ in6; - out4 = in3 ^ in5; - out7 = in2 ^ in6; - out0 = in0 ^ in3 ^ in7; - out6 = in1 ^ in5 ^ in7; - out3 = tmp0 ^ in7; - out5 = tmp0 ^ in0; - out2 = out4 ^ in2 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_22(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in3; - out1 = in0 ^ in4; - out7 = in2 ^ in7; - out4 = in4 ^ in5 ^ in7; - out5 = in0 ^ in5 ^ in6; - out6 = in1 ^ in6 ^ in7; - out3 = in2 ^ in3 ^ in4 ^ in6; - out2 = in1 ^ in3 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_23(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out7 = in2; - out0 = in0 ^ in3; - out4 = in5 ^ in7; - out5 = in0 ^ in6; - out6 = in1 ^ in7; - out3 = in2 ^ in4 ^ in6; - out1 = in0 ^ in1 ^ in4; - out2 = out4 ^ out6 ^ in2 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_24(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in4 ^ in7; - tmp0 = in3 ^ in4; - out0 = in3 ^ in6 ^ in7; - out3 = tmp0 ^ in1; - tmp1 = out0 ^ in5; - out6 = tmp1 ^ out3; - out2 = tmp1 ^ in0; - out7 = tmp1 ^ in2 ^ in3; - out5 = out2 ^ in4; - out4 = tmp0 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_25(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1 ^ in4; - tmp0 = in2 ^ in5; - out1 = out3 ^ in7; - out7 = tmp0 ^ in6; - out6 = out1 ^ in5; - out4 = out7 ^ in3 ^ in7; - out2 = out4 ^ in0; - out0 = tmp0 ^ out2; - out5 = out0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_26(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in3 ^ in6; - tmp0 = in4 ^ in7; - out7 = in2 ^ in5 ^ in7; - tmp1 = out0 ^ in0 ^ in5; - out1 = tmp0 ^ in0; - tmp2 = tmp0 ^ in6; - out2 = tmp1 ^ in1; - out5 = tmp1 ^ in7; - out6 = tmp2 ^ in1; - out4 = tmp2 ^ out7; - out3 = out0 ^ out6 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_27(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out7 = in2 ^ in5; - out0 = in0 ^ in3 ^ in6; - out6 = in1 ^ in4 ^ in7; - out4 = out7 ^ in6; - out2 = out0 ^ out7 ^ in1; - out5 = out0 ^ in7; - out1 = out6 ^ in0; - out3 = out6 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_28(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in3; - out1 = in4 ^ in6; - out0 = in3 ^ in5 ^ in7; - tmp0 = out1 ^ in7; - tmp1 = out0 ^ in4; - out7 = tmp0 ^ in2; - tmp2 = tmp0 ^ in1; - out3 = tmp1 ^ in0; - out6 = tmp1 ^ tmp2; - out4 = tmp2 ^ in3; - out5 = out3 ^ in2 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_29(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in3; - tmp0 = in1 ^ in3; - tmp1 = in4 ^ in6; - tmp2 = in0 ^ in4 ^ in7; - out6 = tmp0 ^ in5; - out4 = tmp0 ^ in6 ^ in7; - out1 = tmp1 ^ in1; - out7 = tmp1 ^ in2; - out3 = tmp2 ^ in5; - out5 = tmp2 ^ in2; - out0 = out3 ^ in3 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in3 ^ in5; - tmp0 = in1 ^ in3; - tmp1 = in0 ^ in4; - out7 = in2 ^ in4 ^ in7; - out3 = tmp1 ^ out0 ^ in2; - out2 = tmp0 ^ in7; - out6 = tmp0 ^ in6; - out1 = tmp1 ^ in6; - out5 = tmp1 ^ out7 ^ in5; - out4 = out1 ^ in0 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1 ^ in6; - out7 = in2 ^ in4; - tmp0 = in0 ^ in5; - tmp1 = in2 ^ in7; - out6 = in1 ^ in3; - out1 = out4 ^ in0 ^ in4; - out3 = tmp0 ^ out7; - out0 = tmp0 ^ in3; - out5 = tmp1 ^ in0; - out2 = tmp1 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = in2 ^ in3 ^ in4; - tmp2 = tmp0 ^ in6; - out4 = tmp1 ^ in1; - out5 = tmp1 ^ in0 ^ in5; - tmp3 = tmp2 ^ in4; - out6 = tmp2 ^ out4; - out7 = tmp3 ^ in7; - out2 = tmp3 ^ out5; - out3 = out6 ^ in0; - out0 = tmp1 ^ out7; - out1 = tmp0 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - out4 = tmp0 ^ in1; - tmp1 = tmp0 ^ in0; - out2 = tmp1 ^ in6; - out5 = tmp1 ^ in4; - tmp2 = out2 ^ in2; - tmp3 = tmp2 ^ in5; - out0 = tmp3 ^ in7; - out7 = tmp3 ^ out5; - out6 = out4 ^ out7 ^ in6; - out3 = tmp2 ^ out6; - out1 = out0 ^ out6 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in7; - out0 = in3 ^ in5 ^ in6; - tmp1 = tmp0 ^ in0; - tmp2 = tmp0 ^ in2; - out1 = tmp1 ^ in6; - out4 = tmp2 ^ in1; - out7 = tmp2 ^ in5; - out3 = out0 ^ out4 ^ in0; - out2 = out3 ^ out7 ^ in7; - out6 = tmp1 ^ out2; - out5 = tmp1 ^ out7 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_2F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in2 ^ in5; - out4 = in1 ^ in2 ^ in7; - out6 = in1 ^ in3 ^ in4; - out5 = tmp0 ^ in2; - tmp2 = tmp0 ^ in6; - out7 = tmp1 ^ in4; - out0 = tmp2 ^ in5; - out2 = tmp2 ^ out4; - out1 = tmp2 ^ out6 ^ in7; - out3 = tmp1 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_30(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in4 ^ in5; - tmp0 = in3 ^ in6; - tmp1 = in4 ^ in7; - out6 = in1 ^ in2 ^ in5; - out3 = tmp0 ^ in5; - out4 = tmp0 ^ in0; - out7 = tmp0 ^ in2; - out0 = tmp1 ^ in3; - out2 = tmp1 ^ out3; - out5 = tmp1 ^ in0 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_31(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in5 ^ in6; - tmp0 = in4 ^ in5; - tmp1 = in0 ^ in3 ^ in4; - tmp2 = out3 ^ in2; - out1 = tmp0 ^ in1; - out0 = tmp1 ^ in7; - out4 = tmp1 ^ in6; - out6 = tmp2 ^ in1; - out2 = tmp2 ^ out0 ^ in0; - out5 = out1 ^ in0 ^ in7; - out7 = tmp0 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_32(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in3 ^ in4; - out7 = in2 ^ in3; - tmp0 = in5 ^ in6; - tmp1 = in0 ^ in7; - out6 = in1 ^ in2; - out1 = in0 ^ in4 ^ in5; - out2 = tmp0 ^ out0 ^ in1; - out3 = tmp0 ^ out7 ^ in7; - out4 = tmp1 ^ in6; - out5 = tmp1 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_33(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - tmp1 = in0 ^ in4; - tmp2 = in1 ^ in5; - out6 = in1 ^ in2 ^ in6; - out7 = tmp0 ^ in7; - out0 = tmp1 ^ in3; - out1 = tmp1 ^ tmp2; - tmp3 = tmp2 ^ in7; - tmp4 = tmp2 ^ in4 ^ in6; - out5 = tmp3 ^ in0; - out3 = tmp3 ^ out6; - out4 = tmp4 ^ out5; - out2 = tmp0 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_34(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in4; - tmp1 = in4 ^ in5; - tmp2 = tmp0 ^ in1; - tmp3 = tmp0 ^ in6; - out1 = tmp1 ^ in7; - tmp4 = tmp1 ^ in2; - out5 = tmp2 ^ in0; - out3 = tmp2 ^ out1; - out0 = tmp3 ^ in7; - out7 = tmp3 ^ tmp4; - out6 = tmp4 ^ in1; - out2 = out3 ^ out5 ^ in3; - out4 = tmp4 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_35(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in6; - tmp1 = in5 ^ in7; - out7 = tmp0 ^ tmp1 ^ in3; - out3 = tmp1 ^ in1; - out1 = out3 ^ in4; - tmp2 = out1 ^ in7; - out5 = tmp2 ^ in0 ^ in3; - out6 = tmp0 ^ tmp2; - out0 = out3 ^ out5 ^ in6; - out4 = tmp0 ^ out0; - out2 = out4 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_36(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0 ^ in2; - tmp0 = in1 ^ in3; - out0 = in3 ^ in4 ^ in6; - out6 = in1 ^ in2 ^ in4; - out5 = tmp0 ^ in0; - tmp1 = out5 ^ in5; - out2 = tmp1 ^ in4; - out3 = tmp1 ^ out4; - out1 = tmp0 ^ out2 ^ in7; - out7 = out3 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_37(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in2 ^ in4; - tmp2 = tmp0 ^ in6; - out3 = tmp0 ^ in5; - out4 = tmp1 ^ in0; - out6 = tmp2 ^ in4; - out1 = out3 ^ out4 ^ in7; - tmp3 = out4 ^ in1 ^ in3; - out7 = tmp3 ^ out1; - out2 = tmp3 ^ in5; - out5 = tmp1 ^ out2; - out0 = tmp2 ^ tmp3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_38(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in3; - tmp0 = in3 ^ in4; - tmp1 = in5 ^ in7; - tmp2 = out3 ^ in1; - out2 = tmp0 ^ in6; - out0 = tmp0 ^ tmp1; - out4 = tmp1 ^ tmp2; - out7 = out2 ^ in2; - out1 = out2 ^ in3 ^ in5; - out6 = out4 ^ in0 ^ in2; - out5 = tmp2 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_39(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0; - tmp0 = in1 ^ in5; - tmp1 = tmp0 ^ in4; - out1 = tmp1 ^ in6; - out5 = out1 ^ in0 ^ in2; - tmp2 = tmp0 ^ out5; - out2 = tmp2 ^ in0 ^ in3; - out7 = out2 ^ in7; - out6 = tmp1 ^ out7; - out4 = tmp2 ^ out6; - out0 = out4 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in0 ^ in2; - tmp2 = in3 ^ in4; - tmp3 = in1 ^ in6; - tmp4 = in3 ^ in7; - out4 = tmp0 ^ in5; - out5 = tmp1 ^ tmp3; - out3 = tmp1 ^ tmp4; - out0 = tmp2 ^ in5; - out7 = tmp2 ^ in2; - tmp5 = tmp3 ^ in4; - out2 = tmp4 ^ tmp5; - out1 = tmp5 ^ out4; - out6 = tmp0 ^ out3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in6; - tmp1 = in2 ^ in7; - tmp2 = tmp0 ^ in3; - out3 = tmp1 ^ in0; - out6 = tmp1 ^ tmp2; - out2 = out6 ^ in4; - out7 = tmp0 ^ out2; - out0 = out3 ^ out7 ^ in5; - out5 = out0 ^ out2 ^ in7; - out1 = tmp2 ^ out0; - out4 = out1 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in2 ^ in7; - tmp2 = in1 ^ in6 ^ in7; - out2 = tmp0 ^ in4; - out3 = tmp0 ^ tmp2; - out4 = tmp1 ^ out3 ^ in5; - out5 = tmp2 ^ out2 ^ in2; - out1 = out4 ^ out5 ^ in6; - out0 = out1 ^ in3; - out7 = tmp1 ^ out0; - out6 = tmp2 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - tmp1 = tmp0 ^ in3; - out2 = tmp1 ^ in4; - tmp2 = out2 ^ in5; - out4 = tmp2 ^ in1 ^ in6; - out5 = out4 ^ in7; - out6 = out5 ^ in0; - out7 = out6 ^ in1; - out0 = tmp0 ^ out7; - out1 = tmp1 ^ out5; - out3 = tmp2 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in5; - tmp1 = tmp0 ^ in4; - out0 = tmp1 ^ in6; - out7 = tmp1 ^ in2; - out6 = out7 ^ in1 ^ in5 ^ in7; - out2 = out6 ^ in0 ^ in2; - out4 = out0 ^ out6 ^ in0; - out5 = tmp0 ^ out4; - out3 = out5 ^ in7; - out1 = out3 ^ out6 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_3F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - out3 = tmp0 ^ in2 ^ in6; - tmp1 = out3 ^ in5 ^ in7; - out4 = tmp1 ^ in4; - out5 = tmp1 ^ in3; - out1 = out4 ^ in2; - out7 = out1 ^ out3 ^ in3; - out2 = tmp0 ^ out7 ^ in5; - tmp2 = out2 ^ in0; - out6 = tmp2 ^ in6; - out0 = tmp1 ^ tmp2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_40(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in3 ^ in7; - tmp0 = in3 ^ in4; - tmp1 = in6 ^ in7; - out4 = tmp0 ^ in2; - out5 = tmp0 ^ in5; - out0 = tmp1 ^ in2; - out7 = tmp1 ^ in1 ^ in5; - out2 = out0 ^ in4; - out3 = out2 ^ out5 ^ in7; - out6 = out3 ^ out4 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_41(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in2 ^ in3; - tmp0 = in5 ^ in6; - tmp1 = in6 ^ in7; - out5 = in3 ^ in4; - out1 = in1 ^ in3 ^ in7; - out6 = in0 ^ in4 ^ in5; - out3 = tmp0 ^ in2; - out7 = tmp0 ^ in1; - out2 = tmp1 ^ in4; - out0 = tmp1 ^ in0 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_42(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in2 ^ in6; - out5 = in3 ^ in5; - out1 = in0 ^ in3 ^ in7; - out7 = in1 ^ in5 ^ in7; - out4 = in2 ^ in4 ^ in7; - out6 = in0 ^ in4 ^ in6; - out2 = out0 ^ in1 ^ in4; - out3 = out5 ^ in6 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_43(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in3; - out7 = in1 ^ in5; - out4 = in2 ^ in7; - out6 = in0 ^ in4; - out0 = in0 ^ in2 ^ in6; - out3 = in5 ^ in6 ^ in7; - out2 = in1 ^ in4 ^ in6; - out1 = in0 ^ in1 ^ in3 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_44(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in3; - out0 = in2 ^ in7; - tmp0 = in4 ^ in7; - out7 = in1 ^ in6 ^ in7; - out6 = in0 ^ in5 ^ in6; - out4 = tmp0 ^ in3 ^ in6; - out3 = out0 ^ in1 ^ in3 ^ in5; - out2 = out0 ^ in0 ^ in4; - out5 = tmp0 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_45(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in1 ^ in3; - out7 = in1 ^ in6; - out5 = in4 ^ in7; - out6 = in0 ^ in5; - out0 = in0 ^ in2 ^ in7; - out4 = in3 ^ in6 ^ in7; - out2 = out5 ^ in0; - out3 = out0 ^ out6 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_46(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in2; - out1 = in0 ^ in3; - out7 = in1 ^ in7; - out4 = in4 ^ in6; - out5 = in5 ^ in7; - out6 = in0 ^ in6; - out3 = in1 ^ in3 ^ in5; - out2 = out4 ^ out6 ^ in1 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_47(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in6; - out7 = in1; - out5 = in7; - out6 = in0; - tmp0 = in0 ^ in1; - out3 = in1 ^ in5; - out0 = in0 ^ in2; - out1 = tmp0 ^ in3; - out2 = tmp0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_48(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - out1 = in3 ^ in6 ^ in7; - out3 = tmp0 ^ in0; - out0 = tmp0 ^ out1 ^ in5; - tmp1 = out0 ^ in4; - out2 = tmp1 ^ in7; - out5 = tmp1 ^ in3; - out4 = out5 ^ in1; - out7 = tmp0 ^ out4; - out6 = tmp1 ^ out3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_49(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in2; - tmp0 = in2 ^ in5; - out2 = in4 ^ in5 ^ in6; - tmp1 = tmp0 ^ out2 ^ in3; - out7 = out2 ^ in1; - out5 = tmp1 ^ in7; - out4 = out5 ^ out7 ^ in6; - out1 = tmp0 ^ out4; - out6 = out1 ^ out7 ^ in0; - out0 = tmp1 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in6; - tmp1 = in3 ^ in7; - out0 = tmp0 ^ in5; - out3 = tmp1 ^ in0; - out5 = tmp1 ^ out0; - out4 = out0 ^ in1 ^ in4; - out1 = out3 ^ in6; - out2 = out4 ^ in7; - out6 = out1 ^ in4; - out7 = tmp0 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in7; - tmp0 = in1 ^ in5; - tmp1 = in2 ^ in6; - tmp2 = out3 ^ in3; - out7 = tmp0 ^ in4; - out4 = tmp0 ^ tmp1; - tmp3 = tmp1 ^ in0; - out6 = tmp2 ^ in4; - out5 = tmp2 ^ tmp3; - out1 = tmp2 ^ in1 ^ in6; - out2 = out7 ^ in6 ^ in7; - out0 = tmp3 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in3 ^ in6; - tmp0 = in2 ^ in5; - tmp1 = out1 ^ in5 ^ in7; - out0 = tmp0 ^ in7; - tmp2 = tmp0 ^ in4; - out6 = tmp1 ^ in0; - out2 = tmp2 ^ in0; - out5 = tmp2 ^ in6; - out3 = tmp0 ^ out6 ^ in1; - out7 = out0 ^ out5 ^ in1; - out4 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in5; - tmp1 = in1 ^ in6; - out4 = in1 ^ in3 ^ in5; - tmp2 = tmp0 ^ in7; - out2 = tmp0 ^ in4; - out1 = tmp1 ^ in3; - out7 = tmp1 ^ in4; - out0 = tmp2 ^ in2; - out6 = tmp2 ^ in3; - out5 = out7 ^ in1 ^ in2; - out3 = tmp1 ^ out0 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in2 ^ in5; - out7 = in1 ^ in4 ^ in7; - out1 = in0 ^ in3 ^ in6; - out5 = out0 ^ in6; - out4 = out7 ^ in5; - out3 = out1 ^ in1; - out6 = out1 ^ in7; - out2 = out4 ^ in0 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_4F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in2 ^ in6; - out7 = in1 ^ in4; - out3 = in0 ^ in1 ^ in6; - out4 = in1 ^ in5 ^ in7; - out0 = in0 ^ in2 ^ in5; - out6 = in0 ^ in3 ^ in7; - out1 = out3 ^ in3; - out2 = out4 ^ in0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_50(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in7; - tmp0 = in3 ^ in5; - out0 = out2 ^ in4 ^ in6; - out1 = tmp0 ^ in7; - tmp1 = tmp0 ^ in6; - out3 = out0 ^ in3; - out7 = tmp1 ^ in1; - tmp2 = tmp1 ^ in0; - out5 = out3 ^ in1 ^ in2; - out4 = tmp2 ^ in2; - out6 = tmp2 ^ out3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_51(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in7; - out3 = in2 ^ in4 ^ in6 ^ in7; - out0 = out3 ^ in0; - out6 = out0 ^ in5; - out4 = out6 ^ in3 ^ in7; - out1 = out0 ^ out4 ^ in1; - out7 = out1 ^ in6; - out5 = out7 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_52(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in1 ^ in2; - tmp0 = in2 ^ in4; - tmp1 = in3 ^ in5; - tmp2 = in3 ^ in6; - tmp3 = in0 ^ in7; - out0 = tmp0 ^ in6; - out6 = tmp0 ^ tmp3; - out7 = tmp1 ^ in1; - out1 = tmp1 ^ tmp3; - out3 = tmp2 ^ in4; - out5 = tmp2 ^ in1 ^ in7; - out4 = tmp2 ^ out1 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_53(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in1; - out3 = in4 ^ in6; - out0 = out3 ^ in0 ^ in2; - out6 = out0 ^ in7; - out4 = out6 ^ in5; - out7 = out0 ^ out4 ^ in1 ^ in3; - out1 = out7 ^ in0; - out5 = out7 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_54(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in3 ^ in5; - tmp0 = in1 ^ in3; - tmp1 = in2 ^ in4; - tmp2 = in0 ^ in7; - out5 = in1 ^ in4 ^ in6; - out4 = tmp2 ^ out1; - out7 = tmp0 ^ in6; - out3 = tmp0 ^ tmp1; - out0 = tmp1 ^ in7; - tmp3 = tmp2 ^ in2; - out2 = tmp3 ^ in6; - out6 = tmp3 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_55(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in3; - tmp1 = in1 ^ in4; - tmp2 = in6 ^ in7; - out7 = tmp0 ^ tmp2; - out1 = tmp0 ^ in5; - out3 = tmp1 ^ in2; - out5 = tmp1 ^ in5 ^ in6; - out2 = tmp2 ^ in0; - out4 = out5 ^ out7 ^ in0; - out6 = out2 ^ in2 ^ in5; - out0 = out5 ^ out6 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_56(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in2 ^ in4; - tmp0 = in0 ^ in2; - out4 = in0 ^ in5; - out7 = in1 ^ in3; - out5 = in1 ^ in6; - out6 = tmp0 ^ in7; - out2 = tmp0 ^ out5; - out1 = out4 ^ in3; - out3 = out7 ^ in4 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_57(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in5; - tmp1 = in1 ^ in7; - out0 = in0 ^ in2 ^ in4; - out5 = in1 ^ in5 ^ in6; - out4 = tmp0 ^ in4; - out1 = tmp0 ^ in1 ^ in3; - out2 = tmp0 ^ out5; - out3 = tmp1 ^ in4; - out7 = tmp1 ^ in3; - out6 = tmp1 ^ out2 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_58(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in5; - tmp0 = in2 ^ in3 ^ in4; - out5 = tmp0 ^ in1; - out6 = tmp0 ^ in0 ^ in5; - out3 = out6 ^ in7; - tmp1 = out2 ^ out5; - out7 = tmp1 ^ in6; - out4 = tmp1 ^ out3 ^ in3; - out0 = out4 ^ out7 ^ in0; - out1 = tmp0 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_59(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in5; - tmp0 = in0 ^ in5 ^ in7; - out3 = tmp0 ^ in2 ^ in4; - out0 = out3 ^ in6; - tmp1 = out0 ^ in7; - out6 = tmp1 ^ in3; - out5 = out6 ^ in0 ^ in1 ^ in6; - out4 = tmp0 ^ out5; - out1 = tmp1 ^ out4; - out7 = out1 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in2 ^ in5; - out5 = tmp0 ^ in3; - out4 = tmp0 ^ in0; - tmp2 = tmp1 ^ in4; - out2 = tmp1 ^ in1 ^ in7; - out7 = tmp2 ^ out5; - out6 = out4 ^ out7 ^ in5; - out0 = tmp2 ^ in6; - out1 = out0 ^ out6 ^ in7; - out3 = tmp1 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - tmp1 = in0 ^ in4; - tmp2 = in1 ^ in5; - out5 = tmp0 ^ tmp2; - tmp3 = tmp1 ^ in6; - out3 = tmp1 ^ in5; - out2 = tmp2 ^ in7; - tmp4 = out3 ^ in2; - out7 = out2 ^ in3 ^ in4; - out0 = tmp4 ^ in6; - out6 = tmp0 ^ tmp3; - out4 = tmp2 ^ tmp4; - out1 = tmp3 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in6; - tmp1 = in0 ^ in2 ^ in5; - out1 = tmp0 ^ in5; - tmp2 = tmp0 ^ in1; - out2 = tmp1 ^ in6; - out6 = tmp1 ^ in3; - out4 = tmp2 ^ in0; - out7 = tmp2 ^ in4; - out3 = tmp1 ^ out7; - out0 = out3 ^ out4 ^ in7; - out5 = out0 ^ in1 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in0 ^ in6; - out2 = tmp1 ^ in5; - tmp2 = out2 ^ in3; - out6 = tmp2 ^ in2; - out1 = tmp0 ^ tmp2; - tmp3 = out1 ^ in4 ^ in5; - out4 = tmp3 ^ in0; - out7 = tmp3 ^ in7; - tmp4 = out4 ^ out6; - out5 = tmp4 ^ in7; - out0 = tmp0 ^ out5; - out3 = tmp1 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = in3 ^ in5; - tmp2 = in1 ^ in7; - out7 = in1 ^ in3 ^ in4; - out0 = tmp0 ^ in4; - tmp3 = tmp1 ^ in0; - out5 = tmp2 ^ in2; - out1 = tmp3 ^ in6; - out6 = tmp0 ^ tmp3; - tmp4 = tmp2 ^ out1; - out3 = tmp4 ^ in4; - out4 = tmp1 ^ tmp4; - out2 = tmp0 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_5F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in5; - tmp1 = in0 ^ in6; - tmp2 = tmp0 ^ in7; - tmp3 = tmp1 ^ in3; - out2 = tmp1 ^ tmp2; - out5 = tmp2 ^ in2; - out6 = tmp3 ^ in2; - out3 = out2 ^ in4; - out4 = out3 ^ in5; - out1 = tmp0 ^ tmp3; - out7 = tmp3 ^ out4; - out0 = out4 ^ out5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_60(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in2 ^ in5; - tmp0 = in3 ^ in6; - out1 = in3 ^ in4 ^ in7; - out7 = out4 ^ in1; - tmp1 = out4 ^ in4; - out0 = tmp0 ^ in2; - out5 = tmp0 ^ in0; - out2 = tmp0 ^ tmp1; - out3 = tmp1 ^ in7; - out6 = out3 ^ out7 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_61(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - out4 = tmp0 ^ in4; - tmp1 = out4 ^ in3; - out3 = tmp1 ^ in7; - out2 = tmp1 ^ in2 ^ in6; - out1 = tmp0 ^ out3 ^ in1; - out0 = out2 ^ out4 ^ in0; - out7 = tmp1 ^ out1; - out6 = out0 ^ out1 ^ in2; - out5 = tmp0 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_62(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in4 ^ in5; - tmp0 = in0 ^ in3 ^ in4; - out1 = tmp0 ^ in7; - out5 = tmp0 ^ in6; - tmp1 = out1 ^ in0; - tmp2 = tmp1 ^ out3; - out4 = tmp2 ^ in2; - tmp3 = tmp2 ^ in1; - out0 = out4 ^ in5 ^ in6; - out7 = tmp3 ^ out0; - out6 = tmp0 ^ tmp3; - out2 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_63(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in4; - tmp1 = in1 ^ in7; - out3 = tmp0 ^ in5; - tmp2 = out3 ^ in6; - out4 = out3 ^ in2 ^ in7; - out5 = tmp2 ^ in0; - tmp3 = out5 ^ in3; - out0 = tmp3 ^ out4; - out2 = tmp1 ^ tmp2; - out6 = tmp1 ^ tmp3; - tmp4 = tmp0 ^ out2; - out1 = tmp4 ^ out5; - out7 = tmp4 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_64(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in2 ^ in3; - out1 = in3 ^ in4; - out7 = in1 ^ in2; - tmp0 = in4 ^ in5; - tmp1 = in0 ^ in7; - out4 = in5 ^ in6 ^ in7; - out2 = tmp0 ^ out0 ^ in0; - out3 = tmp0 ^ out7 ^ in6; - out5 = tmp1 ^ in6; - out6 = tmp1 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_65(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in4 ^ in5; - tmp2 = in6 ^ in7; - out7 = in1 ^ in2 ^ in7; - out1 = in1 ^ in3 ^ in4; - out0 = tmp0 ^ in2; - out2 = tmp0 ^ tmp1; - out4 = tmp1 ^ tmp2; - tmp3 = tmp2 ^ in0; - out3 = out4 ^ out7 ^ in3; - out5 = tmp3 ^ in5; - out6 = tmp3 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_66(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in2 ^ in3; - tmp2 = in0 ^ in4; - out7 = tmp0 ^ in6; - out0 = tmp1 ^ in7; - out1 = tmp2 ^ in3; - tmp3 = tmp2 ^ in6; - tmp4 = out1 ^ in5; - out5 = tmp3 ^ in7; - out4 = tmp3 ^ tmp4; - out2 = tmp0 ^ tmp4 ^ in7; - out6 = tmp1 ^ out2 ^ in4; - out3 = tmp3 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_67(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = tmp0 ^ in1; - tmp2 = tmp0 ^ in7; - out1 = tmp1 ^ in4; - out0 = tmp2 ^ in2; - tmp3 = out1 ^ in7; - out2 = tmp3 ^ in5; - out3 = out2 ^ in0 ^ in6; - out7 = tmp1 ^ out0 ^ in6; - out5 = tmp1 ^ out3; - out4 = tmp2 ^ out5; - out6 = tmp3 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_68(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in4; - tmp1 = in2 ^ in3 ^ in5; - tmp2 = tmp0 ^ in1; - tmp3 = tmp0 ^ in6; - out0 = tmp1 ^ in6; - out6 = tmp2 ^ in0; - out7 = tmp1 ^ tmp2; - out1 = tmp3 ^ in7; - out2 = out1 ^ in2; - out4 = tmp2 ^ out2; - out3 = out4 ^ out6 ^ in3; - out5 = tmp3 ^ out3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_69(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in6 ^ in7; - out2 = tmp0 ^ in3 ^ in4; - out1 = out2 ^ in1; - out3 = out2 ^ in0 ^ in2; - out4 = out1 ^ in2 ^ in3; - out6 = out1 ^ in0 ^ in7; - out7 = out4 ^ in5 ^ in6; - out5 = out4 ^ out6 ^ in5; - out0 = tmp0 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in6; - out3 = in0 ^ in4 ^ in6; - tmp1 = tmp0 ^ in3; - out4 = tmp1 ^ in1; - tmp2 = tmp1 ^ in7; - out2 = out4 ^ in4; - out0 = tmp2 ^ in5; - out5 = tmp2 ^ out3; - out7 = out2 ^ in3 ^ in5; - out1 = tmp0 ^ out5; - out6 = tmp1 ^ out7 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in6; - out2 = tmp0 ^ in1 ^ in3; - out4 = out2 ^ in2; - tmp1 = out2 ^ in0; - out7 = out4 ^ in3 ^ in5 ^ in7; - out1 = tmp1 ^ in7; - out3 = tmp1 ^ in1; - out6 = tmp1 ^ in5; - out0 = tmp1 ^ out7 ^ in6; - out5 = tmp0 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1; - tmp0 = in2 ^ in3; - out5 = in0 ^ in2; - out1 = in3 ^ in4 ^ in6; - tmp1 = out5 ^ in1; - out0 = tmp0 ^ in5; - out6 = tmp0 ^ tmp1; - out3 = tmp1 ^ in4; - out7 = out3 ^ in0; - out2 = out6 ^ out7 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1 ^ in4; - tmp0 = in0 ^ in2; - tmp1 = out4 ^ in3; - out7 = out4 ^ in2 ^ in7; - out5 = tmp0 ^ in5; - out3 = tmp0 ^ tmp1; - out1 = tmp1 ^ in6; - out0 = out5 ^ in3; - out2 = out3 ^ out7 ^ in4; - out6 = out1 ^ in0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in3; - tmp1 = in0 ^ in4; - out4 = tmp0 ^ in7; - out6 = tmp0 ^ in0 ^ in5; - out5 = tmp1 ^ in2; - tmp2 = tmp1 ^ in3; - out3 = tmp2 ^ out4; - out1 = tmp2 ^ in6; - out2 = tmp0 ^ out5; - out0 = out2 ^ out3 ^ in5; - out7 = out1 ^ out2 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_6F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in7; - tmp1 = tmp0 ^ in4; - tmp2 = tmp0 ^ in0 ^ in2; - out4 = tmp1 ^ in1; - out0 = tmp2 ^ in5; - out3 = out4 ^ in0; - out2 = out3 ^ in7; - out1 = out2 ^ in6; - out6 = out1 ^ in4 ^ in5; - out7 = tmp2 ^ out1; - out5 = tmp1 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_70(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in2; - tmp0 = in2 ^ in4; - out2 = in2 ^ in3 ^ in5; - tmp1 = tmp0 ^ in6; - tmp2 = out2 ^ in7; - out0 = tmp1 ^ in3; - out4 = tmp1 ^ in0; - out7 = tmp2 ^ in1; - out6 = out4 ^ in1; - out5 = out7 ^ in0 ^ in2; - out1 = tmp0 ^ tmp2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_71(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in3 ^ in5; - out3 = in2 ^ in3; - tmp0 = in0 ^ in2; - tmp1 = out2 ^ in1; - out4 = tmp0 ^ in6; - tmp2 = tmp0 ^ in1; - out7 = tmp1 ^ in2; - out1 = tmp1 ^ in4 ^ in7; - out0 = out4 ^ in3 ^ in4; - out6 = tmp2 ^ in4; - out5 = tmp2 ^ out3 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_72(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in7; - tmp0 = in0 ^ in4; - tmp1 = tmp0 ^ in3 ^ in7; - out1 = tmp1 ^ in5; - out5 = out1 ^ in1; - tmp2 = tmp0 ^ out5; - out2 = tmp2 ^ in2; - out7 = out2 ^ in6; - out6 = tmp1 ^ out7; - out4 = tmp2 ^ out6; - out0 = out4 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_73(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in3 ^ in7; - out2 = out3 ^ in1 ^ in5; - out1 = out2 ^ in0 ^ in4; - out5 = out1 ^ in5; - out6 = out1 ^ out3 ^ in2; - out0 = out2 ^ out6 ^ in6; - out7 = out0 ^ out1 ^ in3; - out4 = out0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_74(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in4; - tmp1 = in1 ^ in2 ^ in6; - out4 = in0 ^ in4 ^ in7; - out5 = in0 ^ in1 ^ in5; - out0 = tmp0 ^ in2; - out1 = tmp0 ^ in5; - out3 = tmp1 ^ in7; - out6 = tmp1 ^ in0; - out2 = tmp1 ^ out5 ^ in3; - out7 = out3 ^ in3 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_75(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0 ^ in7; - tmp0 = in1 ^ in3; - out5 = in0 ^ in1; - out7 = tmp0 ^ in2; - tmp1 = tmp0 ^ in4; - out6 = out5 ^ in2; - tmp2 = out7 ^ in6; - out1 = tmp1 ^ in5; - out0 = tmp1 ^ out6; - out3 = tmp2 ^ in7; - out2 = tmp2 ^ out6 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_76(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1 ^ in6; - tmp0 = in0 ^ in5; - tmp1 = in3 ^ in7; - tmp2 = tmp0 ^ in4; - tmp3 = tmp1 ^ in2; - out5 = tmp2 ^ in1; - out1 = tmp2 ^ in3; - out0 = tmp3 ^ in4; - out4 = out1 ^ in5; - out7 = tmp3 ^ out3; - out2 = tmp0 ^ out7; - out6 = tmp1 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_77(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0 ^ in3; - tmp0 = in1 ^ in4; - tmp1 = in1 ^ in6; - tmp2 = out4 ^ in5; - out5 = tmp0 ^ in0; - out1 = tmp0 ^ tmp2; - out3 = tmp1 ^ in3; - out2 = tmp1 ^ tmp2 ^ in7; - out7 = out3 ^ in2; - tmp3 = out7 ^ in6; - out6 = tmp2 ^ tmp3; - out0 = tmp3 ^ out5 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_78(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in2 ^ in7; - tmp2 = in0 ^ in5 ^ in6; - out2 = tmp1 ^ in3; - out3 = tmp2 ^ in2; - out5 = out3 ^ in1 ^ in3; - out0 = tmp0 ^ out3 ^ in4; - out1 = tmp1 ^ out0; - out4 = out1 ^ out5 ^ in5; - out7 = tmp0 ^ out4; - out6 = tmp2 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_79(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in3 ^ in7; - tmp0 = in3 ^ in4; - tmp1 = in1 ^ in5; - tmp2 = tmp1 ^ in2; - out4 = tmp2 ^ in0 ^ in7; - tmp3 = out4 ^ in5; - out5 = tmp3 ^ out2 ^ in6; - out7 = tmp0 ^ tmp2; - out6 = tmp0 ^ tmp3; - out3 = tmp1 ^ out5; - out0 = out3 ^ in4; - out1 = tmp3 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - out2 = tmp0 ^ in3; - tmp1 = out2 ^ in4; - out4 = tmp1 ^ in0 ^ in5; - out5 = out4 ^ in6; - out6 = out5 ^ in7; - out7 = out6 ^ in0; - out0 = out7 ^ in1; - out1 = tmp0 ^ out6; - out3 = tmp1 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in1 ^ in3; - tmp0 = in0 ^ in5; - out4 = tmp0 ^ out2 ^ in2; - tmp1 = out4 ^ in4; - out6 = tmp1 ^ in7; - out5 = tmp1 ^ in5 ^ in6; - out0 = out6 ^ in1 ^ in6; - tmp2 = out0 ^ in2; - out1 = tmp2 ^ in1; - out3 = tmp2 ^ in4; - out7 = tmp0 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in5; - tmp1 = tmp0 ^ in4; - out0 = tmp1 ^ in2; - out1 = tmp1 ^ in6; - out7 = out0 ^ in1 ^ in5 ^ in7; - out5 = out1 ^ out7 ^ in0; - out3 = out5 ^ in6; - out6 = tmp0 ^ out5; - out2 = out6 ^ in1; - out4 = out2 ^ out7 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = tmp0 ^ in3; - tmp2 = tmp0 ^ in6; - out7 = tmp1 ^ in4; - tmp3 = tmp2 ^ in0; - out5 = tmp3 ^ in7; - out4 = tmp3 ^ in2 ^ in5; - out2 = tmp1 ^ out5; - out6 = tmp2 ^ out2; - out0 = out4 ^ out7 ^ in6; - out1 = tmp3 ^ out0; - out3 = out6 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in4; - tmp1 = in0 ^ in5; - out1 = tmp0 ^ tmp1 ^ in6; - out3 = tmp1 ^ in1; - out4 = out1 ^ in1 ^ in7; - tmp2 = out4 ^ in3; - out5 = tmp2 ^ in2; - out6 = tmp0 ^ out5; - out7 = tmp1 ^ out4 ^ in2; - out2 = out6 ^ in5 ^ in7; - out0 = tmp2 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_7F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in7; - tmp1 = tmp0 ^ in3 ^ in5; - tmp2 = tmp1 ^ in0; - out0 = tmp2 ^ in4; - out6 = tmp2 ^ in1; - out3 = tmp0 ^ out6; - tmp3 = out3 ^ in6; - out1 = tmp3 ^ in4; - out2 = tmp3 ^ in5; - out4 = tmp3 ^ in7; - out5 = tmp1 ^ out1; - out7 = out0 ^ out4 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_80(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - tmp1 = in4 ^ in5; - out1 = in2 ^ in6 ^ in7; - out5 = tmp0 ^ in4; - tmp2 = tmp0 ^ in1; - out6 = tmp1 ^ in3; - out7 = tmp1 ^ in0 ^ in6; - out4 = tmp2 ^ in7; - out3 = tmp2 ^ out6; - out2 = out3 ^ out5 ^ in6; - out0 = out2 ^ in3 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_81(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in6; - tmp1 = tmp0 ^ in3; - out6 = tmp1 ^ in5; - out5 = out6 ^ in2 ^ in6; - out3 = out5 ^ in1; - out2 = tmp0 ^ out3; - out1 = out3 ^ out6 ^ in7; - out4 = tmp1 ^ out1; - out7 = out2 ^ out4 ^ in0; - out0 = out7 ^ in1 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_82(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1 ^ in2; - tmp0 = in6 ^ in7; - out5 = in2 ^ in3; - out6 = in3 ^ in4; - out7 = in0 ^ in4 ^ in5; - out0 = in1 ^ in5 ^ in6; - out1 = tmp0 ^ in0 ^ in2; - out2 = tmp0 ^ in3 ^ in5; - out3 = tmp0 ^ out0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_83(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in2 ^ in5; - tmp2 = in3 ^ in6; - out4 = in1 ^ in2 ^ in4; - out0 = tmp0 ^ in5 ^ in6; - out5 = tmp1 ^ in3; - tmp3 = tmp1 ^ in7; - out6 = tmp2 ^ in4; - out2 = tmp2 ^ tmp3; - tmp4 = tmp3 ^ out4; - out1 = tmp3 ^ out0; - out3 = tmp4 ^ in3; - out7 = tmp0 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_84(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in2 ^ in6; - out6 = in3 ^ in5; - out0 = in1 ^ in5 ^ in7; - out7 = in0 ^ in4 ^ in6; - out4 = in1 ^ in3 ^ in6; - out5 = in2 ^ in4 ^ in7; - out2 = out6 ^ in0 ^ in1; - out3 = out5 ^ in5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_85(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in6; - tmp1 = in3 ^ in6; - tmp2 = tmp0 ^ in4; - out1 = tmp0 ^ in2; - out6 = tmp1 ^ in5; - out4 = tmp2 ^ in3; - tmp3 = out1 ^ out6; - out2 = tmp3 ^ in0; - out3 = tmp2 ^ tmp3 ^ in7; - out7 = out2 ^ out3 ^ in1; - out5 = tmp1 ^ out3; - out0 = tmp2 ^ out7 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_86(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out6 = in3; - out7 = in0 ^ in4; - out0 = in1 ^ in5; - out5 = in2 ^ in7; - out3 = in4 ^ in5 ^ in6; - out1 = in0 ^ in2 ^ in6; - out4 = in1 ^ in6 ^ in7; - out2 = in0 ^ in3 ^ in5 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_87(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out6 = in3 ^ in6; - tmp0 = in0 ^ in1; - out7 = in0 ^ in4 ^ in7; - out5 = in2 ^ in5 ^ in7; - out3 = out6 ^ in4 ^ in5; - out0 = tmp0 ^ in5; - tmp1 = tmp0 ^ in6; - out2 = out5 ^ in0 ^ in3; - out1 = tmp1 ^ in2; - out4 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_88(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in2 ^ in7; - tmp0 = in5 ^ in6; - out0 = in1 ^ in6 ^ in7; - out6 = in4 ^ in5 ^ in7; - out3 = out0 ^ out1 ^ in0 ^ in4; - out7 = tmp0 ^ in0; - tmp1 = tmp0 ^ in3; - out2 = out0 ^ in3; - out4 = tmp1 ^ in2; - out5 = tmp1 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_89(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in7; - tmp1 = in2 ^ in7; - tmp2 = tmp0 ^ in6; - out1 = tmp1 ^ in1; - out7 = tmp2 ^ in5; - out0 = tmp2 ^ in1; - out2 = out1 ^ in3 ^ in6; - out6 = out7 ^ in0 ^ in4; - out5 = out6 ^ in3; - out3 = tmp0 ^ out2 ^ in4; - out4 = tmp1 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in1 ^ in6; - out7 = in0 ^ in5; - out2 = in3 ^ in6; - out6 = in4 ^ in7; - out1 = in0 ^ in2 ^ in7; - out3 = out0 ^ out6 ^ in0; - out4 = out1 ^ out7 ^ in6; - out5 = out2 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in3 ^ in6; - tmp2 = in5 ^ in7; - tmp3 = tmp0 ^ in7; - out0 = tmp0 ^ in6; - out2 = tmp1 ^ in2; - out5 = tmp1 ^ tmp2; - out7 = tmp2 ^ in0; - tmp4 = tmp3 ^ in4; - out1 = tmp3 ^ in2; - out6 = tmp4 ^ out0; - out4 = out6 ^ in2 ^ in5; - out3 = tmp1 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in2; - out0 = in1 ^ in7; - out7 = in0 ^ in6; - out5 = in4 ^ in6; - out6 = in5 ^ in7; - out2 = out0 ^ in0 ^ in3; - out3 = out5 ^ out7 ^ in2 ^ in7; - out4 = out6 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in1 ^ in2; - tmp0 = in6 ^ in7; - out0 = in0 ^ in1 ^ in7; - out5 = in4 ^ in5 ^ in6; - out6 = tmp0 ^ in5; - out7 = tmp0 ^ in0; - out4 = tmp0 ^ out5 ^ in3; - out2 = out0 ^ in2 ^ in3; - out3 = out2 ^ in1 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in1; - out4 = in5; - out7 = in0; - out5 = in6; - out6 = in7; - out3 = in0 ^ in4; - out1 = in0 ^ in2; - out2 = in0 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_8F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in0 ^ in1; - tmp0 = in0 ^ in3; - out4 = in4 ^ in5; - out7 = in0 ^ in7; - out5 = in5 ^ in6; - out6 = in6 ^ in7; - out1 = out0 ^ in2; - out2 = tmp0 ^ in2; - out3 = tmp0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_90(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in2 ^ in6 ^ in7; - out3 = tmp0 ^ in7; - out1 = tmp1 ^ in5; - tmp2 = out1 ^ in4; - out6 = tmp2 ^ in3; - out5 = out6 ^ in1; - out4 = out5 ^ in0; - out0 = tmp0 ^ tmp2; - out7 = tmp0 ^ out4; - out2 = tmp1 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_91(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in4; - tmp1 = tmp0 ^ in3 ^ in5; - out2 = tmp1 ^ in1; - out6 = tmp1 ^ in7; - tmp2 = out2 ^ in5 ^ in7; - out3 = tmp2 ^ in4; - out5 = tmp2 ^ in6; - out1 = tmp1 ^ out5 ^ in2; - tmp3 = out1 ^ in0; - out4 = tmp3 ^ in3; - out0 = tmp0 ^ tmp3; - out7 = tmp2 ^ tmp3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_92(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1; - tmp0 = in4 ^ in5; - tmp1 = tmp0 ^ in1; - out2 = tmp0 ^ in3 ^ in7; - out0 = tmp1 ^ in6; - out7 = out2 ^ in0; - out4 = out0 ^ in0 ^ in2; - out5 = out4 ^ out7 ^ in5; - out6 = tmp1 ^ out5; - out1 = out6 ^ out7 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_93(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1 ^ in3; - tmp0 = in2 ^ in7; - tmp1 = out3 ^ in6; - tmp2 = tmp0 ^ in4; - out5 = tmp0 ^ tmp1; - out6 = tmp2 ^ in3; - out2 = out6 ^ in5; - out0 = out2 ^ out5 ^ in0; - out7 = tmp1 ^ out0; - out1 = tmp2 ^ out0; - out4 = out1 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_94(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in2 ^ in6; - tmp0 = in1 ^ in4 ^ in5; - out1 = out3 ^ in5; - out5 = tmp0 ^ out3; - out0 = tmp0 ^ in7; - out4 = tmp0 ^ in0 ^ in3; - out6 = out1 ^ in3 ^ in7; - out2 = out4 ^ in6; - out7 = out0 ^ out2 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_95(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - out3 = tmp0 ^ in6; - tmp1 = tmp0 ^ in7; - tmp2 = out3 ^ in0; - out6 = tmp1 ^ in5; - tmp3 = tmp2 ^ in4; - out7 = tmp3 ^ in2; - tmp4 = tmp3 ^ in5; - out2 = tmp4 ^ in1; - tmp5 = out2 ^ in6; - out0 = tmp1 ^ tmp5; - out1 = tmp5 ^ out7; - out4 = tmp2 ^ out1; - out5 = tmp4 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_96(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in6 ^ in7; - tmp0 = in1 ^ in5; - tmp1 = in5 ^ in6; - out6 = out3 ^ in2 ^ in3; - out0 = tmp0 ^ in4; - tmp2 = tmp1 ^ in2; - out4 = out0 ^ in0 ^ in7; - out1 = tmp2 ^ in0; - out5 = tmp2 ^ in1; - out7 = tmp0 ^ out4 ^ in3; - out2 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_97(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in4; - tmp1 = in2 ^ in6; - out3 = in3 ^ in6 ^ in7; - out7 = tmp0 ^ in3; - tmp2 = tmp0 ^ in5; - out5 = tmp1 ^ in1; - out6 = tmp1 ^ out3; - out0 = tmp2 ^ in1; - out2 = tmp2 ^ out3 ^ in2; - tmp3 = out0 ^ in4; - out4 = tmp3 ^ in7; - out1 = tmp1 ^ tmp3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_98(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in5 ^ in7; - tmp1 = in1 ^ in4 ^ in7; - out1 = tmp0 ^ in2; - out0 = tmp1 ^ in6; - out2 = tmp1 ^ in3; - out6 = out0 ^ out1 ^ in1; - out5 = tmp0 ^ out2; - out3 = tmp1 ^ out6 ^ in0; - out7 = out0 ^ out5 ^ in0; - out4 = out6 ^ out7 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_99(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - out5 = in1 ^ in3 ^ in4; - out6 = in2 ^ in4 ^ in5; - out4 = tmp0 ^ in2; - tmp1 = tmp0 ^ in6; - tmp2 = out5 ^ in7; - out7 = tmp1 ^ in5; - out0 = tmp1 ^ tmp2; - out2 = tmp2 ^ in2; - out3 = out0 ^ out6 ^ in3; - out1 = tmp1 ^ out3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9A(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in3 ^ in4; - tmp0 = in0 ^ in5; - tmp1 = in1 ^ in6; - out5 = in1 ^ in3 ^ in5; - tmp2 = tmp0 ^ in7; - out3 = tmp0 ^ tmp1; - out0 = tmp1 ^ in4; - out7 = tmp2 ^ in3; - out1 = tmp2 ^ in2; - out6 = out0 ^ in1 ^ in2; - out4 = out1 ^ in4 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9B(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in1 ^ in3; - tmp0 = in3 ^ in5; - out6 = in2 ^ in4; - out4 = in0 ^ in2 ^ in7; - out7 = tmp0 ^ in0; - out2 = out6 ^ in3; - out1 = out4 ^ in1 ^ in5; - out3 = out7 ^ in1 ^ in6; - out0 = tmp0 ^ out3 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9C(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out1 = in2 ^ in5; - tmp0 = in0 ^ in3 ^ in6; - out3 = out1 ^ in0; - out6 = out1 ^ in6; - out7 = tmp0 ^ in7; - out4 = out7 ^ in4; - out2 = out4 ^ in1; - out0 = tmp0 ^ out2; - out5 = out0 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9D(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out6 = in2 ^ in5; - tmp0 = in0 ^ in3; - out5 = in1 ^ in4 ^ in7; - out1 = out6 ^ in1; - out3 = tmp0 ^ out6; - out7 = tmp0 ^ in6; - out0 = out5 ^ in0; - out4 = out7 ^ in7; - out2 = out5 ^ out7 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9E(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in1 ^ in4; - tmp0 = in0 ^ in5; - out6 = in2 ^ in6; - out7 = in0 ^ in3 ^ in7; - out4 = in0 ^ in4 ^ in6; - out5 = in1 ^ in5 ^ in7; - out1 = tmp0 ^ in2; - out3 = tmp0 ^ in7; - out2 = out4 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_9F(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out6 = in2; - out7 = in0 ^ in3; - tmp0 = in0 ^ in1; - out4 = in0 ^ in6; - out5 = in1 ^ in7; - out1 = tmp0 ^ in2 ^ in5; - out2 = out7 ^ in2 ^ in4 ^ in6; - out3 = out7 ^ in5 ^ in7; - out0 = tmp0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in6; - out2 = tmp0 ^ in7; - tmp1 = tmp0 ^ in5; - out6 = out2 ^ in3 ^ in4; - out0 = tmp1 ^ in3; - tmp2 = out0 ^ in2; - out3 = tmp2 ^ in7; - tmp3 = tmp2 ^ in1; - out5 = tmp3 ^ in0; - out4 = tmp3 ^ out6; - out7 = out5 ^ out6 ^ in1; - out1 = tmp1 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = tmp0 ^ in1; - tmp2 = tmp0 ^ in4; - out4 = tmp1 ^ in7; - out7 = tmp2 ^ in0; - out6 = tmp2 ^ out4 ^ in3; - out3 = out4 ^ in6; - out2 = out3 ^ in5; - out1 = out2 ^ in4; - out5 = out1 ^ out6 ^ in0; - out0 = tmp1 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in6; - tmp0 = in1 ^ in3 ^ in5; - out3 = tmp0 ^ in6; - out4 = tmp0 ^ in2 ^ in4; - out0 = out3 ^ in7; - out6 = out0 ^ in4; - out1 = out0 ^ out4 ^ in0; - out7 = out1 ^ in5; - out5 = out7 ^ in3 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in6; - out3 = in1 ^ in5 ^ in6; - tmp0 = out2 ^ in0; - out4 = out2 ^ out3 ^ in3; - tmp1 = tmp0 ^ in4; - out0 = tmp0 ^ out4 ^ in7; - out5 = tmp1 ^ in3; - out7 = tmp1 ^ in5; - out1 = tmp1 ^ in1 ^ in7; - out6 = tmp1 ^ out0 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in3; - tmp1 = in2 ^ in4; - tmp2 = in2 ^ in5; - tmp3 = in0 ^ in7; - out0 = tmp0 ^ in5; - out6 = tmp0 ^ in6 ^ in7; - out1 = tmp1 ^ in6; - out7 = tmp1 ^ tmp3; - out3 = tmp2 ^ in3; - tmp4 = tmp2 ^ out1; - out2 = tmp3 ^ in1; - out5 = tmp4 ^ out7; - out4 = tmp4 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in2 ^ in5; - tmp0 = in1 ^ in6; - tmp1 = in0 ^ in1; - tmp2 = in2 ^ in4; - out6 = in1 ^ in3 ^ in7; - out4 = tmp0 ^ in5; - out1 = tmp0 ^ tmp2; - out0 = tmp1 ^ in3 ^ in5; - out2 = tmp1 ^ in2 ^ in7; - out7 = tmp2 ^ in0; - out5 = tmp0 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0; - out3 = in3 ^ in5 ^ in7; - out1 = in0 ^ in2 ^ in4 ^ in6; - out0 = out3 ^ in1; - out7 = out1 ^ in7; - out6 = out0 ^ in6; - out5 = out7 ^ in5; - out4 = out6 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in2; - out3 = in5 ^ in7; - out7 = out2 ^ in4 ^ in6; - out6 = out3 ^ in1 ^ in3; - out1 = out7 ^ in1; - out5 = out7 ^ in7; - out0 = out6 ^ in0; - out4 = out6 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in4; - tmp1 = in1 ^ in6; - tmp2 = in0 ^ in2 ^ in7; - out1 = tmp0 ^ in7; - out4 = tmp0 ^ in6; - out0 = tmp1 ^ in3; - out2 = tmp1 ^ in5; - out6 = tmp1 ^ in4; - out7 = tmp2 ^ in5; - out3 = tmp2 ^ out0 ^ in6; - out5 = out7 ^ in2 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_A9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in2 ^ in6; - out6 = in1 ^ in4; - out7 = in0 ^ in2 ^ in5; - out5 = in0 ^ in3 ^ in7; - out2 = out4 ^ in1 ^ in5; - out1 = out6 ^ in2 ^ in7; - out0 = out2 ^ out7 ^ in3; - out3 = out1 ^ in0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - tmp1 = in1 ^ in3; - tmp2 = in6 ^ in7; - out1 = tmp0 ^ in4 ^ in7; - out3 = tmp1 ^ in0; - out0 = tmp1 ^ tmp2; - out2 = tmp2 ^ in5; - out7 = tmp0 ^ out2; - out6 = out1 ^ out7 ^ in1; - out5 = out0 ^ out6 ^ in0; - out4 = out5 ^ out7 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in1; - tmp0 = in1 ^ in4; - tmp1 = in0 ^ in7; - out6 = tmp0 ^ in5; - out1 = tmp0 ^ tmp1 ^ in2; - out5 = tmp1 ^ in3 ^ in4; - out0 = tmp0 ^ out5 ^ in6; - out4 = out0 ^ out3 ^ in2; - out2 = out4 ^ in3 ^ in5; - out7 = tmp1 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in1 ^ in3; - out1 = in2 ^ in4; - tmp0 = in0 ^ in2; - out4 = in4 ^ in7; - out5 = in0 ^ in5; - out6 = in1 ^ in6; - out7 = tmp0 ^ in7; - out3 = tmp0 ^ in3 ^ in6; - out2 = out5 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AD(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in7; - out5 = in0; - out6 = in1; - out7 = in0 ^ in2; - out0 = in0 ^ in1 ^ in3; - out2 = out7 ^ in1 ^ in5; - out1 = in1 ^ in2 ^ in4; - out3 = out7 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in3 ^ in4; - tmp0 = in0 ^ in4; - tmp1 = in0 ^ in7; - out0 = in1 ^ in3 ^ in7; - out1 = tmp0 ^ in2; - out5 = tmp0 ^ in5; - tmp2 = tmp1 ^ in6; - out2 = tmp1 ^ in5; - out3 = tmp2 ^ in3; - out7 = tmp2 ^ in2; - out6 = tmp2 ^ out2 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_AF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in3; - tmp0 = in0 ^ in7; - out5 = in0 ^ in4; - out6 = in1 ^ in5; - out7 = in0 ^ in2 ^ in6; - out0 = tmp0 ^ in1 ^ in3; - out3 = tmp0 ^ in6; - out2 = tmp0 ^ in2 ^ in5; - out1 = out5 ^ in1 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in4; - tmp1 = in3 ^ in6; - out2 = tmp0 ^ in7; - tmp2 = tmp0 ^ tmp1; - out0 = tmp2 ^ in5; - out3 = tmp2 ^ in2; - out6 = out3 ^ in6; - tmp3 = out6 ^ in0 ^ in1; - out7 = tmp3 ^ in5; - out5 = tmp3 ^ out2; - out1 = out0 ^ out5 ^ in0; - out4 = tmp1 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in4; - out2 = tmp0 ^ in2 ^ in7; - tmp1 = out2 ^ in6; - out1 = tmp1 ^ in5; - out3 = tmp1 ^ in7; - out4 = tmp1 ^ in0; - out6 = out3 ^ in3; - out0 = out6 ^ in0 ^ in2 ^ in5; - out5 = tmp1 ^ out0 ^ in1; - out7 = tmp0 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in4; - tmp0 = in4 ^ in7; - tmp1 = in1 ^ in3 ^ in6; - out3 = tmp0 ^ tmp1; - tmp2 = tmp1 ^ in0; - out0 = out3 ^ in5; - out4 = tmp2 ^ in2; - tmp3 = out4 ^ in6; - out5 = tmp0 ^ tmp3; - out1 = tmp3 ^ out0; - tmp4 = out1 ^ in7; - out7 = tmp4 ^ in3; - out6 = tmp2 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in4; - tmp0 = in0 ^ in5; - tmp1 = in1 ^ in6; - out3 = tmp1 ^ in4 ^ in7; - tmp2 = tmp0 ^ out3; - out0 = tmp2 ^ in3; - out1 = tmp2 ^ in2; - out5 = out0 ^ in2 ^ in6; - out7 = tmp1 ^ out5; - out4 = out7 ^ in1 ^ in5 ^ in7; - out6 = tmp0 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0 ^ in1; - out5 = out4 ^ in2; - tmp0 = out4 ^ in4; - out6 = out5 ^ in0 ^ in3; - out7 = tmp0 ^ out6; - out2 = tmp0 ^ in6 ^ in7; - out3 = out7 ^ in0 ^ in7; - out0 = out5 ^ out7 ^ in5; - out1 = out0 ^ out6 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in2 ^ in4; - out4 = tmp0 ^ in4; - out3 = tmp1 ^ in7; - tmp2 = out4 ^ in5; - out7 = out3 ^ in0 ^ in3; - out0 = tmp2 ^ in3; - out2 = tmp0 ^ out3 ^ in6; - out5 = tmp1 ^ tmp2; - out6 = out2 ^ out7 ^ in2; - out1 = tmp0 ^ out0 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in3 ^ in4; - tmp0 = in1 ^ in2; - tmp1 = in0 ^ in4; - tmp2 = in3 ^ in5; - tmp3 = out3 ^ in1 ^ in7; - out5 = tmp0 ^ tmp1; - out6 = tmp0 ^ tmp2; - out2 = tmp1 ^ in6; - out4 = tmp1 ^ tmp3; - out0 = tmp3 ^ in5; - out1 = out2 ^ in2 ^ in5; - out7 = tmp2 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in4; - tmp0 = in0 ^ in4; - out2 = tmp0 ^ in2 ^ in6; - tmp1 = out2 ^ in7; - out1 = out2 ^ in1 ^ in5; - out7 = tmp1 ^ in3; - out5 = out1 ^ in6; - out6 = tmp0 ^ out1 ^ in3; - out0 = tmp1 ^ out6; - out4 = out0 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in4; - tmp1 = in2 ^ in5; - out2 = tmp0 ^ in5; - out4 = tmp1 ^ in0; - tmp2 = tmp1 ^ in7; - out6 = tmp2 ^ out2; - out7 = out4 ^ in3; - out1 = tmp2 ^ in4; - out3 = tmp0 ^ out7; - out0 = out3 ^ out4 ^ in6; - out5 = out0 ^ in0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_B9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - tmp1 = in4 ^ in5; - out4 = tmp0 ^ tmp1; - tmp2 = tmp0 ^ in3 ^ in7; - out3 = out4 ^ in1; - out7 = tmp2 ^ in5; - out2 = out3 ^ in0; - out1 = out2 ^ in7; - out6 = out1 ^ in5 ^ in6; - out0 = tmp2 ^ out6; - out5 = tmp1 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in5 ^ in7; - out2 = tmp0 ^ in4; - tmp1 = out2 ^ in2; - out1 = tmp1 ^ in0; - out6 = tmp1 ^ in1; - out4 = out1 ^ in3 ^ in4; - tmp2 = out4 ^ out6; - out7 = out4 ^ in6 ^ in7; - out5 = tmp2 ^ in6; - out3 = tmp0 ^ tmp2; - out0 = out6 ^ out7 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in2 ^ in4 ^ in5 ^ in7; - tmp0 = out2 ^ in1; - out4 = out2 ^ in0 ^ in3; - out1 = tmp0 ^ in0; - out6 = tmp0 ^ in6; - out3 = out1 ^ in2; - tmp1 = out4 ^ out6 ^ in4; - out0 = tmp1 ^ in7; - out5 = tmp1 ^ in5; - out7 = tmp0 ^ tmp1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - tmp1 = in2 ^ in4; - out0 = in1 ^ in3 ^ in4; - out6 = in1 ^ in2 ^ in7; - out7 = tmp0 ^ in3; - out5 = tmp0 ^ out6 ^ in6; - out1 = tmp1 ^ in5; - tmp2 = out1 ^ out5 ^ in1; - out3 = tmp2 ^ in3; - out4 = tmp1 ^ tmp2; - out2 = tmp2 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BD(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in1 ^ in4; - out0 = tmp0 ^ tmp1; - out7 = tmp0 ^ in2 ^ in7; - out1 = tmp1 ^ in2 ^ in5; - tmp2 = out1 ^ in0; - out2 = tmp2 ^ in6; - out3 = out2 ^ in1 ^ in7; - out4 = out3 ^ in2; - out5 = tmp1 ^ out4; - out6 = tmp2 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3 ^ in6; - out4 = tmp0 ^ in5; - out7 = tmp0 ^ in2; - out3 = out4 ^ in4; - out1 = out3 ^ out7 ^ in0; - out2 = out3 ^ in3 ^ in7; - out0 = out2 ^ out4 ^ in1; - out5 = tmp0 ^ out0; - out6 = out1 ^ out5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_BF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in4; - out3 = tmp0 ^ in5 ^ in6; - out4 = out3 ^ in3; - tmp1 = out3 ^ in7; - out2 = tmp1 ^ in2; - out5 = tmp1 ^ in1; - tmp2 = out2 ^ in5; - out7 = tmp2 ^ in3 ^ in4; - tmp3 = tmp0 ^ out5; - out0 = tmp3 ^ out4; - out1 = tmp2 ^ tmp3; - out6 = tmp3 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in2 ^ in5; - tmp0 = in1 ^ in4; - tmp1 = in3 ^ in6; - out0 = out5 ^ in1; - out4 = tmp0 ^ in7; - out3 = tmp0 ^ tmp1; - out1 = tmp1 ^ in2; - out6 = tmp1 ^ in0; - out7 = out4 ^ in0; - out2 = out4 ^ out5 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in2; - tmp0 = in0 ^ in1; - out4 = in1 ^ in7; - out6 = in0 ^ in3; - out3 = in1 ^ in4 ^ in6; - tmp1 = tmp0 ^ in2; - out7 = tmp0 ^ in4; - out0 = tmp1 ^ in5; - out1 = tmp1 ^ out6 ^ in6; - out2 = out6 ^ out7 ^ in5 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1 ^ in3 ^ in4; - tmp0 = in0 ^ in3 ^ in6; - out5 = in2 ^ in4 ^ in5; - tmp1 = out4 ^ in7; - out1 = tmp0 ^ in2; - out6 = tmp0 ^ in5; - out2 = out5 ^ in3; - out7 = tmp0 ^ tmp1; - out3 = tmp1 ^ in2 ^ in6; - out0 = tmp1 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in1 ^ in3; - tmp0 = in0 ^ in2; - tmp1 = in3 ^ in5; - out5 = in2 ^ in4; - tmp2 = tmp0 ^ out4; - out2 = tmp1 ^ in4; - out6 = tmp1 ^ in0; - out0 = tmp1 ^ tmp2 ^ in7; - out1 = tmp2 ^ in6; - out7 = out1 ^ out5 ^ in3; - out3 = tmp0 ^ out7 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in7; - out3 = tmp0 ^ in4; - tmp1 = tmp0 ^ in2; - out1 = tmp1 ^ in6; - out5 = tmp1 ^ in5; - out4 = out1 ^ out3 ^ in1; - out0 = out4 ^ in4 ^ in5; - out2 = out0 ^ out3 ^ in0; - out7 = out1 ^ out2 ^ in7; - out6 = tmp1 ^ out0 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in4 ^ in7; - tmp0 = in3 ^ in7; - out4 = in1 ^ in2 ^ in6; - out6 = in0 ^ in3 ^ in4; - out5 = tmp0 ^ in2; - out1 = tmp0 ^ out4; - out0 = out4 ^ in0 ^ in5; - out2 = out0 ^ out5 ^ in4; - out7 = tmp0 ^ out2 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in5 ^ in6; - tmp1 = in1 ^ in7; - tmp2 = tmp0 ^ in0; - tmp3 = tmp0 ^ tmp1; - tmp4 = tmp2 ^ in4; - out0 = tmp3 ^ in2; - out6 = tmp4 ^ in3; - out2 = out6 ^ in2; - out7 = tmp1 ^ tmp4; - out3 = tmp2 ^ out2; - tmp5 = out3 ^ in5; - out5 = tmp5 ^ in7; - out4 = tmp3 ^ tmp5; - out1 = tmp4 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in2 ^ in4; - tmp0 = in3 ^ in5; - tmp1 = out3 ^ in7; - out6 = tmp0 ^ in0 ^ in4; - out5 = tmp1 ^ in3; - out2 = out6 ^ in6; - out7 = out2 ^ in1 ^ in3; - out0 = tmp1 ^ out7; - out1 = tmp0 ^ out0; - out4 = out1 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out0 = in1 ^ in2; - out1 = in2 ^ in3; - tmp0 = in5 ^ in6; - tmp1 = in0 ^ in7; - out2 = out1 ^ in1 ^ in4; - out4 = tmp0 ^ in4; - out5 = tmp0 ^ in7; - out6 = tmp1 ^ in6; - out7 = tmp1 ^ in1; - out3 = out2 ^ in0 ^ in2 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_C9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in5 ^ in6; - out7 = in0 ^ in1; - tmp0 = in1 ^ in3; - out5 = in6 ^ in7; - out6 = in0 ^ in7; - out0 = out7 ^ in2; - out3 = out7 ^ in4 ^ in5; - out1 = tmp0 ^ in2; - out2 = tmp0 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in7; - tmp1 = in2 ^ in7; - tmp2 = tmp0 ^ in6; - out0 = tmp1 ^ in1; - tmp3 = tmp1 ^ in3; - out6 = tmp2 ^ in5; - out7 = tmp2 ^ in1; - out2 = tmp3 ^ in4; - out5 = out6 ^ in0 ^ in4; - out4 = out5 ^ in3; - out1 = tmp0 ^ tmp3; - out3 = tmp3 ^ out5 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in7; - tmp1 = in5 ^ in7; - out7 = in0 ^ in1 ^ in6; - out5 = tmp0 ^ in6; - out2 = tmp0 ^ in3; - out6 = tmp1 ^ in0; - out4 = tmp1 ^ in3 ^ in6; - tmp2 = out5 ^ out7 ^ in2; - out1 = tmp2 ^ out2; - out0 = tmp2 ^ in4; - out3 = tmp2 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in5; - tmp1 = in1 ^ in6; - out1 = in2 ^ in3 ^ in7; - out5 = tmp0 ^ in6; - out0 = tmp1 ^ in2; - tmp2 = out5 ^ in0 ^ in7; - out3 = tmp2 ^ in4; - out6 = tmp0 ^ out3; - out7 = tmp1 ^ tmp2 ^ in3; - tmp3 = out1 ^ out6; - out4 = tmp2 ^ tmp3; - out2 = tmp3 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CD(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in3 ^ in6; - tmp0 = in0 ^ in1; - tmp1 = in2 ^ in7; - out6 = in0 ^ in4 ^ in7; - out2 = tmp0 ^ out5 ^ in4; - out7 = tmp0 ^ in5; - out0 = tmp0 ^ in2 ^ in6; - out4 = tmp1 ^ in5; - out1 = tmp1 ^ in1 ^ in3; - out3 = out6 ^ in5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in5; - tmp1 = tmp0 ^ in3; - out4 = tmp1 ^ in4; - tmp2 = out4 ^ in6; - out3 = tmp2 ^ in0; - out5 = tmp2 ^ in2; - out2 = out3 ^ in5 ^ in7; - out6 = tmp1 ^ out2; - out7 = out2 ^ out4 ^ in1; - out1 = tmp2 ^ out6; - out0 = tmp0 ^ out7 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_CF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in6; - tmp1 = in0 ^ in1 ^ in5; - out4 = in2 ^ in3 ^ in5; - out5 = tmp0 ^ in4; - out7 = tmp1 ^ in6; - out1 = tmp1 ^ out4 ^ in7; - tmp2 = out5 ^ in0; - out2 = tmp2 ^ in7; - out3 = tmp2 ^ out4; - out6 = tmp0 ^ out2 ^ in5; - out0 = tmp0 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - tmp1 = in1 ^ in4; - tmp2 = in2 ^ in5; - out7 = tmp0 ^ tmp1; - out0 = tmp1 ^ tmp2; - tmp3 = tmp2 ^ in3; - out1 = tmp3 ^ in6; - tmp4 = out1 ^ in1; - out2 = tmp4 ^ in7; - out3 = out2 ^ in2; - out4 = tmp0 ^ out3; - out5 = tmp3 ^ out3; - out6 = tmp4 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in5 ^ in6; - tmp1 = tmp0 ^ in1; - out1 = tmp1 ^ in2; - out2 = tmp1 ^ in7; - out3 = out2 ^ in3; - out5 = out3 ^ in2; - tmp2 = out3 ^ in0; - out4 = tmp2 ^ in4; - out7 = tmp0 ^ out4; - out6 = tmp2 ^ out1 ^ in6; - out0 = out2 ^ out6 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in5 ^ in6; - out2 = tmp0 ^ in2 ^ in3; - out1 = out2 ^ in0; - out3 = out2 ^ in1; - out4 = out1 ^ in1 ^ in2; - out6 = out1 ^ in6 ^ in7; - out7 = out4 ^ in4 ^ in5; - out5 = out4 ^ out6 ^ in4; - out0 = tmp0 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in3 ^ in5 ^ in6; - tmp0 = out2 ^ in2; - tmp1 = tmp0 ^ in1; - out1 = tmp1 ^ in0; - out3 = tmp1 ^ in3; - out4 = out1 ^ in2 ^ in4; - tmp2 = out4 ^ in5; - out7 = tmp2 ^ in7; - out0 = tmp0 ^ out7; - tmp3 = out0 ^ in0; - out5 = tmp3 ^ in6; - out6 = tmp2 ^ tmp3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in3 ^ in5; - tmp0 = in1 ^ in5; - tmp1 = tmp0 ^ in2; - out4 = tmp1 ^ in0; - tmp2 = tmp1 ^ in6; - out2 = out4 ^ in3 ^ in7; - out0 = tmp2 ^ in4; - out5 = tmp2 ^ out3; - out1 = tmp0 ^ out5 ^ in7; - out6 = tmp0 ^ out2 ^ in4; - out7 = tmp1 ^ out6 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in5; - tmp0 = in0 ^ in4; - tmp1 = tmp0 ^ in1 ^ in5; - out4 = tmp1 ^ in2; - out0 = out4 ^ in6; - tmp2 = tmp0 ^ out0; - out5 = tmp2 ^ in3; - out1 = out5 ^ in7; - out6 = tmp1 ^ out1; - out7 = tmp2 ^ out6; - out2 = out7 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2 ^ in4 ^ in6; - out5 = tmp0 ^ in3; - out0 = tmp0 ^ in5 ^ in7; - out3 = out0 ^ out5 ^ in2; - tmp1 = out3 ^ in0; - out1 = tmp1 ^ in6; - out2 = tmp1 ^ in7; - out4 = tmp1 ^ in1; - out6 = tmp1 ^ in4; - out7 = tmp0 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in3; - out3 = in2 ^ in5 ^ in7; - out2 = tmp0 ^ in5; - tmp1 = tmp0 ^ out3 ^ in1; - out1 = tmp1 ^ in6; - out4 = tmp1 ^ in4; - tmp2 = out1 ^ in4; - out6 = tmp2 ^ in1; - out7 = tmp2 ^ in2; - out0 = tmp2 ^ in3; - out5 = tmp2 ^ in0 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0; - out5 = in1; - tmp0 = in1 ^ in2; - out6 = in0 ^ in2; - out0 = tmp0 ^ in4; - tmp1 = tmp0 ^ in3; - out7 = tmp1 ^ out6; - out2 = tmp1 ^ in6; - out3 = out7 ^ in7; - out1 = tmp1 ^ in1 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_D9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in0 ^ in4; - out5 = in1 ^ in5; - out2 = in1 ^ in3 ^ in6; - out3 = in0 ^ in1 ^ in7; - out6 = in0 ^ in2 ^ in6; - out0 = out4 ^ in1 ^ in2; - out1 = out5 ^ in2 ^ in3; - out7 = out3 ^ in3; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out5 = in1 ^ in4; - tmp0 = in2 ^ in7; - tmp1 = in0 ^ in2 ^ in3; - out0 = tmp0 ^ out5; - out4 = tmp0 ^ tmp1; - out2 = tmp0 ^ in3 ^ in6; - out1 = tmp1 ^ in5; - out3 = tmp1 ^ in1; - out6 = out1 ^ in3; - out7 = out3 ^ in2 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in1 ^ in5; - tmp2 = in3 ^ in7; - out3 = tmp0 ^ in2; - out5 = tmp1 ^ in4; - out6 = tmp1 ^ out3 ^ in6; - out2 = tmp2 ^ in6; - tmp3 = tmp2 ^ in4; - tmp4 = out3 ^ in3; - out4 = tmp3 ^ in0; - out1 = tmp4 ^ in5; - out0 = tmp3 ^ tmp4; - out7 = tmp0 ^ out2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - tmp1 = in0 ^ in3; - out6 = tmp0 ^ in4; - tmp2 = tmp0 ^ in7; - out3 = tmp1 ^ in6; - tmp3 = tmp1 ^ in1; - out1 = tmp1 ^ tmp2 ^ in5; - out4 = tmp2 ^ in6; - out2 = tmp3 ^ in2; - out7 = tmp3 ^ in5; - out5 = tmp2 ^ out2; - out0 = out2 ^ out3 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DD(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in6; - out2 = in0 ^ in1 ^ in3; - out6 = out3 ^ in2 ^ in4; - out7 = out2 ^ in5 ^ in7; - out0 = out6 ^ in1; - out4 = out6 ^ in7; - out5 = out7 ^ in0; - out1 = out5 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3 ^ in6; - tmp1 = in3 ^ in4 ^ in7; - out4 = tmp0 ^ in0; - out5 = tmp1 ^ in1; - out3 = out4 ^ in7; - out2 = out3 ^ in6; - out1 = out2 ^ in5; - out6 = tmp1 ^ out1; - out0 = tmp0 ^ out5; - out7 = out0 ^ out1 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_DF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in3 ^ in7; - tmp0 = out2 ^ in1 ^ in5; - out1 = tmp0 ^ in2; - out7 = tmp0 ^ in6; - out5 = tmp0 ^ in0 ^ in4; - tmp1 = out1 ^ out5 ^ in6; - out4 = tmp1 ^ in3; - out6 = tmp1 ^ in5; - tmp2 = tmp1 ^ in7; - out0 = tmp2 ^ in1; - out3 = tmp2 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1 ^ in7; - tmp0 = in2 ^ in4; - out4 = out3 ^ in3 ^ in5; - out2 = tmp0 ^ in1; - tmp1 = tmp0 ^ in6; - out0 = out4 ^ in2; - out6 = out4 ^ in0; - out1 = tmp1 ^ in3; - out5 = tmp1 ^ in0; - out7 = out5 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in1 ^ in4; - tmp0 = in1 ^ in7; - out3 = tmp0 ^ in3; - tmp1 = out3 ^ in5; - out4 = tmp1 ^ in4; - tmp2 = tmp1 ^ in0; - out0 = tmp2 ^ in2; - out6 = tmp2 ^ in6; - tmp3 = out0 ^ out4 ^ in6; - out5 = tmp3 ^ in5; - out7 = tmp0 ^ tmp3; - out1 = tmp2 ^ out5 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in1 ^ in2; - out4 = in1 ^ in5; - out2 = in2 ^ in4 ^ in7; - out5 = in0 ^ in2 ^ in6; - out0 = out3 ^ in3 ^ in5; - out7 = out3 ^ in0 ^ in4; - out6 = out2 ^ out7 ^ in3; - out1 = out5 ^ in3 ^ in4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in4 ^ in7; - tmp0 = in1 ^ in3; - out3 = tmp0 ^ in2; - tmp1 = out3 ^ in0; - out0 = tmp1 ^ in5; - tmp2 = tmp1 ^ in4; - out1 = tmp2 ^ in6; - tmp3 = tmp2 ^ in3; - out7 = tmp3 ^ in7; - out6 = out1 ^ out2 ^ in2; - tmp4 = tmp0 ^ out0; - out5 = tmp4 ^ in6; - out4 = tmp3 ^ tmp4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in6; - tmp0 = in0 ^ in4; - tmp1 = tmp0 ^ in2 ^ in6; - out2 = tmp1 ^ in1; - out7 = out2 ^ in5; - tmp2 = tmp0 ^ out7; - out4 = tmp2 ^ in3; - out0 = out4 ^ in7; - out6 = tmp1 ^ out0; - out5 = tmp2 ^ out6; - out1 = out5 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in3 ^ in6; - tmp0 = in0 ^ in1; - tmp1 = in5 ^ in7; - out2 = tmp0 ^ in4 ^ in6; - tmp2 = tmp1 ^ out2; - out6 = tmp2 ^ in3; - out7 = tmp2 ^ in2; - out0 = out6 ^ in2 ^ in4; - out5 = out6 ^ in1 ^ in2; - out1 = tmp0 ^ out5 ^ in5; - out4 = tmp1 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in2 ^ in6 ^ in7; - out2 = out3 ^ in0 ^ in4; - out4 = out3 ^ in1 ^ in5; - out1 = out2 ^ in3; - out7 = out2 ^ out4 ^ in2; - out0 = out4 ^ in3 ^ in7; - out5 = out1 ^ in4; - out6 = out0 ^ out2 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in3; - out3 = tmp0 ^ in6 ^ in7; - tmp1 = out3 ^ in0; - out5 = tmp1 ^ in5; - tmp2 = tmp1 ^ in4; - tmp3 = out5 ^ in7; - out1 = tmp2 ^ in1; - out0 = tmp3 ^ in1; - out6 = out1 ^ in2; - out2 = tmp0 ^ tmp2; - tmp4 = tmp3 ^ out6; - out4 = tmp4 ^ in6; - out7 = tmp4 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in3 ^ in6; - tmp0 = in4 ^ in7; - out1 = in2 ^ in3 ^ in4; - out5 = tmp0 ^ in0; - tmp1 = tmp0 ^ in1; - tmp2 = tmp1 ^ in5; - out0 = tmp1 ^ out1; - out2 = tmp2 ^ in2; - out6 = tmp2 ^ out5; - tmp3 = out6 ^ in6; - out3 = tmp3 ^ in7; - out7 = tmp3 ^ in2 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_E9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = in3 ^ in6; - tmp2 = tmp0 ^ in6; - out4 = tmp1 ^ in4; - out6 = tmp2 ^ in5; - out7 = tmp2 ^ in2 ^ in7; - out3 = out6 ^ in3 ^ in7; - out0 = tmp1 ^ out7; - out2 = out3 ^ out4 ^ in0; - out5 = tmp0 ^ out2; - out1 = out0 ^ out5 ^ in5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_EA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in6 ^ in7; - out5 = in0 ^ in7; - out6 = in0 ^ in1; - out0 = in1 ^ in2 ^ in3; - out2 = in2 ^ in4 ^ in5; - out7 = out6 ^ in2; - out1 = out0 ^ out6 ^ in4; - out3 = out7 ^ in5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_EB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in4 ^ in5; - tmp0 = in0 ^ in1; - out4 = in4 ^ in6 ^ in7; - out5 = in0 ^ in5 ^ in7; - out6 = tmp0 ^ in6; - tmp1 = tmp0 ^ in2; - out0 = tmp1 ^ in3; - out7 = tmp1 ^ in7; - out1 = out0 ^ in4; - out3 = out0 ^ in5 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_EC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out3 = in0 ^ in5; - out4 = in2 ^ in3 ^ in7; - out5 = in0 ^ in3 ^ in4; - out6 = out3 ^ in1 ^ in4; - out1 = out4 ^ in4; - out0 = out4 ^ in1 ^ in6; - out2 = out0 ^ out5 ^ in5; - out7 = out2 ^ in4 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_ED(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in2 ^ in4; - tmp1 = in3 ^ in5; - out4 = tmp0 ^ in3 ^ in7; - out3 = tmp1 ^ in0; - out1 = out4 ^ in1; - out5 = out3 ^ in4; - out7 = out1 ^ out5 ^ in6; - out2 = tmp0 ^ out7; - out0 = tmp1 ^ out7; - out6 = out2 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_EE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in2; - tmp0 = in0 ^ in1; - out5 = in0 ^ in3; - tmp1 = tmp0 ^ in2; - out6 = tmp0 ^ in4; - tmp2 = tmp1 ^ out5; - out7 = tmp1 ^ in5; - out1 = tmp2 ^ out6 ^ in7; - out0 = tmp2 ^ in6; - tmp3 = out7 ^ in1; - out3 = tmp3 ^ in7; - out2 = tmp3 ^ in4 ^ in6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_EF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out4 = in2 ^ in4; - tmp0 = in0 ^ in5; - tmp1 = in4 ^ in6; - out5 = tmp0 ^ in3; - out2 = tmp0 ^ tmp1; - out6 = tmp1 ^ in0 ^ in1; - out3 = out5 ^ in2 ^ in7; - out7 = out3 ^ in1 ^ in3; - out0 = out4 ^ out6 ^ in3; - out1 = tmp1 ^ out0 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F0(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in4 ^ in5; - out2 = tmp0 ^ in6; - out3 = tmp1 ^ in1; - tmp2 = tmp1 ^ in7; - out1 = out2 ^ out3 ^ in3; - tmp3 = tmp0 ^ tmp2; - out0 = tmp3 ^ in3; - out5 = tmp3 ^ in0; - out4 = out1 ^ out5 ^ in4; - out7 = out4 ^ in2; - out6 = tmp2 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F1(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in1 ^ in6; - tmp0 = in3 ^ in5; - out3 = tmp0 ^ in1 ^ in4; - tmp1 = out3 ^ in2; - out1 = tmp1 ^ in6; - tmp2 = tmp1 ^ in0; - tmp3 = out1 ^ in5; - out0 = tmp2 ^ in7; - out6 = tmp2 ^ in4; - out7 = tmp3 ^ in0; - out5 = tmp0 ^ out0; - out4 = tmp3 ^ out5 ^ in1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F2(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in4 ^ in5; - out2 = in2 ^ in6 ^ in7; - tmp1 = tmp0 ^ in1; - tmp2 = tmp1 ^ in2; - out0 = tmp2 ^ in3; - out3 = tmp2 ^ in7; - out5 = out3 ^ in0 ^ in4; - tmp3 = tmp0 ^ out5; - out7 = tmp3 ^ in3; - out4 = tmp3 ^ out2; - out1 = out0 ^ out4 ^ in4; - out6 = tmp1 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F3(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in6 ^ in7; - tmp0 = in0 ^ in1; - out4 = tmp0 ^ in6; - tmp1 = tmp0 ^ in2; - out5 = tmp1 ^ in7; - out6 = tmp1 ^ in3; - out7 = out6 ^ in4; - out0 = out7 ^ in5; - out1 = out0 ^ in6; - out3 = out0 ^ in0 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F4(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in1 ^ in2; - tmp0 = out2 ^ in3; - out4 = tmp0 ^ in4; - out5 = out4 ^ in5; - out6 = out5 ^ in6; - out7 = out6 ^ in7; - out0 = out7 ^ in0; - out1 = out0 ^ in1; - out3 = tmp0 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F5(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in1; - tmp0 = out2 ^ in2; - out4 = tmp0 ^ in3; - out5 = out4 ^ in4; - out6 = out5 ^ in5; - out7 = out6 ^ in6; - out0 = out7 ^ in7; - out1 = out0 ^ in0; - out3 = tmp0 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F6(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in7; - out2 = tmp0 ^ in2; - out4 = out2 ^ in1 ^ in4; - out7 = out4 ^ in3 ^ in5; - out5 = out7 ^ in4 ^ in7; - out0 = tmp0 ^ out7 ^ in6; - tmp1 = out0 ^ in1; - out6 = out0 ^ in0 ^ in5; - out3 = tmp1 ^ in3; - out1 = tmp0 ^ tmp1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F7(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in7; - tmp0 = out2 ^ in1; - out4 = tmp0 ^ in2; - out5 = out4 ^ in3 ^ in7; - out6 = out5 ^ in4; - out7 = out6 ^ in5; - out0 = out7 ^ in6; - out1 = out0 ^ in7; - out3 = tmp0 ^ out1; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F8(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in4; - tmp1 = in3 ^ in5; - tmp2 = tmp0 ^ in6; - out4 = tmp0 ^ tmp1; - out1 = tmp1 ^ in2 ^ in4; - out3 = tmp2 ^ in1; - out5 = out3 ^ in5; - out7 = out1 ^ out5 ^ in7; - out6 = tmp1 ^ out7; - out0 = tmp2 ^ out7; - out2 = out6 ^ in0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_F9(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in3 ^ in5; - tmp1 = in0 ^ in6; - out4 = tmp0 ^ in0; - tmp2 = tmp1 ^ in4; - tmp3 = tmp1 ^ in2; - out5 = tmp2 ^ in1; - out3 = out5 ^ in3; - tmp4 = tmp3 ^ out3; - out1 = tmp4 ^ in5; - out0 = tmp4 ^ in0 ^ in7; - out6 = tmp0 ^ out0 ^ in4; - out7 = tmp2 ^ tmp4; - out2 = tmp3 ^ out6; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FA(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in1; - tmp1 = tmp0 ^ in2; - tmp2 = tmp0 ^ in5; - tmp3 = tmp1 ^ in7; - out5 = tmp2 ^ in6; - out6 = tmp3 ^ in6; - out7 = tmp3 ^ in3; - out3 = out6 ^ in4; - out2 = tmp1 ^ out5; - out4 = out2 ^ out3 ^ in1; - out0 = out4 ^ out7 ^ in5; - out1 = tmp2 ^ out0; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FB(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in5 ^ in6; - tmp0 = in0 ^ in1; - out4 = in0 ^ in5 ^ in7; - out5 = tmp0 ^ in6; - tmp1 = tmp0 ^ in2; - out6 = tmp1 ^ in7; - out7 = tmp1 ^ in3; - out0 = out7 ^ in4; - out1 = out0 ^ in5; - out3 = out0 ^ in6 ^ in7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FC(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in1 ^ in2; - tmp1 = in0 ^ in7; - out2 = tmp0 ^ tmp1 ^ in5; - out3 = tmp1 ^ in4; - tmp2 = out2 ^ in6; - out6 = tmp2 ^ in4; - out7 = tmp2 ^ in3; - out4 = out6 ^ in1 ^ in3; - tmp3 = out4 ^ in0; - out1 = tmp3 ^ in6; - out0 = tmp3 ^ in1 ^ in5; - out5 = tmp0 ^ out4; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FD(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in5; - tmp1 = in1 ^ in7; - out2 = tmp0 ^ tmp1; - out6 = out2 ^ in2 ^ in4; - tmp2 = out6 ^ in0; - out1 = tmp2 ^ in3; - out0 = tmp0 ^ out1 ^ in6; - out5 = out0 ^ in2; - tmp3 = out5 ^ in1; - out3 = tmp3 ^ in6; - out7 = tmp2 ^ tmp3; - out4 = tmp1 ^ out7; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FE(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - tmp0 = in0 ^ in2; - out2 = tmp0 ^ in5; - out3 = tmp0 ^ in4; - tmp1 = out3 ^ in6; - out4 = tmp1 ^ in5; - tmp2 = tmp1 ^ in1; - out6 = tmp2 ^ in7; - tmp3 = tmp2 ^ in0; - out0 = tmp3 ^ in3; - tmp4 = out0 ^ out4 ^ in7; - out5 = tmp4 ^ in6; - out7 = tmp4 ^ in2; - out1 = tmp3 ^ out5; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -static void gf8_muladd_FF(uint8_t * out, uint8_t * in, unsigned int width) -{ - unsigned int i; - uint64_t * in_ptr = (uint64_t *)in; - uint64_t * out_ptr = (uint64_t *)out; - - for (i = 0; i < width; i++) - { - uint64_t out0, out1, out2, out3, out4, out5, out6, out7; - uint64_t tmp0, tmp1, tmp2, tmp3; - - uint64_t in0 = out_ptr[0]; - uint64_t in1 = out_ptr[width]; - uint64_t in2 = out_ptr[width * 2]; - uint64_t in3 = out_ptr[width * 3]; - uint64_t in4 = out_ptr[width * 4]; - uint64_t in5 = out_ptr[width * 5]; - uint64_t in6 = out_ptr[width * 6]; - uint64_t in7 = out_ptr[width * 7]; - - out2 = in0 ^ in5; - tmp0 = in4 ^ in7; - tmp1 = out2 ^ in2; - out4 = tmp1 ^ in6; - out7 = tmp1 ^ in1 ^ in3; - out1 = tmp0 ^ out7; - tmp2 = out1 ^ in5; - out6 = tmp2 ^ in3; - tmp3 = tmp2 ^ in7; - out0 = tmp3 ^ in6; - out3 = tmp3 ^ in1; - out5 = tmp0 ^ out0 ^ in2; - - out_ptr[0] = out0 ^ in_ptr[0]; - out_ptr[width] = out1 ^ in_ptr[width]; - out_ptr[width * 2] = out2 ^ in_ptr[width * 2]; - out_ptr[width * 3] = out3 ^ in_ptr[width * 3]; - out_ptr[width * 4] = out4 ^ in_ptr[width * 4]; - out_ptr[width * 5] = out5 ^ in_ptr[width * 5]; - out_ptr[width * 6] = out6 ^ in_ptr[width * 6]; - out_ptr[width * 7] = out7 ^ in_ptr[width * 7]; - - in_ptr++; - out_ptr++; - } -} - -void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in, unsigned int width) = -{ - gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, - gf8_muladd_04, gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, - gf8_muladd_08, gf8_muladd_09, gf8_muladd_0A, gf8_muladd_0B, - gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, gf8_muladd_0F, - gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13, - gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, - gf8_muladd_18, gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, - gf8_muladd_1C, gf8_muladd_1D, gf8_muladd_1E, gf8_muladd_1F, - gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, gf8_muladd_23, - gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27, - gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, - gf8_muladd_2C, gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, - gf8_muladd_30, gf8_muladd_31, gf8_muladd_32, gf8_muladd_33, - gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, gf8_muladd_37, - gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B, - gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, - gf8_muladd_40, gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, - gf8_muladd_44, gf8_muladd_45, gf8_muladd_46, gf8_muladd_47, - gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, gf8_muladd_4B, - gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F, - gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, - gf8_muladd_54, gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, - gf8_muladd_58, gf8_muladd_59, gf8_muladd_5A, gf8_muladd_5B, - gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, gf8_muladd_5F, - gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63, - gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, - gf8_muladd_68, gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, - gf8_muladd_6C, gf8_muladd_6D, gf8_muladd_6E, gf8_muladd_6F, - gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, gf8_muladd_73, - gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77, - gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, - gf8_muladd_7C, gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, - gf8_muladd_80, gf8_muladd_81, gf8_muladd_82, gf8_muladd_83, - gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, gf8_muladd_87, - gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B, - gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, - gf8_muladd_90, gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, - gf8_muladd_94, gf8_muladd_95, gf8_muladd_96, gf8_muladd_97, - gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, gf8_muladd_9B, - gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F, - gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, - gf8_muladd_A4, gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, - gf8_muladd_A8, gf8_muladd_A9, gf8_muladd_AA, gf8_muladd_AB, - gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, gf8_muladd_AF, - gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3, - gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, - gf8_muladd_B8, gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, - gf8_muladd_BC, gf8_muladd_BD, gf8_muladd_BE, gf8_muladd_BF, - gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, gf8_muladd_C3, - gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7, - gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, - gf8_muladd_CC, gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, - gf8_muladd_D0, gf8_muladd_D1, gf8_muladd_D2, gf8_muladd_D3, - gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, gf8_muladd_D7, - gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB, - gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, - gf8_muladd_E0, gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, - gf8_muladd_E4, gf8_muladd_E5, gf8_muladd_E6, gf8_muladd_E7, - gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, gf8_muladd_EB, - gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF, - gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, - gf8_muladd_F4, gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, - gf8_muladd_F8, gf8_muladd_F9, gf8_muladd_FA, gf8_muladd_FB, - gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, gf8_muladd_FF -}; diff --git a/xlators/cluster/ec/src/ec-gf8.c b/xlators/cluster/ec/src/ec-gf8.c new file mode 100644 index 00000000000..039adae5929 --- /dev/null +++ b/xlators/cluster/ec/src/ec-gf8.c @@ -0,0 +1,5882 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec-gf8.h" + +static ec_gf_op_t ec_gf8_mul_00_ops[] = {{EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_00 = {0, + { + 0, + }, + ec_gf8_mul_00_ops}; + +static ec_gf_op_t ec_gf8_mul_01_ops[] = {{EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_01 = {8, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_01_ops}; + +static ec_gf_op_t ec_gf8_mul_02_ops[] = {{EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_02 = {8, + { + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_02_ops}; + +static ec_gf_op_t ec_gf8_mul_03_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_03 = {9, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + }, + ec_gf8_mul_03_ops}; + +static ec_gf_op_t ec_gf8_mul_04_ops[] = { + {EC_GF_OP_XOR3, 8, 6, 7}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_04 = {9, + { + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_04_ops}; + +static ec_gf_op_t ec_gf8_mul_05_ops[] = { + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_05 = {8, + { + 0, + 1, + 2, + 6, + 7, + 3, + 4, + 5, + }, + ec_gf8_mul_05_ops}; + +static ec_gf_op_t ec_gf8_mul_06_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_06 = {9, + { + 7, + 0, + 1, + 2, + 8, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_06_ops}; + +static ec_gf_op_t ec_gf8_mul_07_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_07 = {8, + { + 6, + 0, + 1, + 3, + 2, + 4, + 5, + 7, + }, + ec_gf8_mul_07_ops}; + +static ec_gf_op_t ec_gf8_mul_08_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 6, 7}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_08 = {9, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 8, + }, + ec_gf8_mul_08_ops}; + +static ec_gf_op_t ec_gf8_mul_09_ops[] = { + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_09 = {8, + { + 0, + 1, + 2, + 3, + 5, + 6, + 7, + 4, + }, + ec_gf8_mul_09_ops}; + +static ec_gf_op_t ec_gf8_mul_0A_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0A = {8, + { + 5, + 0, + 1, + 2, + 6, + 7, + 3, + 4, + }, + ec_gf8_mul_0A_ops}; + +static ec_gf_op_t ec_gf8_mul_0B_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 5, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR3, 3, 8, 6}, {EC_GF_OP_XOR2, 1, 9, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0B = {10, + { + 7, + 1, + 5, + 2, + 4, + 3, + 0, + 6, + 8, + 9, + }, + ec_gf8_mul_0B_ops}; + +static ec_gf_op_t ec_gf8_mul_0C_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0C = {9, + { + 5, + 7, + 0, + 1, + 8, + 2, + 3, + 4, + 6, + }, + ec_gf8_mul_0C_ops}; + +static ec_gf_op_t ec_gf8_mul_0D_ops[] = { + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR3, 2, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0D = {9, + { + 5, + 6, + 7, + 3, + 1, + 0, + 2, + 4, + 8, + }, + ec_gf8_mul_0D_ops}; + +static ec_gf_op_t ec_gf8_mul_0E_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0E = {8, + { + 7, + 0, + 6, + 1, + 3, + 2, + 4, + 5, + }, + ec_gf8_mul_0E_ops}; + +static ec_gf_op_t ec_gf8_mul_0F_ops[] = { + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0F = {8, + { + 1, + 0, + 5, + 6, + 7, + 2, + 3, + 4, + }, + ec_gf8_mul_0F_ops}; + +static ec_gf_op_t ec_gf8_mul_10_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_10 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_10_ops}; + +static ec_gf_op_t ec_gf8_mul_11_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_11 = {8, + { + 4, + 1, + 2, + 6, + 0, + 5, + 7, + 3, + }, + ec_gf8_mul_11_ops}; + +static ec_gf_op_t ec_gf8_mul_12_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_12 = {8, + { + 7, + 0, + 1, + 2, + 3, + 5, + 6, + 4, + }, + ec_gf8_mul_12_ops}; + +static ec_gf_op_t ec_gf8_mul_13_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 3, 7}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_13 = {9, + { + 4, + 5, + 2, + 6, + 0, + 1, + 7, + 3, + 8, + }, + ec_gf8_mul_13_ops}; + +static ec_gf_op_t ec_gf8_mul_14_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_14 = {8, + { + 6, + 7, + 0, + 1, + 2, + 4, + 5, + 3, + }, + ec_gf8_mul_14_ops}; + +static ec_gf_op_t ec_gf8_mul_15_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR3, 5, 8, 7}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_15 = {9, + { + 0, + 1, + 2, + 4, + 7, + 6, + 5, + 3, + 8, + }, + ec_gf8_mul_15_ops}; + +static ec_gf_op_t ec_gf8_mul_16_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_16 = {8, + { + 6, + 7, + 4, + 1, + 2, + 3, + 5, + 0, + }, + ec_gf8_mul_16_ops}; + +static ec_gf_op_t ec_gf8_mul_17_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_17 = {8, + { + 5, + 7, + 0, + 1, + 3, + 2, + 4, + 6, + }, + ec_gf8_mul_17_ops}; + +static ec_gf_op_t ec_gf8_mul_18_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_18 = {9, + { + 4, + 5, + 7, + 6, + 0, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_18_ops}; + +static ec_gf_op_t ec_gf8_mul_19_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_19 = {8, + { + 0, + 5, + 2, + 6, + 7, + 1, + 3, + 4, + }, + ec_gf8_mul_19_ops}; + +static ec_gf_op_t ec_gf8_mul_1A_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1A = {8, + { + 7, + 0, + 4, + 5, + 3, + 1, + 2, + 6, + }, + ec_gf8_mul_1A_ops}; + +static ec_gf_op_t ec_gf8_mul_1B_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1B = {8, + { + 7, + 4, + 5, + 6, + 3, + 1, + 2, + 0, + }, + ec_gf8_mul_1B_ops}; + +static ec_gf_op_t ec_gf8_mul_1C_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1C = {8, + { + 5, + 4, + 3, + 0, + 1, + 7, + 2, + 6, + }, + ec_gf8_mul_1C_ops}; + +static ec_gf_op_t ec_gf8_mul_1D_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 4, 2}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1D = {9, + { + 0, + 7, + 5, + 8, + 2, + 3, + 4, + 1, + 6, + }, + ec_gf8_mul_1D_ops}; + +static ec_gf_op_t ec_gf8_mul_1E_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1E = {8, + { + 4, + 7, + 5, + 1, + 6, + 0, + 2, + 3, + }, + ec_gf8_mul_1E_ops}; + +static ec_gf_op_t ec_gf8_mul_1F_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR3, 8, 3, 7}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1F = {9, + { + 1, + 4, + 5, + 6, + 7, + 0, + 3, + 2, + 8, + }, + ec_gf8_mul_1F_ops}; + +static ec_gf_op_t ec_gf8_mul_20_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_20 = {8, + { + 7, + 4, + 5, + 6, + 3, + 0, + 1, + 2, + }, + ec_gf8_mul_20_ops}; + +static ec_gf_op_t ec_gf8_mul_21_ops[] = { + {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 8, 7, 5}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_21 = {10, + { + 0, + 1, + 2, + 7, + 5, + 4, + 3, + 6, + 8, + 9, + }, + ec_gf8_mul_21_ops}; + +static ec_gf_op_t ec_gf8_mul_22_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_22 = {8, + { + 3, + 0, + 5, + 2, + 6, + 4, + 1, + 7, + }, + ec_gf8_mul_22_ops}; + +static ec_gf_op_t ec_gf8_mul_23_ops[] = { + {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_23 = {9, + { + 0, + 4, + 3, + 2, + 5, + 6, + 1, + 8, + 7, + }, + ec_gf8_mul_23_ops}; + +static ec_gf_op_t ec_gf8_mul_24_ops[] = { + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_24 = {8, + { + 6, + 7, + 0, + 1, + 2, + 4, + 5, + 3, + }, + ec_gf8_mul_24_ops}; + +static ec_gf_op_t ec_gf8_mul_25_ops[] = { + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_25 = {8, + { + 2, + 7, + 0, + 1, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_25_ops}; + +static ec_gf_op_t ec_gf8_mul_26_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_26 = {8, + { + 3, + 4, + 1, + 2, + 0, + 5, + 6, + 7, + }, + ec_gf8_mul_26_ops}; + +static ec_gf_op_t ec_gf8_mul_27_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_27 = {8, + { + 3, + 0, + 1, + 2, + 6, + 7, + 4, + 5, + }, + ec_gf8_mul_27_ops}; + +static ec_gf_op_t ec_gf8_mul_28_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_28 = {8, + { + 5, + 6, + 3, + 0, + 1, + 2, + 4, + 7, + }, + ec_gf8_mul_28_ops}; + +static ec_gf_op_t ec_gf8_mul_29_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_29 = {8, + { + 4, + 6, + 3, + 5, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_29_ops}; + +static ec_gf_op_t ec_gf8_mul_2A_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 6, 8, 4}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2A = {9, + { + 3, + 4, + 7, + 2, + 6, + 5, + 1, + 0, + 8, + }, + ec_gf8_mul_2A_ops}; + +static ec_gf_op_t ec_gf8_mul_2B_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2B = {8, + { + 3, + 4, + 7, + 5, + 6, + 0, + 1, + 2, + }, + ec_gf8_mul_2B_ops}; + +static ec_gf_op_t ec_gf8_mul_2C_ops[] = { + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2C = {8, + { + 5, + 6, + 7, + 0, + 2, + 3, + 4, + 1, + }, + ec_gf8_mul_2C_ops}; + +static ec_gf_op_t ec_gf8_mul_2D_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 6}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2D = {9, + { + 7, + 0, + 3, + 5, + 1, + 4, + 2, + 6, + 8, + }, + ec_gf8_mul_2D_ops}; + +static ec_gf_op_t ec_gf8_mul_2E_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2E = {9, + { + 5, + 0, + 7, + 3, + 2, + 6, + 4, + 1, + 8, + }, + ec_gf8_mul_2E_ops}; + +static ec_gf_op_t ec_gf8_mul_2F_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR3, 8, 7, 6}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2F = {9, + { + 6, + 3, + 2, + 5, + 7, + 0, + 1, + 4, + 8, + }, + ec_gf8_mul_2F_ops}; + +static ec_gf_op_t ec_gf8_mul_30_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 7}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_30 = {9, + { + 3, + 4, + 7, + 5, + 0, + 6, + 1, + 2, + 8, + }, + ec_gf8_mul_30_ops}; + +static ec_gf_op_t ec_gf8_mul_31_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_31 = {8, + { + 7, + 1, + 4, + 5, + 6, + 0, + 2, + 3, + }, + ec_gf8_mul_31_ops}; + +static ec_gf_op_t ec_gf8_mul_32_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_32 = {8, + { + 3, + 4, + 6, + 7, + 5, + 0, + 1, + 2, + }, + ec_gf8_mul_32_ops}; + +static ec_gf_op_t ec_gf8_mul_33_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_33 = {8, + { + 5, + 4, + 3, + 0, + 2, + 1, + 6, + 7, + }, + ec_gf8_mul_33_ops}; + +static ec_gf_op_t ec_gf8_mul_34_ops[] = { + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_34 = {8, + { + 7, + 5, + 3, + 0, + 2, + 4, + 1, + 6, + }, + ec_gf8_mul_34_ops}; + +static ec_gf_op_t ec_gf8_mul_35_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_35 = {8, + { + 6, + 7, + 5, + 4, + 2, + 0, + 1, + 3, + }, + ec_gf8_mul_35_ops}; + +static ec_gf_op_t ec_gf8_mul_36_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_36 = {8, + { + 6, + 7, + 4, + 1, + 2, + 3, + 0, + 5, + }, + ec_gf8_mul_36_ops}; + +static ec_gf_op_t ec_gf8_mul_37_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 1}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_37 = {9, + { + 6, + 7, + 2, + 1, + 0, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_37_ops}; + +static ec_gf_op_t ec_gf8_mul_38_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 7}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_38 = {9, + { + 4, + 5, + 6, + 3, + 0, + 1, + 7, + 2, + 8, + }, + ec_gf8_mul_38_ops}; + +static ec_gf_op_t ec_gf8_mul_39_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_39 = {8, + { + 1, + 6, + 3, + 0, + 5, + 2, + 4, + 7, + }, + ec_gf8_mul_39_ops}; + +static ec_gf_op_t ec_gf8_mul_3A_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3A = {8, + { + 3, + 4, + 7, + 0, + 5, + 6, + 1, + 2, + }, + ec_gf8_mul_3A_ops}; + +static ec_gf_op_t ec_gf8_mul_3B_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR3, 8, 7, 3}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3B = {9, + { + 3, + 0, + 1, + 7, + 6, + 2, + 4, + 8, + 5, + }, + ec_gf8_mul_3B_ops}; + +static ec_gf_op_t ec_gf8_mul_3C_ops[] = { + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3C = {8, + { + 3, + 6, + 4, + 1, + 7, + 2, + 0, + 5, + }, + ec_gf8_mul_3C_ops}; + +static ec_gf_op_t ec_gf8_mul_3D_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3D = {8, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + }, + ec_gf8_mul_3D_ops}; + +static ec_gf_op_t ec_gf8_mul_3E_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3E = {8, + { + 6, + 1, + 2, + 7, + 0, + 3, + 5, + 4, + }, + ec_gf8_mul_3E_ops}; + +static ec_gf_op_t ec_gf8_mul_3F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_COPY, 10, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR3, 4, 9, 7}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 3, 10, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3F = {11, + { + 1, + 7, + 6, + 2, + 4, + 3, + 5, + 0, + 8, + 9, + 10, + }, + ec_gf8_mul_3F_ops}; + +static ec_gf_op_t ec_gf8_mul_40_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 7, 6}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_40 = {9, + { + 5, + 7, + 4, + 6, + 2, + 3, + 0, + 1, + 8, + }, + ec_gf8_mul_40_ops}; + +static ec_gf_op_t ec_gf8_mul_41_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_41 = {9, + { + 0, + 7, + 6, + 5, + 3, + 4, + 8, + 1, + 2, + }, + ec_gf8_mul_41_ops}; + +static ec_gf_op_t ec_gf8_mul_42_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_42 = {9, + { + 2, + 7, + 1, + 6, + 4, + 3, + 0, + 5, + 8, + }, + ec_gf8_mul_42_ops}; + +static ec_gf_op_t ec_gf8_mul_43_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_43 = {8, + { + 2, + 6, + 4, + 1, + 7, + 3, + 0, + 5, + }, + ec_gf8_mul_43_ops}; + +static ec_gf_op_t ec_gf8_mul_44_ops[] = { + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_44 = {8, + { + 2, + 3, + 4, + 1, + 6, + 5, + 0, + 7, + }, + ec_gf8_mul_44_ops}; + +static ec_gf_op_t ec_gf8_mul_45_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_45 = {8, + { + 2, + 3, + 0, + 1, + 7, + 4, + 5, + 6, + }, + ec_gf8_mul_45_ops}; + +static ec_gf_op_t ec_gf8_mul_46_ops[] = { + {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_46 = {9, + { + 2, + 0, + 1, + 3, + 4, + 5, + 6, + 7, + 8, + }, + ec_gf8_mul_46_ops}; + +static ec_gf_op_t ec_gf8_mul_47_ops[] = { + {EC_GF_OP_XOR3, 8, 0, 1}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_47 = {9, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 8, + }, + ec_gf8_mul_47_ops}; + +static ec_gf_op_t ec_gf8_mul_48_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_48 = {8, + { + 4, + 5, + 6, + 0, + 1, + 3, + 7, + 2, + }, + ec_gf8_mul_48_ops}; + +static ec_gf_op_t ec_gf8_mul_49_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 6}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR3, 1, 8, 5}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_49 = {9, + { + 7, + 2, + 4, + 0, + 3, + 5, + 1, + 6, + 8, + }, + ec_gf8_mul_49_ops}; + +static ec_gf_op_t ec_gf8_mul_4A_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4A = {8, + { + 5, + 6, + 7, + 0, + 1, + 3, + 4, + 2, + }, + ec_gf8_mul_4A_ops}; + +static ec_gf_op_t ec_gf8_mul_4B_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR3, 8, 3, 7}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4B = {9, + { + 5, + 3, + 6, + 7, + 0, + 2, + 4, + 1, + 8, + }, + ec_gf8_mul_4B_ops}; + +static ec_gf_op_t ec_gf8_mul_4C_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4C = {8, + { + 5, + 3, + 4, + 7, + 0, + 6, + 2, + 1, + }, + ec_gf8_mul_4C_ops}; + +static ec_gf_op_t ec_gf8_mul_4D_ops[] = { + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR3, 9, 3, 1}, + {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 0, 8, 2}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4D = {10, + { + 0, + 9, + 3, + 5, + 6, + 4, + 7, + 1, + 2, + 8, + }, + ec_gf8_mul_4D_ops}; + +static ec_gf_op_t ec_gf8_mul_4E_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4E = {8, + { + 2, + 3, + 0, + 1, + 5, + 6, + 7, + 4, + }, + ec_gf8_mul_4E_ops}; + +static ec_gf_op_t ec_gf8_mul_4F_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4F = {8, + { + 0, + 3, + 5, + 6, + 1, + 2, + 7, + 4, + }, + ec_gf8_mul_4F_ops}; + +static ec_gf_op_t ec_gf8_mul_50_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_50 = {8, + { + 4, + 5, + 7, + 3, + 0, + 1, + 2, + 6, + }, + ec_gf8_mul_50_ops}; + +static ec_gf_op_t ec_gf8_mul_51_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_51 = {8, + { + 0, + 1, + 7, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_51_ops}; + +static ec_gf_op_t ec_gf8_mul_52_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR3, 3, 5, 8}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 9, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_52 = {10, + { + 2, + 3, + 1, + 4, + 6, + 7, + 0, + 5, + 8, + 9, + }, + ec_gf8_mul_52_ops}; + +static ec_gf_op_t ec_gf8_mul_53_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_53 = {8, + { + 2, + 0, + 1, + 4, + 5, + 6, + 7, + 3, + }, + ec_gf8_mul_53_ops}; + +static ec_gf_op_t ec_gf8_mul_54_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_54 = {8, + { + 7, + 3, + 0, + 4, + 2, + 6, + 5, + 1, + }, + ec_gf8_mul_54_ops}; + +static ec_gf_op_t ec_gf8_mul_55_ops[] = { + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_55 = {8, + { + 1, + 5, + 6, + 4, + 3, + 7, + 2, + 0, + }, + ec_gf8_mul_55_ops}; + +static ec_gf_op_t ec_gf8_mul_56_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_56 = {8, + { + 2, + 3, + 0, + 4, + 5, + 6, + 7, + 1, + }, + ec_gf8_mul_56_ops}; + +static ec_gf_op_t ec_gf8_mul_57_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_57 = {8, + { + 2, + 3, + 0, + 1, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_57_ops}; + +static ec_gf_op_t ec_gf8_mul_58_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_58 = {8, + { + 4, + 3, + 2, + 7, + 0, + 1, + 5, + 6, + }, + ec_gf8_mul_58_ops}; + +static ec_gf_op_t ec_gf8_mul_59_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_59 = {8, + { + 7, + 3, + 5, + 6, + 1, + 2, + 0, + 4, + }, + ec_gf8_mul_59_ops}; + +static ec_gf_op_t ec_gf8_mul_5A_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5A = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 5, + 4, + }, + ec_gf8_mul_5A_ops}; + +static ec_gf_op_t ec_gf8_mul_5B_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5B = {8, + { + 6, + 0, + 7, + 5, + 2, + 1, + 3, + 4, + }, + ec_gf8_mul_5B_ops}; + +static ec_gf_op_t ec_gf8_mul_5C_ops[] = { + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5C = {9, + { + 7, + 5, + 2, + 4, + 1, + 0, + 6, + 3, + 8, + }, + ec_gf8_mul_5C_ops}; + +static ec_gf_op_t ec_gf8_mul_5D_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5D = {8, + { + 1, + 3, + 5, + 4, + 6, + 7, + 2, + 0, + }, + ec_gf8_mul_5D_ops}; + +static ec_gf_op_t ec_gf8_mul_5E_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5E = {8, + { + 4, + 3, + 6, + 2, + 5, + 7, + 0, + 1, + }, + ec_gf8_mul_5E_ops}; + +static ec_gf_op_t ec_gf8_mul_5F_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5F = {8, + { + 6, + 1, + 3, + 4, + 5, + 7, + 2, + 0, + }, + ec_gf8_mul_5F_ops}; + +static ec_gf_op_t ec_gf8_mul_60_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_60 = {8, + { + 2, + 3, + 4, + 7, + 5, + 6, + 0, + 1, + }, + ec_gf8_mul_60_ops}; + +static ec_gf_op_t ec_gf8_mul_61_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_61 = {8, + { + 0, + 5, + 6, + 7, + 4, + 2, + 1, + 3, + }, + ec_gf8_mul_61_ops}; + +static ec_gf_op_t ec_gf8_mul_62_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_62 = {8, + { + 2, + 0, + 3, + 4, + 5, + 6, + 7, + 1, + }, + ec_gf8_mul_62_ops}; + +static ec_gf_op_t ec_gf8_mul_63_ops[] = { + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_63 = {8, + { + 3, + 4, + 6, + 5, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_63_ops}; + +static ec_gf_op_t ec_gf8_mul_64_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 8, 7, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_64 = {9, + { + 2, + 3, + 4, + 6, + 5, + 7, + 8, + 1, + 0, + }, + ec_gf8_mul_64_ops}; + +static ec_gf_op_t ec_gf8_mul_65_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_65 = {8, + { + 2, + 5, + 1, + 3, + 4, + 0, + 6, + 7, + }, + ec_gf8_mul_65_ops}; + +static ec_gf_op_t ec_gf8_mul_66_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_66 = {8, + { + 2, + 3, + 1, + 4, + 5, + 7, + 0, + 6, + }, + ec_gf8_mul_66_ops}; + +static ec_gf_op_t ec_gf8_mul_67_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_67 = {8, + { + 2, + 4, + 5, + 6, + 7, + 3, + 1, + 0, + }, + ec_gf8_mul_67_ops}; + +static ec_gf_op_t ec_gf8_mul_68_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_68 = {8, + { + 5, + 7, + 2, + 3, + 0, + 6, + 4, + 1, + }, + ec_gf8_mul_68_ops}; + +static ec_gf_op_t ec_gf8_mul_69_ops[] = { + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_69 = {8, + { + 0, + 1, + 3, + 2, + 4, + 5, + 7, + 6, + }, + ec_gf8_mul_69_ops}; + +static ec_gf_op_t ec_gf8_mul_6A_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6A = {8, + { + 5, + 7, + 4, + 6, + 1, + 2, + 0, + 3, + }, + ec_gf8_mul_6A_ops}; + +static ec_gf_op_t ec_gf8_mul_6B_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6B = {9, + { + 6, + 7, + 2, + 0, + 3, + 1, + 5, + 4, + 8, + }, + ec_gf8_mul_6B_ops}; + +static ec_gf_op_t ec_gf8_mul_6C_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6C = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_6C_ops}; + +static ec_gf_op_t ec_gf8_mul_6D_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR3, 8, 3, 4}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6D = {9, + { + 3, + 6, + 7, + 0, + 4, + 5, + 1, + 2, + 8, + }, + ec_gf8_mul_6D_ops}; + +static ec_gf_op_t ec_gf8_mul_6E_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6E = {8, + { + 5, + 6, + 3, + 1, + 7, + 2, + 0, + 4, + }, + ec_gf8_mul_6E_ops}; + +static ec_gf_op_t ec_gf8_mul_6F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 0, 8, 7}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6F = {9, + { + 2, + 6, + 3, + 7, + 0, + 1, + 4, + 5, + 8, + }, + ec_gf8_mul_6F_ops}; + +static ec_gf_op_t ec_gf8_mul_70_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_70 = {8, + { + 3, + 4, + 5, + 2, + 6, + 0, + 1, + 7, + }, + ec_gf8_mul_70_ops}; + +static ec_gf_op_t ec_gf8_mul_71_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_71 = {8, + { + 4, + 7, + 5, + 3, + 6, + 0, + 2, + 1, + }, + ec_gf8_mul_71_ops}; + +static ec_gf_op_t ec_gf8_mul_72_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_72 = {8, + { + 0, + 5, + 2, + 7, + 4, + 1, + 3, + 6, + }, + ec_gf8_mul_72_ops}; + +static ec_gf_op_t ec_gf8_mul_73_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_73 = {8, + { + 6, + 0, + 1, + 7, + 4, + 5, + 2, + 3, + }, + ec_gf8_mul_73_ops}; + +static ec_gf_op_t ec_gf8_mul_74_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_74 = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_74_ops}; + +static ec_gf_op_t ec_gf8_mul_75_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_75 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_75_ops}; + +static ec_gf_op_t ec_gf8_mul_76_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 8, 6, 2}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 8, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_76 = {9, + { + 2, + 3, + 0, + 6, + 5, + 1, + 7, + 8, + 4, + }, + ec_gf8_mul_76_ops}; + +static ec_gf_op_t ec_gf8_mul_77_ops[] = { + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_77 = {8, + { + 7, + 4, + 3, + 6, + 0, + 1, + 5, + 2, + }, + ec_gf8_mul_77_ops}; + +static ec_gf_op_t ec_gf8_mul_78_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 0, 2}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_78 = {9, + { + 4, + 7, + 3, + 2, + 5, + 1, + 6, + 0, + 8, + }, + ec_gf8_mul_78_ops}; + +static ec_gf_op_t ec_gf8_mul_79_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 7}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_79 = {9, + { + 4, + 5, + 7, + 3, + 1, + 6, + 2, + 0, + 8, + }, + ec_gf8_mul_79_ops}; + +static ec_gf_op_t ec_gf8_mul_7A_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7A = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_7A_ops}; + +static ec_gf_op_t ec_gf8_mul_7B_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR3, 8, 5, 3}, + {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 4, 1, 9}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7B = {10, + { + 1, + 2, + 3, + 4, + 8, + 5, + 6, + 0, + 7, + 9, + }, + ec_gf8_mul_7B_ops}; + +static ec_gf_op_t ec_gf8_mul_7C_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7C = {8, + { + 2, + 4, + 1, + 6, + 3, + 5, + 7, + 0, + }, + ec_gf8_mul_7C_ops}; + +static ec_gf_op_t ec_gf8_mul_7D_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7D = {8, + { + 1, + 0, + 3, + 5, + 6, + 7, + 2, + 4, + }, + ec_gf8_mul_7D_ops}; + +static ec_gf_op_t ec_gf8_mul_7E_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_COPY, 8, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 6, 2, 7}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7E = {9, + { + 5, + 1, + 2, + 0, + 7, + 3, + 4, + 6, + 8, + }, + ec_gf8_mul_7E_ops}; + +static ec_gf_op_t ec_gf8_mul_7F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR3, 9, 7, 5}, {EC_GF_OP_XOR2, 2, 9, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 9, 0}, + {EC_GF_OP_XOR3, 9, 6, 4}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7F = {10, + { + 4, + 1, + 0, + 5, + 6, + 7, + 2, + 3, + 8, + 9, + }, + ec_gf8_mul_7F_ops}; + +static ec_gf_op_t ec_gf8_mul_80_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_80 = {8, + { + 7, + 5, + 6, + 4, + 1, + 2, + 3, + 0, + }, + ec_gf8_mul_80_ops}; + +static ec_gf_op_t ec_gf8_mul_81_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_81 = {8, + { + 2, + 7, + 4, + 1, + 5, + 6, + 3, + 0, + }, + ec_gf8_mul_81_ops}; + +static ec_gf_op_t ec_gf8_mul_82_ops[] = { + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR3, 5, 8, 7}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_82 = {9, + { + 6, + 2, + 7, + 5, + 1, + 3, + 4, + 0, + 8, + }, + ec_gf8_mul_82_ops}; + +static ec_gf_op_t ec_gf8_mul_83_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_83 = {8, + { + 3, + 5, + 6, + 7, + 1, + 2, + 4, + 0, + }, + ec_gf8_mul_83_ops}; + +static ec_gf_op_t ec_gf8_mul_84_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_84 = {8, + { + 7, + 6, + 0, + 4, + 1, + 5, + 3, + 2, + }, + ec_gf8_mul_84_ops}; + +static ec_gf_op_t ec_gf8_mul_85_ops[] = { + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_85 = {8, + { + 7, + 6, + 0, + 3, + 2, + 4, + 5, + 1, + }, + ec_gf8_mul_85_ops}; + +static ec_gf_op_t ec_gf8_mul_86_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_86 = {8, + { + 1, + 2, + 6, + 4, + 5, + 7, + 3, + 0, + }, + ec_gf8_mul_86_ops}; + +static ec_gf_op_t ec_gf8_mul_87_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR3, 5, 8, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_87 = {9, + { + 1, + 2, + 3, + 4, + 5, + 7, + 6, + 0, + 8, + }, + ec_gf8_mul_87_ops}; + +static ec_gf_op_t ec_gf8_mul_88_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_88 = {8, + { + 6, + 7, + 3, + 1, + 2, + 4, + 5, + 0, + }, + ec_gf8_mul_88_ops}; + +static ec_gf_op_t ec_gf8_mul_89_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR3, 8, 5, 2}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_89 = {9, + { + 2, + 1, + 6, + 5, + 7, + 3, + 4, + 0, + 8, + }, + ec_gf8_mul_89_ops}; + +static ec_gf_op_t ec_gf8_mul_8A_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8A = {8, + { + 1, + 2, + 3, + 0, + 6, + 7, + 4, + 5, + }, + ec_gf8_mul_8A_ops}; + +static ec_gf_op_t ec_gf8_mul_8B_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8B = {8, + { + 6, + 1, + 2, + 3, + 5, + 7, + 4, + 0, + }, + ec_gf8_mul_8B_ops}; + +static ec_gf_op_t ec_gf8_mul_8C_ops[] = { + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8C = {8, + { + 1, + 2, + 0, + 7, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_8C_ops}; + +static ec_gf_op_t ec_gf8_mul_8D_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8D = {8, + { + 7, + 1, + 3, + 2, + 4, + 5, + 0, + 6, + }, + ec_gf8_mul_8D_ops}; + +static ec_gf_op_t ec_gf8_mul_8E_ops[] = {{EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8E = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_8E_ops}; + +static ec_gf_op_t ec_gf8_mul_8F_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8F = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_8F_ops}; + +static ec_gf_op_t ec_gf8_mul_90_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_90 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 3, + 2, + }, + ec_gf8_mul_90_ops}; + +static ec_gf_op_t ec_gf8_mul_91_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_COPY, 9, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR3, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_91 = {10, + { + 2, + 3, + 1, + 4, + 0, + 6, + 7, + 5, + 8, + 9, + }, + ec_gf8_mul_91_ops}; + +static ec_gf_op_t ec_gf8_mul_92_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_92 = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 5, + 4, + }, + ec_gf8_mul_92_ops}; + +static ec_gf_op_t ec_gf8_mul_93_ops[] = { + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_93 = {8, + { + 6, + 4, + 5, + 1, + 7, + 2, + 3, + 0, + }, + ec_gf8_mul_93_ops}; + +static ec_gf_op_t ec_gf8_mul_94_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_94 = {8, + { + 7, + 5, + 0, + 2, + 6, + 1, + 3, + 4, + }, + ec_gf8_mul_94_ops}; + +static ec_gf_op_t ec_gf8_mul_95_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_95 = {8, + { + 7, + 6, + 1, + 3, + 0, + 4, + 5, + 2, + }, + ec_gf8_mul_95_ops}; + +static ec_gf_op_t ec_gf8_mul_96_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR3, 8, 0, 4}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_96 = {9, + { + 4, + 0, + 1, + 6, + 7, + 2, + 3, + 5, + 8, + }, + ec_gf8_mul_96_ops}; + +static ec_gf_op_t ec_gf8_mul_97_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 8, 6, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_97 = {9, + { + 4, + 5, + 3, + 6, + 7, + 1, + 2, + 0, + 8, + }, + ec_gf8_mul_97_ops}; + +static ec_gf_op_t ec_gf8_mul_98_ops[] = { + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_98 = {8, + { + 4, + 2, + 3, + 6, + 7, + 5, + 1, + 0, + }, + ec_gf8_mul_98_ops}; + +static ec_gf_op_t ec_gf8_mul_99_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_99 = {8, + { + 6, + 5, + 3, + 7, + 0, + 1, + 4, + 2, + }, + ec_gf8_mul_99_ops}; + +static ec_gf_op_t ec_gf8_mul_9A_ops[] = { + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9A = {9, + { + 6, + 3, + 4, + 0, + 5, + 1, + 2, + 7, + 8, + }, + ec_gf8_mul_9A_ops}; + +static ec_gf_op_t ec_gf8_mul_9B_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_COPY, 9, 5, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 8, 3, 2}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 9, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9B = {10, + { + 4, + 5, + 8, + 6, + 7, + 1, + 2, + 0, + 3, + 9, + }, + ec_gf8_mul_9B_ops}; + +static ec_gf_op_t ec_gf8_mul_9C_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9C = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_9C_ops}; + +static ec_gf_op_t ec_gf8_mul_9D_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9D = {8, + { + 0, + 1, + 2, + 3, + 7, + 4, + 5, + 6, + }, + ec_gf8_mul_9D_ops}; + +static ec_gf_op_t ec_gf8_mul_9E_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_COPY, 8, 7, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9E = {9, + { + 4, + 5, + 3, + 8, + 6, + 0, + 2, + 7, + 1, + }, + ec_gf8_mul_9E_ops}; + +static ec_gf_op_t ec_gf8_mul_9F_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9F = {9, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_9F_ops}; + +static ec_gf_op_t ec_gf8_mul_A0_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A0 = {8, + { + 3, + 1, + 6, + 7, + 5, + 2, + 4, + 0, + }, + ec_gf8_mul_A0_ops}; + +static ec_gf_op_t ec_gf8_mul_A1_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 0, 6}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A1 = {9, + { + 7, + 4, + 1, + 5, + 6, + 0, + 2, + 3, + 8, + }, + ec_gf8_mul_A1_ops}; + +static ec_gf_op_t ec_gf8_mul_A2_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A2 = {8, + { + 7, + 0, + 6, + 3, + 2, + 1, + 4, + 5, + }, + ec_gf8_mul_A2_ops}; + +static ec_gf_op_t ec_gf8_mul_A3_ops[] = { + {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A3 = {9, + { + 3, + 7, + 2, + 6, + 1, + 4, + 0, + 5, + 8, + }, + ec_gf8_mul_A3_ops}; + +static ec_gf_op_t ec_gf8_mul_A4_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A4 = {8, + { + 5, + 6, + 7, + 2, + 4, + 3, + 0, + 1, + }, + ec_gf8_mul_A4_ops}; + +static ec_gf_op_t ec_gf8_mul_A5_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR3, 8, 5, 6}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A5 = {9, + { + 1, + 4, + 2, + 5, + 6, + 7, + 3, + 0, + 8, + }, + ec_gf8_mul_A5_ops}; + +static ec_gf_op_t ec_gf8_mul_A6_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A6 = {8, + { + 1, + 2, + 0, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_A6_ops}; + +static ec_gf_op_t ec_gf8_mul_A7_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A7 = {8, + { + 0, + 1, + 2, + 5, + 6, + 7, + 3, + 4, + }, + ec_gf8_mul_A7_ops}; + +static ec_gf_op_t ec_gf8_mul_A8_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 9, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A8 = {10, + { + 1, + 7, + 5, + 8, + 6, + 3, + 4, + 0, + 2, + 9, + }, + ec_gf8_mul_A8_ops}; + +static ec_gf_op_t ec_gf8_mul_A9_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A9 = {8, + { + 3, + 7, + 6, + 1, + 2, + 0, + 4, + 5, + }, + ec_gf8_mul_A9_ops}; + +static ec_gf_op_t ec_gf8_mul_AA_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AA = {8, + { + 0, + 4, + 5, + 3, + 6, + 7, + 1, + 2, + }, + ec_gf8_mul_AA_ops}; + +static ec_gf_op_t ec_gf8_mul_AB_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_COPY, 9, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR3, 3, 9, 7}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AB = {10, + { + 2, + 3, + 8, + 0, + 5, + 6, + 1, + 4, + 7, + 9, + }, + ec_gf8_mul_AB_ops}; + +static ec_gf_op_t ec_gf8_mul_AC_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AC = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_AC_ops}; + +static ec_gf_op_t ec_gf8_mul_AD_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AD = {9, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 8, + }, + ec_gf8_mul_AD_ops}; + +static ec_gf_op_t ec_gf8_mul_AE_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_COPY, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AE = {9, + { + 7, + 0, + 5, + 6, + 3, + 4, + 1, + 2, + 8, + }, + ec_gf8_mul_AE_ops}; + +static ec_gf_op_t ec_gf8_mul_AF_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AF = {8, + { + 0, + 1, + 2, + 7, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_AF_ops}; + +static ec_gf_op_t ec_gf8_mul_B0_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B0 = {8, + { + 4, + 0, + 7, + 2, + 3, + 1, + 6, + 5, + }, + ec_gf8_mul_B0_ops}; + +static ec_gf_op_t ec_gf8_mul_B1_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR3, 5, 8, 1}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B1 = {9, + { + 2, + 6, + 4, + 7, + 0, + 1, + 3, + 5, + 8, + }, + ec_gf8_mul_B1_ops}; + +static ec_gf_op_t ec_gf8_mul_B2_ops[] = { + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 4, 5}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B2 = {9, + { + 0, + 7, + 4, + 5, + 6, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_B2_ops}; + +static ec_gf_op_t ec_gf8_mul_B3_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 9, 5, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 4}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR3, 1, 9, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B3 = {10, + { + 2, + 3, + 4, + 5, + 1, + 6, + 0, + 7, + 8, + 9, + }, + ec_gf8_mul_B3_ops}; + +static ec_gf_op_t ec_gf8_mul_B4_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B4 = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_B4_ops}; + +static ec_gf_op_t ec_gf8_mul_B5_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR3, 4, 8, 3}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B5 = {9, + { + 3, + 4, + 0, + 7, + 1, + 5, + 6, + 2, + 8, + }, + ec_gf8_mul_B5_ops}; + +static ec_gf_op_t ec_gf8_mul_B6_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B6 = {8, + { + 5, + 3, + 6, + 4, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_B6_ops}; + +static ec_gf_op_t ec_gf8_mul_B7_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B7 = {8, + { + 5, + 0, + 1, + 4, + 2, + 6, + 7, + 3, + }, + ec_gf8_mul_B7_ops}; + +static ec_gf_op_t ec_gf8_mul_B8_ops[] = { + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B8 = {8, + { + 6, + 4, + 5, + 1, + 2, + 0, + 7, + 3, + }, + ec_gf8_mul_B8_ops}; + +static ec_gf_op_t ec_gf8_mul_B9_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR3, 0, 8, 2}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B9 = {9, + { + 6, + 7, + 0, + 2, + 1, + 4, + 5, + 3, + 8, + }, + ec_gf8_mul_B9_ops}; + +static ec_gf_op_t ec_gf8_mul_BA_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BA = {8, + { + 1, + 2, + 4, + 3, + 5, + 6, + 0, + 7, + }, + ec_gf8_mul_BA_ops}; + +static ec_gf_op_t ec_gf8_mul_BB_ops[] = { + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BB = {9, + { + 7, + 2, + 1, + 8, + 3, + 5, + 6, + 4, + 0, + }, + ec_gf8_mul_BB_ops}; + +static ec_gf_op_t ec_gf8_mul_BC_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR3, 2, 8, 4}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BC = {9, + { + 2, + 6, + 3, + 4, + 5, + 1, + 7, + 0, + 8, + }, + ec_gf8_mul_BC_ops}; + +static ec_gf_op_t ec_gf8_mul_BD_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BD = {8, + { + 4, + 5, + 0, + 2, + 7, + 1, + 6, + 3, + }, + ec_gf8_mul_BD_ops}; + +static ec_gf_op_t ec_gf8_mul_BE_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BE = {8, + { + 0, + 6, + 7, + 4, + 5, + 1, + 3, + 2, + }, + ec_gf8_mul_BE_ops}; + +static ec_gf_op_t ec_gf8_mul_BF_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BF = {8, + { + 5, + 6, + 1, + 7, + 3, + 0, + 2, + 4, + }, + ec_gf8_mul_BF_ops}; + +static ec_gf_op_t ec_gf8_mul_C0_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C0 = {8, + { + 1, + 2, + 3, + 4, + 7, + 5, + 6, + 0, + }, + ec_gf8_mul_C0_ops}; + +static ec_gf_op_t ec_gf8_mul_C1_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C1 = {9, + { + 5, + 6, + 7, + 4, + 1, + 2, + 3, + 0, + 8, + }, + ec_gf8_mul_C1_ops}; + +static ec_gf_op_t ec_gf8_mul_C2_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C2 = {8, + { + 7, + 6, + 3, + 0, + 1, + 4, + 5, + 2, + }, + ec_gf8_mul_C2_ops}; + +static ec_gf_op_t ec_gf8_mul_C3_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR3, 0, 2, 6}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 9, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C3 = {10, + { + 5, + 6, + 4, + 7, + 1, + 2, + 3, + 0, + 8, + 9, + }, + ec_gf8_mul_C3_ops}; + +static ec_gf_op_t ec_gf8_mul_C4_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C4 = {8, + { + 0, + 2, + 1, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_C4_ops}; + +static ec_gf_op_t ec_gf8_mul_C5_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C5 = {8, + { + 4, + 3, + 5, + 7, + 6, + 2, + 0, + 1, + }, + ec_gf8_mul_C5_ops}; + +static ec_gf_op_t ec_gf8_mul_C6_ops[] = { + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 9, 5, 4}, + {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C6 = {10, + { + 6, + 3, + 0, + 4, + 5, + 7, + 2, + 1, + 8, + 9, + }, + ec_gf8_mul_C6_ops}; + +static ec_gf_op_t ec_gf8_mul_C7_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C7 = {8, + { + 7, + 0, + 6, + 2, + 5, + 3, + 4, + 1, + }, + ec_gf8_mul_C7_ops}; + +static ec_gf_op_t ec_gf8_mul_C8_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C8 = {8, + { + 1, + 3, + 2, + 4, + 6, + 7, + 5, + 0, + }, + ec_gf8_mul_C8_ops}; + +static ec_gf_op_t ec_gf8_mul_C9_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C9 = {8, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + }, + ec_gf8_mul_C9_ops}; + +static ec_gf_op_t ec_gf8_mul_CA_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CA = {8, + { + 1, + 2, + 5, + 7, + 3, + 4, + 0, + 6, + }, + ec_gf8_mul_CA_ops}; + +static ec_gf_op_t ec_gf8_mul_CB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CB = {8, + { + 2, + 3, + 4, + 5, + 7, + 6, + 0, + 1, + }, + ec_gf8_mul_CB_ops}; + +static ec_gf_op_t ec_gf8_mul_CC_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CC = {8, + { + 2, + 7, + 1, + 0, + 5, + 6, + 3, + 4, + }, + ec_gf8_mul_CC_ops}; + +static ec_gf_op_t ec_gf8_mul_CD_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CD = {8, + { + 0, + 6, + 1, + 2, + 7, + 3, + 4, + 5, + }, + ec_gf8_mul_CD_ops}; + +static ec_gf_op_t ec_gf8_mul_CE_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_COPY, 8, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 3, 6, 8}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 2, 3}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CE = {9, + { + 5, + 7, + 3, + 0, + 2, + 6, + 4, + 1, + 8, + }, + ec_gf8_mul_CE_ops}; + +static ec_gf_op_t ec_gf8_mul_CF_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CF = {8, + { + 3, + 6, + 7, + 0, + 2, + 4, + 5, + 1, + }, + ec_gf8_mul_CF_ops}; + +static ec_gf_op_t ec_gf8_mul_D0_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D0 = {8, + { + 5, + 6, + 7, + 2, + 0, + 3, + 1, + 4, + }, + ec_gf8_mul_D0_ops}; + +static ec_gf_op_t ec_gf8_mul_D1_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D1 = {9, + { + 5, + 6, + 3, + 2, + 0, + 7, + 4, + 1, + 8, + }, + ec_gf8_mul_D1_ops}; + +static ec_gf_op_t ec_gf8_mul_D2_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D2 = {8, + { + 7, + 0, + 2, + 1, + 3, + 4, + 6, + 5, + }, + ec_gf8_mul_D2_ops}; + +static ec_gf_op_t ec_gf8_mul_D3_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D3 = {9, + { + 0, + 3, + 2, + 8, + 4, + 6, + 7, + 1, + 5, + }, + ec_gf8_mul_D3_ops}; + +static ec_gf_op_t ec_gf8_mul_D4_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 1, 7, 8}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D4 = {9, + { + 4, + 1, + 7, + 5, + 0, + 6, + 3, + 2, + 8, + }, + ec_gf8_mul_D4_ops}; + +static ec_gf_op_t ec_gf8_mul_D5_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D5 = {8, + { + 6, + 7, + 4, + 5, + 2, + 3, + 1, + 0, + }, + ec_gf8_mul_D5_ops}; + +static ec_gf_op_t ec_gf8_mul_D6_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D6 = {9, + { + 0, + 6, + 2, + 7, + 1, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_D6_ops}; + +static ec_gf_op_t ec_gf8_mul_D7_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR3, 8, 3, 5}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR3, 6, 7, 8}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D7 = {9, + { + 3, + 4, + 6, + 5, + 0, + 7, + 1, + 2, + 8, + }, + ec_gf8_mul_D7_ops}; + +static ec_gf_op_t ec_gf8_mul_D8_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D8 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_D8_ops}; + +static ec_gf_op_t ec_gf8_mul_D9_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D9 = {8, + { + 1, + 2, + 6, + 7, + 4, + 5, + 0, + 3, + }, + ec_gf8_mul_D9_ops}; + +static ec_gf_op_t ec_gf8_mul_DA_ops[] = { + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR3, 8, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DA = {9, + { + 2, + 5, + 7, + 1, + 0, + 4, + 3, + 6, + 8, + }, + ec_gf8_mul_DA_ops}; + +static ec_gf_op_t ec_gf8_mul_DB_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DB = {9, + { + 7, + 5, + 6, + 2, + 3, + 4, + 1, + 0, + 8, + }, + ec_gf8_mul_DB_ops}; + +static ec_gf_op_t ec_gf8_mul_DC_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DC = {8, + { + 4, + 5, + 2, + 6, + 7, + 1, + 0, + 3, + }, + ec_gf8_mul_DC_ops}; + +static ec_gf_op_t ec_gf8_mul_DD_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DD = {8, + { + 1, + 2, + 3, + 6, + 7, + 0, + 4, + 5, + }, + ec_gf8_mul_DD_ops}; + +static ec_gf_op_t ec_gf8_mul_DE_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DE = {8, + { + 0, + 5, + 2, + 6, + 7, + 1, + 3, + 4, + }, + ec_gf8_mul_DE_ops}; + +static ec_gf_op_t ec_gf8_mul_DF_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR3, 1, 9, 2}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DF = {10, + { + 7, + 2, + 8, + 4, + 3, + 1, + 0, + 6, + 5, + 9, + }, + ec_gf8_mul_DF_ops}; + +static ec_gf_op_t ec_gf8_mul_E0_ops[] = { + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E0 = {8, + { + 2, + 3, + 4, + 7, + 5, + 6, + 0, + 1, + }, + ec_gf8_mul_E0_ops}; + +static ec_gf_op_t ec_gf8_mul_E1_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 7, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR3, 9, 5, 3}, + {EC_GF_OP_XOR2, 0, 9, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E1 = {10, + { + 0, + 7, + 1, + 3, + 4, + 5, + 6, + 2, + 8, + 9, + }, + ec_gf8_mul_E1_ops}; + +static ec_gf_op_t ec_gf8_mul_E2_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E2 = {8, + { + 2, + 3, + 7, + 1, + 5, + 6, + 0, + 4, + }, + ec_gf8_mul_E2_ops}; + +static ec_gf_op_t ec_gf8_mul_E3_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR3, 8, 2, 7}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 4}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E3 = {9, + { + 5, + 4, + 7, + 2, + 1, + 3, + 6, + 0, + 8, + }, + ec_gf8_mul_E3_ops}; + +static ec_gf_op_t ec_gf8_mul_E4_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E4 = {8, + { + 7, + 0, + 1, + 6, + 3, + 4, + 2, + 5, + }, + ec_gf8_mul_E4_ops}; + +static ec_gf_op_t ec_gf8_mul_E5_ops[] = { + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E5 = {9, + { + 4, + 5, + 3, + 6, + 7, + 1, + 0, + 2, + 8, + }, + ec_gf8_mul_E5_ops}; + +static ec_gf_op_t ec_gf8_mul_E6_ops[] = { + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E6 = {8, + { + 5, + 4, + 3, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_E6_ops}; + +static ec_gf_op_t ec_gf8_mul_E7_ops[] = { + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 9, 0, 6}, {EC_GF_OP_XOR2, 4, 9, 0}, + {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E7 = {10, + { + 1, + 4, + 3, + 6, + 7, + 5, + 2, + 0, + 8, + 9, + }, + ec_gf8_mul_E7_ops}; + +static ec_gf_op_t ec_gf8_mul_E8_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E8 = {8, + { + 1, + 4, + 2, + 7, + 3, + 0, + 5, + 6, + }, + ec_gf8_mul_E8_ops}; + +static ec_gf_op_t ec_gf8_mul_E9_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E9 = {9, + { + 6, + 2, + 0, + 3, + 4, + 1, + 5, + 7, + 8, + }, + ec_gf8_mul_E9_ops}; + +static ec_gf_op_t ec_gf8_mul_EA_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EA = {8, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_EA_ops}; + +static ec_gf_op_t ec_gf8_mul_EB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EB = {8, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_EB_ops}; + +static ec_gf_op_t ec_gf8_mul_EC_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EC = {9, + { + 7, + 4, + 3, + 0, + 2, + 5, + 1, + 6, + 8, + }, + ec_gf8_mul_EC_ops}; + +static ec_gf_op_t ec_gf8_mul_ED_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_ED = {8, + { + 5, + 6, + 7, + 0, + 1, + 4, + 3, + 2, + }, + ec_gf8_mul_ED_ops}; + +static ec_gf_op_t ec_gf8_mul_EE_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 2, 3}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EE = {9, + { + 6, + 4, + 5, + 7, + 2, + 3, + 0, + 1, + 8, + }, + ec_gf8_mul_EE_ops}; + +static ec_gf_op_t ec_gf8_mul_EF_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_COPY, 8, 0, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EF = {9, + { + 6, + 4, + 5, + 7, + 2, + 0, + 3, + 1, + 8, + }, + ec_gf8_mul_EF_ops}; + +static ec_gf_op_t ec_gf8_mul_F0_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 8, 3, 6}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F0 = {9, + { + 3, + 4, + 6, + 1, + 2, + 0, + 5, + 7, + 8, + }, + ec_gf8_mul_F0_ops}; + +static ec_gf_op_t ec_gf8_mul_F1_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 9, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 4, 9, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR3, 9, 8, 7}, + {EC_GF_OP_XOR2, 1, 9, 0}, {EC_GF_OP_XOR2, 5, 9, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F1 = {10, + { + 7, + 2, + 6, + 3, + 5, + 1, + 4, + 0, + 8, + 9, + }, + ec_gf8_mul_F1_ops}; + +static ec_gf_op_t ec_gf8_mul_F2_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 4}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F2 = {9, + { + 1, + 0, + 6, + 7, + 4, + 5, + 2, + 3, + 8, + }, + ec_gf8_mul_F2_ops}; + +static ec_gf_op_t ec_gf8_mul_F3_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F3 = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_F3_ops}; + +static ec_gf_op_t ec_gf8_mul_F4_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F4 = {8, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_F4_ops}; + +static ec_gf_op_t ec_gf8_mul_F5_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F5 = {8, + { + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_F5_ops}; + +static ec_gf_op_t ec_gf8_mul_F6_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 9, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR3, 7, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F6 = {10, + { + 0, + 6, + 2, + 7, + 4, + 3, + 5, + 9, + 1, + 8, + }, + ec_gf8_mul_F6_ops}; + +static ec_gf_op_t ec_gf8_mul_F7_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F7 = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 5, + }, + ec_gf8_mul_F7_ops}; + +static ec_gf_op_t ec_gf8_mul_F8_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F8 = {8, + { + 6, + 2, + 0, + 1, + 4, + 5, + 3, + 7, + }, + ec_gf8_mul_F8_ops}; + +static ec_gf_op_t ec_gf8_mul_F9_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR3, 8, 7, 1}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F9 = {9, + { + 4, + 1, + 7, + 6, + 0, + 3, + 5, + 2, + 8, + }, + ec_gf8_mul_F9_ops}; + +static ec_gf_op_t ec_gf8_mul_FA_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FA = {8, + { + 0, + 1, + 2, + 4, + 5, + 6, + 7, + 3, + }, + ec_gf8_mul_FA_ops}; + +static ec_gf_op_t ec_gf8_mul_FB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FB = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_FB_ops}; + +static ec_gf_op_t ec_gf8_mul_FC_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_COPY, 9, 3, 0}, + {EC_GF_OP_XOR3, 8, 5, 7}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR3, 0, 9, 2}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FC = {10, + { + 5, + 6, + 3, + 7, + 1, + 8, + 0, + 4, + 2, + 9, + }, + ec_gf8_mul_FC_ops}; + +static ec_gf_op_t ec_gf8_mul_FD_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_COPY, 8, 7, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FD = {9, + { + 5, + 3, + 7, + 6, + 1, + 2, + 4, + 0, + 8, + }, + ec_gf8_mul_FD_ops}; + +static ec_gf_op_t ec_gf8_mul_FE_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FE = {9, + { + 3, + 4, + 8, + 2, + 5, + 0, + 6, + 1, + 7, + }, + ec_gf8_mul_FE_ops}; + +static ec_gf_op_t ec_gf8_mul_FF_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 9, 0, 0}, + {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 9, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 9, 4, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR3, 3, 8, 5}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FF = {10, + { + 6, + 5, + 0, + 1, + 2, + 4, + 9, + 3, + 7, + 8, + }, + ec_gf8_mul_FF_ops}; + +ec_gf_mul_t *ec_gf8_mul[] = { + &ec_gf8_mul_00, &ec_gf8_mul_01, &ec_gf8_mul_02, &ec_gf8_mul_03, + &ec_gf8_mul_04, &ec_gf8_mul_05, &ec_gf8_mul_06, &ec_gf8_mul_07, + &ec_gf8_mul_08, &ec_gf8_mul_09, &ec_gf8_mul_0A, &ec_gf8_mul_0B, + &ec_gf8_mul_0C, &ec_gf8_mul_0D, &ec_gf8_mul_0E, &ec_gf8_mul_0F, + &ec_gf8_mul_10, &ec_gf8_mul_11, &ec_gf8_mul_12, &ec_gf8_mul_13, + &ec_gf8_mul_14, &ec_gf8_mul_15, &ec_gf8_mul_16, &ec_gf8_mul_17, + &ec_gf8_mul_18, &ec_gf8_mul_19, &ec_gf8_mul_1A, &ec_gf8_mul_1B, + &ec_gf8_mul_1C, &ec_gf8_mul_1D, &ec_gf8_mul_1E, &ec_gf8_mul_1F, + &ec_gf8_mul_20, &ec_gf8_mul_21, &ec_gf8_mul_22, &ec_gf8_mul_23, + &ec_gf8_mul_24, &ec_gf8_mul_25, &ec_gf8_mul_26, &ec_gf8_mul_27, + &ec_gf8_mul_28, &ec_gf8_mul_29, &ec_gf8_mul_2A, &ec_gf8_mul_2B, + &ec_gf8_mul_2C, &ec_gf8_mul_2D, &ec_gf8_mul_2E, &ec_gf8_mul_2F, + &ec_gf8_mul_30, &ec_gf8_mul_31, &ec_gf8_mul_32, &ec_gf8_mul_33, + &ec_gf8_mul_34, &ec_gf8_mul_35, &ec_gf8_mul_36, &ec_gf8_mul_37, + &ec_gf8_mul_38, &ec_gf8_mul_39, &ec_gf8_mul_3A, &ec_gf8_mul_3B, + &ec_gf8_mul_3C, &ec_gf8_mul_3D, &ec_gf8_mul_3E, &ec_gf8_mul_3F, + &ec_gf8_mul_40, &ec_gf8_mul_41, &ec_gf8_mul_42, &ec_gf8_mul_43, + &ec_gf8_mul_44, &ec_gf8_mul_45, &ec_gf8_mul_46, &ec_gf8_mul_47, + &ec_gf8_mul_48, &ec_gf8_mul_49, &ec_gf8_mul_4A, &ec_gf8_mul_4B, + &ec_gf8_mul_4C, &ec_gf8_mul_4D, &ec_gf8_mul_4E, &ec_gf8_mul_4F, + &ec_gf8_mul_50, &ec_gf8_mul_51, &ec_gf8_mul_52, &ec_gf8_mul_53, + &ec_gf8_mul_54, &ec_gf8_mul_55, &ec_gf8_mul_56, &ec_gf8_mul_57, + &ec_gf8_mul_58, &ec_gf8_mul_59, &ec_gf8_mul_5A, &ec_gf8_mul_5B, + &ec_gf8_mul_5C, &ec_gf8_mul_5D, &ec_gf8_mul_5E, &ec_gf8_mul_5F, + &ec_gf8_mul_60, &ec_gf8_mul_61, &ec_gf8_mul_62, &ec_gf8_mul_63, + &ec_gf8_mul_64, &ec_gf8_mul_65, &ec_gf8_mul_66, &ec_gf8_mul_67, + &ec_gf8_mul_68, &ec_gf8_mul_69, &ec_gf8_mul_6A, &ec_gf8_mul_6B, + &ec_gf8_mul_6C, &ec_gf8_mul_6D, &ec_gf8_mul_6E, &ec_gf8_mul_6F, + &ec_gf8_mul_70, &ec_gf8_mul_71, &ec_gf8_mul_72, &ec_gf8_mul_73, + &ec_gf8_mul_74, &ec_gf8_mul_75, &ec_gf8_mul_76, &ec_gf8_mul_77, + &ec_gf8_mul_78, &ec_gf8_mul_79, &ec_gf8_mul_7A, &ec_gf8_mul_7B, + &ec_gf8_mul_7C, &ec_gf8_mul_7D, &ec_gf8_mul_7E, &ec_gf8_mul_7F, + &ec_gf8_mul_80, &ec_gf8_mul_81, &ec_gf8_mul_82, &ec_gf8_mul_83, + &ec_gf8_mul_84, &ec_gf8_mul_85, &ec_gf8_mul_86, &ec_gf8_mul_87, + &ec_gf8_mul_88, &ec_gf8_mul_89, &ec_gf8_mul_8A, &ec_gf8_mul_8B, + &ec_gf8_mul_8C, &ec_gf8_mul_8D, &ec_gf8_mul_8E, &ec_gf8_mul_8F, + &ec_gf8_mul_90, &ec_gf8_mul_91, &ec_gf8_mul_92, &ec_gf8_mul_93, + &ec_gf8_mul_94, &ec_gf8_mul_95, &ec_gf8_mul_96, &ec_gf8_mul_97, + &ec_gf8_mul_98, &ec_gf8_mul_99, &ec_gf8_mul_9A, &ec_gf8_mul_9B, + &ec_gf8_mul_9C, &ec_gf8_mul_9D, &ec_gf8_mul_9E, &ec_gf8_mul_9F, + &ec_gf8_mul_A0, &ec_gf8_mul_A1, &ec_gf8_mul_A2, &ec_gf8_mul_A3, + &ec_gf8_mul_A4, &ec_gf8_mul_A5, &ec_gf8_mul_A6, &ec_gf8_mul_A7, + &ec_gf8_mul_A8, &ec_gf8_mul_A9, &ec_gf8_mul_AA, &ec_gf8_mul_AB, + &ec_gf8_mul_AC, &ec_gf8_mul_AD, &ec_gf8_mul_AE, &ec_gf8_mul_AF, + &ec_gf8_mul_B0, &ec_gf8_mul_B1, &ec_gf8_mul_B2, &ec_gf8_mul_B3, + &ec_gf8_mul_B4, &ec_gf8_mul_B5, &ec_gf8_mul_B6, &ec_gf8_mul_B7, + &ec_gf8_mul_B8, &ec_gf8_mul_B9, &ec_gf8_mul_BA, &ec_gf8_mul_BB, + &ec_gf8_mul_BC, &ec_gf8_mul_BD, &ec_gf8_mul_BE, &ec_gf8_mul_BF, + &ec_gf8_mul_C0, &ec_gf8_mul_C1, &ec_gf8_mul_C2, &ec_gf8_mul_C3, + &ec_gf8_mul_C4, &ec_gf8_mul_C5, &ec_gf8_mul_C6, &ec_gf8_mul_C7, + &ec_gf8_mul_C8, &ec_gf8_mul_C9, &ec_gf8_mul_CA, &ec_gf8_mul_CB, + &ec_gf8_mul_CC, &ec_gf8_mul_CD, &ec_gf8_mul_CE, &ec_gf8_mul_CF, + &ec_gf8_mul_D0, &ec_gf8_mul_D1, &ec_gf8_mul_D2, &ec_gf8_mul_D3, + &ec_gf8_mul_D4, &ec_gf8_mul_D5, &ec_gf8_mul_D6, &ec_gf8_mul_D7, + &ec_gf8_mul_D8, &ec_gf8_mul_D9, &ec_gf8_mul_DA, &ec_gf8_mul_DB, + &ec_gf8_mul_DC, &ec_gf8_mul_DD, &ec_gf8_mul_DE, &ec_gf8_mul_DF, + &ec_gf8_mul_E0, &ec_gf8_mul_E1, &ec_gf8_mul_E2, &ec_gf8_mul_E3, + &ec_gf8_mul_E4, &ec_gf8_mul_E5, &ec_gf8_mul_E6, &ec_gf8_mul_E7, + &ec_gf8_mul_E8, &ec_gf8_mul_E9, &ec_gf8_mul_EA, &ec_gf8_mul_EB, + &ec_gf8_mul_EC, &ec_gf8_mul_ED, &ec_gf8_mul_EE, &ec_gf8_mul_EF, + &ec_gf8_mul_F0, &ec_gf8_mul_F1, &ec_gf8_mul_F2, &ec_gf8_mul_F3, + &ec_gf8_mul_F4, &ec_gf8_mul_F5, &ec_gf8_mul_F6, &ec_gf8_mul_F7, + &ec_gf8_mul_F8, &ec_gf8_mul_F9, &ec_gf8_mul_FA, &ec_gf8_mul_FB, + &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF}; diff --git a/xlators/cluster/ec/src/ec-gf.h b/xlators/cluster/ec/src/ec-gf8.h index 23bca91e3b5..4aca91127fc 100644 --- a/xlators/cluster/ec/src/ec-gf.h +++ b/xlators/cluster/ec/src/ec-gf8.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser @@ -11,13 +11,8 @@ #ifndef __EC_GF8_H__ #define __EC_GF8_H__ -#define EC_GF_BITS 8 -#define EC_GF_MOD 0x11D +#include "ec-galois.h" -#define EC_GF_SIZE (1 << EC_GF_BITS) -#define EC_GF_WORD_SIZE sizeof(uint64_t) - -extern void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in, - unsigned int width); +extern ec_gf_mul_t *ec_gf8_mul[]; #endif /* __EC_GF8_H__ */ diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c index a121bb43e5c..7d991f04aac 100644 --- a/xlators/cluster/ec/src/ec-heal.c +++ b/xlators/cluster/ec/src/ec-heal.c @@ -8,158 +8,164 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" -#include "compat-errno.h" - +#include <glusterfs/defaults.h> +#include <glusterfs/compat-errno.h> +#include <glusterfs/byte-order.h> +#include <glusterfs/syncop.h> +#include <glusterfs/syncop-utils.h> +#include <glusterfs/cluster-syncop.h> + +#include "ec.h" +#include "ec-types.h" +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" #include "ec-method.h" #include "ec-fops.h" - -#include "ec-mem-types.h" -#include "ec-data.h" - -static char *ec_ignore_xattrs[] = { - GF_SELINUX_XATTR_KEY, - QUOTA_SIZE_KEY, - NULL +#include "ec-heald.h" + +#define EC_COUNT(array, max) \ + ({ \ + int __i; \ + int __res = 0; \ + for (__i = 0; __i < max; __i++) \ + if (array[__i]) \ + __res++; \ + __res; \ + }) +#define EC_INTERSECT(dst, src1, src2, max) \ + ({ \ + int __i; \ + for (__i = 0; __i < max; __i++) \ + dst[__i] = src1[__i] && src2[__i]; \ + }) +#define EC_ADJUST_SOURCE(source, sources, max) \ + ({ \ + int __i; \ + if (sources[source] == 0) { \ + source = -1; \ + for (__i = 0; __i < max; __i++) \ + if (sources[__i]) \ + source = __i; \ + } \ + }) +#define IA_EQUAL(f, s, field) \ + (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0) +#define EC_REPLIES_ALLOC(replies, numsubvols) \ + do { \ + int __i = 0; \ + replies = alloca0(numsubvols * sizeof(*replies)); \ + for (__i = 0; __i < numsubvols; __i++) \ + INIT_LIST_HEAD(&replies[__i].entries.list); \ + } while (0) + +struct ec_name_data { + call_frame_t *frame; + unsigned char *participants; + unsigned char *failed_on; + unsigned char *gfidless; + unsigned char *enoent; + unsigned char *same; + char *name; + inode_t *parent; + default_args_cbk_t *replies; + uint32_t heal_pending; }; +static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL}; + static gf_boolean_t -ec_ignorable_key_match (dict_t *dict, char *key, data_t *val, void *mdata) +ec_ignorable_key_match(dict_t *dict, char *key, data_t *val, void *mdata) { - int i = 0; + int i = 0; - if (!key) - goto out; + if (!key) + goto out; - for (i = 0; ec_ignore_xattrs[i]; i++) { - if (!strcmp (key, ec_ignore_xattrs[i])) - return _gf_true; - } + if (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0) + return _gf_true; + + for (i = 0; ec_ignore_xattrs[i]; i++) { + if (!strcmp(key, ec_ignore_xattrs[i])) + return _gf_true; + } out: - return _gf_false; + return _gf_false; } -/* FOP: heal */ - -void ec_heal_exclude(ec_heal_t * heal, uintptr_t mask) +static gf_boolean_t +ec_sh_key_match(dict_t *dict, char *key, data_t *val, void *mdata) { - LOCK(&heal->lock); - - heal->bad &= ~mask; - - UNLOCK(&heal->lock); + return !ec_ignorable_key_match(dict, key, val, mdata); } +/* FOP: heal */ -void ec_heal_lookup_resume(ec_fop_data_t * fop) +void +ec_set_entry_healing(ec_fop_data_t *fop) { - ec_heal_t * heal = fop->data; - ec_cbk_data_t * cbk; - uintptr_t good = 0, bad = 0; + ec_inode_t *ctx = NULL; + loc_t *loc = NULL; - if (heal->lookup != NULL) - { - ec_fop_data_release(heal->lookup); - } - ec_fop_data_acquire(fop); + if (!fop) + return; - list_for_each_entry(cbk, &fop->cbk_list, list) + loc = &fop->loc[0]; + LOCK(&loc->inode->lock); { - if ((cbk->op_ret < 0) && (cbk->op_errno == ENOTCONN)) - { - continue; - } - - if (cbk == fop->answer) - { - if (cbk->op_ret >= 0) - { - heal->iatt = cbk->iatt[0]; - heal->version = cbk->version; - heal->raw_size = cbk->size; - heal->fop->pre_size = cbk->iatt[0].ia_size; - heal->fop->post_size = cbk->iatt[0].ia_size; - heal->fop->have_size = 1; - - if (ec_loc_update(heal->xl, &heal->loc, cbk->inode, - &cbk->iatt[0]) != 0) - { - fop->answer = NULL; - fop->error = EIO; - - bad |= cbk->mask; - - continue; - } - } - - good |= cbk->mask; - } - else - { - bad |= cbk->mask; + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + ctx->heal_count += 1; } } - - /* Heal lookups are not executed concurrently with anything else. So, when - * a lookup finishes, it's safe to access heal->good and heal->bad without - * acquiring any lock. - */ - heal->good = good; - heal->bad = bad; - - heal->lookup = fop; - - ec_resume_parent(fop, fop->answer != NULL ? 0 : fop->error); + UNLOCK(&loc->inode->lock); } -int32_t ec_heal_entry_lookup_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, inode_t * inode, - struct iatt * buf, dict_t * xdata, - struct iatt * postparent) +void +ec_reset_entry_healing(ec_fop_data_t *fop) { - ec_heal_lookup_resume(cookie); - - return 0; -} - -int32_t ec_heal_inode_lookup_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, inode_t * inode, - struct iatt * buf, dict_t * xdata, - struct iatt * postparent) -{ - ec_heal_lookup_resume(cookie); + ec_inode_t *ctx = NULL; + loc_t *loc = NULL; + int32_t heal_count = 0; + if (!fop) + return; - return 0; + loc = &fop->loc[0]; + LOCK(&loc->inode->lock); + { + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + ctx->heal_count += -1; + heal_count = ctx->heal_count; + } + } + UNLOCK(&loc->inode->lock); + GF_ASSERT(heal_count >= 0); } -uintptr_t ec_heal_check(ec_fop_data_t * fop, uintptr_t * pgood) +uintptr_t +ec_heal_check(ec_fop_data_t *fop, uintptr_t *pgood) { - ec_cbk_data_t * cbk; - uintptr_t mask[2] = { 0, 0 }; + ec_cbk_data_t *cbk; + uintptr_t mask[2] = {0, 0}; list_for_each_entry(cbk, &fop->cbk_list, list) { mask[cbk->op_ret >= 0] |= cbk->mask; } - if (pgood != NULL) - { + if (pgood != NULL) { *pgood = mask[1]; } return mask[0]; } -void ec_heal_update(ec_fop_data_t * fop, int32_t is_open) +void +ec_heal_update(ec_fop_data_t *fop, int32_t is_open) { - ec_heal_t * heal = fop->data; + ec_heal_t *heal = fop->data; uintptr_t good, bad; bad = ec_heal_check(fop, &good); @@ -167,8 +173,7 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open) LOCK(&heal->lock); heal->bad &= ~bad; - if (is_open) - { + if (is_open) { heal->open |= good; } @@ -177,9 +182,10 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open) fop->error = 0; } -void ec_heal_avoid(ec_fop_data_t * fop) +void +ec_heal_avoid(ec_fop_data_t *fop) { - ec_heal_t * heal = fop->data; + ec_heal_t *heal = fop->data; uintptr_t bad; bad = ec_heal_check(fop, NULL); @@ -191,1426 +197,3171 @@ void ec_heal_avoid(ec_fop_data_t * fop) UNLOCK(&heal->lock); } -int32_t ec_heal_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) +int32_t +ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_heal_update(cookie, 0); + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + if (op_ret >= 0) { + GF_ASSERT( + ec_set_inode_size(heal->fop, heal->fd->inode, heal->total_size)); + } return 0; } -int32_t ec_heal_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) +void +ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc, off_t offset, + size_t size) { - ec_heal_update(cookie, 0); + struct gf_flock flock; + fop_inodelk_cbk_t cbk = NULL; - return 0; + flock.l_type = type; + flock.l_whence = SEEK_SET; + flock.l_start = offset; + flock.l_len = size; + flock.l_pid = 0; + flock.l_owner.len = 0; + + if (type == F_UNLCK) { + /* Remove inode size information before unlocking it. */ + if (fd == NULL) { + ec_clear_inode_info(heal->fop, heal->loc.inode); + } else { + ec_clear_inode_info(heal->fop, heal->fd->inode); + } + cbk = ec_lock_unlocked; + } else { + /* Otherwise use the callback to update size information. */ + cbk = ec_heal_lock_cbk; + } + + if (fd != NULL) { + ec_finodelk(heal->fop->frame, heal->xl, + &heal->fop->frame->root->lk_owner, heal->fop->mask, + EC_MINIMUM_ALL, cbk, heal, heal->xl->name, fd, F_SETLKW, + &flock, NULL); + } else { + ec_inodelk(heal->fop->frame, heal->xl, + &heal->fop->frame->root->lk_owner, heal->fop->mask, + EC_MINIMUM_ALL, cbk, heal, heal->xl->name, loc, F_SETLKW, + &flock, NULL); + } } -int32_t ec_heal_symlink_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - inode_t * inode, struct iatt * buf, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) +void +ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd, off_t offset, + size_t size) { - ec_heal_update(cookie, 0); - - return 0; + ec_heal_lock(heal, type, use_fd ? heal->fd : NULL, &heal->loc, offset, + size); } -int32_t ec_heal_create_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - fd_t * fd, inode_t * inode, struct iatt * buf, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) +int32_t +ec_heal_xattr_clean(dict_t *dict, char *key, data_t *data, void *arg) { - ec_heal_update(cookie, 1); + dict_t *base = arg; + + if (ec_ignorable_key_match(NULL, key, NULL, NULL)) { + dict_del(dict, key); + return 0; + } + + if (dict_get(base, key) != NULL) + dict_del(dict, key); return 0; } -int32_t ec_heal_setattr_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct iatt * preop_stbuf, - struct iatt * postop_stbuf, - dict_t * xdata) +/******************************************************************** + * ec_wind_xattrop_parallel: + * Helper function to update the extended attributes + * in parallel. + * + *******************************************************************/ +void +ec_wind_xattrop_parallel(call_frame_t *frame, xlator_t *subvol, int child_index, + loc_t *loc, gf_xattrop_flags_t flags, dict_t **dict, + dict_t *xdata) { - ec_heal_update(cookie, 0); - - return 0; + gf_msg_debug("EC", 0, "WIND: on child %d ", child_index); + STACK_WIND_COOKIE( + frame, cluster_xattrop_cbk, (void *)(uintptr_t)child_index, subvol, + subvol->fops->xattrop, loc, flags, dict[child_index], xdata); } -int32_t ec_heal_setxattr_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - dict_t * xdata) +int32_t +ec_heal_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) { + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno); + + gf_msg_debug(fop->xl->name, 0, + "%s: write op_ret %d, op_errno %s" + " at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), op_ret, strerror(op_errno), + heal->offset); + ec_heal_update(cookie, 0); return 0; } -int32_t ec_heal_removexattr_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, dict_t * xdata) +int32_t +ec_heal_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) { - ec_heal_update(cookie, 0); + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno); + + ec_heal_avoid(fop); + + if (op_ret > 0) { + gf_msg_debug(fop->xl->name, 0, + "%s: read succeeded, proceeding " + "to write at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), heal->offset); + ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE, + ec_heal_writev_cbk, heal, heal->fd, vector, count, + heal->offset, 0, iobref, NULL); + } else { + if (op_ret < 0) { + gf_msg_debug(fop->xl->name, 0, + "%s: read failed %s, failing " + "to heal block at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), strerror(op_errno), + heal->offset); + heal->bad = 0; + } + heal->done = 1; + } return 0; } -int32_t ec_heal_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, inode_t * inode, - struct iatt * buf, struct iatt * preparent, - struct iatt * postparent, dict_t * xdata) +void +ec_heal_data_block(ec_heal_t *heal) { - ec_fop_data_t * fop = cookie; - ec_heal_t * heal = fop->data; - uintptr_t good, bad; + ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad); - bad = ec_heal_check(fop, &good); - ec_heal_exclude(heal, good); + if ((heal->good != 0) && (heal->bad != 0) && + (heal->iatt.ia_type == IA_IFREG)) { + ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN, + ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, 0, + NULL); + } +} - if (bad != 0) - { - fop->error = 0; +/* FOP: fheal */ - xdata = fop->xdata; - fop = fop->parent; +void +ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd, + int32_t partial, dict_t *xdata) +{ + ec_fd_t *ctx = ec_fd_get(fd, this); - ec_create(fop->frame, fop->xl, bad, EC_MINIMUM_ONE, - ec_heal_create_cbk, heal, &heal->loc, 0, - st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL), - 0, heal->fd, xdata); + if (ctx != NULL) { + gf_msg_trace("ec", 0, "FHEAL ctx: flags=%X, open=%" PRIXPTR, ctx->flags, + ctx->open); + ec_heal(frame, this, target, fop_flags, func, data, &ctx->loc, partial, + xdata); } - - return 0; } -int32_t ec_heal_target_open_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, fd_t * fd, dict_t * xdata) +/* Common heal code */ +void +ec_mask_to_char_array(uintptr_t mask, unsigned char *array, int numsubvols) { - ec_heal_update(cookie, 1); + int i = 0; - return 0; + for (i = 0; i < numsubvols; i++) + array[i] = ((mask >> i) & 1); } -int32_t ec_heal_source_open_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, - int32_t op_errno, fd_t * fd, dict_t * xdata) +uintptr_t +ec_char_array_to_mask(unsigned char *array, int numsubvols) { - ec_heal_avoid(cookie); + int i = 0; + uintptr_t mask = 0; - return 0; + if (array == NULL) + goto out; + + for (i = 0; i < numsubvols; i++) + if (array[i]) + mask |= (1ULL << i); +out: + return mask; } -int32_t ec_heal_reopen_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - fd_t * fd, dict_t * xdata) +int +ec_heal_entry_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *versions, uint64_t *dirty, + unsigned char *sources, + unsigned char *healed_sinks) { - ec_fop_data_t * fop = cookie; - ec_fd_t * ctx; - uintptr_t good; + uint64_t xattr[EC_VERSION_SIZE] = {0}; + int source = -1; + uint64_t max_version = 0; + int ret = 0; + int i = 0; + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; - ec_heal_check(fop, &good); + if (replies[i].op_ret == -1) + continue; - if (good != 0) - { - LOCK(&fd->lock); + if (source == -1) + source = i; - ctx = __ec_fd_get(fd, fop->xl); - if (ctx != NULL) { - ctx->bad &= ~good; - ctx->open |= good; + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + versions[i] = xattr[EC_DATA_TXN]; + if (max_version < versions[i]) { + max_version = versions[i]; + source = i; + } } - UNLOCK(&fd->lock); + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_DATA_TXN]; + } } - return 0; -} + if (source < 0) + goto out; -int32_t ec_heal_create(ec_heal_t * heal, uintptr_t mask, int32_t try_link) -{ - loc_t loc; - dict_t * xdata; + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; - xdata = dict_new(); - if (xdata == NULL) - { - return ENOMEM; + if (replies[i].op_ret == -1) + continue; + + if (versions[i] == versions[source]) + sources[i] = 1; + else + healed_sinks[i] = 1; } - if (dict_set_static_bin(xdata, "gfid-req", heal->iatt.ia_gfid, - sizeof(uuid_t)) != 0) - { - dict_unref(xdata); +out: + return source; +} - return ENOMEM; +int +ec_adjust_versions(call_frame_t *frame, ec_t *ec, ec_txn_t type, inode_t *inode, + int source, unsigned char *sources, + unsigned char *healed_sinks, uint64_t *versions, + uint64_t *dirty) +{ + int i = 0; + int ret = 0; + int call_count = 0; + dict_t **xattr = NULL; + int op_ret = 0; + loc_t loc = {0}; + gf_boolean_t erase_dirty = _gf_false; + uint64_t *versions_xattr = NULL; + uint64_t *dirty_xattr = NULL; + uint64_t allzero[2] = {0}; + unsigned char *on = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + + /* Allocate the required memory */ + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + EC_REPLIES_ALLOC(replies, ec->nodes); + xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer); + if (!xattr) { + op_ret = -ENOMEM; + goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict_new(); + if (!xattr[i]) { + op_ret = -ENOMEM; + goto out; + } } - if ((heal->iatt.ia_type == IA_IFREG) && try_link) - { - memset(&loc, 0, sizeof(loc)); - loc.inode = heal->loc.inode; - uuid_copy(loc.gfid, heal->iatt.ia_gfid); + /* dirty xattr represents if the file/dir needs heal. Unless all the + * copies are healed, don't erase it */ + if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) == + ec->nodes) + erase_dirty = _gf_true; + else + op_ret = -ENOTCONN; - ec_link(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_link_cbk, heal, &loc, &heal->loc, xdata); + /* Populate the xattr array */ + for (i = 0; i < ec->nodes; i++) { + if (!sources[i] && !healed_sinks[i]) + continue; + versions_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*versions_xattr), + gf_common_mt_pointer); + if (!versions_xattr) { + op_ret = -ENOMEM; + continue; + } - dict_unref(xdata); + versions_xattr[type] = hton64(versions[source] - versions[i]); + ret = dict_set_bin(xattr[i], EC_XATTR_VERSION, versions_xattr, + (sizeof(*versions_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + op_ret = -ENOMEM; + continue; + } - return 0; - } + if (erase_dirty) { + dirty_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*dirty_xattr), + gf_common_mt_pointer); + if (!dirty_xattr) { + op_ret = -ENOMEM; + continue; + } - switch (heal->iatt.ia_type) - { - case IA_IFDIR: - ec_mkdir(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_mkdir_cbk, heal, &heal->loc, - st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL), - 0, xdata); + dirty_xattr[type] = hton64(-dirty[i]); + ret = dict_set_bin(xattr[i], EC_XATTR_DIRTY, dirty_xattr, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + op_ret = -ENOMEM; + continue; + } + } - break; + if (memcmp(versions_xattr, allzero, + (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) { + if (!erase_dirty) { + continue; + } - case IA_IFLNK: - ec_symlink(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_symlink_cbk, heal, heal->symlink, &heal->loc, - 0, xdata); + if (memcmp(dirty_xattr, allzero, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)) == 0) { + continue; + } + } - break; + on[i] = 1; + call_count++; + } - case IA_IFREG: - ec_create(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_create_cbk, heal, &heal->loc, 0, - st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL), - 0, heal->fd, xdata); + /* Update the bricks with xattr */ + if (call_count) { + PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame, + ec_wind_xattrop_parallel, &loc, + GF_XATTROP_ADD_ARRAY64, xattr, NULL); + ret = cluster_fop_success_fill(replies, ec->nodes, output); + } - break; + if (ret < call_count) { + op_ret = -ENOTCONN; + goto out; + } - default: - ec_mknod(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_mknod_cbk, heal, &heal->loc, - st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL), - heal->iatt.ia_rdev, 0, xdata); +out: + /* Cleanup */ + if (xattr) { + for (i = 0; i < ec->nodes; i++) { + if (xattr[i]) + dict_unref(xattr[i]); + } + GF_FREE(xattr); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return op_ret; +} - break; +int +ec_heal_metadata_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *versions, uint64_t *dirty, + unsigned char *sources, + unsigned char *healed_sinks) +{ + uint64_t xattr[EC_VERSION_SIZE] = {0}; + uint64_t max_version = 0; + int same_count = 0; + int max_same_count = 0; + int same_source = -1; + int ret = 0; + int i = 0; + int j = 0; + int *groups = NULL; + struct iatt source_ia = {0}; + struct iatt child_ia = {0}; + + groups = alloca0(ec->nodes * sizeof(*groups)); + for (i = 0; i < ec->nodes; i++) + groups[i] = -1; + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + if (replies[i].op_ret < 0) + continue; + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + versions[i] = xattr[EC_METADATA_TXN]; + } + + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_METADATA_TXN]; + } + if (groups[i] >= 0) /*Already part of group*/ + continue; + groups[i] = i; + same_count = 1; + source_ia = replies[i].stat; + for (j = i + 1; j < ec->nodes; j++) { + if (!replies[j].valid || replies[j].op_ret < 0) + continue; + child_ia = replies[j].stat; + if (!IA_EQUAL(source_ia, child_ia, gfid) || + !IA_EQUAL(source_ia, child_ia, type) || + !IA_EQUAL(source_ia, child_ia, prot) || + !IA_EQUAL(source_ia, child_ia, uid) || + !IA_EQUAL(source_ia, child_ia, gid)) + continue; + if (!are_dicts_equal(replies[i].xdata, replies[j].xdata, + ec_sh_key_match, NULL)) + continue; + groups[j] = i; + same_count++; + } + + if (max_same_count < same_count) { + max_same_count = same_count; + same_source = i; + } } - dict_unref(xdata); + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; + } - return 0; + for (i = 0; i < ec->nodes; i++) { + if (groups[i] == groups[same_source]) + sources[i] = 1; + else if (replies[i].valid && replies[i].op_ret >= 0) + healed_sinks[i] = 1; + } + for (i = 0; i < ec->nodes; i++) { + if (sources[i] && (versions[i] > max_version)) { + same_source = i; + max_version = versions[i]; + } + } + ret = same_source; +out: + return ret; } -int32_t ec_heal_parent_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, - int32_t op_ret, int32_t op_errno, uintptr_t mask, - uintptr_t good, uintptr_t bad, dict_t *xdata) +int +__ec_heal_metadata_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, + default_args_cbk_t *replies, uint64_t *versions, + uint64_t *dirty, unsigned char *sources, + unsigned char *healed_sinks) { - ec_fop_data_t *fop = cookie; - ec_heal_t *heal = fop->data; + loc_t loc = {0}; + unsigned char *output = NULL; + unsigned char *lookup_on = NULL; + int ret = 0; + int source = 0; + default_args_cbk_t *greplies = NULL; + int i = 0; + EC_REPLIES_ALLOC(greplies, ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + output = alloca0(ec->nodes); + lookup_on = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, NULL); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } - /* Even if parent self-heal has failed, we try to heal the current entry */ - ec_heal_create(heal, fop->mask, 0); + memcpy(lookup_on, output, ec->nodes); + /*Use getxattr to get the filtered xattrs which filter internal xattrs*/ + ret = cluster_getxattr(ec->xl_list, lookup_on, ec->nodes, greplies, output, + frame, ec->xl, &loc, NULL, NULL); + for (i = 0; i < ec->nodes; i++) { + if (lookup_on[i] && !output[i]) { + replies[i].valid = 0; + continue; + } + if (replies[i].xdata) { + dict_unref(replies[i].xdata); + replies[i].xdata = NULL; + if (greplies[i].xattr) + replies[i].xdata = dict_ref(greplies[i].xattr); + } + } - return 0; + source = ec_heal_metadata_find_direction(ec, replies, versions, dirty, + sources, healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } + ret = source; +out: + cluster_replies_wipe(greplies, ec->nodes); + loc_wipe(&loc); + return ret; } -void ec_heal_parent(ec_heal_t *heal, uintptr_t mask) +/* Metadata heal */ +int +__ec_removexattr_sinks(call_frame_t *frame, ec_t *ec, inode_t *inode, + int source, unsigned char *sources, + unsigned char *healed_sinks, default_args_cbk_t *replies) { - loc_t parent; - int32_t healing = 0; + int i = 0; + int ret = 0; + loc_t loc = {0}; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); - /* First we try to do a partial heal of the parent directory to avoid - * ENOENT/ENOTDIR errors caused by missing parents */ - if (ec_loc_parent(heal->xl, &heal->loc, &parent) == 0) { - if (!__is_root_gfid(parent.gfid)) { - ec_heal(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_parent_cbk, heal, &parent, 1, NULL); + for (i = 0; i < ec->nodes; i++) { + if (i == source) + continue; + if (!sources[i] && !healed_sinks[i]) + continue; + ret = dict_foreach(replies[i].xdata, ec_heal_xattr_clean, + replies[source].xdata); + if (ret < 0) { + sources[i] = 0; + healed_sinks[i] = 0; + continue; + } - healing = 1; + if (replies[i].xdata->count == 0) { + continue; + } else if (sources[i]) { + /* This can happen if setxattr/removexattr succeeds on + * the bricks but fails to update the version. This + * will make sure that the xattrs are made equal after + * heal*/ + sources[i] = 0; + healed_sinks[i] = 1; } - loc_wipe(&parent); - } - if (!healing) { - ec_heal_create(heal, mask, 0); + ret = syncop_removexattr(ec->xl_list[i], &loc, "", replies[i].xdata, + NULL); + if (ret < 0) + healed_sinks[i] = 0; } + + loc_wipe(&loc); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) + return -ENOTCONN; + return 0; } -void ec_heal_recreate(ec_fop_data_t * fop) +int +__ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, unsigned char *sources, + unsigned char *healed_sinks) { - ec_cbk_data_t * cbk; - ec_heal_t * heal = fop->data; - uintptr_t mask = 0; + loc_t loc = {0}; + int ret = 0; + int source = 0; + default_args_cbk_t *replies = NULL; + default_args_cbk_t *sreplies = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + unsigned char *output = NULL; + dict_t *source_dict = NULL; + struct iatt source_buf = {0}; + + EC_REPLIES_ALLOC(replies, ec->nodes); + EC_REPLIES_ALLOC(sreplies, ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + output = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + source = __ec_heal_metadata_prepare(frame, ec, inode, locked_on, replies, + versions, dirty, sources, healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } - if (heal->iatt.ia_type == IA_INVAL) - { - return; + if ((EC_COUNT(sources, ec->nodes) == ec->nodes) || + (EC_COUNT(healed_sinks, ec->nodes) == 0)) { + ret = 0; + goto erase_dirty; } - list_for_each_entry(cbk, &fop->cbk_list, list) - { - if ((cbk->op_ret >= 0) || (cbk->op_errno == ENOENT) || - (cbk->op_errno == ENOTDIR)) - { - mask |= cbk->mask; - } + source_buf = replies[source].stat; + ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, sreplies, + output, frame, ec->xl, &loc, &source_buf, + GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID, + NULL); + /*In case the operation fails on some of the subvols*/ + memcpy(healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; } - if (mask != 0) - { - ec_heal_parent(heal, mask); + ret = __ec_removexattr_sinks(frame, ec, inode, source, sources, + healed_sinks, replies); + if (ret < 0) + goto out; + + source_dict = dict_ref(replies[source].xdata); + if (dict_foreach_match(source_dict, ec_ignorable_key_match, NULL, + dict_remove_foreach_fn, NULL) == -1) { + ret = -ENOMEM; + goto out; } -} -int32_t ec_heal_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) -{ - ec_heal_update(cookie, 0); - ec_heal_recreate(cookie); + ret = cluster_setxattr(ec->xl_list, healed_sinks, ec->nodes, replies, + output, frame, ec->xl, &loc, source_dict, 0, NULL); - return 0; + EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + +erase_dirty: + ret = ec_adjust_versions(frame, ec, EC_METADATA_TXN, inode, source, sources, + healed_sinks, versions, dirty); +out: + if (source_dict) + dict_unref(source_dict); + + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + cluster_replies_wipe(sreplies, ec->nodes); + return ret; } -int32_t ec_heal_unlink_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct iatt * preparent, struct iatt * postparent, - dict_t * xdata) +int +ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks) { - ec_heal_update(cookie, 0); - ec_heal_recreate(cookie); - - return 0; + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + int ret = 0; + default_args_cbk_t *replies = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_metadata(frame, ec, inode, locked_on, sources, + healed_sinks); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; } -int32_t ec_heal_init(ec_fop_data_t * fop) +/*entry heal*/ +int +__ec_heal_entry_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, uint64_t *versions, + uint64_t *dirty, unsigned char *sources, + unsigned char *healed_sinks) { - ec_t * ec = fop->xl->private; - struct iobuf_pool * pool; - inode_t * inode; - ec_inode_t * ctx; - ec_heal_t * heal = NULL; - int32_t error = 0; + loc_t loc = {0}; + int source = 0; + int ret = 0; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + dict_t *xdata = NULL; - inode = fop->loc[0].inode; - if (inode == NULL) - { - gf_log(fop->xl->name, GF_LOG_WARNING, "Unable to start inode healing " - "because there is not enough " - "information"); + EC_REPLIES_ALLOC(replies, ec->nodes); - return ENODATA; + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + xdata = dict_new(); + if (!xdata) { + ret = -ENOMEM; + goto out; } - heal = GF_MALLOC(sizeof(ec_heal_t), ec_mt_ec_heal_t); - if (heal == NULL) - { - return ENOMEM; + if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) || + dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) { + ret = -ENOMEM; + goto out; } - memset(heal, 0, sizeof(ec_heal_t)); - - if (ec_loc_from_loc(fop->xl, &heal->loc, &fop->loc[0]) != 0) { - error = ENOMEM; + output = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, xdata); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } + source = ec_heal_entry_find_direction(ec, replies, versions, dirty, sources, + healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } + ret = source; +out: + if (xdata) + dict_unref(xdata); + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} +int32_t +ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia, + call_frame_t *frame, xlator_t *this, unsigned char *on) +{ + dict_t *xattr = NULL; + int32_t ret = -1; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + uint64_t dirty[EC_VERSION_SIZE] = {1, 1}; + loc_t newloc = {0}; + + /*Symlinks don't have any data to be healed*/ + if (ia->ia_type == IA_IFLNK) + dirty[EC_DATA_TXN] = 0; + + newloc.inode = inode_ref(loc->inode); + gf_uuid_copy(newloc.gfid, ia->ia_gfid); + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + xattr = dict_new(); + if (!xattr) { + ret = -ENOMEM; goto out; } - LOCK_INIT(&heal->lock); + ret = ec_dict_set_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (ret) + goto out; - heal->xl = fop->xl; - heal->fop = fop; - pool = fop->xl->ctx->iobuf_pool; - heal->size = iobpool_default_pagesize(pool) * ec->fragments; - heal->partial = fop->int32; - fop->heal = heal; + ret = cluster_xattrop(ec->xl_list, on, ec->nodes, replies, output, frame, + ec->xl, &newloc, GF_XATTROP_ADD_ARRAY64, xattr, NULL); - LOCK(&inode->lock); + if (ret < ec->fragments) { + ret = -ENOTCONN; + goto out; + } - ctx = __ec_inode_get(inode, fop->xl); - if (ctx == NULL) - { - error = EIO; +out: + if (xattr) + dict_unref(xattr); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&newloc); + return ret; +} - goto unlock; +/*Name heal*/ +int +ec_delete_stale_name(dict_t *gfid_db, char *key, data_t *d, void *data) +{ + struct ec_name_data *name_data = data; + struct iatt *ia = NULL; + ec_t *ec = NULL; + loc_t loc = {0}; + unsigned char *same = data_to_bin(d); + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + int ret = 0; + int estale_count = 0; + int i = 0; + call_frame_t *frame = name_data->frame; + uuid_t gfid; + + ec = name_data->frame->this->private; + EC_REPLIES_ALLOC(replies, ec->nodes); + if (EC_COUNT(same, ec->nodes) >= ec->fragments) { + ret = 0; + goto out; } - if (list_empty(&ctx->heal)) { - gf_log("ec", GF_LOG_INFO, "Healing '%s', gfid %s", heal->loc.path, - uuid_utoa(heal->loc.gfid)); - } else { - error = EEXIST; + loc.parent = inode_ref(name_data->parent); + loc.inode = inode_new(name_data->parent->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; } - list_add_tail(&heal->list, &ctx->heal); - heal = NULL; + gf_uuid_parse(key, gfid); + gf_uuid_copy(loc.pargfid, name_data->parent->gfid); + loc.name = name_data->name; + output = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, name_data->participants, ec->nodes, + replies, output, name_data->frame, ec->xl, &loc, NULL); -unlock: - UNLOCK(&inode->lock); + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + if (replies[i].op_ret == -1) { + if (replies[i].op_errno == ESTALE || replies[i].op_errno == ENOENT) + estale_count++; + else + name_data->participants[i] = 0; + } else if (gf_uuid_compare(gfid, replies[i].stat.ia_gfid)) { + estale_count++; + gf_msg_debug(ec->xl->name, 0, "%s/%s: different gfid as %s", + uuid_utoa(name_data->parent->gfid), name_data->name, + key); + } + } - if (error == EEXIST) { - LOCK(&fop->lock); + if (estale_count <= ec->redundancy) { + /* We have at least ec->fragments number of fragments, so the + * file is recoverable, so don't delete it*/ - fop->jobs++; - fop->refs++; + /* Please note that the lookup call above could fail with + * ENOTCONN on all subvoumes and still this branch will be + * true, but in those cases conservatively we decide to not + * delete the file until we are sure*/ + ret = 0; + goto out; + } - UNLOCK(&fop->lock); + /*Noway to recover, delete the name*/ + loc_wipe(&loc); + loc.parent = inode_ref(name_data->parent); + gf_uuid_copy(loc.pargfid, loc.parent->gfid); + loc.name = name_data->name; + for (i = 0; i < ec->nodes; i++) { + if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) { + ia = &replies[i].stat; + break; + } + } - error = 0; + if (!ia) { + ret = -ENOTCONN; + goto out; } + if (IA_ISDIR(ia->ia_type)) { + ret = cluster_rmdir(ec->xl_list, same, ec->nodes, replies, output, + frame, ec->xl, &loc, 1, NULL); + gf_msg_debug(ec->xl->name, 0, + "cluster rmdir succeeded on %d " + "nodes", + ret); + } else { + ret = cluster_unlink(ec->xl_list, same, ec->nodes, replies, output, + frame, ec->xl, &loc, 0, NULL); + gf_msg_debug(ec->xl->name, 0, + "cluster unlink succeeded on %d " + "nodes", + ret); + } + + for (i = 0; i < ec->nodes; i++) { + if (output[i]) { + same[i] = 0; + name_data->enoent[i] = 1; + } else { + /*op failed*/ + if (same[i]) + name_data->participants[i] = 0; + } + } + ret = 0; + /*This will help in making decisions about creating names*/ + dict_del(gfid_db, key); out: - GF_FREE(heal); + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s", + uuid_utoa(name_data->parent->gfid), name_data->name, + strerror(-ret)); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} - return error; +int +ec_delete_stale_names(call_frame_t *frame, ec_t *ec, inode_t *parent, + char *name, default_args_cbk_t *replies, dict_t *gfid_db, + unsigned char *enoent, unsigned char *gfidless, + unsigned char *participants) +{ + struct ec_name_data name_data = {0}; + + name_data.enoent = enoent; + name_data.gfidless = gfidless; + name_data.participants = participants; + name_data.name = name; + name_data.parent = parent; + name_data.frame = frame; + name_data.replies = replies; + return dict_foreach(gfid_db, ec_delete_stale_name, &name_data); } -void ec_heal_entrylk(ec_heal_t * heal, entrylk_cmd cmd) +int +_assign_same(dict_t *dict, char *key, data_t *value, void *data) { - loc_t loc; + struct ec_name_data *name_data = data; - if (ec_loc_parent(heal->xl, &heal->loc, &loc) != 0) { - ec_fop_set_error(heal->fop, EIO); + name_data->same = data_to_bin(value); + return 0; +} - return; +int +ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + default_args_cbk_t *lookup_replies, dict_t *gfid_db, + unsigned char *enoent, unsigned char *participants) +{ + int ret = 0; + int i = 0; + struct ec_name_data name_data = {0}; + struct iatt *ia = NULL; + unsigned char *output = 0; + unsigned char *output1 = 0; + unsigned char *on = NULL; + default_args_cbk_t *replies = NULL; + loc_t loc = {0}; + loc_t srcloc = {0}; + unsigned char *link = NULL; + unsigned char *create = NULL; + dict_t *xdata = NULL; + char *linkname = NULL; + ec_config_t config; + + /* There should be just one gfid key */ + EC_REPLIES_ALLOC(replies, ec->nodes); + if (gfid_db->count != 1) { + ret = -EINVAL; + goto out; } - ec_entrylk(heal->fop->frame, heal->xl, -1, EC_MINIMUM_ALL, NULL, NULL, - heal->xl->name, &loc, NULL, cmd, ENTRYLK_WRLCK, NULL); + ret = dict_foreach(gfid_db, _assign_same, &name_data); + if (ret < 0) + goto out; + /*There should at least be one valid success reply with gfid*/ + for (i = 0; i < ec->nodes; i++) + if (name_data.same[i]) + break; - loc_wipe(&loc); -} + if (i == ec->nodes) { + ret = -EINVAL; + goto out; + } -void ec_heal_inodelk(ec_heal_t * heal, int32_t type, int32_t use_fd, - off_t offset, size_t size) -{ - struct gf_flock flock; + ia = &lookup_replies[i].stat; + xdata = dict_new(); + loc.parent = inode_ref(parent); + gf_uuid_copy(loc.pargfid, parent->gfid); + loc.inode = inode_new(parent->table); + if (loc.inode) + srcloc.inode = inode_ref(loc.inode); + gf_uuid_copy(srcloc.gfid, ia->ia_gfid); + if (!loc.inode || !xdata || + dict_set_static_bin(xdata, "gfid-req", ia->ia_gfid, + sizeof(ia->ia_gfid))) { + ret = -ENOMEM; + goto out; + } + loc.name = name; + link = alloca0(ec->nodes); + create = alloca0(ec->nodes); + on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + output1 = alloca0(ec->nodes); - flock.l_type = type; - flock.l_whence = SEEK_SET; - flock.l_start = offset; - flock.l_len = size; - flock.l_pid = 0; - flock.l_owner.len = 0; + for (i = 0; i < ec->nodes; i++) { + if (!lookup_replies[i].valid) + continue; + if (lookup_replies[i].op_ret) + continue; + on[i] = 1; + } + switch (ia->ia_type) { + case IA_IFDIR: + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + (void)cluster_mkdir( + ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl, + &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), 0, xdata); + break; - if (use_fd) - { - ec_finodelk(heal->fop->frame, heal->xl, heal->fop->mask, - EC_MINIMUM_ALL, NULL, NULL, heal->xl->name, heal->fd, - F_SETLKW, &flock, NULL); + case IA_IFLNK: + /*Check for hard links and create/link*/ + ret = cluster_lookup(ec->xl_list, enoent, ec->nodes, replies, + output, frame, ec->xl, &srcloc, NULL); + for (i = 0; i < ec->nodes; i++) { + if (output[i]) { + link[i] = 1; + } else { + if (replies[i].op_errno == ENOENT || + replies[i].op_errno == ESTALE) { + create[i] = 1; + } + } + } + + if (EC_COUNT(link, ec->nodes)) { + cluster_link(ec->xl_list, link, ec->nodes, replies, output1, + frame, ec->xl, &srcloc, &loc, NULL); + } + + if (EC_COUNT(create, ec->nodes)) { + cluster_readlink(ec->xl_list, name_data.same, ec->nodes, + replies, output, frame, ec->xl, &srcloc, 4096, + NULL); + if (EC_COUNT(output, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + + for (i = 0; i < ec->nodes; i++) { + if (output[i]) + break; + } + linkname = alloca0(strlen(replies[i].buf) + 1); + strcpy(linkname, replies[i].buf); + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + cluster_symlink(ec->xl_list, create, ec->nodes, replies, output, + frame, ec->xl, linkname, &loc, 0, xdata); + } + for (i = 0; i < ec->nodes; i++) + if (output1[i]) + output[i] = 1; + break; + case IA_IFREG: + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; + + ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config); + if (ret != 0) { + goto out; + } + + /* Fall through */ + + default: + ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1); + if (ret) + goto out; + ret = cluster_mknod( + ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl, + &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), + makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)), 0, + xdata); + break; } - else - { - ec_inodelk(heal->fop->frame, heal->xl, heal->fop->mask, EC_MINIMUM_ALL, - NULL, NULL, heal->xl->name, &heal->loc, F_SETLKW, &flock, - NULL); + + for (i = 0; i < ec->nodes; i++) { + if (enoent[i] && !output[i]) + participants[i] = 0; } + + ret = 0; +out: + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s", + uuid_utoa(parent->gfid), name, strerror(-ret)); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + loc_wipe(&srcloc); + if (xdata) + dict_unref(xdata); + return ret; } -void ec_heal_lookup(ec_heal_t *heal, uintptr_t mask) +int +__ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + unsigned char *participants) { - dict_t * xdata; - int32_t error = ENOMEM; - + unsigned char *output = NULL; + unsigned char *enoent = NULL; + default_args_cbk_t *replies = NULL; + dict_t *xdata = NULL; + dict_t *gfid_db = NULL; + int ret = 0; + loc_t loc = {0}; + int i = 0; + struct iatt *ia = NULL; + char gfid[64] = {0}; + unsigned char *same = NULL; + unsigned char *gfidless = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + loc.parent = inode_ref(parent); + loc.inode = inode_new(parent->table); + gf_uuid_copy(loc.pargfid, parent->gfid); + loc.name = name; xdata = dict_new(); - if (xdata == NULL) - { + gfid_db = dict_new(); + if (!xdata || !gfid_db || !loc.inode) { + ret = -ENOMEM; goto out; } - if (dict_set_uint64(xdata, "list-xattr", 0) != 0) - { + + ret = dict_set_int32(xdata, GF_GFIDLESS_LOOKUP, 1); + if (ret) { + ret = -ENOMEM; goto out; } - ec_lookup(heal->fop->frame, heal->xl, mask, EC_MINIMUM_MIN, - ec_heal_inode_lookup_cbk, heal, &heal->loc, xdata); + output = alloca0(ec->nodes); + gfidless = alloca0(ec->nodes); + enoent = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, participants, ec->nodes, replies, output, + frame, ec->xl, &loc, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; - error = 0; + if (replies[i].op_ret == -1) { + /*If ESTALE comes here, that means parent dir is not + * present, nothing to do there, so reset participants + * for that brick*/ + if (replies[i].op_errno == ENOENT) + enoent[i] = 1; + else + participants[i] = 0; + continue; + } + ia = &replies[i].stat; + if (gf_uuid_is_null(ia->ia_gfid)) { + if (IA_ISDIR(ia->ia_type) || ia->ia_size == 0) + gfidless[i] = 1; + else + participants[i] = 0; + } else { + uuid_utoa_r(ia->ia_gfid, gfid); + ret = dict_get_bin(gfid_db, gfid, (void **)&same); + if (ret < 0) { + same = alloca0(ec->nodes); + } + same[i] = 1; + if (ret < 0) { + ret = dict_set_static_bin(gfid_db, gfid, same, ec->nodes); + } + if (ret < 0) + goto out; + } + } -out: - if (xdata != NULL) - { - dict_unref(xdata); + ret = ec_delete_stale_names(frame, ec, parent, name, replies, gfid_db, + enoent, gfidless, participants); + + if (gfid_db->count == 0) { + /* All entries seem to be stale entries and deleted, + * nothing more to do.*/ + goto out; + } + + if (gfid_db->count > 1) { + gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL, + "%s/%s: Not able to heal", uuid_utoa(parent->gfid), name); + memset(participants, 0, ec->nodes); + goto out; + } + + EC_INTERSECT(enoent, enoent, participants, ec->nodes); + if (EC_COUNT(enoent, ec->nodes) == 0) { + ret = 0; + goto out; } - ec_fop_set_error(heal->fop, error); + ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent, + participants); + if (ret >= 0) { + /* If ec_create_name() succeeded we return 1 to indicate that a new + * file has been created and it will need to be healed. */ + ret = 1; + } +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + if (xdata) + dict_unref(xdata); + if (gfid_db) + dict_unref(gfid_db); + return ret; } -void ec_heal_remove(ec_heal_t * heal, ec_cbk_data_t * cbk) +int +ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + unsigned char *participants) { - if (cbk->iatt[0].ia_type == IA_IFDIR) - { - // TODO: Remove directory recursively ? - ec_rmdir(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE, - ec_heal_rmdir_cbk, heal, &heal->loc, 0, NULL); + int ret = 0; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + unsigned char *locked_on = NULL; + loc_t loc = {0}; + + loc.parent = inode_ref(parent); + loc.name = name; + loc.inode = inode_new(parent->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; } - else + + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + locked_on = alloca0(ec->nodes); + ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, parent, 0, 0); { - ec_unlink(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE, - ec_heal_unlink_cbk, heal, &heal->loc, 0, NULL); + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s/%s: Skipping " + "heal as only %d number of subvolumes could " + "be locked", + uuid_utoa(parent->gfid), name, ret); + ret = -ENOTCONN; + goto unlock; + } + EC_INTERSECT(participants, participants, locked_on, ec->nodes); + ret = __ec_heal_name(frame, ec, parent, name, participants); } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, parent, 0, 0); +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; } -void ec_heal_remove_others(ec_heal_t * heal) +int +ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) { - struct list_head * item; - ec_cbk_data_t * cbk; - - item = heal->lookup->cbk_list.next; - do - { - item = item->next; - cbk = list_entry(item, ec_cbk_data_t, list); + struct ec_name_data *name_data = data; + xlator_t *this = THIS; + ec_t *ec = this->private; + unsigned char *name_on = alloca0(ec->nodes); + int i = 0; + int ret = 0; + + if (ec->shutdown) { + gf_msg_debug(this->name, 0, + "Cancelling directory heal " + "because EC is stopping."); + return -ENOTCONN; + } + + memcpy(name_on, name_data->participants, ec->nodes); + ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name, + name_on); + + if (ret < 0) { + memset(name_on, 0, ec->nodes); + } else { + name_data->heal_pending += ret; + } - if (cbk->op_ret < 0) - { - if ((cbk->op_errno != ENOENT) && (cbk->op_errno != ENOTDIR) && - (cbk->op_errno != ESTALE)) - { - gf_log(heal->xl->name, GF_LOG_WARNING, "Don't know how to " - "remove inode with " - "error %d", - cbk->op_errno); - } + for (i = 0; i < ec->nodes; i++) + if (name_data->participants[i] && !name_on[i]) + name_data->failed_on[i] = 1; - ec_heal_exclude(heal, cbk->mask); + return 0; +} +int +ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *participants, uint32_t *pending) +{ + int i = 0; + int j = 0; + loc_t loc = {0}; + struct ec_name_data name_data = {0}; + int ret = 0; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + name_data.frame = frame; + name_data.participants = participants; + name_data.failed_on = alloca0(ec->nodes); + name_data.heal_pending = 0; + + for (i = 0; i < ec->nodes; i++) { + if (!participants[i]) continue; + ret = syncop_dir_scan(ec->xl_list[i], &loc, GF_CLIENT_PID_SELF_HEALD, + &name_data, ec_name_heal_handler); + if (ret < 0) { + break; } + for (j = 0; j < ec->nodes; j++) + if (name_data.failed_on[j]) + participants[j] = 0; - ec_heal_remove(heal, cbk); - } while (item->next != &heal->lookup->cbk_list); + if (EC_COUNT(participants, ec->nodes) <= ec->fragments) { + ret = -ENOTCONN; + break; + } + } + *pending += name_data.heal_pending; + + loc_wipe(&loc); + return ret; } -void ec_heal_prepare_others(ec_heal_t * heal) +int +__ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *heal_on, unsigned char *sources, + unsigned char *healed_sinks, uint32_t *pending) { - struct list_head * item; - ec_cbk_data_t * cbk; - - item = heal->lookup->cbk_list.next; - while (item->next != &heal->lookup->cbk_list) + unsigned char *locked_on = NULL; + unsigned char *output = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + unsigned char *participants = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + int source = 0; + int i = 0; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on, + frame, ec->xl, ec->xl->name, inode, 0, 0); { - item = item->next; - cbk = list_entry(item, ec_cbk_data_t, list); + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_entry_prepare(frame, ec, inode, locked_on, versions, + dirty, sources, healed_sinks); + source = ret; + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + if (ret < 0) + goto out; - if (cbk->op_ret < 0) - { - if ((cbk->op_errno == ENOENT) || (cbk->op_errno == ESTALE)) - { - ec_heal_create(heal, cbk->mask, 1); - } - else - { - gf_log(heal->xl->name, GF_LOG_ERROR, "Don't know how to " - "heal error %d", - cbk->op_errno); + participants = alloca0(ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (sources[i] || healed_sinks[i]) + participants[i] = 1; + } + ret = ec_heal_names(frame, ec, inode, participants, pending); - ec_heal_exclude(heal, cbk->mask); - } - } - else - { - if ((heal->iatt.ia_type != cbk->iatt[0].ia_type) || - (uuid_compare(heal->iatt.ia_gfid, cbk->iatt[0].ia_gfid) != 0)) - { - ec_heal_remove(heal, cbk); - } + if (EC_COUNT(participants, ec->nodes) <= ec->fragments) + goto out; + + for (i = 0; i < ec->nodes; i++) { + if (!participants[i]) { + sources[i] = 0; + healed_sinks[i] = 0; } } + + ec_adjust_versions(frame, ec, EC_DATA_TXN, inode, source, sources, + healed_sinks, versions, dirty); +out: + cluster_replies_wipe(replies, ec->nodes); + return ret; } -int32_t ec_heal_readlink_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - const char * path, struct iatt * buf, - dict_t * xdata) +int +ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks, + uint32_t *pending) { - ec_fop_data_t * fop = cookie; - ec_heal_t * heal = fop->data; - - if (op_ret >= 0) + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + char selfheal_domain[1024] = {0}; + int ret = 0; + default_args_cbk_t *replies = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + + sprintf(selfheal_domain, "%s:self-heal", ec->xl->name); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + /*If other processes are already doing the heal, don't block*/ + ret = cluster_tiebreaker_inodelk(ec->xl_list, up_subvols, ec->nodes, + replies, locked_on, frame, ec->xl, + selfheal_domain, inode, 0, 0); { - heal->symlink = gf_strdup(path); - if (heal->symlink != NULL) - { - ec_heal_prepare_others(heal); - } - else - { - ec_fop_set_error(fop, EIO); + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; } + ret = __ec_heal_entry(frame, ec, inode, locked_on, sources, + healed_sinks, pending); } - - return 0; +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, selfheal_domain, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; } -ec_cbk_data_t * ec_heal_lookup_check(ec_heal_t * heal, uintptr_t * pgood, - uintptr_t * pbad) +/*Find direction for data heal and heal info*/ +int +ec_heal_data_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *data_versions, uint64_t *dirty, + uint64_t *size, unsigned char *sources, + unsigned char *healed_sinks, + gf_boolean_t check_ondisksize, int which) { - ec_fop_data_t * fop = heal->lookup; - ec_cbk_data_t * cbk = NULL, * ans = NULL; - uintptr_t good = 0, bad = 0; + uint64_t xattr[EC_VERSION_SIZE] = {0}; + char version_size[128] = {0}; + dict_t *version_size_db = NULL; + unsigned char *same = NULL; + int max_same_count = 0; + int source = 0; + int i = 0; + int ret = 0; + dict_t *dict = NULL; + uint64_t source_size = 0; + + version_size_db = dict_new(); + if (!version_size_db) { + ret = -ENOMEM; + goto out; + } - list_for_each_entry(ans, &fop->cbk_list, list) - { - if ((ans->op_ret < 0) && (ans->op_errno == ENOTCONN)) - { + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) continue; + if (replies[i].op_ret < 0) + continue; + dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata + : replies[i].xattr; + + ret = ec_dict_get_array(dict, EC_XATTR_VERSION, xattr, EC_VERSION_SIZE); + if (ret == 0) { + data_versions[i] = xattr[EC_DATA_TXN]; } - if (ans == fop->answer) - { - good |= ans->mask; - cbk = ans; + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(dict, EC_XATTR_DIRTY, xattr, EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_DATA_TXN]; } - else - { - bad |= ans->mask; + ret = ec_dict_del_number(dict, EC_XATTR_SIZE, &size[i]); + /*Build a db of same metadata and data version and size*/ + snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64, + data_versions[i], size[i]); + + ret = dict_get_bin(version_size_db, version_size, (void **)&same); + if (ret < 0) { + same = alloca0(ec->nodes); } - } - - *pgood = good; - *pbad = bad; - return cbk; -} - -void ec_heal_prepare(ec_heal_t * heal) -{ - ec_cbk_data_t * cbk; - int32_t error = ENOMEM; - - heal->available = heal->good; + same[i] = 1; + if (max_same_count < EC_COUNT(same, ec->nodes)) { + max_same_count = EC_COUNT(same, ec->nodes); + source = i; + } - cbk = heal->lookup->answer; - if (cbk->op_ret < 0) - { - if ((cbk->op_errno == ENOENT) || (cbk->op_errno == ENOTDIR)) - { - ec_heal_remove_others(heal); + if (ret < 0) { + ret = dict_set_static_bin(version_size_db, version_size, same, + ec->nodes); } - else - { - gf_log(heal->xl->name, GF_LOG_ERROR, "Don't know how to heal " - "error %d", - cbk->op_errno); + + if (ret < 0) { + ret = -ENOMEM; + goto out; } } - else - { - if (heal->iatt.ia_type == IA_IFREG) - { - heal->fd = fd_create(heal->loc.inode, heal->fop->frame->root->pid); - if (heal->fd == NULL) - { - gf_log(heal->xl->name, GF_LOG_ERROR, "Unable to create a new " - "file descriptor"); + /* If we don't have ec->fragments number of same version,size it is not + * recoverable*/ + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; + } else { + snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64, + data_versions[source], size[source]); - goto out; - } + ret = dict_get_bin(version_size_db, version_size, (void **)&same); + if (ret < 0) + goto out; + memcpy(sources, same, ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (replies[i].valid && (replies[i].op_ret == 0) && !sources[i]) + healed_sinks[i] = 1; } + } - if (heal->iatt.ia_type == IA_IFLNK) - { - ec_readlink(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE, - ec_heal_readlink_cbk, heal, &heal->loc, - heal->iatt.ia_size, NULL); + /* There could be files with versions, size same but on disk ia_size + * could be different because of disk crashes, mark them as sinks as + * well*/ + + if (check_ondisksize) { + source_size = size[source]; + ec_adjust_size_up(ec, &source_size, _gf_true); + + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + if (replies[i].stat.ia_size != source_size) { + sources[i] = 0; + healed_sinks[i] = 1; + max_same_count--; + } else { + source = i; + } + } } - else - { - ec_heal_prepare_others(heal); + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; } } - error = 0; - + ret = source; out: - ec_fop_set_error(heal->fop, error); + if (version_size_db) + dict_unref(version_size_db); + return ret; } -int32_t ec_heal_open_others(ec_heal_t * heal) +int +__ec_heal_data_prepare(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *locked_on, uint64_t *versions, + uint64_t *dirty, uint64_t *size, unsigned char *sources, + unsigned char *healed_sinks, unsigned char *trim, + struct iatt *stbuf) { - struct list_head * item; - ec_cbk_data_t * cbk; - uintptr_t mask = 0, open = heal->open; + default_args_cbk_t *replies = NULL; + default_args_cbk_t *fstat_replies = NULL; + unsigned char *output = NULL; + unsigned char *fstat_output = NULL; + dict_t *xattrs = NULL; + uint64_t zero_array[2] = {0}; + int source = 0; + int ret = 0; + uint64_t zero_value = 0; + int i = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + EC_REPLIES_ALLOC(fstat_replies, ec->nodes); + output = alloca0(ec->nodes); + fstat_output = alloca0(ec->nodes); + xattrs = dict_new(); + if (!xattrs || + dict_set_static_bin(xattrs, EC_XATTR_VERSION, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xattrs, EC_XATTR_DIRTY, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xattrs, EC_XATTR_SIZE, &zero_value, + sizeof(zero_value))) { + ret = -ENOMEM; + goto out; + } - item = heal->lookup->cbk_list.next; - while (item->next != &heal->lookup->cbk_list) - { - item = item->next; - cbk = list_entry(item, ec_cbk_data_t, list); + ret = cluster_fxattrop(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, + NULL); - if ((cbk->op_ret < 0) || (cbk->iatt[0].ia_type != IA_IFREG) || - (uuid_compare(heal->iatt.ia_gfid, cbk->iatt[0].ia_gfid) != 0)) - { - ec_heal_exclude(heal, cbk->mask); - } - else - { - mask |= cbk->mask & ~heal->open; - } - } + ret = cluster_fstat(ec->xl_list, locked_on, ec->nodes, fstat_replies, + fstat_output, frame, ec->xl, fd, NULL); - if (mask != 0) - { - ec_open(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_target_open_cbk, heal, &heal->loc, O_RDWR | O_TRUNC, - heal->fd, NULL); + for (i = 0; i < ec->nodes; i++) { + output[i] = output[i] && fstat_output[i]; + replies[i].valid = output[i]; + if (output[i]) + replies[i].stat = fstat_replies[i].stat; + } - open |= mask; + if (EC_COUNT(output, ec->nodes) <= ec->fragments) { + ret = -ENOTCONN; + goto out; } - return (open != 0); -} + source = ec_heal_data_find_direction(ec, replies, versions, dirty, size, + sources, healed_sinks, _gf_true, + EC_COMBINE_DICT); + ret = source; + if (ret < 0) + goto out; -uintptr_t ec_heal_needs_data_rebuild(ec_heal_t *heal) -{ - ec_fop_data_t *fop = heal->lookup; - ec_cbk_data_t *cbk = NULL; - uintptr_t bad = 0; + if (stbuf) + *stbuf = replies[source].stat; - if ((heal->fop->error != 0) || (heal->good == 0) || - (heal->iatt.ia_type != IA_IFREG)) { - return 0; + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i]) { + if (replies[i].stat.ia_size) + trim[i] = 1; + } } - list_for_each_entry(cbk, &fop->cbk_list, list) { - if ((cbk->op_ret >= 0) && - ((cbk->size != heal->raw_size) || - (cbk->version != heal->version))) { - bad |= cbk->mask; - } + if (EC_COUNT(sources, ec->nodes) < ec->fragments) { + ret = -ENOTCONN; + goto out; } - return bad; + ret = source; +out: + if (xattrs) + dict_unref(xattrs); + cluster_replies_wipe(replies, ec->nodes); + cluster_replies_wipe(fstat_replies, ec->nodes); + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + } else { + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d", + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes)); + } + return ret; } -void ec_heal_setxattr_others(ec_heal_t * heal) +int +__ec_heal_mark_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd, + uint64_t *versions, unsigned char *healed_sinks) { - ec_cbk_data_t * cbk; - dict_t * xdata; - int32_t error = ENOMEM; + int i = 0; + int ret = 0; + unsigned char *mark = NULL; + dict_t *xattrs = NULL; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + uint64_t versions_xattr[2] = {0}; + + EC_REPLIES_ALLOC(replies, ec->nodes); + xattrs = dict_new(); + if (!xattrs) { + ret = -ENOMEM; + goto out; + } - if ((heal->good != 0) && (heal->bad != 0)) - { - cbk = heal->lookup->answer; - xdata = cbk->xdata; + mark = alloca0(ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (!healed_sinks[i]) + continue; + if ((versions[i] >> EC_SELFHEAL_BIT) & 1) + continue; + mark[i] = 1; + } - if (dict_foreach_match (xdata, ec_ignorable_key_match, NULL, - dict_remove_foreach_fn, NULL) == -1) - goto out; + if (EC_COUNT(mark, ec->nodes) == 0) + return 0; - if ((cbk->iatt[0].ia_type == IA_IFREG) || - (cbk->iatt[0].ia_type == IA_IFDIR)) - { - if (ec_dict_set_number(xdata, EC_XATTR_VERSION, cbk->version) != 0) - { - goto out; - } - if (cbk->iatt[0].ia_type == IA_IFREG) - { - uint64_t dirty; - - dirty = ec_heal_needs_data_rebuild(heal) != 0; - if ((ec_dict_set_number(xdata, EC_XATTR_SIZE, - cbk->iatt[0].ia_size) != 0) || - (ec_dict_set_number(xdata, EC_XATTR_DIRTY, dirty) != 0)) { - goto out; - } - } - } + versions_xattr[EC_DATA_TXN] = hton64(1ULL << EC_SELFHEAL_BIT); + if (dict_set_static_bin(xattrs, EC_XATTR_VERSION, versions_xattr, + sizeof(versions_xattr))) { + ret = -ENOMEM; + goto out; + } - ec_setxattr(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE, - ec_heal_setxattr_cbk, heal, &heal->loc, xdata, 0, NULL); + output = alloca0(ec->nodes); + ret = cluster_fxattrop(ec->xl_list, mark, ec->nodes, replies, output, frame, + ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!output[i]) { + if (mark[i]) + healed_sinks[i] = 0; + continue; + } + versions[i] |= (1ULL << EC_SELFHEAL_BIT); } - error = 0; + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + ret = 0; out: - ec_fop_set_error(heal->fop, error); + cluster_replies_wipe(replies, ec->nodes); + if (xattrs) + dict_unref(xattrs); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; } int32_t -ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data, - void *arg) +ec_manager_heal_block(ec_fop_data_t *fop, int32_t state) { - dict_t *base = arg; + ec_heal_t *heal = fop->data; + heal->fop = fop; - if (ec_ignorable_key_match (NULL, key, NULL, NULL)) { - dict_del (dict, key); - return 0; - } + switch (state) { + case EC_STATE_INIT: + ec_owner_set(fop->frame, fop->frame->root); - if (dict_get (base, key) != NULL) - dict_del (dict, key); + ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0); - return 0; -} + return EC_STATE_HEAL_DATA_COPY; -void ec_heal_removexattr_others(ec_heal_t * heal) -{ - struct list_head * item; - ec_cbk_data_t * cbk; - dict_t * xdata; + case EC_STATE_HEAL_DATA_COPY: + gf_msg_debug(fop->xl->name, 0, "%s: read/write starting", + uuid_utoa(heal->fd->inode->gfid)); + ec_heal_data_block(heal); - if ((heal->good == 0) || (heal->bad == 0)) - { - return; - } + return EC_STATE_HEAL_DATA_UNLOCK; - xdata = heal->lookup->answer->xdata; - item = heal->lookup->cbk_list.next; - while (item->next != &heal->lookup->cbk_list) - { - item = item->next; - cbk = list_entry(item, ec_cbk_data_t, list); + case -EC_STATE_HEAL_DATA_COPY: + case -EC_STATE_HEAL_DATA_UNLOCK: + case EC_STATE_HEAL_DATA_UNLOCK: + ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0); - if (cbk->op_ret >= 0) - { - if (dict_foreach(cbk->xdata, ec_heal_xattr_clean, xdata) == 0) - { - ec_removexattr(heal->fop->frame, heal->xl, cbk->mask, - EC_MINIMUM_ONE, ec_heal_removexattr_cbk, heal, - &heal->loc, "", cbk->xdata); + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0, + (heal->good | heal->bad), heal->good, heal->bad, + 0, NULL); } - } + + return EC_STATE_END; + case -EC_STATE_REPORT: + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1, + fop->error, 0, 0, 0, 0, NULL); + } + + return EC_STATE_END; + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; } } -void ec_heal_attr(ec_heal_t * heal) +/*Takes lock */ +void +ec_heal_block(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, ec_heal_t *heal) { - if ((heal->good != 0) && (heal->bad != 0)) - { - ec_setattr(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE, - ec_heal_setattr_cbk, heal, &heal->loc, &heal->iatt, - GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID | - GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL); + ec_cbk_t callback = {.heal = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(HEAL) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags, + NULL, ec_manager_heal_block, callback, heal); + if (fop == NULL) + goto out; + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL); } } -void ec_heal_open(ec_heal_t * heal) +int32_t +ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, uintptr_t mask, + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) { - heal->bad = ec_heal_needs_data_rebuild(heal); - if (heal->bad == 0) { - return; - } + ec_heal_t *heal = cookie; - if (ec_heal_open_others(heal)) - { - ec_open(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN, - ec_heal_source_open_cbk, heal, &heal->loc, O_RDONLY, heal->fd, - NULL); + if (heal->fop) { + heal->fop->heal = NULL; } + heal->fop = NULL; + heal->error = op_ret < 0 ? op_errno : 0; + syncbarrier_wake(heal->data); + return 0; } -void ec_heal_reopen_fd(ec_heal_t * heal) +int +ec_sync_heal_block(call_frame_t *frame, xlator_t *this, ec_heal_t *heal) { - inode_t * inode; - fd_t * fd; - ec_fd_t *ctx_fd; - ec_inode_t *ctx_inode; - uintptr_t mask; - int32_t flags; + ec_heal_block(frame, this, heal->bad | heal->good, EC_MINIMUM_ONE, + ec_heal_block_done, heal); + syncbarrier_wait(heal->data, 1); + if (heal->error != 0) { + return -heal->error; + } + if (heal->bad == 0) + return -ENOTCONN; + return 0; +} - inode = heal->loc.inode; +int +ec_rebuild_data(call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size, + unsigned char *sources, unsigned char *healed_sinks) +{ + ec_heal_t *heal = NULL; + int ret = 0; + syncbarrier_t barrier; + + if (syncbarrier_init(&barrier)) + return -ENOMEM; + + heal = alloca0(sizeof(*heal)); + heal->fd = fd_ref(fd); + heal->xl = ec->xl; + heal->data = &barrier; + ec_adjust_size_up(ec, &size, _gf_false); + heal->total_size = size; + heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size)); + /* We need to adjust the size to a multiple of the stripe size of the + * volume. Otherwise writes would need to fill gaps (head and/or tail) + * with existent data from the bad bricks. This could be garbage on a + * damaged file or it could fail if there aren't enough bricks. */ + heal->size -= heal->size % ec->stripe_size; + heal->bad = ec_char_array_to_mask(healed_sinks, ec->nodes); + heal->good = ec_char_array_to_mask(sources, ec->nodes); + heal->iatt.ia_type = IA_IFREG; + LOCK_INIT(&heal->lock); - LOCK(&inode->lock); + for (heal->offset = 0; (heal->offset < size) && !heal->done; + heal->offset += heal->size) { + /* We immediately abort any heal if a shutdown request has been + * received to avoid delays. The healing of this file will be + * restarted by another SHD or other client that accesses the + * file. */ + if (ec->shutdown) { + gf_msg_debug(ec->xl->name, 0, + "Cancelling heal because " + "EC is stopping."); + ret = -ENOTCONN; + break; + } - ctx_inode = __ec_inode_get(inode, heal->xl); - if (ctx_inode != NULL) { - ctx_inode->bad &= ~(heal->good | heal->bad); + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d, offset: %" PRIu64 " bsize: %" PRIu64, + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes), heal->offset, + heal->size); + ret = ec_sync_heal_block(frame, ec->xl, heal); + if (ret < 0) + break; } + memset(healed_sinks, 0, ec->nodes); + ec_mask_to_char_array(heal->bad, healed_sinks, ec->nodes); + fd_unref(heal->fd); + LOCK_DESTROY(&heal->lock); + syncbarrier_destroy(heal->data); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; +} - list_for_each_entry(fd, &inode->fd_list, inode_list) - { - ctx_fd = ec_fd_get(fd, heal->xl); - if (ctx_fd != NULL) { - mask = heal->bad & ~ctx_fd->open; - if (mask != 0) - { - UNLOCK(&inode->lock); - - if (heal->iatt.ia_type == IA_IFDIR) - { - ec_opendir(heal->fop->frame, heal->xl, mask, - EC_MINIMUM_ONE, ec_heal_reopen_cbk, NULL, - &heal->loc, fd, NULL); - } - else - { - flags = ctx_fd->flags & ~(O_TRUNC | O_APPEND); +int +__ec_heal_trim_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *healed_sinks, unsigned char *trim, + uint64_t size) +{ + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + int ret = 0; + int i = 0; + off_t trim_offset = 0; - ec_open(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, - ec_heal_reopen_cbk, NULL, &heal->loc, flags, fd, - NULL); - } + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); - LOCK(&inode->lock); - } - } + if (EC_COUNT(trim, ec->nodes) == 0) { + ret = 0; + goto out; + } + trim_offset = size; + ec_adjust_offset_up(ec, &trim_offset, _gf_true); + ret = cluster_ftruncate(ec->xl_list, trim, ec->nodes, replies, output, + frame, ec->xl, fd, trim_offset, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!output[i] && trim[i]) + healed_sinks[i] = 0; } - UNLOCK(&inode->lock); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + +out: + cluster_replies_wipe(replies, ec->nodes); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; } -int32_t ec_heal_writev_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct iatt * prebuf, struct iatt * postbuf, - dict_t * xdata) +int +ec_data_undo_pending(call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr, + uint64_t *versions, uint64_t *dirty, uint64_t *size, + int source, gf_boolean_t erase_dirty, int idx) { - ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno); + uint64_t versions_xattr[2] = {0}; + uint64_t dirty_xattr[2] = {0}; + uint64_t allzero[2] = {0}; + uint64_t size_xattr = 0; + int ret = 0; + + versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_VERSION, versions_xattr, + sizeof(versions_xattr)); + if (ret < 0) + goto out; - ec_heal_update(cookie, 0); + size_xattr = hton64(size[source] - size[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_SIZE, &size_xattr, + sizeof(size_xattr)); + if (ret < 0) + goto out; - return 0; + if (erase_dirty) { + dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_DIRTY, dirty_xattr, + sizeof(dirty_xattr)); + if (ret < 0) + goto out; + } + + if ((memcmp(versions_xattr, allzero, sizeof(allzero)) == 0) && + (memcmp(dirty_xattr, allzero, sizeof(allzero)) == 0) && + (size_xattr == 0)) { + ret = 0; + goto out; + } + + ret = syncop_fxattrop(ec->xl_list[idx], fd, GF_XATTROP_ADD_ARRAY64, xattr, + NULL, NULL, NULL); +out: + return ret; } -int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iovec * vector, int32_t count, - struct iatt * stbuf, struct iobref * iobref, - dict_t * xdata) +int +__ec_fd_data_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *sources, + unsigned char *healed_sinks, uint64_t *versions, + uint64_t *dirty, uint64_t *size) { - ec_fop_data_t * fop = cookie; - ec_heal_t * heal = fop->data; + dict_t *xattr = NULL; + int i = 0; + int ret = 0; + int op_ret = 0; + int source = -1; + gf_boolean_t erase_dirty = _gf_false; + + xattr = dict_new(); + if (!xattr) { + op_ret = -ENOMEM; + goto out; + } - ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno); + /* dirty xattr represents if the file needs heal. Unless all the + * copies are healed, don't erase it */ + if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) == + ec->nodes) + erase_dirty = _gf_true; - ec_heal_avoid(fop); + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + source = i; + break; + } + } - if (op_ret > 0) - { - ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE, - ec_heal_writev_cbk, heal, heal->fd, vector, count, - heal->offset, 0, iobref, NULL); + if (source == -1) { + op_ret = -ENOTCONN; + goto out; } - else - { - heal->done = 1; + + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i]) { + ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty, + size, source, erase_dirty, i); + if (ret < 0) + goto out; + } } - return 0; + if (!erase_dirty) + goto out; + + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty, + size, source, erase_dirty, i); + if (ret < 0) + continue; + } + } +out: + if (xattr) + dict_unref(xattr); + return op_ret; } -void ec_heal_data(ec_heal_t * heal) +int +ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *sources, + unsigned char *healed_sinks, + uint64_t *versions, uint64_t *dirty, + uint64_t *size) { - ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad); - - if ((heal->good != 0) && (heal->bad != 0) && - (heal->iatt.ia_type == IA_IFREG)) + unsigned char *locked_on = NULL; + unsigned char *participants = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + unsigned char *postsh_sources = NULL; + unsigned char *postsh_healed_sinks = NULL; + unsigned char *postsh_trim = NULL; + uint64_t *postsh_versions = NULL; + uint64_t *postsh_dirty = NULL; + uint64_t *postsh_size = NULL; + int ret = 0; + int i = 0; + struct iatt source_buf = {0}; + loc_t loc = {0}; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + participants = alloca0(ec->nodes); + postsh_sources = alloca0(ec->nodes); + postsh_healed_sinks = alloca0(ec->nodes); + postsh_trim = alloca0(ec->nodes); + postsh_versions = alloca0(ec->nodes * sizeof(*postsh_versions)); + postsh_dirty = alloca0(ec->nodes * sizeof(*postsh_dirty)); + postsh_size = alloca0(ec->nodes * sizeof(*postsh_size)); + + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i] || sources[i]) + participants[i] = 1; + } + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, fd->inode, 0, + 0); { - ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN, - ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, - 0, NULL); + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(fd->inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + + ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, postsh_versions, + postsh_dirty, postsh_size, postsh_sources, + postsh_healed_sinks, postsh_trim, + &source_buf); + if (ret < 0) + goto unlock; + + loc.inode = inode_ref(fd->inode); + gf_uuid_copy(loc.gfid, fd->inode->gfid); + ret = cluster_setattr( + ec->xl_list, healed_sinks, ec->nodes, replies, output, frame, + ec->xl, &loc, &source_buf, + GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL); + EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_fd_data_adjust_versions(frame, ec, fd, sources, healed_sinks, + versions, dirty, size); } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, fd->inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; } -void ec_heal_update_dirty(ec_heal_t *heal, uintptr_t mask) +int +__ec_heal_data(call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on, + unsigned char *sources, unsigned char *healed_sinks) { - dict_t *dict; + unsigned char *locked_on = NULL; + unsigned char *output = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + uint64_t *size = NULL; + unsigned char *trim = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + int source = 0; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + trim = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + size = alloca0(ec->nodes * sizeof(*size)); + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on, + frame, ec->xl, ec->xl->name, fd->inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(fd->inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } - dict = dict_new(); - if (dict == NULL) { - ec_fop_set_error(heal->fop, EIO); + ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, versions, dirty, + size, sources, healed_sinks, trim, NULL); + if (ret < 0) + goto unlock; - return; - } + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = __ec_fd_data_adjust_versions( + frame, ec, fd, sources, healed_sinks, versions, dirty, size); + goto unlock; + } - if (ec_dict_set_number(dict, EC_XATTR_DIRTY, -1) != 0) { - dict_unref(dict); - ec_fop_set_error(heal->fop, EIO); + source = ret; + ret = __ec_heal_mark_sinks(frame, ec, fd, versions, healed_sinks); + if (ret < 0) + goto unlock; - return; + ret = __ec_heal_trim_sinks(frame, ec, fd, healed_sinks, trim, + size[source]); } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, fd->inode, 0, 0); + if (ret < 0) + goto out; + + if (EC_COUNT(healed_sinks, ec->nodes) == 0) + goto out; - ec_fxattrop(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, NULL, NULL, - heal->fd, GF_XATTROP_ADD_ARRAY64, dict, NULL); + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d", + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes)); - dict_unref(dict); + ret = ec_rebuild_data(frame, ec, fd, size[source], sources, healed_sinks); + if (ret < 0) + goto out; + + ret = ec_restore_time_and_adjust_versions( + frame, ec, fd, sources, healed_sinks, versions, dirty, size); +out: + cluster_replies_wipe(replies, ec->nodes); + return ret; } -void ec_heal_dispatch(ec_heal_t *heal) +int +ec_heal_data(call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks) { - ec_fop_data_t *fop; - ec_cbk_data_t *cbk; - inode_t *inode; - ec_inode_t *ctx; - ec_heal_t *next = NULL; - struct list_head list; - int32_t error; - - inode = heal->loc.inode; - - INIT_LIST_HEAD(&list); - - LOCK(&inode->lock); - - /* A heal object not belonging to any list means that it has not been fully - * executed. It got its information from a previous heal that was executing - * when this heal started. */ - if (!list_empty(&heal->list)) { - list_del_init(&heal->list); - ctx = __ec_inode_get(inode, heal->xl); - if (ctx != NULL) { - ctx->bad &= ~heal->good; - - if (heal->partial) { - /* Collect all partial heal requests. All of them will receive - * the same answer. 'next' will contain a pointer to the first - * full request (if any) after this partial heal request.*/ - while (!list_empty(&ctx->heal)) { - next = list_entry(ctx->heal.next, ec_heal_t, list); - if (!next->partial) { - break; - } - list_move_tail(&next->list, &list); - } - if (list_empty(&ctx->heal)) { - next = NULL; - } - } else { - /* This is a full heal request, so take all received heal - * requests to answer them now. */ - list_splice_init(&ctx->heal, &list); - } - } + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + fd_t *fd = NULL; + loc_t loc = {0}; + char selfheal_domain[1024] = {0}; + int ret = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + + fd = fd_create(inode, 0); + if (!fd) { + ret = -ENOMEM; + goto out; } - UNLOCK(&inode->lock); - - fop = heal->fop; - error = fop->error; - - cbk = ec_cbk_data_allocate(fop->frame, heal->xl, fop, fop->id, 0, - error == 0 ? 0 : -1, error); - if (cbk != NULL) { - cbk->uintptr[0] = heal->available; - cbk->uintptr[1] = heal->good; - cbk->uintptr[2] = heal->fixed; + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); - ec_combine(cbk, NULL); - - fop->answer = cbk; - } else if (error == 0) { - error = ENOMEM; + ret = cluster_open(ec->xl_list, up_subvols, ec->nodes, replies, output, + frame, ec->xl, &loc, O_RDWR | O_LARGEFILE, fd, NULL); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; } - if (heal->lookup != NULL) - { - ec_fop_data_release(heal->lookup); + fd_bind(fd); + sprintf(selfheal_domain, "%s:self-heal", ec->xl->name); + /*If other processes are already doing the heal, don't block*/ + if (block) { + ret = cluster_inodelk(ec->xl_list, output, ec->nodes, replies, + locked_on, frame, ec->xl, selfheal_domain, inode, + 0, 0); + } else { + ret = cluster_tiebreaker_inodelk(ec->xl_list, output, ec->nodes, + replies, locked_on, frame, ec->xl, + selfheal_domain, inode, 0, 0); } - if (heal->fd != NULL) { - fd_unref(heal->fd); + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_data(frame, ec, fd, locked_on, sources, healed_sinks); } - GF_FREE(heal->symlink); - loc_wipe(&heal->loc); - - LOCK_DESTROY(&heal->lock); +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, selfheal_domain, inode, 0, 0); +out: + if (fd) + fd_unref(fd); + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} - GF_FREE(heal); +int +ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode) +{ + int i = 0; + int ret = 0; + dict_t **xattr = NULL; + loc_t loc = {0}; + uint64_t dirty_xattr[EC_VERSION_SIZE] = {0}; + unsigned char *on = NULL; + default_args_cbk_t *replies = NULL; + dict_t *dict = NULL; + + /* Allocate the required memory */ + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + on = alloca0(ec->nodes); + EC_REPLIES_ALLOC(replies, ec->nodes); + xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer); + if (!xattr) { + ret = -ENOMEM; + goto out; + } + dict = dict_new(); + if (!dict) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict; + on[i] = 1; + } + ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + ret = -ENOMEM; + goto out; + } + PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame, + ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64, + xattr, NULL); +out: + if (dict) { + dict_unref(dict); + } + if (xattr) { + GF_FREE(xattr); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} - ec_fop_set_error(fop, error); +void +ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) +{ + call_frame_t *frame = NULL; + unsigned char *participants = NULL; + unsigned char *msources = NULL; + unsigned char *mhealed_sinks = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + ec_t *ec = NULL; + int ret = 0; + int op_ret = 0; + int op_errno = 0; + intptr_t mgood = 0; + intptr_t mbad = 0; + intptr_t good = 0; + intptr_t bad = 0; + uint32_t pending = 0; + ec_fop_data_t *fop = data; + gf_boolean_t blocking = _gf_false; + ec_heal_need_t need_heal = EC_HEAL_NONEED; + unsigned char *up_subvols = NULL; + char up_bricks[32]; + + ec = this->private; + + /* If it is heal request from getxattr, complete the heal and then + * unwind, if it is ec_heal with NULL as frame then no need to block + * the heal as the caller doesn't care about its completion. In case + * of heald whichever gets tiebreaking inodelk will take care of the + * heal, so no need to block*/ + if (fop->req_frame && !ec->shd.iamshd) + blocking = _gf_true; + + frame = create_frame(this, this->ctx->pool); + if (!frame) + goto out; - /* Resume all pending heal requests, setting the same data obtained by - * this heal execution. */ - while (!list_empty(&list)) { - heal = list_entry(list.next, ec_heal_t, list); - list_del_init(&heal->list); + ec_owner_set(frame, frame->root); + /*Do heal as root*/ + frame->root->uid = 0; + frame->root->gid = 0; + /*Mark the fops as internal*/ + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; + participants = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, participants, ec->nodes); + + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + if (loc->name && strlen(loc->name)) { + ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name, + participants); + if (ret >= 0) { + gf_msg_debug(this->name, 0, + "%s: name heal " + "successful on %" PRIXPTR, + loc->path, + ec_char_array_to_mask(participants, ec->nodes)); + } else { + gf_msg_debug( + this->name, 0, + "%s: name heal " + "failed. ret = %d, subvolumes up = %s", + loc->path, ret, + ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up, ec->nodes)); + } + } - heal->available = cbk->uintptr[0]; - heal->good = cbk->uintptr[1]; - heal->fixed = cbk->uintptr[2]; + /* Mount triggers heal only when it detects that it must need heal, shd + * triggers heals periodically which need not be thorough*/ + if (ec->shd.iamshd && (ret <= 0)) { + ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false, + &need_heal); + + if (need_heal == EC_HEAL_PURGE_INDEX) { + gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL, + "Index entry needs to be purged for: %s ", + uuid_utoa(loc->gfid)); + /* We need to send zero-xattrop so that stale index entry could be + * removed. We need not take lock on this entry to do so as + * xattrop on a brick is atomic. */ + ec_heal_purge_stale_index(frame, ec, loc->inode); + goto out; + } else if (need_heal == EC_HEAL_NONEED) { + gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL, + "Heal is not required for : %s ", uuid_utoa(loc->gfid)); + goto out; + } + } - /* Setting 'done' to 1 avoids executing all heal logic and directly - * reports the result to the caller. */ - heal->done = 1; + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + if (IA_ISREG(loc->inode->ia_type)) { + ret = ec_heal_data(frame, ec, blocking, loc->inode, sources, + healed_sinks); + } else if (IA_ISDIR(loc->inode->ia_type) && !partial) { + ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks, + &pending); + } else { + ret = 0; + memcpy(sources, participants, ec->nodes); + memcpy(healed_sinks, participants, ec->nodes); + } - ec_resume(heal->fop, error); + if (ret == 0) { + good = ec_char_array_to_mask(sources, ec->nodes); + bad = ec_char_array_to_mask(healed_sinks, ec->nodes); + } else { + op_ret = -1; + op_errno = -ret; + } + msources = alloca0(ec->nodes); + mhealed_sinks = alloca0(ec->nodes); + ret = ec_heal_metadata(frame, ec, loc->inode, msources, mhealed_sinks); + if (ret == 0) { + mgood = ec_char_array_to_mask(msources, ec->nodes); + mbad = ec_char_array_to_mask(mhealed_sinks, ec->nodes); + } else { + op_ret = -1; + op_errno = -ret; } - /* If there is a pending full request, resume it. */ - if (next != NULL) { - ec_resume(next->fop, 0); +out: + ec_reset_entry_healing(fop); + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno, + ec_char_array_to_mask(participants, ec->nodes), + mgood & good, mbad & bad, pending, NULL); } + if (frame) + STACK_DESTROY(frame->root); + + return; } -void ec_wind_heal(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +int +ec_synctask_heal_wrap(void *opaque) { - ec_cbk_data_t * cbk; - ec_heal_t *heal = fop->heal; + ec_fop_data_t *fop = opaque; + ec_heal_do(fop->xl, fop, &fop->loc[0], fop->int32); + return 0; +} - ec_trace("WIND", fop, "idx=%d", idx); +int +ec_heal_done(int ret, call_frame_t *heal, void *opaque) +{ + if (opaque) + ec_fop_data_release(opaque); + return 0; +} - cbk = ec_cbk_data_allocate(fop->frame, fop->xl, fop, EC_FOP_HEAL, idx, - fop->error == 0 ? 0 : -1, fop->error); - if (cbk != NULL) - { - cbk->uintptr[0] = heal->available; - cbk->uintptr[1] = heal->good; - cbk->uintptr[2] = heal->bad; +ec_fop_data_t * +__ec_dequeue_heals(ec_t *ec) +{ + ec_fop_data_t *fop = NULL; + + if (list_empty(&ec->heal_waiting)) + goto none; + + if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals)) + goto none; + + fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer); + ec->heal_waiters--; + list_del_init(&fop->healer); + list_add(&fop->healer, &ec->healing); + ec->healers++; + return fop; +none: + gf_msg_debug(ec->xl->name, 0, "Num healers: %d, Num Waiters: %d", + ec->healers, ec->heal_waiters); + return NULL; +} - ec_combine(cbk, NULL); +void +ec_heal_fail(ec_t *ec, ec_fop_data_t *fop) +{ + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0, + 0, 0, NULL); } - - ec_complete(fop); + ec_fop_data_release(fop); } -int32_t ec_manager_heal(ec_fop_data_t * fop, int32_t state) +void +ec_launch_heal(ec_t *ec, ec_fop_data_t *fop) { - ec_cbk_data_t * cbk; - ec_heal_t *heal = fop->heal; + int ret = 0; + call_frame_t *frame = NULL; - switch (state) - { - case EC_STATE_INIT: - ec_owner_set(fop->frame, fop->frame->root); - - fop->error = ec_heal_init(fop); - if (fop->error != 0) - { - return EC_STATE_REPORT; - } + frame = create_frame(ec->xl, ec->xl->ctx->pool); + if (!frame) { + ret = -1; + goto out; + } - return EC_STATE_DISPATCH; + ec_owner_set(frame, frame->root); + /*Do heal as root*/ + frame->root->uid = 0; + frame->root->gid = 0; + /*Mark the fops as internal*/ + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; - case EC_STATE_DISPATCH: - if (heal->done) { - return EC_STATE_HEAL_DISPATCH; - } + ret = synctask_new(ec->xl->ctx->env, ec_synctask_heal_wrap, ec_heal_done, + frame, fop); +out: + if (ret < 0) { + ec_fop_set_error(fop, ENOMEM); + ec_heal_fail(ec, fop); + } - ec_heal_entrylk(heal, ENTRYLK_LOCK); + if (frame) + STACK_DESTROY(frame->root); +} - return EC_STATE_HEAL_ENTRY_LOOKUP; +void +ec_handle_healers_done(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + ec_fop_data_t *heal_fop = NULL; - case EC_STATE_HEAL_ENTRY_LOOKUP: - ec_lookup(fop->frame, heal->xl, fop->mask, EC_MINIMUM_MIN, - ec_heal_entry_lookup_cbk, heal, &heal->loc, NULL); + if (list_empty(&fop->healer)) + return; - return EC_STATE_HEAL_ENTRY_PREPARE; + LOCK(&ec->lock); - case EC_STATE_HEAL_ENTRY_PREPARE: - if (!heal->partial || (heal->iatt.ia_type == IA_IFDIR)) { - ec_heal_prepare(heal); - } + list_del_init(&fop->healer); - if (heal->partial) { - return EC_STATE_HEAL_UNLOCK_ENTRY; - } + do { + ec->healers--; + heal_fop = __ec_dequeue_heals(ec); - return EC_STATE_HEAL_PRE_INODELK_LOCK; + if ((heal_fop != NULL) && ec->shutdown) { + /* This will prevent ec_handle_healers_done() to be + * called recursively. That would be problematic if + * the queue is too big. */ + list_del_init(&heal_fop->healer); - case EC_STATE_HEAL_PRE_INODELK_LOCK: - // Only heal data/metadata if enough information is supplied. - if (uuid_is_null(heal->loc.gfid)) - { - ec_heal_entrylk(heal, ENTRYLK_UNLOCK); + UNLOCK(&ec->lock); - return EC_STATE_HEAL_DISPATCH; - } + ec_fop_set_error(fop, ENOTCONN); + ec_heal_fail(ec, heal_fop); - ec_heal_inodelk(heal, F_WRLCK, 0, 0, 0); + LOCK(&ec->lock); + } + } while ((heal_fop != NULL) && ec->shutdown); - return EC_STATE_HEAL_PRE_INODE_LOOKUP; + UNLOCK(&ec->lock); - case EC_STATE_HEAL_PRE_INODE_LOOKUP: - ec_heal_lookup(heal, heal->fop->mask); + if (heal_fop) + ec_launch_heal(ec, heal_fop); +} - return EC_STATE_HEAL_XATTRIBUTES_REMOVE; +gf_boolean_t +ec_is_entry_healing(ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + int32_t heal_count = 0; + loc_t *loc = NULL; - case EC_STATE_HEAL_XATTRIBUTES_REMOVE: - ec_heal_removexattr_others(heal); + loc = &fop->loc[0]; - return EC_STATE_HEAL_XATTRIBUTES_SET; + LOCK(&loc->inode->lock); + { + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + heal_count = ctx->heal_count; + } + } + UNLOCK(&loc->inode->lock); + GF_ASSERT(heal_count >= 0); + return heal_count; +} - case EC_STATE_HEAL_XATTRIBUTES_SET: - ec_heal_setxattr_others(heal); +void +ec_heal_throttle(xlator_t *this, ec_fop_data_t *fop) +{ + gf_boolean_t can_heal = _gf_true; + ec_t *ec = this->private; + ec_fop_data_t *fop_rel = NULL; - return EC_STATE_HEAL_ATTRIBUTES; + if (fop->req_frame == NULL) { + LOCK(&ec->lock); + { + if ((ec->background_heals > 0) && + (ec->heal_wait_qlen + ec->background_heals) > + (ec->heal_waiters + ec->healers)) { + if (!ec_is_entry_healing(fop)) { + list_add_tail(&fop->healer, &ec->heal_waiting); + ec->heal_waiters++; + ec_set_entry_healing(fop); + } else { + fop_rel = fop; + } + fop = __ec_dequeue_heals(ec); + } else { + can_heal = _gf_false; + } + } + UNLOCK(&ec->lock); + } - case EC_STATE_HEAL_ATTRIBUTES: - ec_heal_attr(heal); + if (can_heal) { + if (fop) { + if (fop->req_frame != NULL) { + ec_set_entry_healing(fop); + } + ec_launch_heal(ec, fop); + } + } else { + gf_msg_debug(this->name, 0, + "Max number of heals are " + "pending, background self-heal rejected"); + ec_fop_set_error(fop, EBUSY); + ec_heal_fail(ec, fop); + } + if (fop_rel) { + ec_heal_done(0, NULL, fop_rel); + } +} - return EC_STATE_HEAL_OPEN; +void +ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc, + int32_t partial, dict_t *xdata) +{ + ec_cbk_t callback = {.heal = func}; + ec_fop_data_t *fop = NULL; + int32_t err = EINVAL; - case EC_STATE_HEAL_OPEN: - ec_heal_open(heal); + gf_msg_trace("ec", 0, "EC(HEAL) %p", frame); - return EC_STATE_HEAL_REOPEN_FD; + VALIDATE_OR_GOTO(this, fail); + GF_VALIDATE_OR_GOTO(this->name, this->private, fail); - case EC_STATE_HEAL_REOPEN_FD: - ec_heal_reopen_fd(heal); + if (!loc || !loc->inode || gf_uuid_is_null(loc->inode->gfid)) + goto fail; - return EC_STATE_HEAL_UNLOCK; + if (frame && frame->local) + goto fail; + fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags, + NULL, NULL, callback, data); - case -EC_STATE_HEAL_XATTRIBUTES_REMOVE: - case -EC_STATE_HEAL_XATTRIBUTES_SET: - case -EC_STATE_HEAL_ATTRIBUTES: - case -EC_STATE_HEAL_OPEN: - case -EC_STATE_HEAL_REOPEN_FD: - case -EC_STATE_HEAL_UNLOCK: - case EC_STATE_HEAL_UNLOCK: - ec_heal_inodelk(heal, F_UNLCK, 0, 0, 0); + err = ENOMEM; - /* Fall through */ + if (fop == NULL) + goto fail; - case -EC_STATE_HEAL_ENTRY_PREPARE: - case -EC_STATE_HEAL_PRE_INODELK_LOCK: - case -EC_STATE_HEAL_PRE_INODE_LOOKUP: - case -EC_STATE_HEAL_UNLOCK_ENTRY: - case EC_STATE_HEAL_UNLOCK_ENTRY: - ec_heal_entrylk(heal, ENTRYLK_UNLOCK); + fop->int32 = partial; - heal->bad = ec_heal_needs_data_rebuild(heal); - if (heal->bad != 0) - { - return EC_STATE_HEAL_DATA_LOCK; - } + if (loc) { + if (loc_copy(&fop->loc[0], loc) != 0) + goto fail; + } - return EC_STATE_HEAL_DISPATCH; + if (xdata) + fop->xdata = dict_ref(xdata); - case EC_STATE_HEAL_DATA_LOCK: - if (heal->done) - { - return EC_STATE_HEAL_POST_INODELK_LOCK; - } + ec_heal_throttle(this, fop); - ec_heal_inodelk(heal, F_WRLCK, 1, heal->offset, heal->size); + return; - return EC_STATE_HEAL_DATA_COPY; +fail: + if (fop) + ec_fop_data_release(fop); + if (func) + func(frame, data, this, -1, err, 0, 0, 0, 0, NULL); +} - case EC_STATE_HEAL_DATA_COPY: - ec_heal_data(heal); +int +ec_replace_heal_done(int ret, call_frame_t *heal, void *opaque) +{ + ec_t *ec = opaque; + gf_boolean_t last_fop = _gf_false; - return EC_STATE_HEAL_DATA_UNLOCK; + if (GF_ATOMIC_DEC(ec->async_fop_count) == 0) { + LOCK(&ec->lock); + { + last_fop = __ec_is_last_fop(ec); + } + UNLOCK(&ec->lock); + } + gf_msg_debug(ec->xl->name, 0, "getxattr on bricks is done ret %d", ret); - case -EC_STATE_HEAL_DATA_COPY: - case -EC_STATE_HEAL_DATA_UNLOCK: - case EC_STATE_HEAL_DATA_UNLOCK: - ec_heal_inodelk(heal, F_UNLCK, 1, heal->offset, heal->size); + if (last_fop) + ec_pending_fops_completed(ec); - heal->offset += heal->size; + return 0; +} - return EC_STATE_HEAL_DATA_LOCK; +int32_t +ec_replace_heal(ec_t *ec, inode_t *inode) +{ + loc_t loc = {0}; + int ret = 0; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + ret = syncop_getxattr(ec->xl, &loc, NULL, EC_XATTR_HEAL, NULL, NULL); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d", + ret); + + /* Once the root inode has been checked, it might have triggered a + * self-heal on it after a replace brick command or for some other + * reason. It can also happen that the volume already had damaged + * files in the index, even if the heal on the root directory failed. + * In both cases we need to wake all index healers to continue + * healing remaining entries that are marked as dirty. */ + ec_shd_index_healer_wake(ec); - case EC_STATE_HEAL_POST_INODELK_LOCK: - ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0); + loc_wipe(&loc); + return ret; +} - return EC_STATE_HEAL_POST_INODE_LOOKUP; +int32_t +ec_replace_brick_heal_wrap(void *opaque) +{ + ec_t *ec = opaque; + inode_table_t *itable = NULL; + int32_t ret = -1; - case EC_STATE_HEAL_POST_INODE_LOOKUP: - heal->fixed = heal->bad; - ec_heal_update_dirty(heal, heal->bad); - ec_heal_lookup(heal, heal->good); + if (ec->xl->itable) + itable = ec->xl->itable; + else + goto out; - return EC_STATE_HEAL_SETATTR; + if (xlator_is_cleanup_starting(ec->xl)) + goto out; - case EC_STATE_HEAL_SETATTR: - ec_setattr(heal->fop->frame, heal->xl, heal->fixed, EC_MINIMUM_ONE, - ec_heal_setattr_cbk, heal, &heal->loc, &heal->iatt, - GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID | - GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL); + ret = ec_replace_heal(ec, itable->root); +out: + return ret; +} - return EC_STATE_HEAL_POST_INODELK_UNLOCK; +int32_t +ec_launch_replace_heal(ec_t *ec) +{ + int ret = -1; - case -EC_STATE_HEAL_SETATTR: - case -EC_STATE_HEAL_POST_INODELK_UNLOCK: - case EC_STATE_HEAL_POST_INODELK_UNLOCK: - ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0); + ret = synctask_new(ec->xl->ctx->env, ec_replace_brick_heal_wrap, + ec_replace_heal_done, NULL, ec); - return EC_STATE_HEAL_DISPATCH; - - case -EC_STATE_HEAL_POST_INODELK_LOCK: - case -EC_STATE_HEAL_POST_INODE_LOOKUP: - case -EC_STATE_HEAL_ENTRY_LOOKUP: - case -EC_STATE_HEAL_DATA_LOCK: - case -EC_STATE_HEAL_DISPATCH: - case EC_STATE_HEAL_DISPATCH: - ec_heal_dispatch(heal); - - return EC_STATE_PREPARE_ANSWER; - - case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d", + ret); + ec_replace_heal_done(-1, NULL, ec); + } - return EC_STATE_REPORT; + return ret; +} - case EC_STATE_REPORT: - cbk = fop->answer; - - GF_ASSERT(cbk != NULL); - - if (fop->id == EC_FOP_HEAL) - { - if (fop->cbks.heal != NULL) - { - fop->cbks.heal(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->uintptr[0], - cbk->uintptr[1], cbk->uintptr[2], - cbk->xdata); - } - } - else - { - if (fop->cbks.fheal != NULL) - { - fop->cbks.fheal(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, cbk->uintptr[0], - cbk->uintptr[1], cbk->uintptr[2], - cbk->xdata); - } - } +int32_t +ec_set_heal_info(dict_t **dict_rsp, char *status) +{ + dict_t *dict = NULL; + int ret = 0; - return EC_STATE_END; + dict = dict_new(); + if (!dict) { + ret = -ENOMEM; + goto out; + } + ret = dict_set_str(dict, "heal-info", status); + if (ret) { + gf_msg(THIS->name, GF_LOG_WARNING, -ret, EC_MSG_HEAL_FAIL, + "Failed to set heal-info key to " + "%s", + status); + dict_unref(dict); + dict = NULL; + } + *dict_rsp = dict; +out: + return ret; +} - case -EC_STATE_INIT: - case -EC_STATE_DISPATCH: - case -EC_STATE_PREPARE_ANSWER: - case -EC_STATE_REPORT: - GF_ASSERT(fop->error != 0); - - if (fop->id == EC_FOP_HEAL) - { - if (fop->cbks.heal != NULL) - { - fop->cbks.heal(fop->req_frame, fop, fop->xl, -1, - fop->error, 0, 0, 0, NULL); +static int32_t +_need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources, + gf_boolean_t self_locked, int32_t lock_count, + ec_heal_need_t *need_heal, uint64_t *versions) +{ + int i = 0; + int source_count = 0; + + source_count = EC_COUNT(sources, ec->nodes); + if (source_count == ec->nodes) { + *need_heal = EC_HEAL_NONEED; + if (self_locked || lock_count == 0) { + for (i = 0; i < ec->nodes; i++) { + if (dirty[i] || (versions[i] != versions[0])) { + *need_heal = EC_HEAL_MUST; + goto out; } } - else - { - if (fop->cbks.fheal != NULL) - { - fop->cbks.fheal(fop->req_frame, fop, fop->xl, -1, - fop->error, 0, 0, 0, NULL); + /* If lock count is 0, all dirty flags are 0 and all the + * versions are macthing then why are we here. It looks + * like something went wrong while removing the index entries + * after completing a successful heal or fop. In this case + * we need to remove this index entry to avoid triggering heal + * in a loop and causing lookups again and again*/ + *need_heal = EC_HEAL_PURGE_INDEX; + } else { + for (i = 0; i < ec->nodes; i++) { + /* Since each lock can only increment the dirty + * count once, if dirty is > 1 it means that + * another operation has left the dirty count + * set and this indicates a problem in the + * inode.*/ + if (dirty[i] > 1) { + *need_heal = EC_HEAL_MUST; + goto out; + } + if (dirty[i] != dirty[0] || (versions[i] != versions[0])) { + *need_heal = EC_HEAL_MAYBE; } } + } + } else { + *need_heal = EC_HEAL_MUST; + } - return EC_STATE_END; +out: + return source_count; +} - default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); +static int32_t +ec_need_metadata_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *meta_versions = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + meta_versions = alloca0(ec->nodes * sizeof(*meta_versions)); + ret = ec_heal_metadata_find_direction(ec, replies, meta_versions, dirty, + sources, healed_sinks); + if (ret < 0 && ret != -EIO) { + goto out; + } - return EC_STATE_END; + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, meta_versions); +out: + return ret; +} + +static int32_t +ec_need_data_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *data_versions = NULL; + uint64_t *size = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + data_versions = alloca0(ec->nodes * sizeof(*data_versions)); + size = alloca0(ec->nodes * sizeof(*size)); + + /* When dd is going on and heal info is called there is a very good + * chance for on disk sizes to mismatch even though nothing is wrong + * we don't need ondisk size check there. But if the file is either + * self-locked or the caller wants a thorough check then make sure to + * perform on disk check also. */ + ret = ec_heal_data_find_direction( + ec, replies, data_versions, dirty, size, sources, healed_sinks, + self_locked || thorough, EC_COMBINE_XDATA); + if (ret < 0 && ret != -EIO) { + goto out; } + + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, data_versions); +out: + return ret; } -void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_heal_cbk_t func, void * data, loc_t * loc, - int32_t partial, dict_t *xdata) +static int32_t +ec_need_entry_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) { - ec_cbk_t callback = { .heal = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *data_versions = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + data_versions = alloca0(ec->nodes * sizeof(*data_versions)); + + ret = ec_heal_entry_find_direction(ec, replies, data_versions, dirty, + sources, healed_sinks); + if (ret < 0 && ret != -EIO) { + goto out; + } - gf_log("ec", GF_LOG_TRACE, "EC(HEAL) %p", frame); + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, data_versions); +out: + return ret; +} - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); +static int32_t +ec_need_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + int ret = 0; - fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_heal, ec_manager_heal, callback, data); - if (fop == NULL) - { + ret = ec_need_metadata_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); + if (ret < 0) + goto out; + + if (*need_heal == EC_HEAL_MUST) goto out; + + if (inode->ia_type == IA_IFREG) { + ret = ec_need_data_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); + } else if (inode->ia_type == IA_IFDIR) { + ret = ec_need_entry_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); } - fop->int32 = partial; +out: + return ret; +} - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); +int32_t +ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + loc_t loc = {0}; + int i = 0; + int ret = 0; + dict_t *xdata = NULL; + uint64_t zero_array[2] = {0}; + uint64_t zero_value = 0; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + int32_t lock_count = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); - goto out; - } + xdata = dict_new(); + if (!xdata || + dict_set_static_bin(xdata, EC_XATTR_VERSION, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xdata, EC_XATTR_DIRTY, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xdata, EC_XATTR_SIZE, &zero_value, + sizeof(zero_value))) { + ret = -ENOMEM; + goto out; } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (!self_locked) { + ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT, ec->xl->name); + if (ret) { + ret = -ENOMEM; goto out; } } - error = 0; + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, xdata); -out: - if (fop != NULL) - { - ec_manager(fop, error); + if (ret != ec->nodes) { + ret = ec->nodes; + *need_heal = EC_HEAL_MUST; + goto out; } - else - { - func(frame, NULL, this, -1, EIO, 0, 0, 0, NULL); + + if (self_locked) + goto need_heal; + + for (i = 0; i < ec->nodes; i++) { + if (!output[i] || !replies[i].xdata) { + continue; + } + if ((dict_get_int32(replies[i].xdata, GLUSTERFS_INODELK_COUNT, + &lock_count) == 0) && + lock_count > 0) { + break; + } } +need_heal: + ret = ec_need_heal(ec, inode, replies, lock_count, self_locked, thorough, + need_heal); +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + if (xdata) { + dict_unref(xdata); + } + return ret; } -/* FOP: fheal */ +int32_t +ec_heal_locked_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + ec_heal_need_t *need_heal) +{ + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0); + if (ret != ec->nodes) { + *need_heal = EC_HEAL_MUST; + goto unlock; + } + ret = ec_heal_inspect(frame, ec, inode, locked_on, _gf_true, _gf_true, + need_heal); +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} -void ec_wind_fheal(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +int32_t +ec_get_heal_info(xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp) { - ec_cbk_data_t * cbk; - ec_heal_t *heal = fop->heal; + int ret = -ENOMEM; + ec_heal_need_t need_heal = EC_HEAL_NONEED; + call_frame_t *frame = NULL; + ec_t *ec = NULL; + unsigned char *up_subvols = NULL; + loc_t loc = { + 0, + }; - ec_trace("WIND", fop, "idx=%d", idx); + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, entry_loc, out); - cbk = ec_cbk_data_allocate(fop->frame, fop->xl, fop, EC_FOP_FHEAL, idx, - fop->error == 0 ? 0 : -1, fop->error); - if (cbk != NULL) - { - cbk->uintptr[0] = heal->available; - cbk->uintptr[1] = heal->good; - cbk->uintptr[2] = heal->bad; + ec = this->private; + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); - ec_combine(cbk, NULL); + if (EC_COUNT(up_subvols, ec->nodes) != ec->nodes) { + need_heal = EC_HEAL_MUST; + goto set_heal; } + frame = create_frame(this, this->ctx->pool); + if (!frame) { + goto out; + } + ec_owner_set(frame, frame->root); + frame->root->uid = 0; + frame->root->gid = 0; + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; - ec_complete(fop); -} - -void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fheal_cbk_t func, void * data, fd_t * fd, - int32_t partial, dict_t *xdata) -{ - ec_fd_t * ctx = ec_fd_get(fd, this); + if (loc_copy(&loc, entry_loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + goto out; + } + if (!loc.inode) { + ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL); + if (ret < 0) + goto out; + } - if (ctx != NULL) - { - gf_log("ec", GF_LOG_DEBUG, "FHEAL ctx: flags=%X, open=%lX, bad=%lX", - ctx->flags, ctx->open, ctx->bad); - ec_heal(frame, this, target, minimum, func, data, &ctx->loc, partial, - xdata); + ret = ec_heal_inspect(frame, ec, loc.inode, up_subvols, _gf_false, + _gf_false, &need_heal); + if (ret == ec->nodes && need_heal != EC_HEAL_MAYBE) { + goto set_heal; + } + need_heal = EC_HEAL_NONEED; + ret = ec_heal_locked_inspect(frame, ec, loc.inode, &need_heal); + if (ret < 0) + goto out; +set_heal: + if (need_heal == EC_HEAL_MUST) { + ret = ec_set_heal_info(dict_rsp, "heal"); + } else { + ret = ec_set_heal_info(dict_rsp, "no-heal"); } +out: + if (frame) { + STACK_DESTROY(frame->root); + } + loc_wipe(&loc); + return ret; } diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c new file mode 100644 index 00000000000..5c1586bc9c5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-heald.c @@ -0,0 +1,681 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/defaults.h> +#include <glusterfs/compat-errno.h> +#include "ec.h" +#include "ec-messages.h" +#include "ec-heald.h" +#include "ec-mem-types.h" +#include <glusterfs/syncop.h> +#include <glusterfs/syncop-utils.h> +#include "protocol-common.h" + +#define NTH_INDEX_HEALER(this, n) \ + (&((((ec_t *)this->private))->shd.index_healers[n])) +#define NTH_FULL_HEALER(this, n) \ + (&((((ec_t *)this->private))->shd.full_healers[n])) + +gf_boolean_t +ec_shd_is_subvol_local(xlator_t *this, int subvol) +{ + ec_t *ec = NULL; + gf_boolean_t is_local = _gf_false; + loc_t loc = { + 0, + }; + + ec = this->private; + loc.inode = this->itable->root; + syncop_is_subvol_local(ec->xl_list[subvol], &loc, &is_local); + return is_local; +} + +char * +ec_subvol_name(xlator_t *this, int subvol) +{ + ec_t *ec = NULL; + + ec = this->private; + if (subvol < 0 || subvol > ec->nodes) + return NULL; + + return ec->xl_list[subvol]->name; +} + +int +__ec_shd_healer_wait(struct subvol_healer *healer) +{ + ec_t *ec = NULL; + struct timespec wait_till = { + 0, + }; + int ret = 0; + + ec = healer->this->private; + +disabled_loop: + wait_till.tv_sec = gf_time() + ec->shd.timeout; + + while (!healer->rerun) { + ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till); + if (ret == ETIMEDOUT) + break; + } + + if (ec->shutdown) { + healer->running = _gf_false; + return -1; + } + + ret = healer->rerun; + healer->rerun = 0; + + if (!ec->shd.enabled || !ec->up) + goto disabled_loop; + + return ret; +} + +int +ec_shd_healer_wait(struct subvol_healer *healer) +{ + int ret = 0; + + pthread_mutex_lock(&healer->mutex); + { + ret = __ec_shd_healer_wait(healer); + } + pthread_mutex_unlock(&healer->mutex); + + return ret; +} + +int +ec_shd_index_inode(xlator_t *this, xlator_t *subvol, inode_t **inode) +{ + loc_t rootloc = { + 0, + }; + int ret = 0; + dict_t *xattr = NULL; + void *index_gfid = NULL; + + *inode = NULL; + rootloc.inode = inode_ref(this->itable->root); + gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid); + + ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_GFID, NULL, + NULL); + if (ret < 0) + goto out; + if (!xattr) { + ret = -EINVAL; + goto out; + } + + ret = dict_get_ptr(xattr, GF_XATTROP_INDEX_GFID, &index_gfid); + if (ret) + goto out; + + gf_msg_debug(this->name, 0, "index-dir gfid for %s: %s", subvol->name, + uuid_utoa(index_gfid)); + + ret = syncop_inode_find(this, subvol, index_gfid, inode, NULL, NULL); + +out: + loc_wipe(&rootloc); + + if (xattr) + dict_unref(xattr); + + return ret; +} + +int +ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name) +{ + loc_t loc = { + 0, + }; + int ret = 0; + + loc.parent = inode_ref(inode); + loc.name = name; + + ret = syncop_unlink(subvol, &loc, NULL, NULL); + + loc_wipe(&loc); + return ret; +} + +static gf_boolean_t +ec_is_heal_completed(char *status) +{ + char *bad_pos = NULL; + char *zero_pos = NULL; + + if (!status) { + return _gf_false; + } + + /*Logic: + * Status will be of the form Good: <binary>, Bad: <binary> + * If heal completes, if we do strchr for '0' it should be present after + * 'Bad:' i.e. strRchr for ':' + * */ + + zero_pos = strchr(status, '0'); + bad_pos = strrchr(status, ':'); + if (!zero_pos || !bad_pos) { + /*malformed status*/ + return _gf_false; + } + + if (zero_pos > bad_pos) { + return _gf_true; + } + + return _gf_false; +} + +int +ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc, + gf_boolean_t full) +{ + dict_t *xdata = NULL; + dict_t *dict = NULL; + uint32_t count; + int32_t ret; + char *heal_status = NULL; + ec_t *ec = healer->this->private; + + GF_ATOMIC_INC(ec->stats.shd.attempted); + ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL, + &xdata); + if (ret == 0) { + if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) { + if (ec_is_heal_completed(heal_status)) { + GF_ATOMIC_INC(ec->stats.shd.completed); + } + } + } + + if (!full && (loc->inode->ia_type == IA_IFDIR)) { + /* If we have just healed a directory, it's possible that + * other index entries have appeared to be healed. */ + if ((xdata != NULL) && + (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) && + (count > 0)) { + /* Force a rerun of the index healer. */ + gf_msg_debug(healer->this->name, 0, "%d more entries to heal", + count); + + healer->rerun = _gf_true; + } + } + + if (xdata != NULL) { + dict_unref(xdata); + } + + if (dict) { + dict_unref(dict); + } + + return ret; +} + +int +ec_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct subvol_healer *healer = data; + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = 0; + + ec = healer->this->private; + if (ec->xl_up_count <= ec->fragments) { + return -ENOTCONN; + } + if (!ec->shd.enabled) + return -EBUSY; + + gf_msg_debug(healer->this->name, 0, "got entry: %s", entry->d_name); + + ret = gf_uuid_parse(entry->d_name, loc.gfid); + if (ret) + return 0; + + /* If this fails with ENOENT/ESTALE index is stale */ + ret = syncop_gfid_to_path(healer->this->itable, subvol, loc.gfid, + (char **)&loc.path); + if (ret < 0) + goto out; + + ret = syncop_inode_find(healer->this, healer->this, loc.gfid, &loc.inode, + NULL, NULL); + if (ret < 0) + goto out; + + ec_shd_selfheal(healer, healer->subvol, &loc, _gf_false); +out: + if (ret == -ENOENT || ret == -ESTALE) { + gf_msg(healer->this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL, + "Purging index for gfid %s:", uuid_utoa(loc.gfid)); + ec_shd_index_purge(subvol, parent->inode, entry->d_name); + } + loc_wipe(&loc); + + return 0; +} + +int +ec_shd_index_sweep(struct subvol_healer *healer) +{ + loc_t loc = {0}; + ec_t *ec = NULL; + int ret = 0; + xlator_t *subvol = NULL; + dict_t *xdata = NULL; + + ec = healer->this->private; + subvol = ec->xl_list[healer->subvol]; + + ret = ec_shd_index_inode(healer->this, subvol, &loc.inode); + if (ret < 0) { + gf_msg(healer->this->name, GF_LOG_WARNING, errno, + EC_MSG_INDEX_DIR_GET_FAIL, "unable to get index-dir on %s", + subvol->name); + goto out; + } + + xdata = dict_new(); + if (!xdata || dict_set_int32(xdata, "get-gfid-type", 1)) { + ret = -ENOMEM; + goto out; + } + + _mask_cancellation(); + ret = syncop_mt_dir_scan(NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD, + healer, ec_shd_index_heal, xdata, + ec->shd.max_threads, ec->shd.wait_qlength); + _unmask_cancellation(); +out: + if (xdata) + dict_unref(xdata); + loc_wipe(&loc); + + return ret; +} + +int +ec_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct subvol_healer *healer = data; + xlator_t *this = healer->this; + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = 0; + + ec = this->private; + + if (this->cleanup_starting) { + return -ENOTCONN; + } + + if (ec->xl_up_count <= ec->fragments) { + return -ENOTCONN; + } + if (!ec->shd.enabled) + return -EBUSY; + + if (gf_uuid_is_null(entry->d_stat.ia_gfid)) { + /* It's possible that an entry has been removed just after + * being seen in a directory but before getting its stat info. + * In this case we'll receive a NULL gfid here. Since the file + * doesn't exist anymore, we can safely ignore it. */ + return 0; + } + + loc.parent = inode_ref(parent->inode); + loc.name = entry->d_name; + gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid); + + /* If this fails with ENOENT/ESTALE index is stale */ + ret = syncop_gfid_to_path(this->itable, subvol, loc.gfid, + (char **)&loc.path); + if (ret < 0) + goto out; + + ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL); + if (ret < 0) + goto out; + + ec_shd_selfheal(healer, healer->subvol, &loc, _gf_true); + + ret = 0; + +out: + loc_wipe(&loc); + return ret; +} + +int +ec_shd_full_sweep(struct subvol_healer *healer, inode_t *inode) +{ + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = -1; + + ec = healer->this->private; + loc.inode = inode; + _mask_cancellation(); + ret = syncop_ftw(ec->xl_list[healer->subvol], &loc, + GF_CLIENT_PID_SELF_HEALD, healer, ec_shd_full_heal); + _unmask_cancellation(); + return ret; +} + +void * +ec_shd_index_healer(void *data) +{ + struct subvol_healer *healer = NULL; + xlator_t *this = NULL; + int run = 0; + + healer = data; + THIS = this = healer->this; + ec_t *ec = this->private; + + for (;;) { + run = ec_shd_healer_wait(healer); + if (run == -1) + break; + + if (ec->xl_up_count > ec->fragments) { + gf_msg_debug(this->name, 0, "starting index sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + ec_shd_index_sweep(healer); + } + gf_msg_debug(this->name, 0, "finished index sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + } + + return NULL; +} + +void * +ec_shd_full_healer(void *data) +{ + struct subvol_healer *healer = NULL; + xlator_t *this = NULL; + loc_t rootloc = {0}; + + int run = 0; + + healer = data; + THIS = this = healer->this; + ec_t *ec = this->private; + + rootloc.inode = this->itable->root; + for (;;) { + run = ec_shd_healer_wait(healer); + if (run < 0) { + break; + } else if (run == 0) { + continue; + } + + if (ec->xl_up_count > ec->fragments) { + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_START, + "starting full sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + + ec_shd_selfheal(healer, healer->subvol, &rootloc, _gf_true); + ec_shd_full_sweep(healer, this->itable->root); + } + + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_STOP, + "finished full sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + } + + return NULL; +} + +int +ec_shd_healer_init(xlator_t *this, struct subvol_healer *healer) +{ + int ret = 0; + + ret = pthread_mutex_init(&healer->mutex, NULL); + if (ret) + goto out; + + ret = pthread_cond_init(&healer->cond, NULL); + if (ret) + goto out; + + healer->this = this; + healer->running = _gf_false; + healer->rerun = _gf_false; +out: + return ret; +} + +int +ec_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer, + void *(threadfn)(void *)) +{ + int ret = 0; + + pthread_mutex_lock(&healer->mutex); + { + if (healer->running) { + pthread_cond_signal(&healer->cond); + } else { + ret = gf_thread_create(&healer->thread, NULL, threadfn, healer, + "ecshd"); + if (ret) + goto unlock; + healer->running = 1; + } + + healer->rerun = 1; + } +unlock: + pthread_mutex_unlock(&healer->mutex); + + return ret; +} + +int +ec_shd_full_healer_spawn(xlator_t *this, int subvol) +{ + if (xlator_is_cleanup_starting(this)) + return -1; + + return ec_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol), + ec_shd_full_healer); +} + +int +ec_shd_index_healer_spawn(xlator_t *this, int subvol) +{ + if (xlator_is_cleanup_starting(this)) + return -1; + + return ec_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol), + ec_shd_index_healer); +} + +void +ec_shd_index_healer_wake(ec_t *ec) +{ + int32_t i; + + for (i = 0; i < ec->nodes; i++) { + if (((ec->xl_up >> i) & 1) != 0) { + ec_shd_index_healer_spawn(ec->xl, i); + } + } +} + +int +ec_selfheal_daemon_init(xlator_t *this) +{ + ec_t *ec = NULL; + ec_self_heald_t *shd = NULL; + int ret = -1; + int i = 0; + + ec = this->private; + shd = &ec->shd; + + shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers), ec->nodes, + ec_mt_subvol_healer_t); + if (!shd->index_healers) + goto out; + + for (i = 0; i < ec->nodes; i++) { + shd->index_healers[i].subvol = i; + ret = ec_shd_healer_init(this, &shd->index_healers[i]); + if (ret) + goto out; + } + + shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), ec->nodes, + ec_mt_subvol_healer_t); + if (!shd->full_healers) + goto out; + + for (i = 0; i < ec->nodes; i++) { + shd->full_healers[i].subvol = i; + ret = ec_shd_healer_init(this, &shd->full_healers[i]); + if (ret) + goto out; + } + + ret = 0; +out: + return ret; +} + +int +ec_heal_op(xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id) +{ + char key[64] = {0}; + int op_ret = 0; + ec_t *ec = NULL; + int i = 0; + GF_UNUSED int ret = 0; + + ec = this->private; + + op_ret = -1; + for (i = 0; i < ec->nodes; i++) { + snprintf(key, sizeof(key), "%d-%d-status", xl_id, i); + + if (((ec->xl_up >> i) & 1) == 0) { + ret = dict_set_str(output, key, "Brick is not connected"); + } else if (!ec->up) { + ret = dict_set_str(output, key, "Disperse subvolume is not up"); + } else if (!ec_shd_is_subvol_local(this, i)) { + ret = dict_set_str(output, key, "Brick is remote"); + } else { + ret = dict_set_str(output, key, "Started self-heal"); + if (op == GF_SHD_OP_HEAL_FULL) { + ec_shd_full_healer_spawn(this, i); + } else if (op == GF_SHD_OP_HEAL_INDEX) { + ec_shd_index_healer_spawn(this, i); + } + op_ret = 0; + } + } + return op_ret; +} + +int +ec_xl_op(xlator_t *this, dict_t *input, dict_t *output) +{ + gf_xl_afr_op_t op = GF_SHD_OP_INVALID; + int ret = 0; + int xl_id = 0; + + ret = dict_get_int32(input, "xl-op", (int32_t *)&op); + if (ret) + goto out; + + ret = dict_get_int32(input, this->name, &xl_id); + if (ret) + goto out; + + ret = dict_set_int32(output, this->name, xl_id); + if (ret) + goto out; + + switch (op) { + case GF_SHD_OP_HEAL_FULL: + ret = ec_heal_op(this, output, op, xl_id); + break; + + case GF_SHD_OP_HEAL_INDEX: + ret = ec_heal_op(this, output, op, xl_id); + break; + + default: + ret = -1; + break; + } +out: + dict_del(output, this->name); + return ret; +} + +void +ec_destroy_healer_object(xlator_t *this, struct subvol_healer *healer) +{ + if (!healer) + return; + + pthread_cond_destroy(&healer->cond); + pthread_mutex_destroy(&healer->mutex); +} + +void +ec_selfheal_daemon_fini(xlator_t *this) +{ + struct subvol_healer *healer = NULL; + ec_self_heald_t *shd = NULL; + ec_t *priv = NULL; + int i = 0; + + priv = this->private; + if (!priv) + return; + + shd = &priv->shd; + if (!shd->iamshd) + return; + + for (i = 0; i < priv->nodes; i++) { + healer = &shd->index_healers[i]; + ec_destroy_healer_object(this, healer); + + healer = &shd->full_healers[i]; + ec_destroy_healer_object(this, healer); + } + + GF_FREE(shd->index_healers); + GF_FREE(shd->full_healers); +} diff --git a/xlators/cluster/ec/src/ec-heald.h b/xlators/cluster/ec/src/ec-heald.h new file mode 100644 index 00000000000..6c7da4edc10 --- /dev/null +++ b/xlators/cluster/ec/src/ec-heald.h @@ -0,0 +1,30 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_HEALD_H__ +#define __EC_HEALD_H__ + +#include "ec-types.h" // for ec_t +#include "glusterfs/dict.h" // for dict_t +#include "glusterfs/globals.h" // for xlator_t + +int +ec_xl_op(xlator_t *this, dict_t *input, dict_t *output); + +int +ec_selfheal_daemon_init(xlator_t *this); + +void +ec_shd_index_healer_wake(ec_t *ec); + +void +ec_selfheal_daemon_fini(xlator_t *this); + +#endif /* __EC_HEALD_H__ */ diff --git a/xlators/cluster/ec/src/ec-helpers.c b/xlators/cluster/ec/src/ec-helpers.c index c580166ef00..48f54475e01 100644 --- a/xlators/cluster/ec/src/ec-helpers.c +++ b/xlators/cluster/ec/src/ec-helpers.c @@ -10,46 +10,29 @@ #include <libgen.h> -#include "byte-order.h" +#include <glusterfs/byte-order.h> +#include "ec.h" #include "ec-mem-types.h" +#include "ec-messages.h" #include "ec-fops.h" +#include "ec-method.h" #include "ec-helpers.h" -#define BACKEND_D_OFF_BITS 63 -#define PRESENT_D_OFF_BITS 63 +static const char *ec_fop_list[] = {[-EC_FOP_HEAL] = "HEAL"}; -#define ONE 1ULL -#define MASK (~0ULL) -#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS)) -#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS)) - -#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1)) -#define SHIFT_BITS (max(0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1))) - -#ifndef ffsll -#define ffsll(x) __builtin_ffsll(x) -#endif - -static const char * ec_fop_list[] = -{ - [-EC_FOP_HEAL] = "HEAL" -}; - -const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits) +const char * +ec_bin(char *str, size_t size, uint64_t value, int32_t digits) { str += size; - if (size-- < 1) - { + if (size-- < 1) { goto failed; } *--str = 0; - while ((value != 0) || (digits > 0)) - { - if (size-- < 1) - { + while ((value != 0) || (digits > 0)) { + if (size-- < 1) { goto failed; } *--str = '0' + (value & 1); @@ -63,21 +46,22 @@ failed: return "<buffer too small>"; } -const char * ec_fop_name(int32_t id) +const char * +ec_fop_name(int32_t id) { - if (id >= 0) - { + if (id >= 0) { return gf_fop_list[id]; } return ec_fop_list[-id]; } -void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...) +void +ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...) { char str1[32], str2[32], str3[32]; - char * msg; - ec_t * ec = fop->xl->private; + char *msg; + ec_t *ec = fop->xl->private; va_list args; int32_t ret; @@ -85,80 +69,28 @@ void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...) ret = vasprintf(&msg, fmt, args); va_end(args); - if (ret < 0) - { + if (ret < 0) { msg = "<memory allocation error>"; } - gf_log("ec", GF_LOG_TRACE, "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] " - "frame=%p/%p, min/exp=%d/%d, err=%d state=%d " - "{%s:%s:%s} %s", - event, ec_fop_name(fop->id), fop, fop->parent, fop->refs, - fop->winds, fop->jobs, fop->req_frame, fop->frame, fop->minimum, - fop->expected, fop->error, fop->state, - ec_bin(str1, sizeof(str1), fop->mask, ec->nodes), - ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes), - ec_bin(str3, sizeof(str3), fop->bad, ec->nodes), msg); + gf_msg_trace("ec", 0, + "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] " + "frame=%p/%p, min/exp=%d/%d, err=%d state=%d " + "{%s:%s:%s} %s", + event, ec_fop_name(fop->id), fop, fop->parent, fop->refs, + fop->winds, fop->jobs, fop->req_frame, fop->frame, + fop->minimum, fop->expected, fop->error, fop->state, + ec_bin(str1, sizeof(str1), fop->mask, ec->nodes), + ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes), + ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg); - if (ret >= 0) - { + if (ret >= 0) { free(msg); } } -uint64_t ec_itransform(ec_t * ec, int32_t idx, uint64_t offset) -{ - int32_t bits; - - if (offset == -1ULL) - { - return -1ULL; - } - - bits = ec->bits_for_nodes; - if ((offset & ~(PRESENT_MASK >> (bits + 1))) != 0) - { - return TOP_BIT | ((offset >> SHIFT_BITS) & (MASK << bits)) | idx; - } - - return (offset * ec->nodes) + idx; -} - -uint64_t ec_deitransform(ec_t * ec, int32_t * idx, uint64_t offset) -{ - uint64_t mask = 0; - - if ((offset & TOP_BIT) != 0) - { - mask = MASK << ec->bits_for_nodes; - - *idx = offset & ~mask; - return ((offset & ~TOP_BIT) & mask) << SHIFT_BITS; - } - - *idx = offset % ec->nodes; - - return offset / ec->nodes; -} - -int32_t ec_bits_count(uint64_t n) -{ - n -= (n >> 1) & 0x5555555555555555ULL; - n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL); - n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - n += n >> 8; - n += n >> 16; - n += n >> 32; - - return n & 0xFF; -} - -int32_t ec_bits_index(uint64_t n) -{ - return ffsll(n) - 1; -} - -int32_t ec_bits_consume(uint64_t * n) +int32_t +ec_bits_consume(uint64_t *n) { uint64_t tmp; @@ -166,24 +98,21 @@ int32_t ec_bits_consume(uint64_t * n) tmp &= -tmp; *n ^= tmp; - return ffsll(tmp) - 1; + return gf_bits_index(tmp); } -size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count, - off_t offset, size_t size) +size_t +ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset, + size_t size) { int32_t i = 0; size_t total = 0, len = 0; - while (i < count) - { - if (offset < vector[i].iov_len) - { - while ((i < count) && (size > 0)) - { + while (i < count) { + if (offset < vector[i].iov_len) { + while ((i < count) && (size > 0)) { len = size; - if (len > vector[i].iov_len - offset) - { + if (len > vector[i].iov_len - offset) { len = vector[i].iov_len - offset; } memcpy(dst, vector[i++].iov_base + offset, len); @@ -203,30 +132,161 @@ size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count, return total; } -int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value) +int32_t +ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr) +{ + struct iobref *iobref = NULL; + struct iobuf *iobuf = NULL; + int32_t ret = -ENOMEM; + + iobuf = iobuf_get_page_aligned(xl->ctx->iobuf_pool, size, + EC_METHOD_WORD_SIZE); + if (iobuf == NULL) { + goto out; + } + + iobref = *piobref; + if (iobref == NULL) { + iobref = iobref_new(); + if (iobref == NULL) { + goto out; + } + } + + ret = iobref_add(iobref, iobuf); + if (ret != 0) { + if (iobref != *piobref) { + iobref_unref(iobref); + } + iobref = NULL; + + goto out; + } + + GF_ASSERT(EC_ALIGN_CHECK(iobuf->ptr, EC_METHOD_WORD_SIZE)); + + *ptr = iobuf->ptr; + +out: + if (iobuf != NULL) { + iobuf_unref(iobuf); + } + + if (iobref != NULL) { + *piobref = iobref; + } + + return ret; +} + +int32_t +ec_dict_set_array(dict_t *dict, char *key, uint64_t value[], int32_t size) { - uint64_t * ptr; + int ret = -1; + uint64_t *ptr = NULL; + int32_t vindex; + + if (value == NULL) { + return -EINVAL; + } + + ptr = GF_MALLOC(sizeof(uint64_t) * size, gf_common_mt_char); + if (ptr == NULL) { + return -ENOMEM; + } + for (vindex = 0; vindex < size; vindex++) { + ptr[vindex] = hton64(value[vindex]); + } + ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t) * size); + if (ret) + GF_FREE(ptr); + return ret; +} + +int32_t +ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size) +{ + void *ptr; + int32_t len; + int32_t vindex; + int32_t old_size = 0; + int32_t err; + + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + + if (len > (size * sizeof(uint64_t)) || (len % sizeof(uint64_t))) { + return -EINVAL; + } + + /* 3.6 version ec would have stored version in 64 bit. In that case treat + * metadata versions same as data*/ + old_size = min(size, len / sizeof(uint64_t)); + for (vindex = 0; vindex < old_size; vindex++) { + value[vindex] = ntoh64(*((uint64_t *)ptr + vindex)); + } + + if (old_size < size) { + for (vindex = old_size; vindex < size; vindex++) { + value[vindex] = value[old_size - 1]; + } + } + + return 0; +} + +int32_t +ec_dict_del_array(dict_t *dict, char *key, uint64_t value[], int32_t size) +{ + int ret = 0; + + ret = ec_dict_get_array(dict, key, value, size); + if (ret == 0) + dict_del(dict, key); + + return ret; +} + +int32_t +ec_dict_set_number(dict_t *dict, char *key, uint64_t value) +{ + int ret = -1; + uint64_t *ptr; ptr = GF_MALLOC(sizeof(value), gf_common_mt_char); - if (ptr == NULL) - { - return -1; + if (ptr == NULL) { + return -ENOMEM; } *ptr = hton64(value); - return dict_set_bin(dict, key, ptr, sizeof(value)); + ret = dict_set_bin(dict, key, ptr, sizeof(value)); + if (ret) + GF_FREE(ptr); + + return ret; } -int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value) +int32_t +ec_dict_del_number(dict_t *dict, char *key, uint64_t *value) { - void * ptr; - int32_t len; + void *ptr; + int32_t len, err; - if ((dict == NULL) || (dict_get_ptr_and_len(dict, key, &ptr, &len) != 0) || - (len != sizeof(uint64_t))) - { - return -1; + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + if (len != sizeof(uint64_t)) { + return -EINVAL; } *value = ntoh64(*(uint64_t *)ptr); @@ -236,22 +296,24 @@ int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value) return 0; } -int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config) +int32_t +ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config) { - uint64_t * ptr, data; + int ret = -1; + uint64_t *ptr, data; - if (config->version > EC_CONFIG_VERSION) - { - gf_log("ec", GF_LOG_ERROR, "Trying to store an unsupported config " - "version (%u)", config->version); + if (config->version > EC_CONFIG_VERSION) { + gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION, + "Trying to store an unsupported config " + "version (%u)", + config->version); - return -1; + return -EINVAL; } ptr = GF_MALLOC(sizeof(uint64_t), gf_common_mt_char); - if (ptr == NULL) - { - return -1; + if (ptr == NULL) { + return -ENOMEM; } data = ((uint64_t)config->version) << 56; @@ -263,30 +325,51 @@ int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config) *ptr = hton64(data); - return dict_set_bin(dict, key, ptr, sizeof(uint64_t)); + ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t)); + if (ret) + GF_FREE(ptr); + + return ret; } -int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config) +int32_t +ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config) { - void * ptr; + void *ptr; uint64_t data; - int32_t len; + int32_t len, err; - if ((dict == NULL) || (dict_get_ptr_and_len(dict, key, &ptr, &len) != 0) || - (len != sizeof(uint64_t))) - { - return -1; + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + if (len != sizeof(uint64_t)) { + return -EINVAL; } data = ntoh64(*(uint64_t *)ptr); + /* Currently we need to get the config xattr for entries of type IA_INVAL. + * These entries can later become IA_DIR entries (after inode_link()), + * which don't have a config xattr. However, since the xattr is requested + * using an xattrop() fop, it will always return a config full of 0's + * instead of saying that it doesn't exist. + * + * We need to filter out this case and consider that a config xattr == 0 is + * the same as a non-existent xattr. Otherwise ec_config_check() will fail. + */ + if (data == 0) { + return -ENODATA; + } config->version = (data >> 56) & 0xff; - if (config->version > EC_CONFIG_VERSION) - { - gf_log("ec", GF_LOG_ERROR, "Found an unsupported config version (%u)", - config->version); + if (config->version > EC_CONFIG_VERSION) { + gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION, + "Found an unsupported config version (%u)", config->version); - return -1; + return -EINVAL; } config->algorithm = (data >> 48) & 0xff; @@ -300,43 +383,43 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config) return 0; } -int32_t ec_loc_gfid_check(xlator_t * xl, uuid_t dst, uuid_t src) +gf_boolean_t +ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src) { - if (uuid_is_null(src)) - { - return 1; + if (gf_uuid_is_null(src)) { + return _gf_true; } - if (uuid_is_null(dst)) - { - uuid_copy(dst, src); + if (gf_uuid_is_null(dst)) { + gf_uuid_copy(dst, src); - return 1; + return _gf_true; } - if (uuid_compare(dst, src) != 0) - { - gf_log(xl->name, GF_LOG_WARNING, "Mismatching GFID's in loc"); + if (gf_uuid_compare(dst, src) != 0) { + gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_GFID_MISMATCH, + "Mismatching GFID's in loc"); - return 0; + return _gf_false; } - return 1; + return _gf_true; } -int32_t ec_loc_setup_inode(xlator_t *xl, loc_t *loc) +int32_t +ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc) { - int32_t ret = -1; + int32_t ret = -EINVAL; if (loc->inode != NULL) { if (!ec_loc_gfid_check(xl, loc->gfid, loc->inode->gfid)) { goto out; } - } else if (loc->parent != NULL) { - if (!uuid_is_null(loc->gfid)) { - loc->inode = inode_find(loc->parent->table, loc->gfid); - } else if (loc->path != NULL) { - loc->inode = inode_resolve(loc->parent->table, (char *)loc->path); + } else if (table != NULL) { + if (!gf_uuid_is_null(loc->gfid)) { + loc->inode = inode_find(table, loc->gfid); + } else if (loc->path && strchr(loc->path, '/')) { + loc->inode = inode_resolve(table, (char *)loc->path); } } @@ -346,50 +429,64 @@ out: return ret; } -int32_t ec_loc_setup_parent(xlator_t *xl, loc_t *loc) +int32_t +ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc) { char *path, *parent; - int32_t ret = -1; + int32_t ret = -EINVAL; if (loc->parent != NULL) { if (!ec_loc_gfid_check(xl, loc->pargfid, loc->parent->gfid)) { goto out; } - } else if (loc->inode != NULL) { - if (!uuid_is_null(loc->pargfid)) { - loc->parent = inode_find(loc->inode->table, loc->pargfid); - } else if (loc->path != NULL) { + } else if (table != NULL) { + if (!gf_uuid_is_null(loc->pargfid)) { + loc->parent = inode_find(table, loc->pargfid); + } else if (loc->path && strchr(loc->path, '/')) { path = gf_strdup(loc->path); if (path == NULL) { - gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'", - loc->path); + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", loc->path); + + ret = -ENOMEM; goto out; } parent = dirname(path); - loc->parent = inode_resolve(loc->inode->table, parent); + loc->parent = inode_resolve(table, parent); + if (loc->parent != NULL) { + gf_uuid_copy(loc->pargfid, loc->parent->gfid); + } GF_FREE(path); } } + /* If 'pargfid' has not been determined, clear 'name' to avoid resolutions + based on <gfid:pargfid>/name. */ + if (gf_uuid_is_null(loc->pargfid)) { + loc->name = NULL; + } + ret = 0; out: return ret; } -int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc) +int32_t +ec_loc_setup_path(xlator_t *xl, loc_t *loc) { - uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; char *name; - int32_t ret = -1; + int32_t ret = -EINVAL; if (loc->path != NULL) { name = strrchr(loc->path, '/'); if (name == NULL) { - gf_log(xl->name, GF_LOG_ERROR, "Invalid path '%s' in loc", - loc->path); - + /* Allow gfid paths: <gfid:...> */ + if (strncmp(loc->path, "<gfid:", 6) == 0) { + ret = 0; + } goto out; } if (name == loc->path) { @@ -407,8 +504,8 @@ int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc) if (loc->name != NULL) { if (strcmp(loc->name, name) != 0) { - gf_log(xl->name, GF_LOG_ERROR, "Invalid name '%s' in loc", - loc->name); + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_LOC_NAME, + "Invalid name '%s' in loc", loc->name); goto out; } @@ -423,45 +520,58 @@ out: return ret; } -int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent) +int32_t +ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent) { + inode_table_t *table = NULL; char *str = NULL; - int32_t ret = -1; + int32_t ret = -ENOMEM; memset(parent, 0, sizeof(loc_t)); if (loc->parent != NULL) { + table = loc->parent->table; parent->inode = inode_ref(loc->parent); + } else if (loc->inode != NULL) { + table = loc->inode->table; } - if (!uuid_is_null(loc->pargfid)) { - uuid_copy(parent->gfid, loc->pargfid); + if (!gf_uuid_is_null(loc->pargfid)) { + gf_uuid_copy(parent->gfid, loc->pargfid); } - if (loc->path != NULL) { + if (loc->path && strchr(loc->path, '/')) { str = gf_strdup(loc->path); if (str == NULL) { - gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'", - loc->path); + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", loc->path); goto out; } parent->path = gf_strdup(dirname(str)); if (parent->path == NULL) { - gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'", - dirname(str)); + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", dirname(str)); goto out; } } - if ((ec_loc_setup_path(xl, parent) != 0) || - (ec_loc_setup_inode(xl, parent) != 0) || - (ec_loc_setup_parent(xl, parent) != 0)) { + ret = ec_loc_setup_path(xl, parent); + if (ret == 0) { + ret = ec_loc_setup_inode(xl, table, parent); + } + if (ret == 0) { + ret = ec_loc_setup_parent(xl, table, parent); + } + if (ret != 0) { goto out; } if ((parent->inode == NULL) && (parent->path == NULL) && - uuid_is_null(parent->gfid)) { - gf_log(xl->name, GF_LOG_ERROR, "Parent inode missing for loc_t"); + gf_uuid_is_null(parent->gfid)) { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_LOC_PARENT_INODE_MISSING, + "Parent inode missing for loc_t"); + + ret = -EINVAL; goto out; } @@ -471,25 +581,32 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent) out: GF_FREE(str); - if (ret != 0) - { + if (ret != 0) { loc_wipe(parent); } return ret; } -int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, - struct iatt *iatt) +int32_t +ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt) { - int32_t ret = -1; + inode_table_t *table = NULL; + int32_t ret = -EINVAL; - if ((inode != NULL) && (loc->inode != inode)) { - if (loc->inode != NULL) { - inode_unref(loc->inode); + if (inode != NULL) { + table = inode->table; + if (loc->inode != inode) { + if (loc->inode != NULL) { + inode_unref(loc->inode); + } + loc->inode = inode_ref(inode); + gf_uuid_copy(loc->gfid, inode->gfid); } - loc->inode = inode_ref(inode); - uuid_copy(loc->gfid, inode->gfid); + } else if (loc->inode != NULL) { + table = loc->inode->table; + } else if (loc->parent != NULL) { + table = loc->parent->table; } if (iatt != NULL) { @@ -498,22 +615,26 @@ int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, } } - if ((ec_loc_setup_path(xl, loc) != 0) || - (ec_loc_setup_inode(xl, loc) != 0) || - (ec_loc_setup_parent(xl, loc) != 0)) { + ret = ec_loc_setup_path(xl, loc); + if (ret == 0) { + ret = ec_loc_setup_inode(xl, table, loc); + } + if (ret == 0) { + ret = ec_loc_setup_parent(xl, table, loc); + } + if (ret != 0) { goto out; } - ret = 0; - out: return ret; } -int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd) +int32_t +ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd) { - ec_fd_t * ctx; - int32_t ret = -1; + ec_fd_t *ctx; + int32_t ret = -ENOMEM; memset(loc, 0, sizeof(*loc)); @@ -524,12 +645,11 @@ int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd) } } - if (ec_loc_update(xl, loc, fd->inode, NULL) != 0) { + ret = ec_loc_update(xl, loc, fd->inode, NULL); + if (ret != 0) { goto out; } - ret = 0; - out: if (ret != 0) { loc_wipe(loc); @@ -538,9 +658,10 @@ out: return ret; } -int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src) +int32_t +ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src) { - int32_t ret = -1; + int32_t ret = -ENOMEM; memset(dst, 0, sizeof(*dst)); @@ -548,12 +669,11 @@ int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src) goto out; } - if (ec_loc_update(xl, dst, NULL, NULL) != 0) { + ret = ec_loc_update(xl, dst, NULL, NULL); + if (ret != 0) { goto out; } - ret = 0; - out: if (ret != 0) { loc_wipe(dst); @@ -562,50 +682,62 @@ out: return ret; } -void ec_owner_set(call_frame_t * frame, void * owner) +void +ec_owner_set(call_frame_t *frame, void *owner) { set_lk_owner_from_ptr(&frame->root->lk_owner, owner); } -void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner) +void +ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner) +{ + lk_owner_copy(&frame->root->lk_owner, owner); +} + +static void +ec_stripe_cache_init(ec_t *ec, ec_inode_t *ctx) { - frame->root->lk_owner.len = owner->len; - memcpy(frame->root->lk_owner.data, owner->data, owner->len); + ec_stripe_list_t *stripe_cache = NULL; + + stripe_cache = &(ctx->stripe_cache); + if (stripe_cache->max == 0) { + stripe_cache->max = ec->stripe_cache; + } } -ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl) +ec_inode_t * +__ec_inode_get(inode_t *inode, xlator_t *xl) { - ec_inode_t * ctx = NULL; + ec_inode_t *ctx = NULL; uint64_t value = 0; - if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) - { + if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) { ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_inode_t); - if (ctx != NULL) - { + if (ctx != NULL) { memset(ctx, 0, sizeof(*ctx)); INIT_LIST_HEAD(&ctx->heal); - + INIT_LIST_HEAD(&ctx->stripe_cache.lru); + ctx->heal_count = 0; value = (uint64_t)(uintptr_t)ctx; - if (__inode_ctx_set(inode, xl, &value) != 0) - { + if (__inode_ctx_set(inode, xl, &value) != 0) { GF_FREE(ctx); return NULL; } } - } - else - { + } else { ctx = (ec_inode_t *)(uintptr_t)value; } + if (ctx) + ec_stripe_cache_init(xl->private, ctx); return ctx; } -ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl) +ec_inode_t * +ec_inode_get(inode_t *inode, xlator_t *xl) { - ec_inode_t * ctx = NULL; + ec_inode_t *ctx = NULL; LOCK(&inode->lock); @@ -616,34 +748,47 @@ ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl) return ctx; } -ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl) +ec_fd_t * +__ec_fd_get(fd_t *fd, xlator_t *xl) { - ec_fd_t * ctx = NULL; + int i = 0; + ec_fd_t *ctx = NULL; + ec_inode_t *ictx = NULL; uint64_t value = 0; + ec_t *ec = xl->private; - if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) - { - ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_fd_t); - if (ctx != NULL) - { + if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) { + ctx = GF_MALLOC(sizeof(*ctx) + (sizeof(ec_fd_status_t) * ec->nodes), + ec_mt_ec_fd_t); + if (ctx != NULL) { memset(ctx, 0, sizeof(*ctx)); + for (i = 0; i < ec->nodes; i++) { + if (fd_is_anonymous(fd)) { + ctx->fd_status[i] = EC_FD_OPENED; + } else { + ctx->fd_status[i] = EC_FD_NOT_OPENED; + } + } + value = (uint64_t)(uintptr_t)ctx; - if (__fd_ctx_set(fd, xl, value) != 0) - { + if (__fd_ctx_set(fd, xl, value) != 0) { GF_FREE(ctx); - return NULL; } + /* Only refering bad-version so no need for lock + * */ + ictx = __ec_inode_get(fd->inode, xl); + if (ictx) { + ctx->bad_version = ictx->bad_version; + } } - } - else - { + } else { ctx = (ec_fd_t *)(uintptr_t)value; } /* Treat anonymous fd specially */ - if (fd->anonymous) { + if (fd->anonymous && ctx) { /* Mark the fd open for all subvolumes. */ ctx->open = -1; /* Try to populate ctx->loc with fd->inode information. */ @@ -653,9 +798,10 @@ ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl) return ctx; } -ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl) +ec_fd_t * +ec_fd_get(fd_t *fd, xlator_t *xl) { - ec_fd_t * ctx = NULL; + ec_fd_t *ctx = NULL; LOCK(&fd->lock); @@ -666,48 +812,56 @@ ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl) return ctx; } -uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale) +gf_boolean_t +ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data) { - off_t head, tmp; - - tmp = *offset; - head = tmp % ec->stripe_size; - tmp -= head; - if (scale) - { - tmp /= ec->fragments; - } - - *offset = tmp; + if (key && (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0)) + return _gf_true; - return head; + return _gf_false; } -uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale) +void +ec_filter_internal_xattrs(dict_t *xattr) { - size += ec->stripe_size - 1; - size -= size % ec->stripe_size; - if (scale) - { - size /= ec->fragments; - } - - return size; + dict_foreach_match(xattr, ec_is_internal_xattr, NULL, + dict_remove_foreach_fn, NULL); } gf_boolean_t -ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data) +ec_is_data_fop(glusterfs_fop_t fop) +{ + switch (fop) { + case GF_FOP_WRITE: + case GF_FOP_TRUNCATE: + case GF_FOP_FTRUNCATE: + case GF_FOP_FALLOCATE: + case GF_FOP_DISCARD: + case GF_FOP_ZEROFILL: + return _gf_true; + default: + return _gf_false; + } + return _gf_false; +} +/* +gf_boolean_t +ec_is_metadata_fop (int32_t lock_kind, glusterfs_fop_t fop) { - if (key && - (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0)) - return _gf_true; + if (lock_kind == EC_LOCK_ENTRY) { + return _gf_false; + } + switch (fop) { + case GF_FOP_SETATTR: + case GF_FOP_FSETATTR: + case GF_FOP_SETXATTR: + case GF_FOP_FSETXATTR: + case GF_FOP_REMOVEXATTR: + case GF_FOP_FREMOVEXATTR: + return _gf_true; + default: + return _gf_false; + } return _gf_false; -} - -void -ec_filter_internal_xattrs (dict_t *xattr) -{ - dict_foreach_match (xattr, ec_is_internal_xattr, NULL, - dict_remove_foreach_fn, NULL); -} +}*/ diff --git a/xlators/cluster/ec/src/ec-helpers.h b/xlators/cluster/ec/src/ec-helpers.h index 5f5d9382532..015db514e05 100644 --- a/xlators/cluster/ec/src/ec-helpers.h +++ b/xlators/cluster/ec/src/ec-helpers.h @@ -11,50 +11,190 @@ #ifndef __EC_HELPERS_H__ #define __EC_HELPERS_H__ -#include "ec-data.h" - -const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits); -const char * ec_fop_name(int32_t id); -void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...); -uint64_t ec_itransform(ec_t * ec, int32_t idx, uint64_t offset); -uint64_t ec_deitransform(ec_t * ec, int32_t * idx, uint64_t offset); -int32_t ec_bits_count(uint64_t n); -int32_t ec_bits_index(uint64_t n); -int32_t ec_bits_consume(uint64_t * n); -size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count, - off_t offset, size_t size); - -int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value); -int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value); -int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config); -int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config); - -int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent); -int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, - struct iatt *iatt); - -int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd); -int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src); - -void ec_owner_set(call_frame_t * frame, void * owner); -void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner); - -ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl); -ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl); -ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl); -ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl); - -uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale); -uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale); - -static inline int32_t ec_is_power_of_2(uint32_t value) +#include "ec-types.h" + +#define EC_ERR(_x) ((void *)-(intptr_t)(_x)) +#define EC_IS_ERR(_x) (((uintptr_t)(_x) & ~0xfffULL) == ~0xfffULL) +#define EC_GET_ERR(_x) ((int32_t)(intptr_t)(_x)) + +#define EC_ALIGN_CHECK(_ptr, _align) ((((uintptr_t)(_ptr)) & ((_align)-1)) == 0) + +const char * +ec_bin(char *str, size_t size, uint64_t value, int32_t digits); +const char * +ec_fop_name(int32_t id); +void +ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...); +int32_t +ec_bits_consume(uint64_t *n); +size_t +ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset, + size_t size); +int32_t +ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr); +int32_t +ec_dict_set_array(dict_t *dict, char *key, uint64_t *value, int32_t size); +int32_t +ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size); + +int32_t +ec_dict_del_array(dict_t *dict, char *key, uint64_t *value, int32_t size); +int32_t +ec_dict_set_number(dict_t *dict, char *key, uint64_t value); +int32_t +ec_dict_del_number(dict_t *dict, char *key, uint64_t *value); +int32_t +ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config); +int32_t +ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config); + +int32_t +ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent); +int32_t +ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt); + +int32_t +ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd); +int32_t +ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src); + +void +ec_owner_set(call_frame_t *frame, void *owner); +void +ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner); + +ec_inode_t * +__ec_inode_get(inode_t *inode, xlator_t *xl); +ec_inode_t * +ec_inode_get(inode_t *inode, xlator_t *xl); +ec_fd_t * +__ec_fd_get(fd_t *fd, xlator_t *xl); +ec_fd_t * +ec_fd_get(fd_t *fd, xlator_t *xl); + +static inline uint32_t +ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale) +{ + uint64_t head, tmp; + + tmp = *value; + head = tmp % ec->stripe_size; + tmp -= head; + + if (scale) { + tmp /= ec->fragments; + } + + *value = tmp; + + return (uint32_t)head; +} + +/* This function can cause an overflow if the passed value is too near to the + * uint64_t limit. If this happens, it returns the tail in negative form and + * the value is set to UINT64_MAX. */ +static inline int32_t +ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale) +{ + uint64_t tmp; + int32_t tail; + + tmp = *value; + /* We first adjust the value down. This never causes overflow. */ + tail = ec_adjust_size_down(ec, &tmp, scale); + + /* If the value was already aligned, tail will be 0 and nothing else + * needs to be done. */ + if (tail != 0) { + /* Otherwise, we need to compute the real tail and adjust the + * returned value to the next stripe. */ + tail = ec->stripe_size - tail; + if (scale) { + tmp += ec->fragment_size; + } else { + tmp += ec->stripe_size; + /* If no scaling is requested there's a possibility of + * overflow. */ + if (tmp < ec->stripe_size) { + tmp = UINT64_MAX; + tail = -tail; + } + } + } + + *value = tmp; + + return tail; +} + +/* This function is equivalent to ec_adjust_size_down() but with a potentially + * different parameter size (off_t vs uint64_t). */ +static inline uint32_t +ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale) +{ + off_t head, tmp; + + tmp = *value; + head = tmp % ec->stripe_size; + tmp -= head; + + if (scale) { + tmp /= ec->fragments; + } + + *value = tmp; + + return (uint32_t)head; +} + +/* This function is equivalent to ec_adjust_size_up() but with a potentially + * different parameter size (off_t vs uint64_t). */ +static inline int32_t +ec_adjust_offset_up(ec_t *ec, off_t *value, gf_boolean_t scale) +{ + uint64_t tail, tmp; + + /* An offset is a signed type that can only have positive values, so + * we take advantage of this to avoid overflows. We simply convert it + * to an unsigned integer and operate normally. This won't cause an + * overflow. Overflow is only checked when converting back to an + * off_t. */ + tmp = *value; + tail = ec->stripe_size; + tail -= (tmp + tail - 1) % tail + 1; + tmp += tail; + if (scale) { + /* If we are scaling, we'll never get an overflow. */ + tmp /= ec->fragments; + } else { + /* Check if there has been an overflow. */ + if ((off_t)tmp < 0) { + tmp = GF_OFF_MAX; + tail = -tail; + } + } + + *value = (off_t)tmp; + + return (int32_t)tail; +} + +static inline int32_t +ec_is_power_of_2(uint32_t value) { return (value != 0) && ((value & (value - 1)) == 0); } gf_boolean_t -ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data); +ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data); void -ec_filter_internal_xattrs (dict_t *xattr); +ec_filter_internal_xattrs(dict_t *xattr); + +gf_boolean_t +ec_is_data_fop(glusterfs_fop_t fop); + +int32_t +ec_launch_replace_heal(ec_t *ec); + #endif /* __EC_HELPERS_H__ */ diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c index d126d48eb12..dad5f4d7018 100644 --- a/xlators/cluster/ec/src/ec-inode-read.c +++ b/xlators/cluster/ec/src/ec-inode-read.c @@ -8,9 +8,8 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" - +#include "ec.h" +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" @@ -19,10 +18,12 @@ /* FOP: access */ -int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -32,28 +33,27 @@ int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno)) - { - if (fop->cbks.access != NULL) - { - fop->cbks.access(fop->req_frame, fop, this, op_ret, op_errno, - xdata); - } + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ACCESS, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_access(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -62,75 +62,112 @@ void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->int32, fop->xdata); } -int32_t ec_manager_access(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_access(ec_fop_data_t *fop, int32_t state) { - switch (state) - { + ec_cbk_data_t *cbk = NULL; + + switch (state) { case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + case EC_STATE_DISPATCH: ec_dispatch_one(fop); + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, NULL)) { + return EC_STATE_DISPATCH; + } + return EC_STATE_REPORT; + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->cbks.access != NULL) { + if (cbk) { + fop->cbks.access(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + } + return EC_STATE_LOCK_REUSE; + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: - if (fop->cbks.access != NULL) - { + if (fop->cbks.access != NULL) { fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); - case EC_STATE_REPORT: return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_access_cbk_t func, void * data, - loc_t * loc, int32_t mask, dict_t * xdata) +void +ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc, + int32_t mask, dict_t *xdata) { - ec_cbk_t callback = { .access = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.access = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(ACCESS) %p", frame); + gf_msg_trace("ec", 0, "EC(ACCESS) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, 0, target, minimum, - ec_wind_access, ec_manager_access, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_access, + ec_manager_access, callback, data); + if (fop == NULL) { goto out; } fop->int32 = mask; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -139,25 +176,22 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: getxattr */ -int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_getxattr(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_dict_compare(dst->dict, src->dict)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching dictionary in " - "answers of 'GF_FOP_GETXATTR'"); + if (!ec_dict_compare(dst->dict, src->dict)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_DICT_MISMATCH, + "Mismatching dictionary in " + "answers of 'GF_FOP_GETXATTR'"); return 0; } @@ -165,12 +199,12 @@ int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * dict, - dict_t * xdata) +int32_t +ec_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -180,34 +214,30 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_GETXATTR, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (dict != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (dict != NULL) { cbk->dict = dict_ref(dict); - if (cbk->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -217,15 +247,15 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_getxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -234,23 +264,44 @@ void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->str[0], fop->xdata); } -int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state) +void +ec_handle_special_xattrs(ec_fop_data_t *fop) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk = NULL; + /* Stime may not be available on all the bricks, so even if some of the + * subvols succeed the operation, treat it as answer.*/ + if (fop->str[0] && fnmatch(GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) { + if (!fop->answer || (fop->answer->op_ret < 0)) { + list_for_each_entry(cbk, &fop->cbk_list, list) + { + if (cbk->op_ret >= 0) { + fop->answer = cbk; + break; + } + } + } + } +} + +int32_t +ec_manager_getxattr(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: /* clear-locks commands must be done without any locks acquired to avoid interferences. */ if ((fop->str[0] == NULL) || (strncmp(fop->str[0], GF_XATTR_CLRLK_CMD, - strlen(GF_XATTR_CLRLK_CMD)) != 0)) { + SLEN(GF_XATTR_CLRLK_CMD)) != 0)) { if (fop->fd == NULL) { - ec_lock_prepare_inode(fop, &fop->loc[0], 0); + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); } else { - ec_lock_prepare_fd(fop, fop->fd, 0); + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); } ec_lock(fop); } @@ -258,41 +309,35 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: - ec_dispatch_all(fop); + if (fop->minimum == EC_MINIMUM_ALL) { + ec_dispatch_all(fop); + } else { + ec_dispatch_one(fop); + } return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA) || - ((cbk->op_ret >= 0) && !ec_dict_combine(cbk, - EC_COMBINE_DICT))) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } + ec_handle_special_xattrs(fop); + if (fop->minimum == EC_MINIMUM_ALL) { + cbk = ec_fop_prepare_answer(fop, _gf_true); + } else { + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { + } + if (cbk != NULL) { + int32_t err; + + err = ec_dict_combine(cbk, EC_COMBINE_DICT); + if (!ec_cbk_set_error(cbk, -err, _gf_true)) { if (cbk->xdata != NULL) - ec_filter_internal_xattrs (cbk->xdata); + ec_filter_internal_xattrs(cbk->xdata); if (cbk->dict != NULL) - ec_filter_internal_xattrs (cbk->dict); + ec_filter_internal_xattrs(cbk->dict); } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -301,8 +346,7 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.getxattr != NULL) - { + if (fop->cbks.getxattr != NULL) { fop->cbks.getxattr(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->dict, cbk->xdata); } @@ -316,10 +360,9 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.getxattr != NULL) - { - fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL, NULL); + if (fop->cbks.getxattr != NULL) { + fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); } return EC_STATE_LOCK_REUSE; @@ -337,24 +380,44 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, - int32_t op_ret, int32_t op_errno, uintptr_t mask, - uintptr_t good, uintptr_t bad, dict_t *xdata) +int32_t +ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, + int32_t op_ret, int32_t op_errno, uintptr_t mask, + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) { - ec_fop_data_t *fop = cookie; - fop_getxattr_cbk_t func = fop->data; + fop_getxattr_cbk_t func = cookie; ec_t *ec = xl->private; dict_t *dict = NULL; char *str; char bin1[65], bin2[65]; + /* We try to return the 'pending' information in xdata, but if this cannot + * be set, we will ignore it silently. We prefer to report the success or + * failure of the heal itself. */ + if (xdata == NULL) { + xdata = dict_new(); + } else { + dict_ref(xdata); + } + if (xdata != NULL) { + if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) { + /* dict_set_uint32() is marked as 'warn_unused_result' and gcc + * enforces to check the result in this case. However we don't + * really care if it succeeded or not. We'll just do the same. + * + * This empty 'if' avoids the warning, and it will be removed by + * the optimizer. */ + } + } + if (op_ret >= 0) { dict = dict_new(); if (dict == NULL) { @@ -374,7 +437,7 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, goto out; } - if (dict_set_str(dict, EC_XATTR_HEAL, str) != 0) { + if (dict_set_dynstr(dict, EC_XATTR_HEAL, str) != 0) { GF_FREE(str); dict_unref(dict); dict = NULL; @@ -388,25 +451,28 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, } out: - func(frame, NULL, xl, op_ret, op_errno, dict, NULL); + func(frame, NULL, xl, op_ret, op_errno, dict, xdata); if (dict != NULL) { dict_unref(dict); } + if (xdata != NULL) { + dict_unref(xdata); + } return 0; } void -ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_getxattr_cbk_t func, void *data, - loc_t *loc, const char *name, dict_t *xdata) +ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc, + const char *name, dict_t *xdata) { - ec_cbk_t callback = { .getxattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.getxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(GETXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(GETXATTR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); @@ -414,47 +480,49 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, /* Special handling of an explicit self-heal request */ if ((name != NULL) && (strcmp(name, EC_XATTR_HEAL) == 0)) { - ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, - func, loc, 0, NULL); + ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, func, + loc, 0, NULL); return; } - fop = ec_fop_data_allocate(frame, this, GF_FOP_GETXATTR, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_getxattr, ec_manager_getxattr, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate( + frame, this, GF_FOP_GETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_getxattr, ec_manager_getxattr, callback, data); + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (name != NULL) - { - fop->str[0] = gf_strdup(name); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (name != NULL) { + /* In case of list-node-uuids xattr, set flag to indicate + * the same and use node-uuid xattr for winding fop */ + if (XATTR_IS_NODE_UUID_LIST(name)) { + fop->int32 = 1; + fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY); + } else { + fop->str[0] = gf_strdup(name); + } + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -464,20 +532,20 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL, NULL); + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: fgetxattr */ -int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * dict, - dict_t * xdata) +int32_t +ec_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -487,34 +555,30 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FGETXATTR, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (dict != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (dict != NULL) { cbk->dict = dict_ref(dict); - if (cbk->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -524,8 +588,7 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } @@ -533,7 +596,7 @@ out: } void -ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx) +ec_wind_fgetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -543,59 +606,54 @@ ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx) } void -ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_fgetxattr_cbk_t func, void *data, - fd_t *fd, const char *name, dict_t *xdata) +ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd, + const char *name, dict_t *xdata) { - ec_cbk_t callback = { .fgetxattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fgetxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FGETXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(FGETXATTR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FGETXATTR, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fgetxattr, ec_manager_getxattr, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate( + frame, this, GF_FOP_FGETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_fgetxattr, ec_manager_getxattr, callback, data); + if (fop == NULL) { goto out; } fop->use_fd = 1; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (name != NULL) - { + if (name != NULL) { fop->str[0] = gf_strdup(name); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -605,21 +663,21 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL, NULL); + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: open */ -int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_open(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (dst->fd != src->fd) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching fd in answers " - "of 'GF_FOP_OPEN': %p <-> %p", + if (dst->fd != src->fd) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH, + "Mismatching fd in answers " + "of 'GF_FOP_OPEN': %p <-> %p", dst->fd, src->fd); return 0; @@ -628,12 +686,12 @@ int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, fd_t * fd, - dict_t * xdata) +int32_t +ec_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, fd_t *fd, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -643,52 +701,51 @@ int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPEN, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (fd != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (fd != NULL) { cbk->fd = fd_ref(fd); - if (cbk->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (cbk->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, + EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } ec_combine(cbk, ec_combine_open); + + ec_update_fd_status(fd, this, idx, op_ret); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_open(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -697,10 +754,10 @@ void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->int32, fop->fd, fop->xdata); } -int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno, - struct iatt *prebuf, struct iatt *postbuf, - dict_t *xdata) +int32_t +ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) { ec_fop_data_t *fop = cookie; int32_t error = 0; @@ -717,25 +774,35 @@ int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, return 0; } -int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_open(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; - ec_fd_t * ctx; + ec_cbk_data_t *cbk; + ec_fd_t *ctx; + int32_t err; - switch (state) - { + switch (state) { case EC_STATE_INIT: LOCK(&fop->fd->lock); ctx = __ec_fd_get(fop->fd, fop->xl); - if ((ctx == NULL) || - (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) { + if (ctx == NULL) { UNLOCK(&fop->fd->lock); - fop->error = EIO; + fop->error = ENOMEM; return EC_STATE_REPORT; } + if (!ctx->loc.inode) { + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { + UNLOCK(&fop->fd->lock); + + fop->error = -err; + + return EC_STATE_REPORT; + } + } ctx->flags = fop->int32; @@ -750,7 +817,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) fop->uint32 = fop->int32 & O_TRUNC; fop->int32 &= ~(O_APPEND | O_TRUNC); - /* Fall through */ + /* Fall through */ case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -758,51 +825,33 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + int32_t err; + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->fd->inode, + NULL); + if (!ec_cbk_set_error(cbk, -err, _gf_true)) { + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; } - } - if (cbk->op_ret >= 0) { - if (ec_loc_update(fop->xl, &fop->loc[0], cbk->fd->inode, - NULL) != 0) { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } else { - LOCK(&fop->fd->lock); - - ctx = __ec_fd_get(fop->fd, fop->xl); - if (ctx != NULL) { - ctx->open |= cbk->mask; - } - - UNLOCK(&fop->fd->lock); - - /* If O_TRUNC was specified, call ftruncate to - effectively trunc the file with appropriate locks - acquired. We don't use ctx->flags because self-heal - can use the same fd with different flags. */ - if (fop->uint32 != 0) { - ec_sleep(fop); - ec_ftruncate(fop->req_frame, fop->xl, cbk->mask, - fop->minimum, ec_open_truncate_cbk, - fop, cbk->fd, 0, NULL); - } + + UNLOCK(&fop->fd->lock); + + /* If O_TRUNC was specified, call ftruncate to + effectively trunc the file with appropriate locks + acquired. We don't use ctx->flags because self-heal + can use the same fd with different flags. */ + if (fop->uint32 != 0) { + ec_sleep(fop); + ec_ftruncate(fop->req_frame, fop->xl, cbk->mask, + fop->minimum, ec_open_truncate_cbk, fop, + cbk->fd, 0, NULL); } } - if (cbk->op_ret < 0) { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); } return EC_STATE_REPORT; @@ -812,8 +861,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.open != NULL) - { + if (fop->cbks.open != NULL) { fop->cbks.open(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->fd, cbk->xdata); } @@ -826,8 +874,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.open != NULL) - { + if (fop->cbks.open != NULL) { fop->cbks.open(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } @@ -835,64 +882,61 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_open_cbk_t func, void * data, loc_t * loc, - int32_t flags, fd_t * fd, dict_t * xdata) +void +ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc, + int32_t flags, fd_t *fd, dict_t *xdata) { - ec_cbk_t callback = { .open = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.open = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(OPEN) %p", frame); + gf_msg_trace("ec", 0, "EC(OPEN) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_open, ec_manager_open, + fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_open, ec_manager_open, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } fop->int32 = flags; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -901,25 +945,22 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: readlink */ -int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_readlink(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 1)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_READLINK'"); + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_READLINK'"); return 0; } @@ -927,11 +968,13 @@ int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_readlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, const char * path, - struct iatt * buf, dict_t * xdata) +int32_t +ec_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, const char *path, + struct iatt *buf, dict_t *xdata) { - ec_fop_data_t * fop = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -941,33 +984,34 @@ int32_t ec_readlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - if (op_ret > 0) - { - ec_iatt_rebuild(fop->xl->private, buf, 1, 1); - } + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); - if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno)) - { - if (fop->cbks.readlink != NULL) - { - fop->cbks.readlink(fop->req_frame, fop, this, op_ret, op_errno, - path, buf, xdata); + if (cbk->op_ret >= 0) { + cbk->iatt[0] = *buf; + cbk->str = gf_strdup(path); + if (!cbk->str) { + ec_cbk_set_error(cbk, ENOMEM, _gf_true); + } } + ec_combine(cbk, NULL); } out: if (fop != NULL) - { ec_complete(fop); - } return 0; } -void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_readlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -976,75 +1020,114 @@ void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->size, fop->xdata); } -int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_readlink(ec_fop_data_t *fop, int32_t state) { - switch (state) - { + ec_cbk_data_t *cbk = NULL; + + switch (state) { case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + return EC_STATE_DISPATCH; + case EC_STATE_DISPATCH: ec_dispatch_one(fop); + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + + if ((cbk != NULL) && (cbk->op_ret >= 0)) { + ec_iatt_rebuild(fop->xl->private, &cbk->iatt[0], 1, 1); + } + return EC_STATE_REPORT; + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->cbks.readlink != NULL) { + fop->cbks.readlink(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->str, &cbk->iatt[0], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: - if (fop->cbks.readlink != NULL) - { - fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL, NULL, NULL); + if (fop->cbks.readlink != NULL) { + fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); } + return EC_STATE_LOCK_REUSE; - case EC_STATE_REPORT: - return EC_STATE_END; + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readlink_cbk_t func, void * data, - loc_t * loc, size_t size, dict_t * xdata) +void +ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc, + size_t size, dict_t *xdata) { - ec_cbk_t callback = { .readlink = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.readlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(READLINK) %p", frame); + gf_msg_trace("ec", 0, "EC(READLINK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_READLINK, 0, target, - minimum, ec_wind_readlink, ec_manager_readlink, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate( + frame, this, GF_FOP_READLINK, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_readlink, ec_manager_readlink, callback, data); + if (fop == NULL) { goto out; } fop->size = size; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1053,95 +1136,82 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: readv */ -int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk) +int32_t +ec_readv_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk) { - ec_cbk_data_t * ans = NULL; - struct iobref * iobref = NULL; - struct iobuf * iobuf = NULL; - uint8_t * buff = NULL, * ptr; - size_t fsize = 0, size = 0, max = 0; - int32_t i = 0; - - if (cbk->op_ret < 0) - { + struct iovec vector[1]; + ec_cbk_data_t *ans = NULL; + struct iobref *iobref = NULL; + void *ptr; + uint64_t fsize = 0, size = 0, max = 0; + int32_t pos, err = -ENOMEM; + + if (cbk->op_ret < 0) { + err = -cbk->op_errno; + goto out; } - cbk->iatt[0].ia_size = fop->pre_size; + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &cbk->iatt[0].ia_size)); - if (cbk->op_ret > 0) - { - struct iovec vector[1]; - uint8_t * blocks[cbk->count]; + if (cbk->op_ret > 0) { + void *blocks[cbk->count]; uint32_t values[cbk->count]; fsize = cbk->op_ret; size = fsize * ec->fragments; - buff = GF_MALLOC(size, gf_common_mt_char); - if (buff == NULL) - { - goto out; - } - ptr = buff; - for (i = 0, ans = cbk; ans != NULL; i++, ans = ans->next) - { - values[i] = ans->idx; - blocks[i] = ptr; - ptr += ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize); + for (ans = cbk; ans != NULL; ans = ans->next) { + pos = gf_bits_count(cbk->mask & ((1 << ans->idx) - 1)); + values[pos] = ans->idx + 1; + blocks[pos] = ans->vector[0].iov_base; + if ((ans->int32 != 1) || + !EC_ALIGN_CHECK(blocks[pos], EC_METHOD_WORD_SIZE)) { + if (iobref == NULL) { + err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr); + if (err != 0) { + goto out; + } + } + ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize); + blocks[pos] = ptr; + ptr += fsize; + } } - iobref = iobref_new(); - if (iobref == NULL) - { + err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr); + if (err != 0) { goto out; } - iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, size); - if (iobuf == NULL) - { - goto out; - } - if (iobref_add(iobref, iobuf) != 0) - { + + err = ec_method_decode(&ec->matrix, fsize, cbk->mask, values, blocks, + ptr); + if (err != 0) { goto out; } - vector[0].iov_base = iobuf->ptr; - vector[0].iov_len = ec_method_decode(fsize, ec->fragments, values, - blocks, iobuf->ptr); - - iobuf_unref(iobuf); - - GF_FREE(buff); - buff = NULL; - - vector[0].iov_base += fop->head; - vector[0].iov_len -= fop->head; + vector[0].iov_base = ptr + fop->head; + vector[0].iov_len = size - fop->head; max = fop->offset * ec->fragments + size; - if (max > cbk->iatt[0].ia_size) - { + if (max > cbk->iatt[0].ia_size) { max = cbk->iatt[0].ia_size; } max -= fop->offset * ec->fragments + fop->head; - if (max > fop->user_size) - { + if (max > fop->user_size) { max = fop->user_size; } size -= fop->head; - if (size > max) - { + if (size > max) { vector[0].iov_len -= size - max; size = max; } @@ -1154,50 +1224,36 @@ int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk) GF_FREE(cbk->vector); cbk->vector = iov_dup(vector, 1); - if (cbk->vector == NULL) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - - return 0; + if (cbk->vector == NULL) { + return -ENOMEM; } } - return 1; + return 0; out: - if (iobuf != NULL) - { - iobuf_unref(iobuf); - } - if (iobref != NULL) - { + if (iobref != NULL) { iobref_unref(iobref); } - GF_FREE(buff); - return 0; + return err; } -int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_readv(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (src->dirty) { - return 0; - } - - if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching vector in " - "answers of 'GF_FOP_READ'"); + if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_VECTOR_MISMATCH, + "Mismatching vector in " + "answers of 'GF_FOP_READ'"); return 0; } - if (!ec_iatt_combine(dst->iatt, src->iatt, 1)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_READ'"); + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_READ'"); return 0; } @@ -1205,14 +1261,14 @@ int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iovec * vector, - int32_t count, struct iatt * stbuf, - struct iobref * iobref, dict_t * xdata) +int32_t +ec_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iovec *vector, int32_t count, + struct iatt *stbuf, struct iobref *iobref, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - ec_t * ec = this->private; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + ec_t *ec = this->private; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1222,76 +1278,68 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_READ, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { + if (cbk != NULL) { + if (op_ret >= 0) { cbk->int32 = count; - if (count > 0) - { + if (count > 0) { cbk->vector = iov_dup(vector, count); - if (cbk->vector == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a " - "vector list."); + if (cbk->vector == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a " + "vector list."); goto out; } cbk->int32 = count; } - if (stbuf != NULL) - { + if (stbuf != NULL) { cbk->iatt[0] = *stbuf; } - if (iobref != NULL) - { + if (iobref != NULL) { cbk->buffers = iobref_ref(iobref); - if (cbk->buffers == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "buffer."); + if (cbk->buffers == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL, + "Failed to reference a " + "buffer."); goto out; } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; + if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) { + ec_cbk_set_error(cbk, EIO, _gf_true); } ec_combine(cbk, ec_combine_readv); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_readv(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1300,67 +1348,49 @@ void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->size, fop->offset, fop->uint32, fop->xdata); } -int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_readv(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + ec_t *ec = fop->xl->private; - switch (state) - { + switch (state) { case EC_STATE_INIT: fop->user_size = fop->size; - fop->head = ec_adjust_offset(fop->xl->private, &fop->offset, 1); - fop->size = ec_adjust_size(fop->xl->private, fop->size + fop->head, - 1); + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + fop->size += fop->head; + ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true); - /* Fall through */ + /* Fall through */ case EC_STATE_LOCK: - ec_lock_prepare_fd(fop, fop->fd, 0); + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset, + fop->size); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: + if (ec->read_mask) { + fop->mask &= ec->read_mask; + } ec_dispatch_min(fop); return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, - cbk->count); - - if (!ec_readv_rebuild(fop->xl->private, fop, cbk)) - { - ec_fop_set_error(fop, EIO); - } + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count); + + err = ec_readv_rebuild(fop->xl->private, fop, cbk); + if (err != 0) { + ec_cbk_set_error(cbk, -err, _gf_true); } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -1369,8 +1399,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.readv != NULL) - { + if (fop->cbks.readv != NULL) { fop->cbks.readv(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->vector, cbk->int32, &cbk->iatt[0], cbk->buffers, cbk->xdata); @@ -1380,14 +1409,12 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.readv != NULL) - { + if (fop->cbks.readv != NULL) { fop->cbks.readv(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, 0, NULL, NULL, NULL); } @@ -1407,32 +1434,32 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_readv_cbk_t func, void * data, fd_t * fd, - size_t size, off_t offset, uint32_t flags, dict_t * xdata) +void +ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, uint32_t flags, dict_t *xdata) { - ec_cbk_t callback = { .readv = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.readv = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(READ) %p", frame); + gf_msg_trace("ec", 0, "EC(READ) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_UPDATE_FD, - target, minimum, ec_wind_readv, + fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_readv, ec_manager_readv, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } @@ -1442,24 +1469,22 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->offset = offset; fop->uint32 = flags; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1468,25 +1493,227 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, 0, NULL, NULL, NULL); + } +} + +/* FOP: seek */ + +int32_t +ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, off_t offset, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + ec_t *ec = this->private; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SEEK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + cbk->offset = offset; + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + } + + if ((op_ret > 0) && ((cbk->offset % ec->fragment_size) != 0)) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + + ec_combine(cbk, NULL); } - else - { - func(frame, NULL, this, -1, EIO, NULL, 0, NULL, NULL, NULL); + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_seek_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->seek, fop->fd, + fop->offset, fop->seek, fop->xdata); +} + +int32_t +ec_manager_seek(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + uint64_t size; + + switch (state) { + case EC_STATE_INIT: + fop->user_size = fop->offset; + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset, + EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT( + ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, &size)); + + if (fop->user_size >= size) { + ec_fop_set_error(fop, ENXIO); + + return EC_STATE_REPORT; + } + + ec_dispatch_one(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + if ((cbk != NULL) && (cbk->op_ret >= 0)) { + ec_t *ec = fop->xl->private; + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &size)); + + cbk->offset *= ec->fragments; + if (cbk->offset < fop->user_size) { + cbk->offset = fop->user_size; + } + if (cbk->offset > size) { + cbk->offset = size; + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.seek != NULL) { + fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->offset, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.seek != NULL) { + fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0, + NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd, + off_t offset, gf_seek_what_t what, dict_t *xdata) +{ + ec_cbk_t callback = {.seek = func}; + ec_fop_data_t *fop = NULL; + int32_t error = EIO; + + gf_msg_trace("ec", 0, "EC(SEEK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_SEEK, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_seek, ec_manager_seek, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->offset = offset; + fop->seek = what; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, EIO, 0, NULL); } } /* FOP: stat */ -int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_stat(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 1)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_STAT'"); + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_STAT'"); return 0; } @@ -1494,12 +1721,12 @@ int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * buf, - dict_t * xdata) +int32_t +ec_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1509,27 +1736,23 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STAT, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (buf != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { cbk->iatt[0] = *buf; } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1539,15 +1762,15 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_stat(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1556,29 +1779,23 @@ void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->xdata); } -int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_stat(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - if (fop->fd == NULL) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 0); - } - else - { - ec_lock_prepare_fd(fop, fop->fd, 0); + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); } ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -1587,33 +1804,18 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, - cbk->count); + cbk = ec_fop_prepare_answer(fop, _gf_true); - cbk->iatt[0].ia_size = fop->pre_size; + if (cbk != NULL) { + if (cbk->iatt[0].ia_type == IA_IFREG) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -1622,18 +1824,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_STAT) - { - if (fop->cbks.stat != NULL) - { + if (fop->id == GF_FOP_STAT) { + if (fop->cbks.stat != NULL) { fop->cbks.stat(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0], cbk->xdata); } - } - else - { - if (fop->cbks.fstat != NULL) - { + } else { + if (fop->cbks.fstat != NULL) { fop->cbks.fstat(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0], cbk->xdata); } @@ -1643,24 +1840,18 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_STAT) - { - if (fop->cbks.stat != NULL) - { - fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL, NULL); + if (fop->id == GF_FOP_STAT) { + if (fop->cbks.stat != NULL) { + fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); } - } - else - { - if (fop->cbks.fstat != NULL) - { + } else { + if (fop->cbks.fstat != NULL) { fop->cbks.fstat(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL); } @@ -1681,51 +1872,49 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_stat_cbk_t func, void * data, loc_t * loc, - dict_t * xdata) +void +ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) { - ec_cbk_t callback = { .stat = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.stat = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(STAT) %p", frame); + gf_msg_trace("ec", 0, "EC(STAT) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_stat, ec_manager_stat, callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_stat, ec_manager_stat, + callback, data); + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1734,24 +1923,21 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } /* FOP: fstat */ -int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * buf, - dict_t * xdata) +int32_t +ec_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1761,27 +1947,23 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSTAT, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (buf != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { cbk->iatt[0] = *buf; } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1791,15 +1973,15 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fstat(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1808,48 +1990,46 @@ void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->xdata); } -void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fstat_cbk_t func, void * data, fd_t * fd, - dict_t * xdata) +void +ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd, + dict_t *xdata) { - ec_cbk_t callback = { .fstat = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fstat = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FSTAT) %p", frame); + gf_msg_trace("ec", 0, "EC(FSTAT) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fstat, ec_manager_stat, callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_fstat, + ec_manager_stat, callback, data); + if (fop == NULL) { goto out; } fop->use_fd = 1; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1858,12 +2038,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c index ffca89c5f14..9b5fe2a7fdc 100644 --- a/xlators/cluster/ec/src/ec-inode-write.c +++ b/xlators/cluster/ec/src/ec-inode-write.c @@ -8,24 +8,114 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" - +#include "ec-messages.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" #include "ec-method.h" #include "ec-fops.h" +#include "ec-mem-types.h" -/* FOP: removexattr */ +int32_t +ec_update_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_cbk_data_t *cbk = NULL; + ec_fop_data_t *parent = fop->parent; + int i = 0; -int32_t ec_removexattr_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - dict_t * xdata) + ec_trace("UPDATE_WRITEV_CBK", cookie, "ret=%d, errno=%d, parent-fop=%s", + op_ret, op_errno, ec_fop_name(parent->id)); + + if (op_ret < 0) { + ec_fop_set_error(parent, op_errno); + goto out; + } + cbk = ec_cbk_data_allocate(parent->frame, this, parent, parent->id, 0, + op_ret, op_errno); + if (!cbk) { + ec_fop_set_error(parent, ENOMEM); + goto out; + } + + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (prebuf) + cbk->iatt[i++] = *prebuf; + + if (postbuf) + cbk->iatt[i++] = *postbuf; + + LOCK(&parent->lock); + { + parent->good &= fop->good; + + if (gf_bits_count(parent->good) < parent->minimum) { + __ec_fop_set_error(parent, EIO); + } else if (fop->error == 0 && parent->answer == NULL) { + parent->answer = cbk; + } + } + UNLOCK(&parent->lock); +out: + return 0; +} + +static int32_t +ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset, uint64_t size) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; + struct iobref *iobref = NULL; + struct iobuf *iobuf = NULL; + struct iovec vector; + int32_t err = -ENOMEM; + + iobref = iobref_new(); + if (iobref == NULL) { + goto out; + } + iobuf = iobuf_get(fop->xl->ctx->iobuf_pool); + if (iobuf == NULL) { + goto out; + } + err = iobref_add(iobref, iobuf); + if (err != 0) { + goto out; + } + + if (fop->locks[0].lock) + ec_lock_update_good(fop->locks[0].lock, fop); + vector.iov_base = iobuf->ptr; + vector.iov_len = size; + memset(vector.iov_base, 0, vector.iov_len); + + ec_writev(fop->frame, fop->xl, mask, fop->minimum, ec_update_writev_cbk, + NULL, fop->fd, &vector, 1, offset, 0, iobref, NULL); + + err = 0; + +out: + if (iobuf != NULL) { + iobuf_unref(iobuf); + } + if (iobref != NULL) { + iobref_unref(iobref); + } + + return err; +} + +int +ec_inode_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, + int op_ret, int op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int i = 0; + int idx = 0; VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); @@ -33,39 +123,48 @@ int32_t ec_removexattr_cbk(call_frame_t * frame, void * cookie, GF_VALIDATE_OR_GOTO(this->name, this->private, out); fop = frame->local; + idx = (int32_t)(uintptr_t)cookie; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_REMOVEXATTR, idx, - op_ret, op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (!cbk) + goto out; - goto out; - } - } + if (op_ret < 0) + goto out; - ec_combine(cbk, NULL); - } + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (prestat) + cbk->iatt[i++] = *prestat; + + if (poststat) + cbk->iatt[i++] = *poststat; out: - if (fop != NULL) - { - ec_complete(fop); - } + if (cbk) + ec_combine(cbk, ec_combine_write); + if (fop) + ec_complete(fop); return 0; } +/* FOP: removexattr */ + +int32_t +ec_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); +} -void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_removexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -74,21 +173,54 @@ void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->str[0], fop->xdata); } -int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state) +void +ec_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) { - ec_cbk_data_t * cbk; + ec_fop_data_t *fop = cookie; + switch (fop->id) { + case GF_FOP_SETXATTR: + if (fop->cbks.setxattr) { + QUORUM_CBK(fop->cbks.setxattr, fop, frame, cookie, this, op_ret, + op_errno, xdata); + } + break; + case GF_FOP_REMOVEXATTR: + if (fop->cbks.removexattr) { + QUORUM_CBK(fop->cbks.removexattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + case GF_FOP_FSETXATTR: + if (fop->cbks.fsetxattr) { + QUORUM_CBK(fop->cbks.fsetxattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + case GF_FOP_FREMOVEXATTR: + if (fop->cbks.fremovexattr) { + QUORUM_CBK(fop->cbks.fremovexattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + } +} - switch (state) - { +int32_t +ec_manager_xattr(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - if (fop->fd == NULL) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 1); - } - else - { - ec_lock_prepare_fd(fop, fop->fd, 1); + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], + EC_UPDATE_META | EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO, + 0, EC_RANGE_FULL); } ec_lock(fop); @@ -100,26 +232,7 @@ int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); - } + ec_fop_prepare_answer(fop, _gf_false); return EC_STATE_REPORT; @@ -128,24 +241,8 @@ int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_REMOVEXATTR) - { - if (fop->cbks.removexattr != NULL) - { - fop->cbks.removexattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - cbk->xdata); - } - } - else - { - if (fop->cbks.fremovexattr != NULL) - { - fop->cbks.fremovexattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - cbk->xdata); - } - } + ec_xattr_cbk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); return EC_STATE_LOCK_REUSE; @@ -156,22 +253,7 @@ int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_REMOVEXATTR) - { - if (fop->cbks.removexattr != NULL) - { - fop->cbks.removexattr(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL); - } - } - else - { - if (fop->cbks.fremovexattr != NULL) - { - fop->cbks.fremovexattr(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL); - } - } + ec_xattr_cbk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); return EC_STATE_LOCK_REUSE; @@ -188,63 +270,58 @@ int32_t ec_manager_removexattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } void -ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_removexattr_cbk_t func, void *data, - loc_t *loc, const char *name, dict_t *xdata) +ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_removexattr_cbk_t func, void *data, + loc_t *loc, const char *name, dict_t *xdata) { - ec_cbk_t callback = { .removexattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.removexattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(REMOVEXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(REMOVEXATTR) %p", frame); - VALIDATE_OR_GOTO (this, out); - GF_VALIDATE_OR_GOTO (this->name, frame, out); - GF_VALIDATE_OR_GOTO (this->name, this->private, out); + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_removexattr, ec_manager_removexattr, + fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, 0, target, + fop_flags, ec_wind_removexattr, ec_manager_xattr, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (name != NULL) - { + if (name != NULL) { fop->str[0] = gf_strdup(name); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -254,61 +331,24 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target, out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL); + func(frame, NULL, this, -1, error, NULL); } } /* FOP: fremovexattr */ -int32_t ec_fremovexattr_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - dict_t * xdata) +int32_t +ec_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FREMOVEXATTR, idx, - op_ret, op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, NULL); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); } -void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fremovexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -318,59 +358,54 @@ void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) } void -ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_fremovexattr_cbk_t func, void *data, - fd_t *fd, const char *name, dict_t *xdata) +ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data, + fd_t *fd, const char *name, dict_t *xdata) { - ec_cbk_t callback = { .fremovexattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fremovexattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FREMOVEXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(FREMOVEXATTR) %p", frame); - VALIDATE_OR_GOTO (this, out); - GF_VALIDATE_OR_GOTO (this->name, frame, out); - GF_VALIDATE_OR_GOTO (this->name, this->private, out); + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fremovexattr, ec_manager_removexattr, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, 0, target, + fop_flags, ec_wind_fremovexattr, + ec_manager_xattr, callback, data); + if (fop == NULL) { goto out; } fop->use_fd = 1; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (name != NULL) - { + if (name != NULL) { fop->str[0] = gf_strdup(name); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -380,87 +415,25 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target, out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL); + func(frame, NULL, this, -1, error, NULL); } } /* FOP: setattr */ -int32_t ec_combine_setattr(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) -{ - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_SETATTR'"); - - return 0; - } - - return 1; -} - -int32_t ec_setattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iatt * preop_stbuf, struct iatt * postop_stbuf, - dict_t * xdata) +int32_t +ec_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SETATTR, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (preop_stbuf != NULL) - { - cbk->iatt[0] = *preop_stbuf; - } - if (postop_stbuf != NULL) - { - cbk->iatt[1] = *postop_stbuf; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_setattr); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); } -void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_setattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -469,29 +442,24 @@ void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], &fop->iatt, fop->int32, fop->xdata); } -int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_setattr(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - if (fop->fd == NULL) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 1); - } - else - { - ec_lock_prepare_fd(fop, fop->fd, 1); + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], + EC_UPDATE_META | EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO, + 0, EC_RANGE_FULL); } ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_get_size_version(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -500,34 +468,18 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, - cbk->count); - - cbk->iatt[0].ia_size = fop->pre_size; - cbk->iatt[1].ia_size = fop->pre_size; + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + if (cbk->iatt[0].ia_type == IA_IFREG) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -536,24 +488,17 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_SETATTR) - { - if (fop->cbks.setattr != NULL) - { - fop->cbks.setattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + if (fop->id == GF_FOP_SETATTR) { + if (fop->cbks.setattr != NULL) { + QUORUM_CBK(fop->cbks.setattr, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); } - } - else - { - if (fop->cbks.fsetattr != NULL) - { - fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + } else { + if (fop->cbks.fsetattr != NULL) { + QUORUM_CBK(fop->cbks.fsetattr, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); } } @@ -561,24 +506,18 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_SETATTR) - { - if (fop->cbks.setattr != NULL) - { + if (fop->id == GF_FOP_SETATTR) { + if (fop->cbks.setattr != NULL) { fop->cbks.setattr(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } - } - else - { - if (fop->cbks.fsetattr != NULL) - { + } else { + if (fop->cbks.fsetattr != NULL) { fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -599,59 +538,54 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_setattr_cbk_t func, void * data, - loc_t * loc, struct iatt * stbuf, int32_t valid, - dict_t * xdata) +void +ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) { - ec_cbk_t callback = { .setattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.setattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(SETATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(SETATTR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_setattr, ec_manager_setattr, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target, + fop_flags, ec_wind_setattr, ec_manager_setattr, + callback, data); + if (fop == NULL) { goto out; } fop->int32 = valid; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (stbuf != NULL) - { + if (stbuf != NULL) { fop->iatt = *stbuf; } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -660,134 +594,162 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: fsetattr */ -int32_t ec_fsetattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iatt * preop_stbuf, struct iatt * postop_stbuf, - dict_t * xdata) +int32_t +ec_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_fsetattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fsetattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsetattr, + fop->fd, &fop->iatt, fop->int32, fop->xdata); +} + +void +ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + ec_cbk_t callback = {.fsetattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSETATTR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = frame->local; + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, 0, target, + fop_flags, ec_wind_fsetattr, ec_manager_setattr, + callback, data); + if (fop == NULL) { + goto out; + } - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + fop->use_fd = 1; - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETATTR, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (preop_stbuf != NULL) - { - cbk->iatt[0] = *preop_stbuf; - } - if (postop_stbuf != NULL) - { - cbk->iatt[1] = *postop_stbuf; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + fop->int32 = valid; - goto out; - } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; } + } + if (stbuf != NULL) { + fop->iatt = *stbuf; + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); - ec_combine(cbk, ec_combine_setattr); + goto out; + } } + error = 0; + out: - if (fop != NULL) - { - ec_complete(fop); + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } +} - return 0; +/* FOP: setxattr */ + +int32_t +ec_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); } -void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_setxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); - STACK_WIND_COOKIE(fop->frame, ec_fsetattr_cbk, (void *)(uintptr_t)idx, - ec->xl_list[idx], ec->xl_list[idx]->fops->fsetattr, - fop->fd, &fop->iatt, fop->int32, fop->xdata); + STACK_WIND_COOKIE(fop->frame, ec_setxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->setxattr, + &fop->loc[0], fop->dict, fop->int32, fop->xdata); } -void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fsetattr_cbk_t func, void * data, - fd_t * fd, struct iatt * stbuf, int32_t valid, dict_t * xdata) +void +ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc, + dict_t *dict, int32_t flags, dict_t *xdata) { - ec_cbk_t callback = { .fsetattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.setxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FSETATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(SETXATTR) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fsetattr, ec_manager_setattr, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, 0, target, + fop_flags, ec_wind_setxattr, ec_manager_xattr, + callback, data); + if (fop == NULL) { goto out; } - fop->use_fd = 1; - - fop->int32 = valid; + fop->int32 = flags; - if (fd != NULL) - { - fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (stbuf != NULL) - { - fop->iatt = *stbuf; + if (dict != NULL) { + fop->dict = dict_copy_with_ref(dict, NULL); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -796,23 +758,21 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } -/* FOP: setxattr */ +/* FOP: fsetxattr */ -int32_t ec_setxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -822,20 +782,18 @@ int32_t ec_setxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SETXATTR, idx, op_ret, + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret, op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -845,68 +803,182 @@ int32_t ec_setxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this, } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fsetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); - STACK_WIND_COOKIE(fop->frame, ec_setxattr_cbk, (void *)(uintptr_t)idx, - ec->xl_list[idx], ec->xl_list[idx]->fops->setxattr, - &fop->loc[0], fop->dict, fop->int32, fop->xdata); + STACK_WIND_COOKIE(fop->frame, ec_fsetxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsetxattr, + fop->fd, fop->dict, fop->int32, fop->xdata); } -int32_t ec_manager_setxattr(ec_fop_data_t * fop, int32_t state) +void +ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd, + dict_t *dict, int32_t flags, dict_t *xdata) { - ec_cbk_data_t * cbk; + ec_cbk_t callback = {.fsetxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - switch (state) - { + gf_msg_trace("ec", 0, "EC(FSETXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, 0, target, + fop_flags, ec_wind_fsetxattr, ec_manager_xattr, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = flags; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (dict != NULL) { + fop->dict = dict_copy_with_ref(dict, NULL); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/********************************************************************* + * + * File Operation : fallocate + * + *********************************************************************/ + +int32_t +ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf, + postbuf, xdata); +} + +void +ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fallocate_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fallocate, + fop->fd, fop->int32, fop->offset, fop->size, fop->xdata); +} + +int32_t +ec_manager_fallocate(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + + switch (state) { case EC_STATE_INIT: - case EC_STATE_LOCK: - if (fop->fd == NULL) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 1); + if (fop->size == 0) { + ec_fop_set_error(fop, EINVAL); + return EC_STATE_REPORT; } - else - { - ec_lock_prepare_fd(fop, fop->fd, 1); + if (fop->int32 & + (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE | + FALLOC_FL_ZERO_RANGE | FALLOC_FL_PUNCH_HOLE)) { + ec_fop_set_error(fop, ENOTSUP); + return EC_STATE_REPORT; } + fop->user_size = fop->offset + fop->size; + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + fop->size += fop->head; + ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fop->offset, fop->size); ec_lock(fop); return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: + ec_dispatch_all(fop); return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + LOCK(&fop->locks[0].lock->loc.inode->lock); { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; + GF_ASSERT(__ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + + /*If mode has FALLOC_FL_KEEP_SIZE keep the size */ + if (fop->int32 & FALLOC_FL_KEEP_SIZE) { + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } else if (fop->user_size > cbk->iatt[0].ia_size) { + cbk->iatt[1].ia_size = fop->user_size; + + /* This shouldn't fail because we have the inode + * locked. */ + GF_ASSERT(__ec_set_inode_size( + fop, fop->locks[0].lock->loc.inode, + cbk->iatt[1].ia_size)); + } else { + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; } } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - } - else - { - ec_fop_set_error(fop, EIO); + UNLOCK(&fop->locks[0].lock->loc.inode->lock); } return EC_STATE_REPORT; @@ -916,22 +988,10 @@ int32_t ec_manager_setxattr(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_SETXATTR) - { - if (fop->cbks.setxattr != NULL) - { - fop->cbks.setxattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, cbk->xdata); - } - } - else - { - if (fop->cbks.fsetxattr != NULL) - { - fop->cbks.fsetxattr(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - cbk->xdata); - } + if (fop->cbks.fallocate != NULL) { + QUORUM_CBK(fop->cbks.fallocate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); } return EC_STATE_LOCK_REUSE; @@ -943,21 +1003,9 @@ int32_t ec_manager_setxattr(ec_fop_data_t * fop, int32_t state) case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_SETXATTR) - { - if (fop->cbks.setxattr != NULL) - { - fop->cbks.setxattr(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL); - } - } - else - { - if (fop->cbks.fsetxattr != NULL) - { - fop->cbks.fsetxattr(fop->req_frame, fop, fop->xl, -1, - fop->error, NULL); - } + if (fop->cbks.fallocate != NULL) { + fop->cbks.fallocate(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); } return EC_STATE_LOCK_REUSE; @@ -975,67 +1023,56 @@ int32_t ec_manager_setxattr(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } void -ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_setxattr_cbk_t func, void *data, - loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata) +ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd, + int32_t mode, off_t offset, size_t len, dict_t *xdata) { - ec_cbk_t callback = { .setxattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fallocate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(SETXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(FALLOCATE) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_setxattr, ec_manager_setxattr, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FALLOCATE, 0, target, + fop_flags, ec_wind_fallocate, + ec_manager_fallocate, callback, data); + if (fop == NULL) { goto out; } - fop->int32 = flags; - - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + fop->use_fd = 1; + fop->int32 = mode; + fop->offset = offset; + fop->size = len; + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (dict != NULL) - { - fop->dict = dict_ref(dict); - if (fop->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - goto out; - } - } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } @@ -1044,302 +1081,315 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL); + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } -/* FOP: fsetxattr */ +/********************************************************************* + * + * File Operation : Discard + * + *********************************************************************/ +void +ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask) +{ + ec_t *ec = fop->xl->private; + off_t off_head = 0; + off_t off_tail = 0; + uint64_t size_head = 0; + uint64_t size_tail = 0; + int error = 0; + + off_head = fop->offset * ec->fragments - fop->int32; + if (fop->size == 0) { + error = ec_update_write(fop, mask, off_head, fop->user_size); + } else { + size_head = fop->int32; + size_tail = (off_head + fop->user_size) % ec->stripe_size; + off_tail = off_head + fop->user_size - size_tail; + if (size_head) { + error = ec_update_write(fop, mask, off_head, size_head); + if (error) { + goto out; + } + } + if (size_tail) { + error = ec_update_write(fop, mask, off_tail, size_tail); + } + } +out: + if (error) + ec_fop_set_error(fop, -error); +} + +void +ec_discard_adjust_offset_size(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + + fop->user_size = fop->size; + /* If discard length covers at least a fragment on brick, we will + * perform discard operation(when fop->size is non-zero) else we just + * write zeros. + */ + fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true); + fop->frag_range.first = fop->offset; + if (fop->size < fop->int32) { + fop->size = 0; + } else { + fop->size -= fop->int32; + ec_adjust_size_down(ec, &fop->size, _gf_true); + } + fop->frag_range.last = fop->offset + fop->size; +} int32_t -ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno, dict_t *xdata) +ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf, + postbuf, xdata); +} + +void +ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); - VALIDATE_OR_GOTO (this, out); - GF_VALIDATE_OR_GOTO (this->name, frame, out); - GF_VALIDATE_OR_GOTO (this->name, frame->local, out); - GF_VALIDATE_OR_GOTO (this->name, this->private, out); + STACK_WIND_COOKIE(fop->frame, ec_discard_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->discard, + fop->fd, fop->offset, fop->size, fop->xdata); +} - fop = frame->local; +int32_t +ec_manager_discard(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + off_t fl_start = 0; + uint64_t fl_size = 0; + + switch (state) { + case EC_STATE_INIT: + if ((fop->size <= 0) || (fop->offset < 0)) { + ec_fop_set_error(fop, EINVAL); + return EC_STATE_REPORT; + } + /* Because of the head/tail writes, "discard" happens on the + * remaining regions, but we need to compute region including + * head/tail writes so compute them separately*/ + fl_start = fop->offset; + fl_size = fop->size; + fl_size += ec_adjust_offset_down(fop->xl->private, &fl_start, + _gf_true); + ec_adjust_size_up(fop->xl->private, &fl_size, _gf_true); - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_discard_adjust_offset_size(fop); - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + /* Fall through */ - goto out; + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fl_start, fl_size); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + + /* Dispatch discard fop only if we have whole fragment + * to deallocate */ + if (fop->size) { + ec_dispatch_all(fop); + return EC_STATE_DELAYED_START; + } else { + /* Assume discard to have succeeded on all bricks */ + ec_succeed_all(fop); } - } - ec_combine(cbk, NULL); - } + /* Fall through */ -out: - if (fop != NULL) - { - ec_complete(fop); - } + case EC_STATE_DELAYED_START: - return 0; -} + if (fop->size) { + if (fop->answer && fop->answer->op_ret == 0) + ec_update_discard_write(fop, fop->answer->mask); + } else { + ec_update_discard_write(fop, fop->mask); + } -void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx) -{ - ec_trace("WIND", fop, "idx=%d", idx); + return EC_STATE_PREPARE_ANSWER; - STACK_WIND_COOKIE(fop->frame, ec_fsetxattr_cbk, (void *)(uintptr_t)idx, - ec->xl_list[idx], ec->xl_list[idx]->fops->fsetxattr, - fop->fd, fop->dict, fop->int32, fop->xdata); + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.discard != NULL) { + QUORUM_CBK(fop->cbks.discard, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_DELAYED_START: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.discard != NULL) { + fop->cbks.discard(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } } void -ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target, - int32_t minimum, fop_fsetxattr_cbk_t func, void *data, - fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata) +ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd, + off_t offset, size_t len, dict_t *xdata) { - ec_cbk_t callback = { .fsetxattr = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.discard = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FSETXATTR) %p", frame); + gf_msg_trace("ec", 0, "EC(DISCARD) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fsetxattr, ec_manager_setxattr, + fop = ec_fop_data_allocate(frame, this, GF_FOP_DISCARD, 0, target, + fop_flags, ec_wind_discard, ec_manager_discard, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } fop->use_fd = 1; + fop->offset = offset; + fop->size = len; - fop->int32 = flags; - - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); - - goto out; - } } - if (dict != NULL) - { - fop->dict = dict_ref(dict); - if (fop->dict == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - goto out; - } - } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } } error = 0; out: if (fop != NULL) { - ec_manager (fop, error); + ec_manager(fop, error); } else { - func (frame, NULL, this, -1, error, NULL); + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } -/* FOP: truncate */ +/********************************************************************* + * + * File Operation : truncate + * + *********************************************************************/ -int32_t ec_truncate_write(ec_fop_data_t * fop, uintptr_t mask) +int32_t +ec_update_truncate_write(ec_fop_data_t *fop, uintptr_t mask) { - ec_t * ec = fop->xl->private; - struct iobref * iobref = NULL; - struct iobuf * iobuf = NULL; - struct iovec vector; - int32_t ret = 0; - - iobref = iobref_new(); - if (iobref == NULL) - { - goto out; - } - iobuf = iobuf_get(fop->xl->ctx->iobuf_pool); - if (iobuf == NULL) - { - goto out; - } - if (iobref_add(iobref, iobuf) != 0) - { - goto out; - } - - vector.iov_base = iobuf->ptr; - vector.iov_len = fop->offset * ec->fragments - fop->user_size; - - memset(iobuf->ptr, 0, vector.iov_len); - - iobuf = NULL; - - ec_writev(fop->frame, fop->xl, mask, fop->minimum, NULL, NULL, fop->fd, - &vector, 1, fop->user_size, 0, iobref, NULL); - - ret = 1; - -out: - if (iobuf != NULL) - { - iobuf_unref(iobuf); - } - if (iobref != NULL) - { - iobref_unref(iobref); - } - - return ret; + ec_t *ec = fop->xl->private; + uint64_t size = fop->offset * ec->fragments - fop->user_size; + return ec_update_write(fop, mask, fop->user_size, size); } -int32_t ec_truncate_open_cbk(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - fd_t * fd, dict_t * xdata) +int32_t +ec_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) { - ec_fop_data_t * fop = cookie; - - if (op_ret >= 0) - { - if (!ec_truncate_write(fop->parent, fop->answer->mask)) - { - fop->error = EIO; + ec_fop_data_t *fop = cookie; + int32_t err; + + fop->parent->good &= fop->good; + if (op_ret >= 0) { + fd_bind(fd); + err = ec_update_truncate_write(fop->parent, fop->answer->mask); + if (err != 0) { + ec_fop_set_error(fop->parent, -err); } } return 0; } -int32_t ec_truncate_clean(ec_fop_data_t * fop) +int32_t +ec_truncate_clean(ec_fop_data_t *fop) { - if (fop->fd == NULL) - { + if (fop->fd == NULL) { fop->fd = fd_create(fop->loc[0].inode, fop->frame->root->pid); - if (fop->fd == NULL) - { - return 0; + if (fop->fd == NULL) { + return -ENOMEM; } ec_open(fop->frame, fop->xl, fop->answer->mask, fop->minimum, - ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, - NULL); - - return 1; - } - else - { - return ec_truncate_write(fop, fop->answer->mask); - } -} - -int32_t ec_combine_truncate(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) -{ - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_TRUNCATE'"); + ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, NULL); return 0; + } else { + return ec_update_truncate_write(fop, fop->answer->mask); } - - return 1; } -int32_t ec_truncate_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * prebuf, - struct iatt * postbuf, dict_t * xdata) +int32_t +ec_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_TRUNCATE, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (prebuf != NULL) - { - cbk->iatt[0] = *prebuf; - } - if (postbuf != NULL) - { - cbk->iatt[1] = *postbuf; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_truncate); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); } -void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_truncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1348,34 +1398,38 @@ void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx) &fop->loc[0], fop->offset, fop->xdata); } -int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_truncate(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + off_t offset_down; - switch (state) - { + switch (state) { case EC_STATE_INIT: fop->user_size = fop->offset; - fop->offset = ec_adjust_size(fop->xl->private, fop->offset, 1); + ec_adjust_offset_up(fop->xl->private, &fop->offset, _gf_true); + fop->frag_range.first = fop->offset; + fop->frag_range.last = UINT64_MAX; - /* Fall through */ + /* Fall through */ case EC_STATE_LOCK: - if (fop->id == GF_FOP_TRUNCATE) - { - ec_lock_prepare_inode(fop, &fop->loc[0], 1); - } - else - { - ec_lock_prepare_fd(fop, fop->fd, 1); + offset_down = fop->user_size; + ec_adjust_offset_down(fop->xl->private, &offset_down, _gf_true); + + if (fop->id == GF_FOP_TRUNCATE) { + ec_lock_prepare_inode( + fop, &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + offset_down, EC_RANGE_FULL); + } else { + ec_lock_prepare_fd( + fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + offset_down, EC_RANGE_FULL); } ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_prepare_update(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -1384,43 +1438,30 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, - cbk->count); - - cbk->iatt[0].ia_size = fop->pre_size; - cbk->iatt[1].ia_size = fop->user_size; - fop->post_size = fop->user_size; - if ((fop->pre_size > fop->post_size) && - (fop->user_size != fop->offset)) - { - if (!ec_truncate_clean(fop)) - { - ec_fop_set_error(fop, EIO); - } + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + /* Inode size doesn't need to be updated under locks, because + * conflicting operations won't be in-flight + */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = fop->user_size; + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_set_inode_size(fop, fop->locks[0].lock->loc.inode, + fop->user_size)); + if ((cbk->iatt[0].ia_size > cbk->iatt[1].ia_size) && + (fop->user_size != fop->offset)) { + err = ec_truncate_clean(fop); + if (err != 0) { + ec_cbk_set_error(cbk, -err, _gf_false); } } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -1429,24 +1470,17 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_TRUNCATE) - { - if (fop->cbks.truncate != NULL) - { - fop->cbks.truncate(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + if (fop->id == GF_FOP_TRUNCATE) { + if (fop->cbks.truncate != NULL) { + QUORUM_CBK(fop->cbks.truncate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); } - } - else - { - if (fop->cbks.ftruncate != NULL) - { - fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, - &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + } else { + if (fop->cbks.ftruncate != NULL) { + QUORUM_CBK(fop->cbks.ftruncate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); } } @@ -1454,24 +1488,18 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state) case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_TRUNCATE) - { - if (fop->cbks.truncate != NULL) - { + if (fop->id == GF_FOP_TRUNCATE) { + if (fop->cbks.truncate != NULL) { fop->cbks.truncate(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } - } - else - { - if (fop->cbks.ftruncate != NULL) - { + } else { + if (fop->cbks.ftruncate != NULL) { fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -1492,54 +1520,51 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_truncate_cbk_t func, void * data, - loc_t * loc, off_t offset, dict_t * xdata) +void +ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc, + off_t offset, dict_t *xdata) { - ec_cbk_t callback = { .truncate = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.truncate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(TRUNCATE) %p", frame); + gf_msg_trace("ec", 0, "EC(TRUNCATE) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_truncate, ec_manager_truncate, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, 0, target, + fop_flags, ec_wind_truncate, ec_manager_truncate, + callback, data); + if (fop == NULL) { goto out; } fop->offset = offset; - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1548,77 +1573,26 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: ftruncate */ -int32_t ec_ftruncate_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, - struct iatt * prebuf, struct iatt * postbuf, - dict_t * xdata) +int32_t +ec_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - int32_t idx = (int32_t)(uintptr_t)cookie; - - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); - - fop = frame->local; - - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); - - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FTRUNCATE, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (prebuf != NULL) - { - cbk->iatt[0] = *prebuf; - } - if (postbuf != NULL) - { - cbk->iatt[1] = *postbuf; - } - } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); - - goto out; - } - } - - ec_combine(cbk, ec_combine_truncate); - } - -out: - if (fop != NULL) - { - ec_complete(fop); - } - - return 0; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); } -void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_ftruncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1627,26 +1601,25 @@ void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->fd, fop->offset, fop->xdata); } -void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_ftruncate_cbk_t func, void * data, - fd_t * fd, off_t offset, dict_t * xdata) +void +ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd, + off_t offset, dict_t *xdata) { - ec_cbk_t callback = { .ftruncate = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.ftruncate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FTRUNCATE) %p", frame); + gf_msg_trace("ec", 0, "EC(FTRUNCATE) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_ftruncate, ec_manager_truncate, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, 0, target, + fop_flags, ec_wind_ftruncate, + ec_manager_truncate, callback, data); + if (fop == NULL) { goto out; } @@ -1654,24 +1627,22 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->offset = offset; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1680,35 +1651,100 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } /* FOP: writev */ +static ec_stripe_t * +ec_allocate_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache) +{ + ec_stripe_t *stripe = NULL; + + if (stripe_cache->count >= stripe_cache->max) { + GF_ASSERT(!list_empty(&stripe_cache->lru)); + stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru); + list_move_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.evicts); + } else { + stripe = GF_MALLOC(sizeof(ec_stripe_t) + ec->stripe_size, + ec_mt_ec_stripe_t); + if (stripe != NULL) { + stripe_cache->count++; + list_add_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.allocs); + } else { + GF_ATOMIC_INC(ec->stats.stripe_cache.errors); + } + } + + return stripe; +} -int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct iovec * vector, int32_t count, - struct iatt * stbuf, struct iobref * iobref, - dict_t * xdata) +static void +ec_write_stripe_data(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) { - ec_t * ec = this->private; - ec_fop_data_t * fop = frame->local; - size_t size, base, tmp; + off_t base; - if (op_ret >= 0) - { + base = fop->size - ec->stripe_size; + memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size); + stripe->frag_offset = fop->frag_range.last - ec->fragment_size; +} + +static void +ec_add_stripe_in_cache(ec_t *ec, ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + ec_stripe_t *stripe = NULL; + ec_stripe_list_t *stripe_cache = NULL; + gf_boolean_t failed = _gf_true; + + LOCK(&fop->fd->inode->lock); + + ctx = __ec_inode_get(fop->fd->inode, fop->xl); + if (ctx == NULL) { + goto out; + } + + stripe_cache = &ctx->stripe_cache; + if (stripe_cache->max > 0) { + stripe = ec_allocate_stripe(ec, stripe_cache); + if (stripe == NULL) { + goto out; + } + + ec_write_stripe_data(ec, fop, stripe); + } + + failed = _gf_false; + +out: + UNLOCK(&fop->fd->inode->lock); + + if (failed) { + gf_msg(ec->xl->name, GF_LOG_DEBUG, ENOMEM, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to create and add stripe in cache"); + } +} + +int32_t +ec_writev_merge_tail(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) +{ + ec_t *ec = this->private; + ec_fop_data_t *fop = frame->local; + uint64_t size, base, tmp; + + if (op_ret >= 0) { tmp = 0; size = fop->size - fop->user_size - fop->head; base = ec->stripe_size - size; - if (op_ret > base) - { + if (op_ret > base) { tmp = min(op_ret - base, size); ec_iov_copy_to(fop->vector[0].iov_base + fop->size - size, vector, count, base, tmp); @@ -1716,46 +1752,44 @@ int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie, size -= tmp; } - if (size > 0) - { + if (size > 0) { memset(fop->vector[0].iov_base + fop->size - size, 0, size); } - } + if (ec->stripe_cache) { + ec_add_stripe_in_cache(ec, fop); + } + } return 0; } -int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct iovec * vector, int32_t count, - struct iatt * stbuf, struct iobref * iobref, - dict_t * xdata) +int32_t +ec_writev_merge_head(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) { - ec_t * ec = this->private; - ec_fop_data_t * fop = frame->local; - size_t size, base; + ec_t *ec = this->private; + ec_fop_data_t *fop = frame->local; + uint64_t size, base; - if (op_ret >= 0) - { + if (op_ret >= 0) { size = fop->head; base = 0; - if (op_ret > 0) - { + if (op_ret > 0) { base = min(op_ret, size); ec_iov_copy_to(fop->vector[0].iov_base, vector, count, 0, base); size -= base; } - if (size > 0) - { + if (size > 0) { memset(fop->vector[0].iov_base + base, 0, size); } size = fop->size - fop->user_size - fop->head; - if ((size > 0) && (fop->size == ec->stripe_size)) - { + if ((size > 0) && (fop->size == ec->stripe_size)) { ec_writev_merge_tail(frame, cookie, this, op_ret, op_errno, vector, count, stbuf, iobref, xdata); } @@ -1764,260 +1798,375 @@ int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie, return 0; } -void ec_writev_start(ec_fop_data_t *fop) +static int +ec_make_internal_fop_xdata(dict_t **xdata) { - ec_t *ec = fop->xl->private; - struct iobref *iobref = NULL; - struct iobuf *iobuf = NULL; - void *ptr = NULL; - ec_fd_t *ctx; - fd_t *fd; - size_t tail; - uid_t uid; - gid_t gid; + dict_t *dict = NULL; - fd = fd_anonymous(fop->fd->inode); - if (fd == NULL) { - ec_fop_set_error(fop, EIO); + if (*xdata) + return 0; - return; - } + dict = dict_new(); + if (!dict) + goto out; - uid = fop->frame->root->uid; - fop->frame->root->uid = 0; - gid = fop->frame->root->gid; - fop->frame->root->gid = 0; + if (dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes")) + goto out; - ctx = ec_fd_get(fop->fd, fop->xl); - if (ctx != NULL) { - if ((ctx->flags & O_APPEND) != 0) { - fop->offset = fop->pre_size; + *xdata = dict; + return 0; +out: + if (dict) + dict_unref(dict); + return -1; +} + +static int32_t +ec_writev_prepare_buffers(ec_t *ec, ec_fop_data_t *fop) +{ + struct iobref *iobref = NULL; + struct iovec *iov; + void *ptr; + int32_t err; + + fop->user_size = iov_length(fop->vector, fop->int32); + fop->head = ec_adjust_offset_down(ec, &fop->offset, _gf_false); + fop->frag_range.first = fop->offset / ec->fragments; + fop->size = fop->user_size + fop->head; + ec_adjust_size_up(ec, &fop->size, _gf_false); + fop->frag_range.last = fop->frag_range.first + fop->size / ec->fragments; + + if ((fop->int32 != 1) || (fop->head != 0) || (fop->size > fop->user_size) || + !EC_ALIGN_CHECK(fop->vector[0].iov_base, EC_METHOD_WORD_SIZE)) { + err = ec_buffer_alloc(ec->xl, fop->size, &iobref, &ptr); + if (err != 0) { + goto out; } + + ec_iov_copy_to(ptr + fop->head, fop->vector, fop->int32, 0, + fop->user_size); + + fop->vector[0].iov_base = ptr; + fop->vector[0].iov_len = fop->size; + + iobref_unref(fop->buffers); + fop->buffers = iobref; } - fop->user_size = iov_length(fop->vector, fop->int32); - fop->head = ec_adjust_offset(ec, &fop->offset, 0); - fop->size = ec_adjust_size(ec, fop->user_size + fop->head, 0); + if (fop->int32 != 2) { + iov = GF_MALLOC(VECTORSIZE(2), gf_common_mt_iovec); + if (iov == NULL) { + err = -ENOMEM; - iobref = iobref_new(); - if (iobref == NULL) { - goto out; + goto out; + } + iov[0].iov_base = fop->vector[0].iov_base; + iov[0].iov_len = fop->vector[0].iov_len; + + GF_FREE(fop->vector); + fop->vector = iov; } - iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, fop->size); - if (iobuf == NULL) { + + fop->vector[1].iov_len = fop->size / ec->fragments; + err = ec_buffer_alloc(ec->xl, fop->vector[1].iov_len * ec->nodes, + &fop->buffers, &fop->vector[1].iov_base); + if (err != 0) { goto out; } - if (iobref_add(iobref, iobuf) != 0) { - goto out; + + err = 0; + +out: + return err; +} + +static void +ec_merge_stripe_head_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) +{ + uint32_t head, size; + + head = fop->head; + memcpy(fop->vector[0].iov_base, stripe->data, head); + + size = ec->stripe_size - head; + if (size > fop->user_size) { + head += fop->user_size; + size = ec->stripe_size - head; + memcpy(fop->vector[0].iov_base + head, stripe->data + head, size); } +} - ptr = iobuf->ptr + fop->head; - ec_iov_copy_to(ptr, fop->vector, fop->int32, 0, fop->user_size); +static void +ec_merge_stripe_tail_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) +{ + uint32_t head, tail; + off_t offset; - fop->vector[0].iov_base = iobuf->ptr; - fop->vector[0].iov_len = fop->size; + offset = fop->user_size + fop->head; + tail = fop->size - offset; + head = ec->stripe_size - tail; - iobuf_unref(iobuf); + memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail); +} - iobref_unref(fop->buffers); - fop->buffers = iobref; +static ec_stripe_t * +ec_get_stripe_from_cache_locked(ec_t *ec, ec_fop_data_t *fop, + uint64_t frag_offset) +{ + ec_inode_t *ctx = NULL; + ec_stripe_t *stripe = NULL; + ec_stripe_list_t *stripe_cache = NULL; - if (fop->head > 0) - { - ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN, ec_writev_merge_head, - NULL, fd, ec->stripe_size, fop->offset, 0, NULL); + ctx = __ec_inode_get(fop->fd->inode, fop->xl); + if (ctx == NULL) { + GF_ATOMIC_INC(ec->stats.stripe_cache.errors); + return NULL; } - tail = fop->size - fop->user_size - fop->head; - if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) + + stripe_cache = &ctx->stripe_cache; + list_for_each_entry(stripe, &stripe_cache->lru, lru) { - if (fop->pre_size > fop->offset + fop->head + fop->user_size) - { - ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN, - ec_writev_merge_tail, NULL, fd, ec->stripe_size, - fop->offset + fop->size - ec->stripe_size, 0, NULL); - } - else - { - memset(fop->vector[0].iov_base + fop->size - tail, 0, tail); + if (stripe->frag_offset == frag_offset) { + list_move_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.hits); + return stripe; } } - fop->frame->root->uid = uid; - fop->frame->root->gid = gid; + GF_ATOMIC_INC(ec->stats.stripe_cache.misses); - fd_unref(fd); + return NULL; +} - return; +static gf_boolean_t +ec_get_and_merge_stripe(ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which) +{ + uint64_t frag_offset; + ec_stripe_t *stripe = NULL; + gf_boolean_t found = _gf_false; -out: - if (iobuf != NULL) { - iobuf_unref(iobuf); - } - if (iobref != NULL) { - iobref_unref(iobref); + if (!ec->stripe_cache) { + return found; } - fop->frame->root->uid = uid; - fop->frame->root->gid = gid; + LOCK(&fop->fd->inode->lock); + if (which == EC_STRIPE_HEAD) { + frag_offset = fop->frag_range.first; + stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset); + if (stripe) { + ec_merge_stripe_head_locked(ec, fop, stripe); + found = _gf_true; + } + } - fd_unref(fd); + if (which == EC_STRIPE_TAIL) { + frag_offset = fop->frag_range.last - ec->fragment_size; + stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset); + if (stripe) { + ec_merge_stripe_tail_locked(ec, fop, stripe); + found = _gf_true; + } + } + UNLOCK(&fop->fd->inode->lock); - ec_fop_set_error(fop, EIO); + return found; } -int32_t ec_combine_writev(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +static uintptr_t +ec_get_lock_good_mask(inode_t *inode, xlator_t *xl) { - if (!ec_iatt_combine(dst->iatt, src->iatt, 2)) + ec_lock_t *lock = NULL; + ec_inode_t *ictx = NULL; + LOCK(&inode->lock); { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in " - "answers of 'GF_FOP_WRITE'"); - - return 0; + ictx = __ec_inode_get(inode, xl); + if (ictx) + lock = ictx->inode_lock; } - - return 1; + UNLOCK(&inode->lock); + if (lock) + return lock->good_mask; + return 0; } -int32_t ec_writev_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct iatt * prebuf, - struct iatt * postbuf, dict_t * xdata) +void +ec_writev_start(ec_fop_data_t *fop) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; - ec_t * ec = this->private; - int32_t idx = (int32_t)(uintptr_t)cookie; + ec_t *ec = fop->xl->private; + ec_fd_t *ctx; + fd_t *fd; + dict_t *xdata = NULL; + uint64_t tail, current; + int32_t err = -ENOMEM; + gf_boolean_t found_stripe = _gf_false; - VALIDATE_OR_GOTO(this, out); - GF_VALIDATE_OR_GOTO(this->name, frame, out); - GF_VALIDATE_OR_GOTO(this->name, frame->local, out); - GF_VALIDATE_OR_GOTO(this->name, this->private, out); + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, ¤t)); - fop = frame->local; + fd = fd_anonymous(fop->fd->inode); + if (fd == NULL) { + goto failed; + } - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + fop->frame->root->uid = 0; + fop->frame->root->gid = 0; - cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_WRITE, idx, op_ret, - op_errno); - if (cbk != NULL) - { - if (op_ret >= 0) - { - if (prebuf != NULL) - { - cbk->iatt[0] = *prebuf; + ctx = ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + if ((ctx->flags & O_APPEND) != 0) { + /* Appending writes take full locks so size won't change because + * of any parallel operations + */ + fop->offset = current; + } + } + + err = ec_writev_prepare_buffers(ec, fop); + if (err != 0) { + goto failed_fd; + } + tail = fop->size - fop->user_size - fop->head; + if (fop->head > 0) { + if (current > fop->offset) { + found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_HEAD); + if (!found_stripe) { + if (ec_make_internal_fop_xdata(&xdata)) { + err = -ENOMEM; + goto failed_xdata; + } + ec_readv(fop->frame, fop->xl, + ec_get_lock_good_mask(fop->fd->inode, fop->xl), + EC_MINIMUM_MIN, ec_writev_merge_head, NULL, fd, + ec->stripe_size, fop->offset, 0, xdata); } - if (postbuf != NULL) - { - cbk->iatt[1] = *postbuf; + } else { + memset(fop->vector[0].iov_base, 0, fop->head); + memset(fop->vector[0].iov_base + fop->size - tail, 0, tail); + if (ec->stripe_cache && (fop->size <= ec->stripe_size)) { + ec_add_stripe_in_cache(ec, fop); } } - if (xdata != NULL) - { - cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + } - goto out; + if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) { + /* Current locking scheme will make sure the 'current' below will + * never decrease while the fop is in progress, so the checks will + * work as expected + */ + if (current > fop->offset + fop->head + fop->user_size) { + found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_TAIL); + if (!found_stripe) { + if (ec_make_internal_fop_xdata(&xdata)) { + err = -ENOMEM; + goto failed_xdata; + } + ec_readv(fop->frame, fop->xl, + ec_get_lock_good_mask(fop->fd->inode, fop->xl), + EC_MINIMUM_MIN, ec_writev_merge_tail, NULL, fd, + ec->stripe_size, + fop->offset + fop->size - ec->stripe_size, 0, xdata); + } + } else { + memset(fop->vector[0].iov_base + fop->size - tail, 0, tail); + if (ec->stripe_cache) { + ec_add_stripe_in_cache(ec, fop); } } + } - if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } + err = 0; - ec_combine(cbk, ec_combine_writev); +failed_xdata: + if (xdata) { + dict_unref(xdata); } +failed_fd: + fd_unref(fd); +failed: + ec_fop_set_error(fop, -err); +} -out: - if (fop != NULL) - { - ec_complete(fop); +int32_t +ec_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *prestat, struct iatt *poststat, + dict_t *xdata) +{ + ec_t *ec = NULL; + if (this && this->private) { + ec = this->private; + if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) { + op_ret = -1; + op_errno = EIO; + } } - - return 0; + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); } -void ec_wind_writev(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_writev(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); struct iovec vector[1]; - struct iobref * iobref = NULL; - struct iobuf * iobuf = NULL; - ssize_t size = 0, bufsize = 0; - - iobref = iobref_new(); - if (iobref == NULL) - { - goto out; - } - - size = fop->vector[0].iov_len; - bufsize = size / ec->fragments; + size_t size; - iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, bufsize); - if (iobuf == NULL) - { - goto out; - } - if (iobref_add(iobref, iobuf) != 0) - { - goto out; - } - - ec_method_encode(size, ec->fragments, idx, fop->vector[0].iov_base, - iobuf->ptr); + size = fop->vector[1].iov_len; - vector[0].iov_base = iobuf->ptr; - vector[0].iov_len = bufsize; - - iobuf_unref(iobuf); + vector[0].iov_base = fop->vector[1].iov_base + idx * size; + vector[0].iov_len = size; STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx, - ec->xl_list[idx], ec->xl_list[idx]->fops->writev, - fop->fd, vector, 1, fop->offset / ec->fragments, - fop->uint32, iobref, fop->xdata); - - iobref_unref(iobref); + ec->xl_list[idx], ec->xl_list[idx]->fops->writev, fop->fd, + vector, 1, fop->offset / ec->fragments, fop->uint32, + fop->buffers, fop->xdata); +} - return; +static void +ec_writev_encode(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + void *blocks[ec->nodes]; + uint32_t i; -out: - if (iobuf != NULL) - { - iobuf_unref(iobuf); - } - if (iobref != NULL) - { - iobref_unref(iobref); + blocks[0] = fop->vector[1].iov_base; + for (i = 1; i < ec->nodes; i++) { + blocks[i] = blocks[i - 1] + fop->vector[1].iov_len; } - - ec_writev_cbk(fop->frame, (void *)(uintptr_t)idx, fop->xl, -1, EIO, NULL, - NULL, NULL); + ec_method_encode(&ec->matrix, fop->vector[0].iov_len, + fop->vector[0].iov_base, blocks); } -int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_writev(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; + ec_fd_t *ctx = NULL; + ec_t *ec = fop->xl->private; + off_t fl_start = 0; + uint64_t fl_size = LONG_MAX; - switch (state) - { + switch (state) { case EC_STATE_INIT: case EC_STATE_LOCK: - ec_lock_prepare_fd(fop, fop->fd, 1); + ctx = ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + if ((ctx->flags & O_APPEND) == 0) { + off_t user_size = 0; + off_t head = 0; + + fl_start = fop->offset; + user_size = iov_length(fop->vector, fop->int32); + head = ec_adjust_offset_down(ec, &fl_start, _gf_true); + fl_size = user_size + head; + ec_adjust_size_up(ec, &fl_size, _gf_true); + } + } + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fl_start, fl_size); ec_lock(fop); - return EC_STATE_GET_SIZE_AND_VERSION; - - case EC_STATE_GET_SIZE_AND_VERSION: - ec_prepare_update(fop); - return EC_STATE_DISPATCH; case EC_STATE_DISPATCH: @@ -2026,62 +2175,60 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state) return EC_STATE_DELAYED_START; case EC_STATE_DELAYED_START: + /* Restore uid, gid if they were changed to do some partial + * reads. */ + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + ec_writev_encode(fop); + ec_dispatch_all(fop); return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if (cbk != NULL) - { - if (!ec_dict_combine(cbk, EC_COMBINE_XDATA)) - { - if (cbk->op_ret >= 0) - { - cbk->op_ret = -1; - cbk->op_errno = EIO; - } - } - if (cbk->op_ret < 0) - { - ec_fop_set_error(fop, cbk->op_errno); - } - else - { - ec_t * ec = fop->xl->private; - size_t size; + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_t *ec = fop->xl->private; + uint64_t size; - ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, - cbk->count); + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + /* This shouldn't fail because we have the inode locked. */ + LOCK(&fop->fd->inode->lock); + { + GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; size = fop->offset + fop->head + fop->user_size; - if (size > fop->pre_size) - { - fop->post_size = size; + if (size > cbk->iatt[0].ia_size) { + /* Only update inode size if this is a top level fop. + * Otherwise this is an internal write and the top + * level fop should take care of the real inode size. + */ + if (fop->parent == NULL) { + /* This shouldn't fail because we have the inode + * locked. */ + GF_ASSERT( + __ec_set_inode_size(fop, fop->fd->inode, size)); + } + cbk->iatt[1].ia_size = size; } + } + UNLOCK(&fop->fd->inode->lock); - cbk->iatt[0].ia_size = fop->pre_size; - cbk->iatt[1].ia_size = fop->post_size; - + if (fop->error == 0) { cbk->op_ret *= ec->fragments; - if (cbk->op_ret < fop->head) - { + if (cbk->op_ret < fop->head) { cbk->op_ret = 0; - } - else - { + } else { cbk->op_ret -= fop->head; } - if (cbk->op_ret > fop->user_size) - { + if (cbk->op_ret > fop->user_size) { cbk->op_ret = fop->user_size; } } } - else - { - ec_fop_set_error(fop, EIO); - } return EC_STATE_REPORT; @@ -2090,25 +2237,30 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.writev != NULL) - { - fop->cbks.writev(fop->req_frame, fop, fop->xl, cbk->op_ret, - cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], - cbk->xdata); + if (fop->cbks.writev != NULL) { + QUORUM_CBK(fop->cbks.writev, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); } return EC_STATE_LOCK_REUSE; + case -EC_STATE_DELAYED_START: + /* We have failed while doing partial reads. We need to restore + * original uid, gid. */ + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + /* Fall through */ + case -EC_STATE_INIT: case -EC_STATE_LOCK: - case -EC_STATE_GET_SIZE_AND_VERSION: case -EC_STATE_DISPATCH: case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.writev != NULL) - { + if (fop->cbks.writev != NULL) { fop->cbks.writev(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, NULL, NULL); } @@ -2128,34 +2280,33 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state) return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_writev_cbk_t func, void * data, fd_t * fd, - struct iovec * vector, int32_t count, off_t offset, - uint32_t flags, struct iobref * iobref, dict_t * xdata) +void +ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata) { - ec_cbk_t callback = { .writev = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.writev = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(WRITE) %p", frame); + gf_msg_trace("ec", 0, "EC(WRITE) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, - EC_FLAG_UPDATE_FD_INODE, target, minimum, + fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, fop_flags, ec_wind_writev, ec_manager_writev, callback, data); - if (fop == NULL) - { + if (fop == NULL) { goto out; } @@ -2165,47 +2316,43 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->use_fd = 1; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (count > 0) - { + if (count > 0) { fop->vector = iov_dup(vector, count); - if (fop->vector == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a " - "vector list."); + if (fop->vector == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a " + "vector list."); goto out; } fop->int32 = count; } - if (iobref != NULL) - { + if (iobref != NULL) { fop->buffers = iobref_ref(iobref); - if (fop->buffers == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "buffer."); + if (fop->buffers == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL, + "Failed to reference a " + "buffer."); goto out; } } - if (xdata != NULL) - { - fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -2214,12 +2361,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); } } diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c index 2e301631b3f..601960d6154 100644 --- a/xlators/cluster/ec/src/ec-locks.c +++ b/xlators/cluster/ec/src/ec-locks.c @@ -8,71 +8,110 @@ cases as published by the Free Software Foundation. */ -#include "xlator.h" -#include "defaults.h" - #include "ec-helpers.h" #include "ec-common.h" #include "ec-combine.h" -#include "ec-method.h" #include "ec-fops.h" +#include "ec-messages.h" #define EC_LOCK_MODE_NONE 0 -#define EC_LOCK_MODE_INC 1 -#define EC_LOCK_MODE_ALL 2 +#define EC_LOCK_MODE_INC 1 +#define EC_LOCK_MODE_ALL 2 -int32_t ec_lock_check(ec_fop_data_t * fop, ec_cbk_data_t ** cbk, - uintptr_t * mask) +int32_t +ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask) { - ec_t * ec = fop->xl->private; - ec_cbk_data_t * ans = NULL; - uintptr_t locked = 0, notlocked = 0; + ec_t *ec = fop->xl->private; + ec_cbk_data_t *ans = NULL; + ec_cbk_data_t *cbk = NULL; + uintptr_t locked = 0; + int32_t good = 0; + int32_t eagain = 0; + int32_t estale = 0; int32_t error = -1; + /* There are some errors that we'll handle in an special way while trying + * to acquire a lock. + * + * EAGAIN: If it's found during a parallel non-blocking lock request, we + * consider that there's contention on the inode, so we consider + * the acquisition a failure and try again with a sequential + * blocking lock request. This will ensure that we get a lock on + * as many bricks as possible (ignoring EAGAIN here would cause + * unnecessary triggers of self-healing). + * + * If it's found during a sequential blocking lock request, it's + * considered an error. Lock will only succeed if there are + * enough other bricks locked. + * + * ESTALE: This can appear during parallel or sequential lock request if + * the inode has just been unlinked. We consider this error is + * not recoverable, but we also don't consider it as fatal. So, + * if it happens during parallel lock, we won't attempt a + * sequential one unless there are EAGAIN errors on other + * bricks (and are enough to form a quorum), but if we reach + * quorum counting the ESTALE bricks, we consider the whole + * result of the operation is ESTALE instead of EIO. + */ + list_for_each_entry(ans, &fop->cbk_list, list) { - if (ans->op_ret >= 0) - { - if (locked != 0) - { + if (ans->op_ret >= 0) { + if (locked != 0) { error = EIO; } locked |= ans->mask; - *cbk = ans; - } - else if (ans->op_errno == EAGAIN) - { - notlocked |= ans->mask; + good = ans->count; + cbk = ans; + } else if (ans->op_errno == ESTALE) { + estale += ans->count; + } else if ((ans->op_errno == EAGAIN) && + (fop->uint32 != EC_LOCK_MODE_INC)) { + eagain += ans->count; } } - if (error == -1) - { - if (ec_bits_count(locked | notlocked) >= ec->fragments) - { - if (notlocked == 0) - { - fop->answer = *cbk; + if (error == -1) { + /* If we have enough quorum with succeeded and EAGAIN answers, we + * ignore for now any ESTALE answer. If there are EAGAIN answers, + * we retry with a sequential blocking lock request if needed. + * Otherwise we succeed. */ + if ((good + eagain) >= ec->fragments) { + if (eagain == 0) { + if (fop->answer == NULL) { + fop->answer = cbk; + } - ec_update_bad(fop, locked); + ec_update_good(fop, locked); error = 0; - } - else - { - if (fop->uint32 == EC_LOCK_MODE_NONE) - { - error = EAGAIN; - } - else - { - fop->uint32 = EC_LOCK_MODE_INC; + } else { + switch (fop->uint32) { + case EC_LOCK_MODE_NONE: + error = EAGAIN; + break; + case EC_LOCK_MODE_ALL: + fop->uint32 = EC_LOCK_MODE_INC; + break; + default: + /* This shouldn't happen because eagain cannot be > 0 + * when fop->uint32 is EC_LOCK_MODE_INC. */ + error = EIO; + break; } } - } - else - { - error = EIO; + } else { + /* We have been unable to find enough candidates that will be able + * to take the lock. If we have quorum on some answer, we return + * it. Otherwise we check if ESTALE answers allow us to reach + * quorum. If so, we return ESTALE. */ + if (fop->answer && fop->answer->op_ret < 0) { + error = fop->answer->op_errno; + } else if ((good + eagain + estale) >= ec->fragments) { + error = ESTALE; + } else { + error = EIO; + } } } @@ -81,47 +120,26 @@ int32_t ec_lock_check(ec_fop_data_t * fop, ec_cbk_data_t ** cbk, return error; } -uintptr_t ec_lock_handler(ec_fop_data_t * fop, ec_cbk_data_t * cbk, - ec_combine_f combine) +int32_t +ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - uintptr_t mask = 0; - - if (fop->uint32 == EC_LOCK_MODE_INC) - { - if (cbk->op_ret < 0) - { - if (cbk->op_errno != ENOTCONN) - { - mask = fop->mask & ~fop->remaining & ~cbk->mask; - fop->remaining = 0; - } - } - } - - ec_combine(cbk, combine); - - return mask; -} - -int32_t ec_lock_unlocked(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - dict_t * xdata) -{ - if (op_ret < 0) - { - gf_log(this->name, GF_LOG_WARNING, "Failed to unlock an entry/inode"); + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED, + "Failed to unlock an entry/inode"); } return 0; } -int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie, - xlator_t * this, int32_t op_ret, int32_t op_errno, - struct gf_flock * flock, dict_t * xdata) +int32_t +ec_lock_lk_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct gf_flock *flock, + dict_t *xdata) { - if (op_ret < 0) - { - gf_log(this->name, GF_LOG_WARNING, "Failed to unlock an lk"); + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_LK_UNLOCK_FAILED, + "Failed to unlock an lk"); } return 0; @@ -129,11 +147,12 @@ int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie, /* FOP: entrylk */ -int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -143,46 +162,36 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ENTRYLK, idx, op_ret, op_errno); - if (cbk != NULL) - { - uintptr_t mask; - - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - mask = ec_lock_handler(fop, cbk, NULL); - if (mask != 0) - { - ec_entrylk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked, - NULL, fop->str[0], &fop->loc[0], fop->str[1], - ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata); - } + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_entrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -192,20 +201,19 @@ void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->entrylk_type, fop->xdata); } -int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_entrylk(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: - if (fop->entrylk_cmd == ENTRYLK_LOCK) - { + if (fop->entrylk_cmd == ENTRYLK_LOCK) { fop->uint32 = EC_LOCK_MODE_ALL; fop->entrylk_cmd = ENTRYLK_LOCK_NB; } - /* Fall through */ + /* Fall through */ case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -213,46 +221,37 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if ((cbk == NULL) || - ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN))) - { + case -EC_STATE_PREPARE_ANSWER: + if (fop->entrylk_cmd != ENTRYLK_UNLOCK) { uintptr_t mask; - fop->error = ec_lock_check(fop, &cbk, &mask); - if (fop->error != 0) - { - if (mask != 0) - { - if (fop->id == GF_FOP_ENTRYLK) - { - ec_entrylk(fop->req_frame, fop->xl, mask, 1, - ec_lock_unlocked, NULL, fop->str[0], - &fop->loc[0], fop->str[1], - ENTRYLK_UNLOCK, fop->entrylk_type, - fop->xdata); - } - else - { - ec_fentrylk(fop->req_frame, fop->xl, mask, 1, + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + if (fop->id == GF_FOP_ENTRYLK) { + ec_entrylk( + fop->frame, fop->xl, mask, 1, ec_lock_unlocked, + NULL, fop->str[0], &fop->loc[0], fop->str[1], + ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata); + } else { + ec_fentrylk(fop->frame, fop->xl, mask, 1, ec_lock_unlocked, NULL, fop->str[0], fop->fd, fop->str[1], ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata); } } - if (fop->error > 0) - { - return EC_STATE_REPORT; - } + if (fop->error < 0) { + fop->error = 0; - fop->error = 0; + fop->entrylk_cmd = ENTRYLK_LOCK; - fop->entrylk_cmd = ENTRYLK_LOCK; + ec_dispatch_inc(fop); - ec_dispatch_inc(fop); - - return EC_STATE_PREPARE_ANSWER; + return EC_STATE_PREPARE_ANSWER; + } } + } else { + ec_fop_prepare_answer(fop, _gf_true); } return EC_STATE_REPORT; @@ -262,125 +261,104 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_ENTRYLK) - { - if (fop->cbks.entrylk != NULL) - { - fop->cbks.entrylk(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, cbk->xdata); + if (fop->id == GF_FOP_ENTRYLK) { + if (fop->cbks.entrylk != NULL) { + fop->cbks.entrylk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); } - } - else - { - if (fop->cbks.fentrylk != NULL) - { + } else { + if (fop->cbks.fentrylk != NULL) { fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->xdata); } } - ec_wait_winds(fop); - return EC_STATE_END; case -EC_STATE_INIT: case -EC_STATE_DISPATCH: - case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_ENTRYLK) - { - if (fop->cbks.entrylk != NULL) - { + if (fop->id == GF_FOP_ENTRYLK) { + if (fop->cbks.entrylk != NULL) { fop->cbks.entrylk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } - } - else - { - if (fop->cbks.fentrylk != NULL) - { + } else { + if (fop->cbks.fentrylk != NULL) { fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } } - ec_wait_winds(fop); - return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_entrylk_cbk_t func, void * data, - const char * volume, loc_t * loc, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata) +void +ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_entrylk_cbk_t func, void *data, + const char *volume, loc_t *loc, const char *basename, + entrylk_cmd cmd, entrylk_type type, dict_t *xdata) { - ec_cbk_t callback = { .entrylk = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.entrylk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(ENTRYLK) %p", frame); + gf_msg_trace("ec", 0, "EC(ENTRYLK) %p", frame); - VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_entrylk, ec_manager_entrylk, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target, + fop_flags, ec_wind_entrylk, ec_manager_entrylk, + callback, data); + if (fop == NULL) { goto out; } fop->entrylk_cmd = cmd; fop->entrylk_type = type; - if (volume != NULL) - { + if (volume != NULL) { fop->str[0] = gf_strdup(volume); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (basename != NULL) - { + if (basename != NULL) { fop->str[1] = gf_strdup(basename); - if (fop->str[1] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[1] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -389,23 +367,21 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: fentrylk */ -int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -415,46 +391,36 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FENTRYLK, idx, op_ret, op_errno); - if (cbk != NULL) - { - uintptr_t mask; - - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - mask = ec_lock_handler(fop, cbk, NULL); - if (mask != 0) - { - ec_fentrylk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked, - NULL, fop->str[0], fop->fd, fop->str[1], - ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata); - } + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_fentrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -464,27 +430,25 @@ void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->entrylk_type, fop->xdata); } -void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_fentrylk_cbk_t func, void * data, - const char * volume, fd_t * fd, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata) +void +ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data, + const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) { - ec_cbk_t callback = { .fentrylk = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.fentrylk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FENTRYLK) %p", frame); + gf_msg_trace("ec", 0, "EC(FENTRYLK) %p", frame); - VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_fentrylk, ec_manager_entrylk, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, 0, target, + fop_flags, ec_wind_fentrylk, ec_manager_entrylk, + callback, data); + if (fop == NULL) { goto out; } @@ -493,44 +457,40 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->entrylk_cmd = cmd; fop->entrylk_type = type; - if (volume != NULL) - { + if (volume != NULL) { fop->str[0] = gf_strdup(volume); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (basename != NULL) - { + if (basename != NULL) { fop->str[1] = gf_strdup(basename); - if (fop->str[1] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[1] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -539,23 +499,21 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: inodelk */ -int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -565,56 +523,36 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_INODELK, idx, op_ret, op_errno); - if (cbk != NULL) - { - uintptr_t mask; - - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - mask = ec_lock_handler(fop, cbk, NULL); - if (mask != 0) - { - ec_t * ec = fop->xl->private; - struct gf_flock flock; - - flock.l_type = F_UNLCK; - flock.l_whence = fop->flock.l_whence; - flock.l_start = fop->flock.l_start * ec->fragments; - flock.l_len = fop->flock.l_len * ec->fragments; - flock.l_pid = 0; - flock.l_owner.len = 0; - - ec_inodelk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked, - NULL, fop->str[0], &fop->loc[0], F_SETLK, &flock, - fop->xdata); - } + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_inodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -624,24 +562,22 @@ void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->xdata); } -int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_inodelk(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: - fop->flock.l_len += ec_adjust_offset(fop->xl->private, - &fop->flock.l_start, 1); - fop->flock.l_len = ec_adjust_size(fop->xl->private, - fop->flock.l_len, 1); - if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) - { + fop->flock.l_len += ec_adjust_offset_down( + fop->xl->private, &fop->flock.l_start, _gf_true); + ec_adjust_offset_up(fop->xl->private, &fop->flock.l_len, _gf_true); + if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) { fop->uint32 = EC_LOCK_MODE_ALL; fop->int32 = F_SETLK; } - /* Fall through */ + /* Fall through */ case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -649,18 +585,14 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if ((cbk == NULL) || - ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN))) - { + case -EC_STATE_PREPARE_ANSWER: + if (fop->flock.l_type != F_UNLCK) { uintptr_t mask; - fop->error = ec_lock_check(fop, &cbk, &mask); - if (fop->error != 0) - { - if (mask != 0) - { - ec_t * ec = fop->xl->private; + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + ec_t *ec = fop->xl->private; struct gf_flock flock; flock.l_type = F_UNLCK; @@ -670,33 +602,31 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state) flock.l_pid = 0; flock.l_owner.len = 0; - if (fop->id == GF_FOP_INODELK) - { - ec_inodelk(fop->req_frame, fop->xl, mask, 1, + if (fop->id == GF_FOP_INODELK) { + ec_inodelk(fop->frame, fop->xl, + &fop->frame->root->lk_owner, mask, 1, ec_lock_unlocked, NULL, fop->str[0], &fop->loc[0], F_SETLK, &flock, fop->xdata); - } - else - { - ec_finodelk(fop->req_frame, fop->xl, mask, 1, + } else { + ec_finodelk(fop->frame, fop->xl, + &fop->frame->root->lk_owner, mask, 1, ec_lock_unlocked, NULL, fop->str[0], fop->fd, F_SETLK, &flock, fop->xdata); } } - if (fop->error > 0) - { - return EC_STATE_REPORT; - } + if (fop->error < 0) { + fop->error = 0; - fop->error = 0; + fop->int32 = F_SETLKW; - fop->int32 = F_SETLKW; + ec_dispatch_inc(fop); - ec_dispatch_inc(fop); - - return EC_STATE_PREPARE_ANSWER; + return EC_STATE_PREPARE_ANSWER; + } } + } else { + ec_fop_prepare_answer(fop, _gf_true); } return EC_STATE_REPORT; @@ -706,128 +636,108 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->id == GF_FOP_INODELK) - { - if (fop->cbks.inodelk != NULL) - { - fop->cbks.inodelk(fop->req_frame, fop, fop->xl, - cbk->op_ret, cbk->op_errno, cbk->xdata); + if (fop->id == GF_FOP_INODELK) { + if (fop->cbks.inodelk != NULL) { + fop->cbks.inodelk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); } - } - else - { - if (fop->cbks.finodelk != NULL) - { + } else { + if (fop->cbks.finodelk != NULL) { fop->cbks.finodelk(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, cbk->xdata); } } - ec_wait_winds(fop); - return EC_STATE_END; case -EC_STATE_INIT: case -EC_STATE_DISPATCH: - case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->id == GF_FOP_INODELK) - { - if (fop->cbks.inodelk != NULL) - { + if (fop->id == GF_FOP_INODELK) { + if (fop->cbks.inodelk != NULL) { fop->cbks.inodelk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } - } - else - { - if (fop->cbks.finodelk != NULL) - { + } else { + if (fop->cbks.finodelk != NULL) { fop->cbks.finodelk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); } } - ec_wait_winds(fop); - return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_inodelk_cbk_t func, void * data, - const char * volume, loc_t * loc, int32_t cmd, - struct gf_flock * flock, dict_t * xdata) +void +ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func, + void *data, const char *volume, loc_t *loc, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) { - ec_cbk_t callback = { .inodelk = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.inodelk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(INODELK) %p", frame); + gf_msg_trace("ec", 0, "EC(INODELK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, - EC_FLAG_UPDATE_LOC_INODE, target, minimum, - ec_wind_inodelk, ec_manager_inodelk, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target, + fop_flags, ec_wind_inodelk, ec_manager_inodelk, + callback, data); + if (fop == NULL) { goto out; } fop->int32 = cmd; + ec_owner_copy(fop->frame, owner); - if (volume != NULL) - { + if (volume != NULL) { fop->str[0] = gf_strdup(volume); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (loc != NULL) - { - if (loc_copy(&fop->loc[0], loc) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location."); + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); goto out; } } - if (flock != NULL) - { + if (flock != NULL) { fop->flock.l_type = flock->l_type; fop->flock.l_whence = flock->l_whence; fop->flock.l_start = flock->l_start; fop->flock.l_len = flock->l_len; fop->flock.l_pid = flock->l_pid; fop->flock.l_owner.len = flock->l_owner.len; - if (flock->l_owner.len > 0) - { + if (flock->l_owner.len > 0) { memcpy(fop->flock.l_owner.data, flock->l_owner.data, flock->l_owner.len); } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -836,23 +746,21 @@ void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: finodelk */ -int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, dict_t * xdata) +int32_t +ec_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -862,56 +770,36 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FINODELK, idx, op_ret, op_errno); - if (cbk != NULL) - { - uintptr_t mask; - - if (xdata != NULL) - { + if (cbk != NULL) { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - mask = ec_lock_handler(fop, cbk, NULL); - if (mask != 0) - { - ec_t * ec = fop->xl->private; - struct gf_flock flock; - - flock.l_type = F_UNLCK; - flock.l_whence = fop->flock.l_whence; - flock.l_start = fop->flock.l_start * ec->fragments; - flock.l_len = fop->flock.l_len * ec->fragments; - flock.l_pid = 0; - flock.l_owner.len = 0; - - ec_finodelk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked, - NULL, fop->str[0], fop->fd, F_SETLK, &flock, - fop->xdata); - } + ec_combine(cbk, NULL); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_finodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -921,76 +809,71 @@ void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->xdata); } -void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_finodelk_cbk_t func, void * data, - const char * volume, fd_t * fd, int32_t cmd, - struct gf_flock * flock, dict_t * xdata) +void +ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func, + void *data, const char *volume, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) { - ec_cbk_t callback = { .finodelk = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.finodelk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(FINODELK) %p", frame); + gf_msg_trace("ec", 0, "EC(FINODELK) %p", frame); VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, - EC_FLAG_UPDATE_FD_INODE, target, minimum, - ec_wind_finodelk, ec_manager_inodelk, callback, - data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, 0, target, + fop_flags, ec_wind_finodelk, ec_manager_inodelk, + callback, data); + if (fop == NULL) { goto out; } fop->use_fd = 1; fop->int32 = cmd; + ec_owner_copy(fop->frame, owner); - if (volume != NULL) - { + if (volume != NULL) { fop->str[0] = gf_strdup(volume); - if (fop->str[0] == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string."); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); goto out; } } - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (flock != NULL) - { + if (flock != NULL) { fop->flock.l_type = flock->l_type; fop->flock.l_whence = flock->l_whence; fop->flock.l_start = flock->l_start; fop->flock.l_len = flock->l_len; fop->flock.l_pid = flock->l_pid; fop->flock.l_owner.len = flock->l_owner.len; - if (flock->l_owner.len > 0) - { + if (flock->l_owner.len > 0) { memcpy(fop->flock.l_owner.data, flock->l_owner.data, flock->l_owner.len); } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -999,25 +882,22 @@ void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL); + } else { + func(frame, NULL, this, -1, error, NULL); } } /* FOP: lk */ -int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst, - ec_cbk_data_t * src) +int32_t +ec_combine_lk(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) { - if (!ec_flock_compare(&dst->flock, &src->flock)) - { - gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching lock in " - "answers of 'GF_FOP_LK'"); + if (!ec_flock_compare(&dst->flock, &src->flock)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_LOCK_MISMATCH, + "Mismatching lock in " + "answers of 'GF_FOP_LK'"); return 0; } @@ -1025,12 +905,12 @@ int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst, return 1; } -int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, - int32_t op_ret, int32_t op_errno, struct gf_flock * flock, - dict_t * xdata) +int32_t +ec_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct gf_flock *flock, dict_t *xdata) { - ec_fop_data_t * fop = NULL; - ec_cbk_data_t * cbk = NULL; + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; int32_t idx = (int32_t)(uintptr_t)cookie; VALIDATE_OR_GOTO(this, out); @@ -1040,72 +920,50 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this, fop = frame->local; - ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, - frame, op_ret, op_errno); + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LK, idx, op_ret, op_errno); - if (cbk != NULL) - { - uintptr_t mask; - - if (op_ret >= 0) - { - if (flock != NULL) - { + if (cbk != NULL) { + if (op_ret >= 0) { + if (flock != NULL) { cbk->flock.l_type = flock->l_type; cbk->flock.l_whence = flock->l_whence; cbk->flock.l_start = flock->l_start; cbk->flock.l_len = flock->l_len; cbk->flock.l_pid = flock->l_pid; cbk->flock.l_owner.len = flock->l_owner.len; - if (flock->l_owner.len > 0) - { + if (flock->l_owner.len > 0) { memcpy(cbk->flock.l_owner.data, flock->l_owner.data, flock->l_owner.len); } } } - if (xdata != NULL) - { + if (xdata != NULL) { cbk->xdata = dict_ref(xdata); - if (cbk->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } } - mask = ec_lock_handler(fop, cbk, ec_combine_lk); - if (mask != 0) - { - ec_t * ec = fop->xl->private; - struct gf_flock flock; - - flock.l_type = F_UNLCK; - flock.l_whence = fop->flock.l_whence; - flock.l_start = fop->flock.l_start * ec->fragments; - flock.l_len = fop->flock.l_len * ec->fragments; - flock.l_pid = 0; - flock.l_owner.len = 0; - - ec_lk(fop->req_frame, fop->xl, mask, 1, ec_lock_lk_unlocked, NULL, - fop->fd, F_SETLK, &flock, fop->xdata); - } + ec_combine(cbk, ec_combine_lk); } out: - if (fop != NULL) - { + if (fop != NULL) { ec_complete(fop); } return 0; } -void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) +void +ec_wind_lk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) { ec_trace("WIND", fop, "idx=%d", idx); @@ -1114,24 +972,19 @@ void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx) fop->int32, &fop->flock, fop->xdata); } -int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state) +int32_t +ec_manager_lk(ec_fop_data_t *fop, int32_t state) { - ec_cbk_data_t * cbk; + ec_cbk_data_t *cbk; - switch (state) - { + switch (state) { case EC_STATE_INIT: - fop->flock.l_len += ec_adjust_offset(fop->xl->private, - &fop->flock.l_start, 1); - fop->flock.l_len = ec_adjust_size(fop->xl->private, - fop->flock.l_len, 1); - if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) - { + if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) { fop->uint32 = EC_LOCK_MODE_ALL; fop->int32 = F_SETLK; } - /* Fall through */ + /* Fall through */ case EC_STATE_DISPATCH: ec_dispatch_all(fop); @@ -1139,44 +992,38 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state) return EC_STATE_PREPARE_ANSWER; case EC_STATE_PREPARE_ANSWER: - cbk = fop->answer; - if ((cbk == NULL) || - ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN))) - { + case -EC_STATE_PREPARE_ANSWER: + if (fop->flock.l_type != F_UNLCK) { uintptr_t mask; - fop->error = ec_lock_check(fop, &cbk, &mask); - if (fop->error != 0) - { - if (mask != 0) - { - ec_t * ec = fop->xl->private; - struct gf_flock flock; + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + struct gf_flock flock = {0}; flock.l_type = F_UNLCK; flock.l_whence = fop->flock.l_whence; - flock.l_start = fop->flock.l_start * ec->fragments; - flock.l_len = fop->flock.l_len * ec->fragments; - flock.l_pid = 0; - flock.l_owner.len = 0; + flock.l_start = fop->flock.l_start; + flock.l_len = fop->flock.l_len; + flock.l_pid = fop->flock.l_pid; + lk_owner_copy(&flock.l_owner, &fop->flock.l_owner); - ec_lk(fop->req_frame, fop->xl, mask, 1, - ec_lock_lk_unlocked, NULL, fop->fd, F_SETLK, - &flock, fop->xdata); - } - if (fop->error > 0) - { - return EC_STATE_REPORT; + ec_lk(fop->frame, fop->xl, mask, 1, ec_lock_lk_unlocked, + NULL, fop->fd, F_SETLK, &flock, fop->xdata); } - fop->error = 0; + if (fop->error < 0) { + fop->error = 0; - fop->int32 = F_SETLKW; + fop->int32 = F_SETLKW; - ec_dispatch_inc(fop); + ec_dispatch_inc(fop); - return EC_STATE_PREPARE_ANSWER; + return EC_STATE_PREPARE_ANSWER; + } } + } else { + ec_fop_prepare_answer(fop, _gf_true); } return EC_STATE_REPORT; @@ -1186,59 +1033,50 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state) GF_ASSERT(cbk != NULL); - if (fop->cbks.lk != NULL) - { + if (fop->cbks.lk != NULL) { fop->cbks.lk(fop->req_frame, fop, fop->xl, cbk->op_ret, cbk->op_errno, &cbk->flock, cbk->xdata); } - ec_wait_winds(fop); - return EC_STATE_END; case -EC_STATE_INIT: case -EC_STATE_DISPATCH: - case -EC_STATE_PREPARE_ANSWER: case -EC_STATE_REPORT: GF_ASSERT(fop->error != 0); - if (fop->cbks.lk != NULL) - { - fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, - NULL, NULL); + if (fop->cbks.lk != NULL) { + fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, + NULL); } - ec_wait_winds(fop); - return EC_STATE_END; default: - gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s", - state, ec_fop_name(fop->id)); + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); return EC_STATE_END; } } -void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target, - int32_t minimum, fop_lk_cbk_t func, void * data, fd_t * fd, - int32_t cmd, struct gf_flock * flock, dict_t * xdata) +void +ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags, + fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) { - ec_cbk_t callback = { .lk = func }; - ec_fop_data_t * fop = NULL; - int32_t error = EIO; + ec_cbk_t callback = {.lk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; - gf_log("ec", GF_LOG_TRACE, "EC(LK) %p", frame); + gf_msg_trace("ec", 0, "EC(LK) %p", frame); - VALIDATE_OR_GOTO(this, out); GF_VALIDATE_OR_GOTO(this->name, frame, out); GF_VALIDATE_OR_GOTO(this->name, this->private, out); - fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, EC_FLAG_UPDATE_FD_INODE, - target, minimum, ec_wind_lk, ec_manager_lk, - callback, data); - if (fop == NULL) - { + fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, fop_flags, + ec_wind_lk, ec_manager_lk, callback, data); + if (fop == NULL) { goto out; } @@ -1246,38 +1084,34 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target, fop->int32 = cmd; - if (fd != NULL) - { + if (fd != NULL) { fop->fd = fd_ref(fd); - if (fop->fd == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "file descriptor."); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); goto out; } } - if (flock != NULL) - { + if (flock != NULL) { fop->flock.l_type = flock->l_type; fop->flock.l_whence = flock->l_whence; fop->flock.l_start = flock->l_start; fop->flock.l_len = flock->l_len; fop->flock.l_pid = flock->l_pid; fop->flock.l_owner.len = flock->l_owner.len; - if (flock->l_owner.len > 0) - { + if (flock->l_owner.len > 0) { memcpy(fop->flock.l_owner.data, flock->l_owner.data, flock->l_owner.len); } } - if (xdata != NULL) - { + if (xdata != NULL) { fop->xdata = dict_ref(xdata); - if (fop->xdata == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to reference a " - "dictionary."); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); goto out; } @@ -1286,12 +1120,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target, error = 0; out: - if (fop != NULL) - { + if (fop != NULL) { ec_manager(fop, error); - } - else - { - func(frame, NULL, this, -1, EIO, NULL, NULL); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); } } diff --git a/xlators/cluster/ec/src/ec-mem-types.h b/xlators/cluster/ec/src/ec-mem-types.h index 8a66fb912a5..3252c4c1c58 100644 --- a/xlators/cluster/ec/src/ec-mem-types.h +++ b/xlators/cluster/ec/src/ec-mem-types.h @@ -11,15 +11,19 @@ #ifndef __EC_MEM_TYPES_H__ #define __EC_MEM_TYPES_H__ -#include "mem-types.h" +#include <glusterfs/mem-types.h> -enum gf_ec_mem_types_ -{ +enum gf_ec_mem_types_ { ec_mt_ec_t = gf_common_mt_end + 1, ec_mt_xlator_t, ec_mt_ec_inode_t, ec_mt_ec_fd_t, - ec_mt_ec_heal_t, + ec_mt_subvol_healer_t, + ec_mt_ec_gf_t, + ec_mt_ec_code_t, + ec_mt_ec_code_builder_t, + ec_mt_ec_matrix_t, + ec_mt_ec_stripe_t, ec_mt_end }; diff --git a/xlators/cluster/ec/src/ec-messages.h b/xlators/cluster/ec/src/ec-messages.h new file mode 100644 index 00000000000..72e98f11286 --- /dev/null +++ b/xlators/cluster/ec/src/ec-messages.h @@ -0,0 +1,61 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _EC_MESSAGES_H_ +#define _EC_MESSAGES_H_ + +#include <glusterfs/glfs-message-id.h> + +/* To add new message IDs, append new identifiers at the end of the list. + * + * Never remove a message ID. If it's not used anymore, you can rename it or + * leave it as it is, but not delete it. This is to prevent reutilization of + * IDs by other messages. + * + * The component name must match one of the entries defined in + * glfs-message-id.h. + */ + +GLFS_MSGID(EC, EC_MSG_INVALID_CONFIG, EC_MSG_HEAL_FAIL, + EC_MSG_DICT_COMBINE_FAIL, EC_MSG_STIME_COMBINE_FAIL, + EC_MSG_INVALID_DICT_NUMS, EC_MSG_IATT_COMBINE_FAIL, + EC_MSG_INVALID_FORMAT, EC_MSG_DICT_GET_FAILED, + EC_MSG_UNHANDLED_STATE, EC_MSG_FILE_DESC_REF_FAIL, + EC_MSG_LOC_COPY_FAIL, EC_MSG_BUF_REF_FAIL, EC_MSG_DICT_REF_FAIL, + EC_MSG_LK_UNLOCK_FAILED, EC_MSG_UNLOCK_FAILED, + EC_MSG_LOC_PARENT_INODE_MISSING, EC_MSG_INVALID_LOC_NAME, + EC_MSG_NO_MEMORY, EC_MSG_GFID_MISMATCH, EC_MSG_UNSUPPORTED_VERSION, + EC_MSG_FD_CREATE_FAIL, EC_MSG_READDIRP_REQ_PREP_FAIL, + EC_MSG_LOOKUP_REQ_PREP_FAIL, EC_MSG_INODE_REF_FAIL, + EC_MSG_LOOKUP_READAHEAD_FAIL, EC_MSG_FRAME_MISMATCH, + EC_MSG_XLATOR_MISMATCH, EC_MSG_VECTOR_MISMATCH, EC_MSG_IATT_MISMATCH, + EC_MSG_FD_MISMATCH, EC_MSG_DICT_MISMATCH, EC_MSG_INDEX_DIR_GET_FAIL, + EC_MSG_PREOP_LOCK_FAILED, EC_MSG_CHILDS_INSUFFICIENT, + EC_MSG_OP_EXEC_UNAVAIL, EC_MSG_UNLOCK_DELAY_FAILED, + EC_MSG_SIZE_VERS_UPDATE_FAIL, EC_MSG_INVALID_REQUEST, + EC_MSG_INVALID_LOCK_TYPE, EC_MSG_SIZE_VERS_GET_FAIL, + EC_MSG_FILE_SIZE_GET_FAIL, EC_MSG_FOP_MISMATCH, + EC_MSG_SUBVOL_ID_DICT_SET_FAIL, EC_MSG_SUBVOL_BUILD_FAIL, + EC_MSG_XLATOR_INIT_FAIL, EC_MSG_NO_PARENTS, EC_MSG_TIMER_CREATE_FAIL, + EC_MSG_TOO_MANY_SUBVOLS, EC_MSG_DATA_UNAVAILABLE, + EC_MSG_INODE_REMOVE_FAIL, EC_MSG_INVALID_REDUNDANCY, + EC_MSG_XLATOR_PARSE_OPT_FAIL, EC_MSG_OP_FAIL_ON_SUBVOLS, + EC_MSG_INVALID_INODE, EC_MSG_LOCK_MISMATCH, EC_MSG_XDATA_MISMATCH, + EC_MSG_HEALING_INFO, EC_MSG_HEAL_SUCCESS, EC_MSG_FULL_SWEEP_START, + EC_MSG_FULL_SWEEP_STOP, EC_MSG_INVALID_FOP, EC_MSG_EC_UP, + EC_MSG_EC_DOWN, EC_MSG_SIZE_XATTR_GET_FAIL, + EC_MSG_VER_XATTR_GET_FAIL, EC_MSG_CONFIG_XATTR_GET_FAIL, + EC_MSG_CONFIG_XATTR_INVALID, EC_MSG_EXTENSION, EC_MSG_EXTENSION_NONE, + EC_MSG_EXTENSION_UNKNOWN, EC_MSG_EXTENSION_UNSUPPORTED, + EC_MSG_EXTENSION_FAILED, EC_MSG_NO_GF, EC_MSG_MATRIX_FAILED, + EC_MSG_DYN_CREATE_FAILED, EC_MSG_DYN_CODEGEN_FAILED, + EC_MSG_THREAD_CLEANUP_FAILED, EC_MSG_FD_BAD); + +#endif /* !_EC_MESSAGES_H_ */ diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c index faab0115cdd..55faed0b193 100644 --- a/xlators/cluster/ec/src/ec-method.c +++ b/xlators/cluster/ec/src/ec-method.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser @@ -11,149 +11,423 @@ #include <string.h> #include <inttypes.h> -#include "ec-gf.h" +#include "ec-types.h" +#include "ec-mem-types.h" +#include "ec-galois.h" +#include "ec-code.h" #include "ec-method.h" +#include "ec-helpers.h" -static uint32_t GfPow[EC_GF_SIZE << 1]; -static uint32_t GfLog[EC_GF_SIZE << 1]; +static void +ec_method_matrix_normal(ec_gf_t *gf, uint32_t *matrix, uint32_t columns, + uint32_t *values, uint32_t count) +{ + uint32_t i, j, v, tmp; + + columns--; + for (i = 0; i < count; i++) { + v = *values++; + *matrix++ = tmp = ec_gf_exp(gf, v, columns); + for (j = 0; j < columns; j++) { + *matrix++ = tmp = ec_gf_div(gf, tmp, v); + } + } +} + +static void +ec_method_matrix_inverse(ec_gf_t *gf, uint32_t *matrix, uint32_t *values, + uint32_t count) +{ + uint32_t a[count]; + uint32_t i, j, p, last, tmp; + + last = count - 1; + for (i = 0; i < last; i++) { + a[i] = 1; + } + a[i] = values[0]; + for (i = last; i > 0; i--) { + for (j = i - 1; j < last; j++) { + a[j] = a[j + 1] ^ ec_gf_mul(gf, values[i], a[j]); + } + a[j] = ec_gf_mul(gf, values[i], a[j]); + } + for (i = 0; i < count; i++) { + p = a[0]; + matrix += count; + *matrix = tmp = p ^ values[i]; + for (j = 1; j < last; j++) { + matrix += count; + *matrix = tmp = a[j] ^ ec_gf_mul(gf, values[i], tmp); + p = tmp ^ ec_gf_mul(gf, values[i], p); + } + for (j = 0; j < last; j++) { + *matrix = ec_gf_div(gf, *matrix, p); + matrix -= count; + } + *matrix = ec_gf_div(gf, 1, p); + matrix++; + } +} -void ec_method_initialize(void) +static void +ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix, + uintptr_t mask, uint32_t *rows, gf_boolean_t inverse) { uint32_t i; - GfPow[0] = 1; - GfLog[0] = EC_GF_SIZE; - for (i = 1; i < EC_GF_SIZE; i++) - { - GfPow[i] = GfPow[i - 1] << 1; - if (GfPow[i] >= EC_GF_SIZE) - { - GfPow[i] ^= EC_GF_MOD; + matrix->refs = 1; + matrix->mask = mask; + matrix->code = list->code; + matrix->columns = list->columns; + INIT_LIST_HEAD(&matrix->lru); + + if (inverse) { + matrix->rows = list->columns; + ec_method_matrix_inverse(matrix->code->gf, matrix->values, rows, + matrix->rows); + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].values = matrix->values + i * matrix->columns; + matrix->row_data[i].func.interleaved = ec_code_build_interleaved( + matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values, + matrix->columns); + } + } else { + matrix->rows = list->rows; + ec_method_matrix_normal(matrix->code->gf, matrix->values, + matrix->columns, rows, matrix->rows); + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].values = matrix->values + i * matrix->columns; + matrix->row_data[i].func.linear = ec_code_build_linear( + matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values, + matrix->columns); } - GfPow[i + EC_GF_SIZE - 1] = GfPow[i]; - GfLog[GfPow[i] + EC_GF_SIZE - 1] = GfLog[GfPow[i]] = i; } } -static uint32_t ec_method_mul(uint32_t a, uint32_t b) +static void +ec_method_matrix_release(ec_matrix_t *matrix) { - if (a && b) - { - return GfPow[GfLog[a] + GfLog[b]]; + uint32_t i; + + for (i = 0; i < matrix->rows; i++) { + if (matrix->row_data[i].func.linear != NULL) { + ec_code_release(matrix->code, &matrix->row_data[i].func); + matrix->row_data[i].func.linear = NULL; + } } - return 0; } -static uint32_t ec_method_div(uint32_t a, uint32_t b) +static void +ec_method_matrix_destroy(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + list_del_init(&matrix->lru); + + ec_method_matrix_release(matrix); + + mem_put(matrix); + + list->count--; +} + +static void +ec_method_matrix_unref(ec_matrix_list_t *list, ec_matrix_t *matrix) { - if (b) - { - if (a) - { - return GfPow[EC_GF_SIZE - 1 + GfLog[a] - GfLog[b]]; + if (--matrix->refs == 0) { + list_add_tail(&matrix->lru, &list->lru); + if (list->count > list->max) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + ec_method_matrix_destroy(list, matrix); } - return 0; } - return EC_GF_SIZE; } -size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row, - uint8_t * in, uint8_t * out) +static ec_matrix_t * +ec_method_matrix_lookup(ec_matrix_list_t *list, uintptr_t mask, uint32_t *pos) { - uint32_t i, j; + ec_matrix_t *matrix; + uint32_t i, j, k; - size /= EC_METHOD_CHUNK_SIZE * columns; - row++; - for (j = 0; j < size; j++) - { - ec_gf_muladd[0](out, in, EC_METHOD_WIDTH); - in += EC_METHOD_CHUNK_SIZE; - for (i = 1; i < columns; i++) - { - ec_gf_muladd[row](out, in, EC_METHOD_WIDTH); - in += EC_METHOD_CHUNK_SIZE; + i = 0; + j = list->count; + while (i < j) { + k = (i + j) >> 1; + matrix = list->objects[k]; + if (matrix->mask == mask) { + *pos = k; + return matrix; + } + if (matrix->mask < mask) { + i = k + 1; + } else { + j = k; } - out += EC_METHOD_CHUNK_SIZE; } + *pos = i; - return size * EC_METHOD_CHUNK_SIZE; + return NULL; } -size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows, - uint8_t ** in, uint8_t * out) +static void +ec_method_matrix_remove(ec_matrix_list_t *list, uintptr_t mask) { - uint32_t i, j, k, off, last, value; - uint32_t f; - uint8_t inv[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS + 1]; - uint8_t mtx[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS]; - uint8_t dummy[EC_METHOD_CHUNK_SIZE]; + uint32_t pos; - size /= EC_METHOD_CHUNK_SIZE; + if (ec_method_matrix_lookup(list, mask, &pos) != NULL) { + list->count--; + if (pos < list->count) { + memmove(list->objects + pos, list->objects + pos + 1, + sizeof(ec_matrix_t *) * (list->count - pos)); + } + } +} + +static void +ec_method_matrix_insert(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + uint32_t pos; - memset(inv, 0, sizeof(inv)); - memset(mtx, 0, sizeof(mtx)); - memset(dummy, 0, sizeof(dummy)); - for (i = 0; i < columns; i++) - { - inv[i][i] = 1; - inv[i][columns] = 1; + GF_ASSERT(ec_method_matrix_lookup(list, matrix->mask, &pos) == NULL); + + if (pos < list->count) { + memmove(list->objects + pos + 1, list->objects + pos, + sizeof(ec_matrix_t *) * (list->count - pos)); } - for (i = 0; i < columns; i++) - { - mtx[i][columns - 1] = 1; - for (j = columns - 1; j > 0; j--) - { - mtx[i][j - 1] = ec_method_mul(mtx[i][j], rows[i] + 1); - } + list->objects[pos] = matrix; + list->count++; +} + +static ec_matrix_t * +ec_method_matrix_get(ec_matrix_list_t *list, uintptr_t mask, uint32_t *rows) +{ + ec_matrix_t *matrix; + uint32_t pos; + + LOCK(&list->lock); + + matrix = ec_method_matrix_lookup(list, mask, &pos); + if (matrix != NULL) { + list_del_init(&matrix->lru); + matrix->refs++; + + goto out; } - for (i = 0; i < columns; i++) - { - f = mtx[i][i]; - for (j = 0; j < columns; j++) - { - mtx[i][j] = ec_method_div(mtx[i][j], f); - inv[i][j] = ec_method_div(inv[i][j], f); + if ((list->count >= list->max) && !list_empty(&list->lru)) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + list_del_init(&matrix->lru); + + ec_method_matrix_remove(list, matrix->mask); + + ec_method_matrix_release(matrix); + } else { + matrix = mem_get0(list->pool); + if (matrix == NULL) { + matrix = EC_ERR(ENOMEM); + goto out; } - for (j = 0; j < columns; j++) - { - if (i != j) - { - f = mtx[j][i]; - for (k = 0; k < columns; k++) - { - mtx[j][k] ^= ec_method_mul(mtx[i][k], f); - inv[j][k] ^= ec_method_mul(inv[i][k], f); - } - } + matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->columns); + } + + ec_method_matrix_init(list, matrix, mask, rows, _gf_true); + + if (list->count < list->max) { + ec_method_matrix_insert(list, matrix); + } else { + matrix->mask = 0; + } + +out: + UNLOCK(&list->lock); + + return matrix; +} + +static void +ec_method_matrix_put(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + LOCK(&list->lock); + + ec_method_matrix_unref(list, matrix); + + UNLOCK(&list->lock); +} + +static int32_t +ec_method_setup(xlator_t *xl, ec_matrix_list_t *list, const char *gen) +{ + ec_matrix_t *matrix; + uint32_t values[list->rows]; + uint32_t i; + int32_t err; + + matrix = GF_MALLOC(sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->rows + + sizeof(uint32_t) * list->columns * list->rows, + ec_mt_ec_matrix_t); + if (matrix == NULL) { + err = -ENOMEM; + goto failed; + } + memset(matrix, 0, sizeof(ec_matrix_t)); + matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->rows); + + list->code = ec_code_create(list->gf, ec_code_detect(xl, gen)); + if (EC_IS_ERR(list->code)) { + err = EC_GET_ERR(list->code); + list->code = NULL; + goto failed_matrix; + } + + for (i = 0; i < list->rows; i++) { + values[i] = i + 1; + } + ec_method_matrix_init(list, matrix, 0, values, _gf_false); + + list->encode = matrix; + + return 0; + +failed_matrix: + GF_FREE(matrix); +failed: + return err; +} + +int32_t +ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns, + uint32_t rows, uint32_t max, const char *gen) +{ + list->columns = columns; + list->rows = rows; + list->max = max; + list->stripe = EC_METHOD_CHUNK_SIZE * list->columns; + INIT_LIST_HEAD(&list->lru); + int32_t err; + + list->pool = mem_pool_new_fn(xl->ctx, + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * columns + + sizeof(uint32_t) * columns * columns, + 128, "ec_matrix_t"); + if (list->pool == NULL) { + err = -ENOMEM; + goto failed; + } + + list->objects = GF_MALLOC(sizeof(ec_matrix_t *) * max, ec_mt_ec_matrix_t); + if (list->objects == NULL) { + err = -ENOMEM; + goto failed_pool; + } + + list->gf = ec_gf_prepare(EC_GF_BITS, EC_GF_MOD); + if (EC_IS_ERR(list->gf)) { + err = EC_GET_ERR(list->gf); + goto failed_objects; + } + + err = ec_method_setup(xl, list, gen); + if (err != 0) { + goto failed_gf; + } + + LOCK_INIT(&list->lock); + + return 0; + +failed_gf: + ec_gf_destroy(list->gf); +failed_objects: + GF_FREE(list->objects); +failed_pool: + mem_pool_destroy(list->pool); +failed: + list->pool = NULL; + list->objects = NULL; + list->gf = NULL; + + return err; +} + +void +ec_method_fini(ec_matrix_list_t *list) +{ + ec_matrix_t *matrix; + + if (list->encode == NULL) { + return; + } + + while (!list_empty(&list->lru)) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + ec_method_matrix_destroy(list, matrix); + } + + GF_ASSERT(list->count == 0); + + if (list->pool) /*Init was successful*/ + LOCK_DESTROY(&list->lock); + + ec_method_matrix_release(list->encode); + GF_FREE(list->encode); + + ec_code_destroy(list->code); + ec_gf_destroy(list->gf); + GF_FREE(list->objects); + + if (list->pool) + mem_pool_destroy(list->pool); +} + +int32_t +ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen) +{ + /* TODO: Allow changing code generator */ + + return 0; +} + +void +ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out) +{ + ec_matrix_t *matrix; + uint64_t pos; + uint32_t i; + + matrix = list->encode; + for (pos = 0; pos < size; pos += list->stripe) { + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].func.linear( + out[i], in, pos, matrix->row_data[i].values, list->columns); + out[i] += EC_METHOD_CHUNK_SIZE; } } - off = 0; - for (f = 0; f < size; f++) - { - for (i = 0; i < columns; i++) - { - last = 0; - j = 0; - do - { - while (inv[i][j] == 0) - { - j++; - } - if (j < columns) - { - value = ec_method_div(last, inv[i][j]); - last = inv[i][j]; - ec_gf_muladd[value](out, in[j] + off, EC_METHOD_WIDTH); - j++; - } - } while (j < columns); - ec_gf_muladd[last](out, dummy, EC_METHOD_WIDTH); +} + +int32_t +ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask, + uint32_t *rows, void **in, void *out) +{ + ec_matrix_t *matrix; + uint64_t pos; + uint32_t i; + + matrix = ec_method_matrix_get(list, mask, rows); + if (EC_IS_ERR(matrix)) { + return EC_GET_ERR(matrix); + } + for (pos = 0; pos < size; pos += EC_METHOD_CHUNK_SIZE) { + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].func.interleaved( + out, in, pos, matrix->row_data[i].values, list->columns); out += EC_METHOD_CHUNK_SIZE; } - off += EC_METHOD_CHUNK_SIZE; } - return size * EC_METHOD_CHUNK_SIZE * columns; + ec_method_matrix_put(list, matrix); + + return 0; } diff --git a/xlators/cluster/ec/src/ec-method.h b/xlators/cluster/ec/src/ec-method.h index 29b46e10443..f91233b2f88 100644 --- a/xlators/cluster/ec/src/ec-method.h +++ b/xlators/cluster/ec/src/ec-method.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser @@ -11,22 +11,38 @@ #ifndef __EC_METHOD_H__ #define __EC_METHOD_H__ -#include "ec-gf.h" +#include "ec-types.h" +#include "ec-galois.h" + +#define EC_GF_BITS 8 +#define EC_GF_MOD 0x11D + +#define EC_GF_SIZE (1 << EC_GF_BITS) /* Determines the maximum size of the matrix used to encode/decode data */ #define EC_METHOD_MAX_FRAGMENTS 16 /* Determines the maximum number of usable elements in the Galois Field */ -#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1) +#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1) #define EC_METHOD_WORD_SIZE 64 #define EC_METHOD_CHUNK_SIZE (EC_METHOD_WORD_SIZE * EC_GF_BITS) -#define EC_METHOD_WIDTH (EC_METHOD_WORD_SIZE / EC_GF_WORD_SIZE) -void ec_method_initialize(void); -size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row, - uint8_t * in, uint8_t * out); -size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows, - uint8_t ** in, uint8_t * out); +int32_t +ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns, + uint32_t rows, uint32_t max, const char *gen); + +void +ec_method_fini(ec_matrix_list_t *list); + +int32_t +ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen); + +void +ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out); + +int32_t +ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask, + uint32_t *rows, void **in, void *out); #endif /* __EC_METHOD_H__ */ diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h new file mode 100644 index 00000000000..de9b89bb2c9 --- /dev/null +++ b/xlators/cluster/ec/src/ec-types.h @@ -0,0 +1,690 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_TYPES_H__ +#define __EC_TYPES_H__ + +#include <glusterfs/xlator.h> +#include <glusterfs/timer.h> +#include "libxlator.h" +#include <glusterfs/atomic.h> + +#define EC_GF_MAX_REGS 16 + +enum _ec_heal_need; +typedef enum _ec_heal_need ec_heal_need_t; + +enum _ec_stripe_part; +typedef enum _ec_stripe_part ec_stripe_part_t; + +enum _ec_read_policy; +typedef enum _ec_read_policy ec_read_policy_t; + +struct _ec_config; +typedef struct _ec_config ec_config_t; + +struct _ec_fd; +typedef struct _ec_fd ec_fd_t; + +struct _ec_fragment_range; +typedef struct _ec_fragment_range ec_fragment_range_t; + +struct _ec_inode; +typedef struct _ec_inode ec_inode_t; + +union _ec_cbk; +typedef union _ec_cbk ec_cbk_t; + +struct _ec_lock; +typedef struct _ec_lock ec_lock_t; + +struct _ec_lock_link; +typedef struct _ec_lock_link ec_lock_link_t; + +struct _ec_fop_data; +typedef struct _ec_fop_data ec_fop_data_t; + +struct _ec_cbk_data; +typedef struct _ec_cbk_data ec_cbk_data_t; + +enum _ec_gf_opcode; +typedef enum _ec_gf_opcode ec_gf_opcode_t; + +struct _ec_gf_op; +typedef struct _ec_gf_op ec_gf_op_t; + +struct _ec_gf_mul; +typedef struct _ec_gf_mul ec_gf_mul_t; + +struct _ec_gf; +typedef struct _ec_gf ec_gf_t; + +struct _ec_code_gen; +typedef struct _ec_code_gen ec_code_gen_t; + +struct _ec_code; +typedef struct _ec_code ec_code_t; + +struct _ec_code_arg; +typedef struct _ec_code_arg ec_code_arg_t; + +struct _ec_code_op; +typedef struct _ec_code_op ec_code_op_t; + +struct _ec_code_builder; +typedef struct _ec_code_builder ec_code_builder_t; + +struct _ec_code_chunk; +typedef struct _ec_code_chunk ec_code_chunk_t; + +struct _ec_stripe; +typedef struct _ec_stripe ec_stripe_t; + +struct _ec_stripe_list; +typedef struct _ec_stripe_list ec_stripe_list_t; + +struct _ec_code_space; +typedef struct _ec_code_space ec_code_space_t; + +typedef void (*ec_code_func_linear_t)(void *dst, void *src, uint64_t offset, + uint32_t *values, uint32_t count); + +typedef void (*ec_code_func_interleaved_t)(void *dst, void **src, + uint64_t offset, uint32_t *values, + uint32_t count); + +union _ec_code_func; +typedef union _ec_code_func ec_code_func_t; + +struct _ec_matrix_row; +typedef struct _ec_matrix_row ec_matrix_row_t; + +struct _ec_matrix; +typedef struct _ec_matrix ec_matrix_t; + +struct _ec_matrix_list; +typedef struct _ec_matrix_list ec_matrix_list_t; + +struct _ec_heal; +typedef struct _ec_heal ec_heal_t; + +struct _ec_self_heald; +typedef struct _ec_self_heald ec_self_heald_t; + +struct _ec_statistics; +typedef struct _ec_statistics ec_statistics_t; + +struct _ec; +typedef struct _ec ec_t; + +typedef void (*ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t); +typedef int32_t (*ec_handler_f)(ec_fop_data_t *, int32_t); +typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t); + +enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX }; + +enum _ec_heal_need { + EC_HEAL_NONEED, + EC_HEAL_MAYBE, + EC_HEAL_MUST, + EC_HEAL_PURGE_INDEX +}; + +enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL }; + +/* Enumartions to indicate FD status. */ +typedef enum { EC_FD_NOT_OPENED, EC_FD_OPENED, EC_FD_OPENING } ec_fd_status_t; + +struct _ec_config { + uint32_t version; + uint8_t algorithm; + uint8_t gf_word_size; + uint8_t bricks; + uint8_t redundancy; + uint32_t chunk_size; +}; + +struct _ec_fd { + loc_t loc; + uintptr_t open; + int32_t flags; + uint64_t bad_version; + ec_fd_status_t fd_status[0]; +}; + +struct _ec_stripe { + struct list_head lru; /* LRU list member */ + uint64_t frag_offset; /* Fragment offset of this stripe */ + char data[]; /* Contents of the stripe */ +}; + +struct _ec_stripe_list { + struct list_head lru; + uint32_t count; + uint32_t max; +}; + +struct _ec_inode { + ec_lock_t *inode_lock; + gf_boolean_t have_info; + gf_boolean_t have_config; + gf_boolean_t have_version; + gf_boolean_t have_size; + int32_t heal_count; + ec_config_t config; + uint64_t pre_version[2]; + uint64_t post_version[2]; + uint64_t pre_size; + uint64_t post_size; + uint64_t dirty[2]; + struct list_head heal; + ec_stripe_list_t stripe_cache; + uint64_t bad_version; +}; + +typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, + int32_t, uintptr_t, uintptr_t, uintptr_t, + uint32_t, dict_t *); +typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, + int32_t, uintptr_t, uintptr_t, uintptr_t, + uint32_t, dict_t *); + +union _ec_cbk { + fop_access_cbk_t access; + fop_create_cbk_t create; + fop_discard_cbk_t discard; + fop_entrylk_cbk_t entrylk; + fop_fentrylk_cbk_t fentrylk; + fop_fallocate_cbk_t fallocate; + fop_flush_cbk_t flush; + fop_fsync_cbk_t fsync; + fop_fsyncdir_cbk_t fsyncdir; + fop_getxattr_cbk_t getxattr; + fop_fgetxattr_cbk_t fgetxattr; + fop_heal_cbk_t heal; + fop_fheal_cbk_t fheal; + fop_inodelk_cbk_t inodelk; + fop_finodelk_cbk_t finodelk; + fop_link_cbk_t link; + fop_lk_cbk_t lk; + fop_lookup_cbk_t lookup; + fop_mkdir_cbk_t mkdir; + fop_mknod_cbk_t mknod; + fop_open_cbk_t open; + fop_opendir_cbk_t opendir; + fop_readdir_cbk_t readdir; + fop_readdirp_cbk_t readdirp; + fop_readlink_cbk_t readlink; + fop_readv_cbk_t readv; + fop_removexattr_cbk_t removexattr; + fop_fremovexattr_cbk_t fremovexattr; + fop_rename_cbk_t rename; + fop_rmdir_cbk_t rmdir; + fop_setattr_cbk_t setattr; + fop_fsetattr_cbk_t fsetattr; + fop_setxattr_cbk_t setxattr; + fop_fsetxattr_cbk_t fsetxattr; + fop_stat_cbk_t stat; + fop_fstat_cbk_t fstat; + fop_statfs_cbk_t statfs; + fop_symlink_cbk_t symlink; + fop_truncate_cbk_t truncate; + fop_ftruncate_cbk_t ftruncate; + fop_unlink_cbk_t unlink; + fop_writev_cbk_t writev; + fop_xattrop_cbk_t xattrop; + fop_fxattrop_cbk_t fxattrop; + fop_zerofill_cbk_t zerofill; + fop_seek_cbk_t seek; + fop_ipc_cbk_t ipc; +}; + +struct _ec_lock { + ec_inode_t *ctx; + gf_timer_t *timer; + + /* List of owners of this lock. All fops added to this list are running + * concurrently. */ + struct list_head owners; + + /* List of fops waiting to be an owner of the lock. Fops are added to this + * list when the current owner has an incompatible access (conflicting lock) + * or the lock is not acquired yet. */ + struct list_head waiting; + + /* List of fops that will wait until the next unlock/lock cycle. This + * happens when the currently acquired lock is decided to be released as + * soon as possible. In this case, all frozen fops will be continued only + * after the lock is reacquired. */ + struct list_head frozen; + + uintptr_t mask; + uintptr_t good_mask; + uintptr_t healing; + uint32_t refs_owners; /* Refs for fops owning the lock */ + uint32_t refs_pending; /* Refs assigned to fops being prepared */ + uint32_t waiting_flags; /*Track xattrop/dirty marking*/ + gf_boolean_t acquired; + gf_boolean_t contention; + gf_boolean_t unlock_now; + gf_boolean_t release; + gf_boolean_t query; + fd_t *fd; + loc_t loc; + union { + entrylk_type type; + struct gf_flock flock; + }; +}; + +struct _ec_lock_link { + ec_lock_t *lock; + ec_fop_data_t *fop; + struct list_head owner_list; + struct list_head wait_list; + gf_boolean_t update[2]; + gf_boolean_t dirty[2]; + gf_boolean_t optimistic_changelog; + loc_t *base; + uint64_t size; + uint32_t waiting_flags; + off_t fl_start; + off_t fl_end; +}; + +/* This structure keeps a range of fragment offsets affected by a fop. Since + * real file offsets can be difficult to handle correctly because of overflows, + * we use the 'scaled' offset, which corresponds to the offset of the fragment + * seen by the bricks, which is always smaller and cannot overflow. */ +struct _ec_fragment_range { + uint64_t first; /* Address of the first affected fragment as seen by the + bricks (offset on brick) */ + uint64_t last; /* Address of the first non affected fragment as seen by + the bricks (offset on brick) */ +}; + +/* EC xlator data structure to collect all the data required to perform + * the file operation.*/ +struct _ec_fop_data { + int32_t id; /* ID of the file operation */ + int32_t refs; + int32_t state; + uint32_t minimum; /* Minimum number of successful + operation required to conclude a + fop as successful */ + int32_t expected; + int32_t winds; + int32_t jobs; + int32_t error; + ec_fop_data_t *parent; + xlator_t *xl; /* points to EC xlator */ + call_frame_t *req_frame; /* frame of the calling xlator */ + call_frame_t *frame; /* frame used by this fop */ + struct list_head cbk_list; /* sorted list of groups of answers */ + struct list_head answer_list; /* list of answers */ + struct list_head pending_list; /* member of ec_t.pending_fops */ + ec_cbk_data_t *answer; /* accepted answer */ + int32_t lock_count; + int32_t locked; + gf_lock_t lock; + ec_lock_link_t locks[2]; + int32_t first_lock; + + uint32_t fop_flags; /* Flags passed by the caller. */ + uint32_t flags; /* Internal flags. */ + uint32_t first; + uintptr_t mask; + uintptr_t healing; /*Dispatch is done but call is successful only + if fop->minimum number of subvolumes succeed + which are not healing*/ + uintptr_t remaining; + uintptr_t received; /* Mask of responses */ + uintptr_t good; + + uid_t uid; + gid_t gid; + + ec_wind_f wind; /* Function to wind to */ + ec_handler_f handler; /* FOP manager function */ + ec_resume_f resume; + ec_cbk_t cbks; /* Callback function for this FOP */ + void *data; + ec_heal_t *heal; + struct list_head healer; + + uint64_t user_size; + uint32_t head; + + int32_t use_fd; /* Indicates whether this FOP uses FD or + not */ + + dict_t *xdata; + dict_t *dict; + int32_t int32; + uint32_t uint32; + uint64_t size; + off_t offset; + mode_t mode[2]; + entrylk_cmd entrylk_cmd; + entrylk_type entrylk_type; + gf_xattrop_flags_t xattrop_flags; + dev_t dev; + inode_t *inode; + fd_t *fd; /* FD of the file on which FOP is + being carried upon */ + struct iatt iatt; + char *str[2]; + loc_t loc[2]; /* Holds the location details for + the file */ + struct gf_flock flock; + struct iovec *vector; + struct iobref *buffers; + gf_seek_what_t seek; + ec_fragment_range_t frag_range; /* This will hold the range of stripes + affected by the fop. */ + char *errstr; /*String of fop name, path and gfid + to be used in gf_msg. */ +}; + +struct _ec_cbk_data { + struct list_head list; /* item in the sorted list of groups */ + struct list_head answer_list; /* item in the list of answers */ + ec_fop_data_t *fop; + ec_cbk_data_t *next; /* next answer in the same group */ + uint32_t idx; + int32_t op_ret; + int32_t op_errno; + int32_t count; + uintptr_t mask; + + dict_t *xdata; + dict_t *dict; + int32_t int32; + uintptr_t uintptr[3]; + uint64_t size; + uint64_t version[2]; + inode_t *inode; + fd_t *fd; + struct statvfs statvfs; + struct iatt iatt[5]; + struct gf_flock flock; + struct iovec *vector; + struct iobref *buffers; + char *str; + gf_dirent_t entries; + off_t offset; + gf_seek_what_t what; +}; + +enum _ec_gf_opcode { + EC_GF_OP_LOAD, + EC_GF_OP_STORE, + EC_GF_OP_COPY, + EC_GF_OP_XOR2, + EC_GF_OP_XOR3, + EC_GF_OP_XORM, + EC_GF_OP_END +}; + +struct _ec_gf_op { + ec_gf_opcode_t op; + uint32_t arg1; + uint32_t arg2; + uint32_t arg3; +}; + +struct _ec_gf_mul { + uint32_t regs; + uint32_t map[EC_GF_MAX_REGS]; + ec_gf_op_t *ops; +}; + +struct _ec_gf { + uint32_t bits; + uint32_t size; + uint32_t mod; + uint32_t min_ops; + uint32_t max_ops; + uint32_t avg_ops; + uint32_t *log; + uint32_t *pow; + ec_gf_mul_t **table; +}; + +struct _ec_code_gen { + char *name; + char **flags; + uint32_t width; + + void (*prolog)(ec_code_builder_t *builder); + void (*epilog)(ec_code_builder_t *builder); + void (*load)(ec_code_builder_t *builder, uint32_t reg, uint32_t offset, + uint32_t bit); + void (*store)(ec_code_builder_t *builder, uint32_t reg, uint32_t bit); + void (*copy)(ec_code_builder_t *builder, uint32_t dst, uint32_t src); + void (*xor2)(ec_code_builder_t *builder, uint32_t dst, uint32_t src); + void (*xor3)(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2); + void (*xorm)(ec_code_builder_t *builder, uint32_t dst, uint32_t offset, + uint32_t bit); +}; + +struct _ec_code { + gf_lock_t lock; + struct list_head spaces; + ec_gf_t *gf; + ec_code_gen_t *gen; +}; + +struct _ec_code_arg { + uint32_t value; +}; + +struct _ec_code_op { + ec_gf_opcode_t op; + ec_code_arg_t arg1; + ec_code_arg_t arg2; + ec_code_arg_t arg3; +}; + +struct _ec_code_builder { + ec_code_t *code; + uint64_t address; + uint8_t *data; + uint32_t size; + int32_t error; + uint32_t regs; + uint32_t bits; + uint32_t width; + uint32_t count; + uint32_t base; + uint32_t map[EC_GF_MAX_REGS]; + gf_boolean_t linear; + uint64_t loop; + ec_code_op_t ops[0]; +}; + +struct _ec_code_chunk { + struct list_head list; + size_t size; + ec_code_space_t *space; +}; + +struct _ec_code_space { + struct list_head list; + struct list_head chunks; + ec_code_t *code; + void *exec; + size_t size; +}; + +union _ec_code_func { + ec_code_func_linear_t linear; + ec_code_func_interleaved_t interleaved; +}; + +struct _ec_matrix_row { + ec_code_func_t func; + uint32_t *values; +}; + +struct _ec_matrix { + struct list_head lru; + uint32_t refs; + uint32_t columns; + uint32_t rows; + uintptr_t mask; + ec_code_t *code; + uint32_t *values; + ec_matrix_row_t row_data[0]; +}; + +struct _ec_matrix_list { + struct list_head lru; + gf_lock_t lock; + uint32_t columns; + uint32_t rows; + uint32_t max; + uint32_t count; + uint32_t stripe; + struct mem_pool *pool; + ec_gf_t *gf; + ec_code_t *code; + ec_matrix_t *encode; + ec_matrix_t **objects; +}; + +struct _ec_heal { + struct list_head list; + gf_lock_t lock; + xlator_t *xl; + ec_fop_data_t *fop; + void *data; + ec_fop_data_t *lookup; + loc_t loc; + struct iatt iatt; + char *symlink; + fd_t *fd; + int32_t partial; + int32_t done; + int32_t error; + gf_boolean_t nameheal; + uintptr_t available; + uintptr_t good; + uintptr_t bad; + uintptr_t open; + uintptr_t fixed; + uint64_t offset; + uint64_t size; + uint64_t total_size; + uint64_t version[2]; + uint64_t raw_size; +}; + +struct subvol_healer { + xlator_t *this; + int subvol; + gf_boolean_t running; + gf_boolean_t rerun; + pthread_mutex_t mutex; + pthread_cond_t cond; + pthread_t thread; +}; + +struct _ec_self_heald { + gf_boolean_t iamshd; + gf_boolean_t enabled; + int timeout; + uint32_t max_threads; + uint32_t wait_qlength; + struct subvol_healer *index_healers; + struct subvol_healer *full_healers; +}; + +struct _ec_statistics { + struct { + gf_atomic_t hits; /* Cache hits. */ + gf_atomic_t misses; /* Cache misses. */ + gf_atomic_t updates; /* Number of times an existing stripe has + been updated with new content. */ + gf_atomic_t invals; /* Number of times an existing stripe has + been invalidated because of truncates + or discards. */ + gf_atomic_t evicts; /* Number of times that an existing entry + has been evicted to make room for newer + entries. */ + gf_atomic_t allocs; /* Number of memory allocations made to + store stripes. */ + gf_atomic_t errors; /* Number of errors that have caused extra + requests. (Basically memory allocation + errors). */ + } stripe_cache; + struct { + gf_atomic_t attempted; /*Number of heals attempted on + files/directories*/ + gf_atomic_t completed; /*Number of heals complted on files/directories*/ + } shd; +}; + +struct _ec { + xlator_t *xl; + int32_t healers; + int32_t heal_waiters; + int32_t nodes; /* Total number of bricks(n) */ + int32_t bits_for_nodes; + int32_t fragments; /* Data bricks(k) */ + int32_t redundancy; /* Redundant bricks(m) */ + uint32_t fragment_size; /* Size of fragment/chunk on a + brick. */ + uint32_t stripe_size; /* (fragment_size * fragments) + maximum size of user data + stored in one stripe. */ + int32_t up; /* Represents whether EC volume is + up or not. */ + uint32_t idx; + uint32_t xl_up_count; /* Number of UP bricks. */ + uintptr_t xl_up; /* Bit flag representing UP + bricks */ + uint32_t xl_notify_count; /* Number of notifications. */ + uintptr_t xl_notify; /* Bit flag representing + notification for bricks. */ + uintptr_t node_mask; + uintptr_t read_mask; /*Stores user defined read-mask*/ + gf_atomic_t async_fop_count; /* Number of on going asynchronous fops. */ + xlator_t **xl_list; + gf_lock_t lock; + gf_timer_t *timer; + gf_boolean_t shutdown; + gf_boolean_t eager_lock; + gf_boolean_t other_eager_lock; + gf_boolean_t optimistic_changelog; + gf_boolean_t parallel_writes; + uint32_t stripe_cache; + uint32_t quorum_count; + uint32_t background_heals; + uint32_t heal_wait_qlen; + uint32_t self_heal_window_size; /* max size of read/writes */ + uint32_t eager_lock_timeout; + uint32_t other_eager_lock_timeout; + struct list_head pending_fops; + struct list_head heal_waiting; + struct list_head healing; + struct mem_pool *fop_pool; + struct mem_pool *cbk_pool; + struct mem_pool *lock_pool; + ec_self_heald_t shd; + char vol_uuid[UUID_SIZE + 1]; + dict_t *leaf_to_subvolid; + ec_read_policy_t read_policy; + ec_matrix_list_t matrix; + ec_statistics_t stats; +}; + +#endif /* __EC_TYPES_H__ */ diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c index 219494b961e..7344be4968d 100644 --- a/xlators/cluster/ec/src/ec.c +++ b/xlators/cluster/ec/src/ec.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser @@ -8,58 +8,63 @@ cases as published by the Free Software Foundation. */ -#include "defaults.h" -#include "statedump.h" -#include "compat-errno.h" +#include <glusterfs/defaults.h> +#include <glusterfs/statedump.h> +#include <glusterfs/compat-errno.h> +#include <glusterfs/upcall-utils.h> +#include "ec.h" +#include "ec-messages.h" #include "ec-mem-types.h" +#include "ec-types.h" #include "ec-helpers.h" #include "ec-common.h" #include "ec-fops.h" #include "ec-method.h" -#include "ec.h" +#include "ec-code.h" +#include "ec-heald.h" +#include <glusterfs/events.h> -#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS -/* The maximum number of nodes is derived from the maximum allowed fragments - * using the rule that redundancy cannot be equal or greater than the number - * of fragments. - */ -#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES) +static char *ec_read_policies[EC_READ_POLICY_MAX + 1] = { + [EC_ROUND_ROBIN] = "round-robin", + [EC_GFID_HASH] = "gfid-hash", + [EC_READ_POLICY_MAX] = NULL}; #define EC_INTERNAL_XATTR_OR_GOTO(name, xattr, op_errno, label) \ - do { \ - if (ec_is_internal_xattr (NULL, (char *)name, NULL, NULL)) { \ - op_errno = EPERM; \ - goto label; \ - } \ - if (name && (strlen (name) == 0) && xattr) { \ - /* Bulk [f]removexattr/[f]setxattr */ \ - GF_IF_INTERNAL_XATTR_GOTO (EC_XATTR_PREFIX"*", xattr, \ - op_errno, label); \ - } \ - } while (0) - -int32_t ec_parse_options(xlator_t * this) -{ - ec_t * ec = this->private; + do { \ + if (ec_is_internal_xattr(NULL, (char *)name, NULL, NULL)) { \ + op_errno = EPERM; \ + goto label; \ + } \ + if (name && (strlen(name) == 0) && xattr) { \ + /* Bulk [f]removexattr/[f]setxattr */ \ + GF_IF_INTERNAL_XATTR_GOTO(EC_XATTR_PREFIX "*", xattr, op_errno, \ + label); \ + } \ + } while (0) + +int32_t +ec_parse_options(xlator_t *this) +{ + ec_t *ec = this->private; int32_t error = EINVAL; uintptr_t mask; GF_OPTION_INIT("redundancy", ec->redundancy, int32, out); ec->fragments = ec->nodes - ec->redundancy; if ((ec->redundancy < 1) || (ec->redundancy >= ec->fragments) || - (ec->fragments > EC_MAX_FRAGMENTS)) - { - gf_log(this->name, GF_LOG_ERROR, "Invalid redundancy (must be between " - "1 and %d)", (ec->nodes - 1) / 2); + (ec->fragments > EC_MAX_FRAGMENTS)) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REDUNDANCY, + "Invalid redundancy (must be between " + "1 and %d)", + (ec->nodes - 1) / 2); goto out; } ec->bits_for_nodes = 1; mask = 2; - while (ec->nodes > mask) - { + while (ec->nodes > mask) { ec->bits_for_nodes++; mask <<= 1; } @@ -67,9 +72,10 @@ int32_t ec_parse_options(xlator_t * this) ec->fragment_size = EC_METHOD_CHUNK_SIZE; ec->stripe_size = ec->fragment_size * ec->fragments; - gf_log("ec", GF_LOG_DEBUG, "Initialized with: nodes=%u, fragments=%u, " - "stripe_size=%u, node_mask=%lX", - ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask); + gf_msg_debug("ec", 0, + "Initialized with: nodes=%u, fragments=%u, " + "stripe_size=%u, node_mask=%" PRIxFAST32, + ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask); error = 0; @@ -77,28 +83,28 @@ out: return error; } -int32_t ec_prepare_childs(xlator_t * this) +int32_t +ec_prepare_childs(xlator_t *this) { - ec_t * ec = this->private; - xlator_list_t * child = NULL; + ec_t *ec = this->private; + xlator_list_t *child = NULL; int32_t count = 0; - for (child = this->children; child != NULL; child = child->next) - { + for (child = this->children; child != NULL; child = child->next) { count++; } - if (count > EC_MAX_NODES) - { - gf_log(this->name, GF_LOG_ERROR, "Too many subvolumes"); + if (count > EC_MAX_NODES) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_TOO_MANY_SUBVOLS, + "Too many subvolumes"); return EINVAL; } ec->nodes = count; ec->xl_list = GF_CALLOC(count, sizeof(ec->xl_list[0]), ec_mt_xlator_t); - if (ec->xl_list == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Allocation of xlator list failed"); + if (ec->xl_list == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Allocation of xlator list failed"); return ENOMEM; } @@ -106,24 +112,51 @@ int32_t ec_prepare_childs(xlator_t * this) ec->xl_up_count = 0; count = 0; - for (child = this->children; child != NULL; child = child->next) - { + for (child = this->children; child != NULL; child = child->next) { ec->xl_list[count++] = child->xlator; } return 0; } -void __ec_destroy_private(xlator_t * this) +/* This function transforms the subvol to subvol-id*/ +static int +_subvol_to_subvolid(dict_t *this, char *key, data_t *value, void *data) +{ + ec_t *ec = data; + xlator_t *subvol = NULL; + int i = 0; + int ret = -1; + + subvol = data_to_ptr(value); + for (i = 0; i < ec->nodes; i++) { + if (ec->xl_list[i] == subvol) { + ret = dict_set_int32(this, key, i); + /* -1 stops dict_foreach and returns -1*/ + if (ret < 0) + ret = -1; + goto out; + } + } +out: + return ret; +} + +int +ec_subvol_to_subvol_id_transform(ec_t *ec, dict_t *leaf_to_subvolid) { - ec_t * ec = this->private; + return dict_foreach(leaf_to_subvolid, _subvol_to_subvolid, ec); +} - if (ec != NULL) - { +void +__ec_destroy_private(xlator_t *this) +{ + ec_t *ec = this->private; + + if (ec != NULL) { LOCK(&ec->lock); - if (ec->timer != NULL) - { + if (ec->timer != NULL) { gf_timer_call_cancel(this->ctx, ec->timer); ec->timer = NULL; } @@ -141,39 +174,41 @@ void __ec_destroy_private(xlator_t * this) sleep(2); this->private = NULL; - if (ec->xl_list != NULL) - { + if (ec->xl_list != NULL) { GF_FREE(ec->xl_list); ec->xl_list = NULL; } - if (ec->fop_pool != NULL) - { + if (ec->fop_pool != NULL) { mem_pool_destroy(ec->fop_pool); } - if (ec->cbk_pool != NULL) - { + if (ec->cbk_pool != NULL) { mem_pool_destroy(ec->cbk_pool); } - if (ec->lock_pool != NULL) - { + if (ec->lock_pool != NULL) { mem_pool_destroy(ec->lock_pool); } LOCK_DESTROY(&ec->lock); + if (ec->leaf_to_subvolid) + dict_unref(ec->leaf_to_subvolid); + + ec_method_fini(&ec->matrix); + GF_FREE(ec); } } -int32_t mem_acct_init(xlator_t * this) +int32_t +mem_acct_init(xlator_t *this) { - if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Memory accounting initialization " - "failed."); + if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Memory accounting initialization " + "failed."); return -1; } @@ -181,229 +216,583 @@ int32_t mem_acct_init(xlator_t * this) return 0; } -int32_t -reconfigure (xlator_t *this, dict_t *options) +void +ec_configure_background_heal_opts(ec_t *ec, int background_heals, + int heal_wait_qlen) { - ec_t *ec = this->private; + if (background_heals == 0) { + ec->heal_wait_qlen = 0; + } else { + ec->heal_wait_qlen = heal_wait_qlen; + } + ec->background_heals = background_heals; +} - GF_OPTION_RECONF ("self-heal-daemon", ec->shd, options, bool, failed); - GF_OPTION_RECONF ("iam-self-heal-daemon", ec->iamshd, options, - bool, failed); +int +ec_assign_read_policy(ec_t *ec, char *read_policy) +{ + int read_policy_idx = -1; - return 0; -failed: + read_policy_idx = gf_get_index_by_elem(ec_read_policies, read_policy); + if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX) return -1; + + ec->read_policy = read_policy_idx; + return 0; } -glusterfs_event_t -ec_get_event_from_state (ec_t *ec) -{ - int down_count = 0; +int32_t +reconfigure(xlator_t *this, dict_t *options) +{ + ec_t *ec = this->private; + char *read_policy = NULL; + char *extensions = NULL; + uint32_t heal_wait_qlen = 0; + uint32_t background_heals = 0; + int32_t ret = -1; + int32_t err; + + GF_OPTION_RECONF("cpu-extensions", extensions, options, str, failed); + + GF_OPTION_RECONF("self-heal-daemon", ec->shd.enabled, options, bool, + failed); + GF_OPTION_RECONF("iam-self-heal-daemon", ec->shd.iamshd, options, bool, + failed); + GF_OPTION_RECONF("eager-lock", ec->eager_lock, options, bool, failed); + GF_OPTION_RECONF("other-eager-lock", ec->other_eager_lock, options, bool, + failed); + GF_OPTION_RECONF("eager-lock-timeout", ec->eager_lock_timeout, options, + uint32, failed); + GF_OPTION_RECONF("other-eager-lock-timeout", ec->other_eager_lock_timeout, + options, uint32, failed); + GF_OPTION_RECONF("background-heals", background_heals, options, uint32, + failed); + GF_OPTION_RECONF("heal-wait-qlength", heal_wait_qlen, options, uint32, + failed); + GF_OPTION_RECONF("self-heal-window-size", ec->self_heal_window_size, + options, uint32, failed); + GF_OPTION_RECONF("heal-timeout", ec->shd.timeout, options, int32, failed); + ec_configure_background_heal_opts(ec, background_heals, heal_wait_qlen); + GF_OPTION_RECONF("shd-max-threads", ec->shd.max_threads, options, uint32, + failed); + GF_OPTION_RECONF("shd-wait-qlength", ec->shd.wait_qlength, options, uint32, + failed); + + GF_OPTION_RECONF("read-policy", read_policy, options, str, failed); + + GF_OPTION_RECONF("optimistic-change-log", ec->optimistic_changelog, options, + bool, failed); + GF_OPTION_RECONF("parallel-writes", ec->parallel_writes, options, bool, + failed); + GF_OPTION_RECONF("stripe-cache", ec->stripe_cache, options, uint32, failed); + GF_OPTION_RECONF("quorum-count", ec->quorum_count, options, uint32, failed); + ret = 0; + if (ec_assign_read_policy(ec, read_policy)) { + ret = -1; + } - if (ec->xl_up_count >= ec->fragments) { - /* If ec is up but some subvolumes are yet to notify, give - * grace time for other subvols to notify to prevent start of - * I/O which may result in self-heals */ - if (ec->timer && ec->xl_notify_count < ec->nodes) - return GF_EVENT_MAXVAL; + err = ec_method_update(this, &ec->matrix, extensions); + if (err != 0) { + ret = -1; + } - return GF_EVENT_CHILD_UP; - } else { - down_count = ec->xl_notify_count - ec->xl_up_count; - if (down_count > ec->redundancy) - return GF_EVENT_CHILD_DOWN; - } +failed: + return ret; +} - return GF_EVENT_MAXVAL; +glusterfs_event_t +ec_get_event_from_state(ec_t *ec) +{ + int down_count = 0; + + if (ec->xl_up_count >= ec->fragments) { + /* If ec is up but some subvolumes are yet to notify, give + * grace time for other subvols to notify to prevent start of + * I/O which may result in self-heals */ + if (ec->xl_notify_count < ec->nodes) + return GF_EVENT_MAXVAL; + + return GF_EVENT_CHILD_UP; + } else { + down_count = ec->xl_notify_count - ec->xl_up_count; + if (down_count > ec->redundancy) + return GF_EVENT_CHILD_DOWN; + } + + return GF_EVENT_MAXVAL; } void -ec_up (xlator_t *this, ec_t *ec) +ec_up(xlator_t *this, ec_t *ec) { - if (ec->timer != NULL) { - gf_timer_call_cancel (this->ctx, ec->timer); - ec->timer = NULL; - } + char str1[32], str2[32]; + + if (ec->timer != NULL) { + gf_timer_call_cancel(this->ctx, ec->timer); + ec->timer = NULL; + } + + ec->up = 1; + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP, + "Going UP : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); - ec->up = 1; - gf_log(this->name, GF_LOG_INFO, "Going UP"); + gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name); } void -ec_down (xlator_t *this, ec_t *ec) +ec_down(xlator_t *this, ec_t *ec) { - if (ec->timer != NULL) { - gf_timer_call_cancel(this->ctx, ec->timer); - ec->timer = NULL; - } + char str1[32], str2[32]; + + if (ec->timer != NULL) { + gf_timer_call_cancel(this->ctx, ec->timer); + ec->timer = NULL; + } + + ec->up = 0; + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN, + "Going DOWN : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); - ec->up = 0; - gf_log(this->name, GF_LOG_INFO, "Going DOWN"); + gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name); } void -ec_notify_cbk (void *data) +ec_notify_cbk(void *data) { - ec_t *ec = data; - glusterfs_event_t event = GF_EVENT_MAXVAL; + ec_t *ec = data; + glusterfs_event_t event = GF_EVENT_MAXVAL; + gf_boolean_t propagate = _gf_false; + gf_boolean_t launch_heal = _gf_false; - LOCK(&ec->lock); - { - if (!ec->timer) { - /* - * Either child_up/child_down is already sent to parent - * This is a spurious wake up. - */ - goto unlock; - } - - gf_timer_call_cancel (ec->xl->ctx, ec->timer); - ec->timer = NULL; - - event = ec_get_event_from_state (ec); - /* If event is still MAXVAL then enough subvolumes didn't - * notify, treat it as CHILD_DOWN. */ - if (event == GF_EVENT_MAXVAL) { - event = GF_EVENT_CHILD_DOWN; - ec->xl_notify = (1ULL << ec->nodes) - 1ULL; - ec->xl_notify_count = ec->nodes; - } else if (event == GF_EVENT_CHILD_UP) { - /* Rest of the bricks are still not coming up, - * notify that ec is up. Files/directories will be - * healed as in when they come up. */ - ec_up (ec->xl, ec); - } - - /* CHILD_DOWN should not come here as no grace period is given - * for notifying CHILD_DOWN. */ - - default_notify (ec->xl, event, NULL); + LOCK(&ec->lock); + { + if (!ec->timer) { + /* + * Either child_up/child_down is already sent to parent + * This is a spurious wake up. + */ + goto unlock; + } + + gf_timer_call_cancel(ec->xl->ctx, ec->timer); + ec->timer = NULL; + + /* The timeout has expired, so any subvolume that has not + * already reported its state, will be considered to be down. + * We mark as if all bricks had reported. */ + ec->xl_notify = (1ULL << ec->nodes) - 1ULL; + ec->xl_notify_count = ec->nodes; + + /* Since we have marked all subvolumes as notified, it's + * guaranteed that ec_get_event_from_state() will return + * CHILD_UP or CHILD_DOWN, but not MAXVAL. */ + event = ec_get_event_from_state(ec); + if (event == GF_EVENT_CHILD_UP) { + /* We are ready to bring the volume up. If there are + * still bricks DOWN, they will be healed when they + * come up. */ + ec_up(ec->xl, ec); + + if (ec->shd.iamshd && !ec->shutdown) { + launch_heal = _gf_true; + GF_ATOMIC_INC(ec->async_fop_count); + } } + + propagate = _gf_true; + } unlock: - UNLOCK(&ec->lock); + UNLOCK(&ec->lock); + + if (launch_heal) { + /* We have just brought the volume UP, so we trigger + * a self-heal check on the root directory. */ + ec_launch_replace_heal(ec); + } + if (propagate) { + default_notify(ec->xl, event, NULL); + } } void -ec_launch_notify_timer (xlator_t *this, ec_t *ec) +ec_launch_notify_timer(xlator_t *this, ec_t *ec) +{ + struct timespec delay = { + 0, + }; + + gf_msg_debug(this->name, 0, "Initiating child-down timer"); + delay.tv_sec = 10; + delay.tv_nsec = 0; + ec->timer = gf_timer_call_after(this->ctx, delay, ec_notify_cbk, ec); + if (ec->timer == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_TIMER_CREATE_FAIL, + "Cannot create timer " + "for delayed initialization"); + } +} + +gf_boolean_t +ec_disable_delays(ec_t *ec) { - struct timespec delay = {0, }; + ec->shutdown = _gf_true; - gf_log (this->name, GF_LOG_DEBUG, "Initiating child-down timer"); - delay.tv_sec = 10; - delay.tv_nsec = 0; - ec->timer = gf_timer_call_after (this->ctx, delay, ec_notify_cbk, ec); - if (ec->timer == NULL) { - gf_log(this->name, GF_LOG_ERROR, "Cannot create timer " - "for delayed initialization"); - } + return __ec_is_last_fop(ec); } void -ec_handle_up (xlator_t *this, ec_t *ec, int32_t idx) +ec_cleanup_healer_object(ec_t *ec) { - if (((ec->xl_notify >> idx) & 1) == 0) { - ec->xl_notify |= 1ULL << idx; - ec->xl_notify_count++; + struct subvol_healer *healer = NULL; + ec_self_heald_t *shd = NULL; + void *res = NULL; + int i = 0; + gf_boolean_t is_join = _gf_false; + + shd = &ec->shd; + if (!shd->iamshd) + return; + + for (i = 0; i < ec->nodes; i++) { + healer = &shd->index_healers[i]; + pthread_mutex_lock(&healer->mutex); + { + healer->rerun = 1; + if (healer->running) { + pthread_cond_signal(&healer->cond); + is_join = _gf_true; + } + } + pthread_mutex_unlock(&healer->mutex); + if (is_join) { + pthread_join(healer->thread, &res); + is_join = _gf_false; } - if (((ec->xl_up >> idx) & 1) == 0) { /* Duplicate event */ - ec->xl_up |= 1ULL << idx; - ec->xl_up_count++; + healer = &shd->full_healers[i]; + pthread_mutex_lock(&healer->mutex); + { + healer->rerun = 1; + if (healer->running) { + pthread_cond_signal(&healer->cond); + is_join = _gf_true; + } + } + pthread_mutex_unlock(&healer->mutex); + if (is_join) { + pthread_join(healer->thread, &res); + is_join = _gf_false; } + } } - void -ec_handle_down (xlator_t *this, ec_t *ec, int32_t idx) +ec_pending_fops_completed(ec_t *ec) { - if (((ec->xl_notify >> idx) & 1) == 0) { - ec->xl_notify |= 1ULL << idx; - ec->xl_notify_count++; - } + if (ec->shutdown) { + default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL); + } +} - if (((ec->xl_up >> idx) & 1) != 0) { /* Duplicate event */ - gf_log(this->name, GF_LOG_DEBUG, "Child %d is DOWN", idx); +static gf_boolean_t +ec_set_up_state(ec_t *ec, uintptr_t index_mask, uintptr_t new_state) +{ + uintptr_t current_state = 0; - ec->xl_up ^= 1ULL << idx; - ec->xl_up_count--; - } + if (xlator_is_cleanup_starting(ec->xl)) + return _gf_false; + + if ((ec->xl_notify & index_mask) == 0) { + ec->xl_notify |= index_mask; + ec->xl_notify_count++; + } + current_state = ec->xl_up & index_mask; + if (current_state != new_state) { + ec->xl_up ^= index_mask; + ec->xl_up_count += (current_state ? -1 : 1); + + return _gf_true; + } + + return _gf_false; +} + +static gf_boolean_t +ec_upcall(ec_t *ec, struct gf_upcall *upcall) +{ + struct gf_upcall_cache_invalidation *ci = NULL; + struct gf_upcall_inodelk_contention *lc = NULL; + inode_t *inode; + inode_table_t *table; + + switch (upcall->event_type) { + case GF_UPCALL_CACHE_INVALIDATION: + ci = upcall->data; + ci->flags |= UP_INVAL_ATTR; + return _gf_true; + + case GF_UPCALL_INODELK_CONTENTION: + lc = upcall->data; + if (strcmp(lc->domain, ec->xl->name) != 0) { + /* The lock is not owned by EC, ignore it. */ + return _gf_true; + } + table = ((xlator_t *)ec->xl->graph->top)->itable; + if (table == NULL) { + /* Self-heal daemon doesn't have an inode table on the top + * xlator because it doesn't need it. In this case we should + * use the inode table managed by EC itself where all inodes + * being healed should be present. However self-heal doesn't + * use eager-locking and inodelk's are already released as + * soon as possible. In this case we can safely ignore these + * notifications. */ + return _gf_false; + } + inode = inode_find(table, upcall->gfid); + /* If inode is not found, it means that it's already released, + * so we can ignore it. Probably it has been released and + * destroyed while the contention notification was being sent. + */ + if (inode != NULL) { + ec_lock_release(ec, inode); + inode_unref(inode); + } + + return _gf_false; + + default: + return _gf_true; + } } int32_t -notify (xlator_t *this, int32_t event, void *data, ...) +ec_notify(xlator_t *this, int32_t event, void *data, void *data2) { - ec_t * ec = this->private; + ec_t *ec = this->private; int32_t idx = 0; int32_t error = 0; glusterfs_event_t old_event = GF_EVENT_MAXVAL; - glusterfs_event_t new_event = GF_EVENT_MAXVAL; - - LOCK (&ec->lock); + dict_t *input = NULL; + dict_t *output = NULL; + gf_boolean_t propagate = _gf_true; + gf_boolean_t needs_shd_check = _gf_false; + int32_t orig_event = event; + uintptr_t mask = 0; + + gf_msg_trace(this->name, 0, "NOTIFY(%d): %p, %p", event, data, data2); + + if (event == GF_EVENT_UPCALL) { + propagate = ec_upcall(ec, data); + goto done; + } - if (event == GF_EVENT_PARENT_UP) { - /* - * Start a timer which sends appropriate event to parent - * xlator to prevent the 'mount' syscall from hanging. - */ - ec_launch_notify_timer (this, ec); - goto unlock; + if (event == GF_EVENT_TRANSLATOR_OP) { + if (!ec->up) { + error = -1; + } else { + input = data; + output = data2; + error = ec_xl_op(this, input, output); } + goto out; + } - for (idx = 0; idx < ec->nodes; idx++) { - if (ec->xl_list[idx] == data) - break; + for (idx = 0; idx < ec->nodes; idx++) { + if (ec->xl_list[idx] == data) { + break; } + } - gf_log (this->name, GF_LOG_TRACE, "NOTIFY(%d): %p, %d", - event, data, idx); + LOCK(&ec->lock); - if (idx < ec->nodes) { /* CHILD_* events */ + if (event == GF_EVENT_PARENT_UP) { + /* + * Start a timer which sends appropriate event to parent + * xlator to prevent the 'mount' syscall from hanging. + */ + ec_launch_notify_timer(this, ec); + goto unlock; + } else if (event == GF_EVENT_PARENT_DOWN) { + /* If there aren't pending fops running after we have waken up + * them, we immediately propagate the notification. */ + propagate = ec_disable_delays(ec); + ec_cleanup_healer_object(ec); + goto unlock; + } - old_event = ec_get_event_from_state (ec); + if (idx < ec->nodes) { /* CHILD_* events */ + old_event = ec_get_event_from_state(ec); + + mask = 1ULL << idx; + if (event == GF_EVENT_CHILD_UP) { + /* We need to trigger a selfheal if a brick changes + * to UP state. */ + if (ec_set_up_state(ec, mask, mask) && ec->shd.iamshd && + !ec->shutdown) { + needs_shd_check = _gf_true; + } + } else if (event == GF_EVENT_CHILD_DOWN) { + ec_set_up_state(ec, mask, 0); + } - if (event == GF_EVENT_CHILD_UP) { - ec_handle_up (this, ec, idx); - } else if (event == GF_EVENT_CHILD_DOWN) { - ec_handle_down (this, ec, idx); - } + event = ec_get_event_from_state(ec); - new_event = ec_get_event_from_state (ec); + if (event == GF_EVENT_CHILD_UP) { + if (!ec->up) { + ec_up(this, ec); + } + } else { + /* If the volume is not UP, it's irrelevant if one + * brick has come up. We cannot heal anything. */ + needs_shd_check = _gf_false; - if (new_event == GF_EVENT_CHILD_UP && !ec->up) { - ec_up (this, ec); - } else if (new_event == GF_EVENT_CHILD_DOWN && ec->up) { - ec_down (this, ec); - } + if ((event == GF_EVENT_CHILD_DOWN) && ec->up) { + ec_down(this, ec); + } + } - if ((new_event == old_event) && (new_event != GF_EVENT_MAXVAL)) - new_event = GF_EVENT_CHILD_MODIFIED; + if (event != GF_EVENT_MAXVAL) { + if (event == old_event) { + if (orig_event == GF_EVENT_CHILD_UP) + event = GF_EVENT_SOME_DESCENDENT_UP; + else /* orig_event has to be GF_EVENT_CHILD_DOWN */ + event = GF_EVENT_SOME_DESCENDENT_DOWN; + } + } else { + propagate = _gf_false; + needs_shd_check = _gf_false; + } - event = GF_EVENT_MAXVAL;/* Take care of notifying inside lock */ - if (new_event != GF_EVENT_MAXVAL) - error = default_notify (this, new_event, data); + if (needs_shd_check) { + GF_ATOMIC_INC(ec->async_fop_count); } + } unlock: - UNLOCK (&ec->lock); + UNLOCK(&ec->lock); - if (event != GF_EVENT_MAXVAL) - return default_notify (this, event, data); +done: + if (needs_shd_check) { + ec_launch_replace_heal(ec); + } + if (propagate) { + error = default_notify(this, event, data); + } - return error; +out: + return error; } int32_t -init (xlator_t *this) +notify(xlator_t *this, int32_t event, void *data, ...) { - ec_t *ec = NULL; + int ret = -1; + va_list ap; + void *data2 = NULL; - if (this->parents == NULL) - { - gf_log(this->name, GF_LOG_WARNING, "Volume does not have parents."); + va_start(ap, data); + data2 = va_arg(ap, dict_t *); + va_end(ap); + ret = ec_notify(this, event, data, data2); + + return ret; +} + +static void +ec_statistics_init(ec_t *ec) +{ + GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0); + GF_ATOMIC_INIT(ec->stats.shd.attempted, 0); + GF_ATOMIC_INIT(ec->stats.shd.completed, 0); +} + +static int +ec_assign_read_mask(ec_t *ec, char *read_mask_str) +{ + char *mask = NULL; + char *maskptr = NULL; + char *saveptr = NULL; + char *id_str = NULL; + int id = 0; + int ret = 0; + uintptr_t read_mask = 0; + + if (!read_mask_str) { + ec->read_mask = 0; + ret = 0; + goto out; + } + + mask = gf_strdup(read_mask_str); + if (!mask) { + ret = -1; + goto out; + } + maskptr = mask; + + for (;;) { + id_str = strtok_r(maskptr, ":", &saveptr); + if (id_str == NULL) + break; + if (gf_string2int(id_str, &id)) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "In read-mask \"%s\" id %s is not a valid integer", + read_mask_str, id_str); + ret = -1; + goto out; + } + + if ((id < 0) || (id >= ec->nodes)) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "In read-mask \"%s\" id %d is not in range [0 - %d]", + read_mask_str, id, ec->nodes - 1); + ret = -1; + goto out; + } + read_mask |= (1UL << id); + maskptr = NULL; + } + + if (gf_bits_count(read_mask) < ec->fragments) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "read-mask \"%s\" should contain at least %d ids", read_mask_str, + ec->fragments); + ret = -1; + goto out; + } + ec->read_mask = read_mask; + ret = 0; +out: + GF_FREE(mask); + return ret; +} + +int32_t +init(xlator_t *this) +{ + ec_t *ec = NULL; + char *read_policy = NULL; + char *extensions = NULL; + int32_t err; + char *read_mask_str = NULL; + + if (this->parents == NULL) { + gf_msg(this->name, GF_LOG_WARNING, 0, EC_MSG_NO_PARENTS, + "Volume does not have parents."); } ec = GF_MALLOC(sizeof(*ec), ec_mt_ec_t); - if (ec == NULL) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to allocate private memory."); + if (ec == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate private memory."); return -1; } @@ -414,36 +803,104 @@ init (xlator_t *this) ec->xl = this; LOCK_INIT(&ec->lock); + GF_ATOMIC_INIT(ec->async_fop_count, 0); + INIT_LIST_HEAD(&ec->pending_fops); + INIT_LIST_HEAD(&ec->heal_waiting); + INIT_LIST_HEAD(&ec->healing); + ec->fop_pool = mem_pool_new(ec_fop_data_t, 1024); ec->cbk_pool = mem_pool_new(ec_cbk_data_t, 4096); ec->lock_pool = mem_pool_new(ec_lock_t, 1024); if ((ec->fop_pool == NULL) || (ec->cbk_pool == NULL) || - (ec->lock_pool == NULL)) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to create memory pools."); + (ec->lock_pool == NULL)) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to create memory pools."); goto failed; } - if (ec_prepare_childs(this) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to initialize xlator"); + if (ec_prepare_childs(this) != 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "Failed to initialize xlator"); goto failed; } - if (ec_parse_options(this) != 0) - { - gf_log(this->name, GF_LOG_ERROR, "Failed to parse xlator options"); + if (ec_parse_options(this) != 0) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_PARSE_OPT_FAIL, + "Failed to parse xlator options"); + + goto failed; + } + + GF_OPTION_INIT("cpu-extensions", extensions, str, failed); + + err = ec_method_init(this, &ec->matrix, ec->fragments, ec->nodes, + ec->nodes * 2, extensions); + if (err != 0) { + gf_msg(this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED, + "Failed to initialize matrix management"); goto failed; } - ec_method_initialize(); - GF_OPTION_INIT ("self-heal-daemon", ec->shd, bool, failed); - GF_OPTION_INIT ("iam-self-heal-daemon", ec->iamshd, bool, failed); + GF_OPTION_INIT("self-heal-daemon", ec->shd.enabled, bool, failed); + GF_OPTION_INIT("iam-self-heal-daemon", ec->shd.iamshd, bool, failed); + GF_OPTION_INIT("eager-lock", ec->eager_lock, bool, failed); + GF_OPTION_INIT("other-eager-lock", ec->other_eager_lock, bool, failed); + GF_OPTION_INIT("eager-lock-timeout", ec->eager_lock_timeout, uint32, + failed); + GF_OPTION_INIT("other-eager-lock-timeout", ec->other_eager_lock_timeout, + uint32, failed); + GF_OPTION_INIT("background-heals", ec->background_heals, uint32, failed); + GF_OPTION_INIT("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed); + GF_OPTION_INIT("self-heal-window-size", ec->self_heal_window_size, uint32, + failed); + ec_configure_background_heal_opts(ec, ec->background_heals, + ec->heal_wait_qlen); + GF_OPTION_INIT("read-policy", read_policy, str, failed); + if (ec_assign_read_policy(ec, read_policy)) + goto failed; + + GF_OPTION_INIT("heal-timeout", ec->shd.timeout, int32, failed); + GF_OPTION_INIT("shd-max-threads", ec->shd.max_threads, uint32, failed); + GF_OPTION_INIT("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed); + GF_OPTION_INIT("optimistic-change-log", ec->optimistic_changelog, bool, + failed); + GF_OPTION_INIT("parallel-writes", ec->parallel_writes, bool, failed); + GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed); + GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed); + GF_OPTION_INIT("ec-read-mask", read_mask_str, str, failed); + + if (ec_assign_read_mask(ec, read_mask_str)) + goto failed; + + this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this); + if (!this->itable) + goto failed; + + if (ec->shd.iamshd) + ec_selfheal_daemon_init(this); + gf_msg_debug(this->name, 0, "Disperse translator initialized."); + + ec->leaf_to_subvolid = dict_new(); + if (!ec->leaf_to_subvolid) + goto failed; + if (glusterfs_reachable_leaves(this, ec->leaf_to_subvolid)) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_BUILD_FAIL, + "Failed to build subvol " + "dictionary"); + goto failed; + } - gf_log(this->name, GF_LOG_DEBUG, "Disperse translator initialized."); + if (ec_subvol_to_subvol_id_transform(ec, ec->leaf_to_subvolid) < 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_ID_DICT_SET_FAIL, + "Failed to build subvol-id " + "dictionary"); + goto failed; + } + + ec_statistics_init(ec); return 0; @@ -453,13 +910,16 @@ failed: return -1; } -void fini(xlator_t * this) +void +fini(xlator_t *this) { + ec_selfheal_daemon_fini(this); __ec_destroy_private(this); } -int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc, - int32_t mask, dict_t * xdata) +int32_t +ec_gf_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, + dict_t *xdata) { ec_access(frame, this, -1, EC_MINIMUM_ONE, default_access_cbk, NULL, loc, mask, xdata); @@ -467,9 +927,9 @@ int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc, - int32_t flags, mode_t mode, mode_t umask, fd_t * fd, - dict_t * xdata) +int32_t +ec_gf_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) { ec_create(frame, this, -1, EC_MINIMUM_MIN, default_create_cbk, NULL, loc, flags, mode, umask, fd, xdata); @@ -477,45 +937,58 @@ int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_discard(call_frame_t * frame, xlator_t * this, fd_t * fd, - off_t offset, size_t len, dict_t * xdata) +int32_t +ec_gf_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + size_t len, dict_t *xdata) { - default_discard_failure_cbk(frame, ENOTSUP); + ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk, NULL, fd, + offset, len, xdata); return 0; } -int32_t ec_gf_entrylk(call_frame_t * frame, xlator_t * this, - const char * volume, loc_t * loc, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata) +int32_t +ec_gf_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, + loc_t *loc, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) { - ec_entrylk(frame, this, -1, EC_MINIMUM_ALL, default_entrylk_cbk, NULL, - volume, loc, basename, cmd, type, xdata); + uint32_t fop_flags = EC_MINIMUM_ALL; + + if (cmd == ENTRYLK_UNLOCK) + fop_flags = EC_MINIMUM_ONE; + ec_entrylk(frame, this, -1, fop_flags, default_entrylk_cbk, NULL, volume, + loc, basename, cmd, type, xdata); return 0; } -int32_t ec_gf_fentrylk(call_frame_t * frame, xlator_t * this, - const char * volume, fd_t * fd, const char * basename, - entrylk_cmd cmd, entrylk_type type, dict_t * xdata) +int32_t +ec_gf_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, + fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) { - ec_fentrylk(frame, this, -1, EC_MINIMUM_ALL, default_fentrylk_cbk, NULL, - volume, fd, basename, cmd, type, xdata); + uint32_t fop_flags = EC_MINIMUM_ALL; + + if (cmd == ENTRYLK_UNLOCK) + fop_flags = EC_MINIMUM_ONE; + ec_fentrylk(frame, this, -1, fop_flags, default_fentrylk_cbk, NULL, volume, + fd, basename, cmd, type, xdata); return 0; } -int32_t ec_gf_fallocate(call_frame_t * frame, xlator_t * this, fd_t * fd, - int32_t keep_size, off_t offset, size_t len, - dict_t * xdata) +int32_t +ec_gf_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, + off_t offset, size_t len, dict_t *xdata) { - default_fallocate_failure_cbk(frame, ENOTSUP); + ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk, NULL, + fd, mode, offset, len, xdata); return 0; } -int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd, - dict_t * xdata) +int32_t +ec_gf_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) { ec_flush(frame, this, -1, EC_MINIMUM_MIN, default_flush_cbk, NULL, fd, xdata); @@ -523,8 +996,9 @@ int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd, - int32_t datasync, dict_t * xdata) +int32_t +ec_gf_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, + dict_t *xdata) { ec_fsync(frame, this, -1, EC_MINIMUM_MIN, default_fsync_cbk, NULL, fd, datasync, xdata); @@ -532,74 +1006,139 @@ int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_fsyncdir(call_frame_t * frame, xlator_t * this, fd_t * fd, - int32_t datasync, dict_t * xdata) +int32_t +ec_gf_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, + dict_t *xdata) { - ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, - fd, datasync, xdata); + ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, fd, + datasync, xdata); return 0; } +int +ec_marker_populate_args(call_frame_t *frame, int type, int *gauge, + xlator_t **subvols) +{ + xlator_t *this = frame->this; + ec_t *ec = this->private; + + memcpy(subvols, ec->xl_list, sizeof(*subvols) * ec->nodes); + + if (type == MARKER_XTIME_TYPE) { + /*Don't error out on ENOENT/ENOTCONN */ + gauge[MCNT_NOTFOUND] = 0; + gauge[MCNT_ENOTCONN] = 0; + } + + return ec->nodes; +} + int32_t -ec_gf_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, - const char *name, dict_t *xdata) +ec_handle_heal_commands(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) { - int error = 0; + dict_t *dict_rsp = NULL; + int op_ret = -1; + int op_errno = ENOMEM; - if (name && strcmp (name, EC_XATTR_HEAL) != 0) { - EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out); - } + if (!name || strcmp(name, GF_HEAL_INFO)) + return -1; - ec_getxattr (frame, this, -1, EC_MINIMUM_MIN, default_getxattr_cbk, - NULL, loc, name, xdata); + op_errno = -ec_get_heal_info(this, loc, &dict_rsp); + if (op_errno <= 0) { + op_errno = op_ret = 0; + } - return 0; -out: - error = ENODATA; - STACK_UNWIND_STRICT (getxattr, frame, -1, error, NULL, NULL); - return 0; + STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict_rsp, NULL); + if (dict_rsp) + dict_unref(dict_rsp); + return 0; } int32_t -ec_gf_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, - const char *name, dict_t *xdata) +ec_gf_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) { - int error = 0; + int error = 0; + ec_t *ec = this->private; + int32_t fop_flags = EC_MINIMUM_ONE; + if (name && strcmp(name, EC_XATTR_HEAL) != 0) { EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out); + } - ec_fgetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fgetxattr_cbk, - NULL, fd, name, xdata); + if (ec_handle_heal_commands(frame, this, loc, name, xdata) == 0) return 0; -out: - error = ENODATA; - STACK_UNWIND_STRICT (fgetxattr, frame, -1, error, NULL, NULL); + + if (cluster_handle_marker_getxattr(frame, loc, name, ec->vol_uuid, NULL, + ec_marker_populate_args) == 0) return 0; + + if (name && ((fnmatch(GF_XATTR_STIME_PATTERN, name, 0) == 0) || + XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) { + fop_flags = EC_MINIMUM_ALL; + } + + ec_getxattr(frame, this, -1, fop_flags, default_getxattr_cbk, NULL, loc, + name, xdata); + + return 0; +out: + error = ENODATA; + STACK_UNWIND_STRICT(getxattr, frame, -1, error, NULL, NULL); + return 0; +} + +int32_t +ec_gf_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name, + dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out); + + ec_fgetxattr(frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk, NULL, + fd, name, xdata); + return 0; +out: + error = ENODATA; + STACK_UNWIND_STRICT(fgetxattr, frame, -1, error, NULL, NULL); + return 0; } -int32_t ec_gf_inodelk(call_frame_t * frame, xlator_t * this, - const char * volume, loc_t * loc, int32_t cmd, - struct gf_flock * flock, dict_t * xdata) +int32_t +ec_gf_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, + loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata) { - ec_inodelk(frame, this, -1, EC_MINIMUM_ALL, default_inodelk_cbk, NULL, - volume, loc, cmd, flock, xdata); + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + + ec_inodelk(frame, this, &frame->root->lk_owner, -1, fop_flags, + default_inodelk_cbk, NULL, volume, loc, cmd, flock, xdata); return 0; } -int32_t ec_gf_finodelk(call_frame_t * frame, xlator_t * this, - const char * volume, fd_t * fd, int32_t cmd, - struct gf_flock * flock, dict_t * xdata) +int32_t +ec_gf_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, + fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata) { - ec_finodelk(frame, this, -1, EC_MINIMUM_ALL, default_finodelk_cbk, NULL, - volume, fd, cmd, flock, xdata); + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + ec_finodelk(frame, this, &frame->root->lk_owner, -1, fop_flags, + default_finodelk_cbk, NULL, volume, fd, cmd, flock, xdata); return 0; } -int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc, - loc_t * newloc, dict_t * xdata) +int32_t +ec_gf_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, + dict_t *xdata) { ec_link(frame, this, -1, EC_MINIMUM_MIN, default_link_cbk, NULL, oldloc, newloc, xdata); @@ -607,17 +1146,22 @@ int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc, return 0; } -int32_t ec_gf_lk(call_frame_t * frame, xlator_t * this, fd_t * fd, - int32_t cmd, struct gf_flock * flock, dict_t * xdata) +int32_t +ec_gf_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) { - ec_lk(frame, this, -1, EC_MINIMUM_ALL, default_lk_cbk, NULL, fd, cmd, - flock, xdata); + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + ec_lk(frame, this, -1, fop_flags, default_lk_cbk, NULL, fd, cmd, flock, + xdata); return 0; } -int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc, - dict_t * xdata) +int32_t +ec_gf_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) { ec_lookup(frame, this, -1, EC_MINIMUM_MIN, default_lookup_cbk, NULL, loc, xdata); @@ -625,8 +1169,9 @@ int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc, - mode_t mode, mode_t umask, dict_t * xdata) +int32_t +ec_gf_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, + mode_t umask, dict_t *xdata) { ec_mkdir(frame, this, -1, EC_MINIMUM_MIN, default_mkdir_cbk, NULL, loc, mode, umask, xdata); @@ -634,8 +1179,9 @@ int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc, - mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata) +int32_t +ec_gf_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, + dev_t rdev, mode_t umask, dict_t *xdata) { ec_mknod(frame, this, -1, EC_MINIMUM_MIN, default_mknod_cbk, NULL, loc, mode, rdev, umask, xdata); @@ -643,17 +1189,19 @@ int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_open(call_frame_t * frame, xlator_t * this, loc_t * loc, - int32_t flags, fd_t * fd, dict_t * xdata) +int32_t +ec_gf_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + fd_t *fd, dict_t *xdata) { - ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, - flags, fd, xdata); + ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, flags, + fd, xdata); return 0; } -int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc, - fd_t * fd, dict_t * xdata) +int32_t +ec_gf_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, + dict_t *xdata) { ec_opendir(frame, this, -1, EC_MINIMUM_MIN, default_opendir_cbk, NULL, loc, fd, xdata); @@ -661,8 +1209,9 @@ int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd, - size_t size, off_t offset, dict_t * xdata) +int32_t +ec_gf_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, dict_t *xdata) { ec_readdir(frame, this, -1, EC_MINIMUM_ONE, default_readdir_cbk, NULL, fd, size, offset, xdata); @@ -670,17 +1219,19 @@ int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_readdirp(call_frame_t * frame, xlator_t * this, fd_t * fd, - size_t size, off_t offset, dict_t * xdata) +int32_t +ec_gf_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, dict_t *xdata) { - ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, - fd, size, offset, xdata); + ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, fd, + size, offset, xdata); return 0; } -int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc, - size_t size, dict_t * xdata) +int32_t +ec_gf_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, + dict_t *xdata) { ec_readlink(frame, this, -1, EC_MINIMUM_ONE, default_readlink_cbk, NULL, loc, size, xdata); @@ -688,60 +1239,63 @@ int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_readv(call_frame_t * frame, xlator_t * this, fd_t * fd, - size_t size, off_t offset, uint32_t flags, dict_t * xdata) +int32_t +ec_gf_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, uint32_t flags, dict_t *xdata) { - ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, - size, offset, flags, xdata); + ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, size, + offset, flags, xdata); return 0; } int32_t -ec_gf_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc, - const char *name, dict_t *xdata) +ec_gf_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) { - int error = 0; + int error = 0; - EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out); + EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out); - ec_removexattr (frame, this, -1, EC_MINIMUM_MIN, - default_removexattr_cbk, NULL, loc, name, xdata); + ec_removexattr(frame, this, -1, EC_MINIMUM_MIN, default_removexattr_cbk, + NULL, loc, name, xdata); - return 0; + return 0; out: - STACK_UNWIND_STRICT (removexattr, frame, -1, error, NULL); - return 0; + STACK_UNWIND_STRICT(removexattr, frame, -1, error, NULL); + return 0; } int32_t -ec_gf_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd, - const char *name, dict_t *xdata) +ec_gf_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, + const char *name, dict_t *xdata) { - int error = 0; + int error = 0; - EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out); + EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out); - ec_fremovexattr (frame, this, -1, EC_MINIMUM_MIN, - default_fremovexattr_cbk, NULL, fd, name, xdata); + ec_fremovexattr(frame, this, -1, EC_MINIMUM_MIN, default_fremovexattr_cbk, + NULL, fd, name, xdata); - return 0; + return 0; out: - STACK_UNWIND_STRICT (fremovexattr, frame, -1, error, NULL); - return 0; + STACK_UNWIND_STRICT(fremovexattr, frame, -1, error, NULL); + return 0; } -int32_t ec_gf_rename(call_frame_t * frame, xlator_t * this, loc_t * oldloc, - loc_t * newloc, dict_t * xdata) +int32_t +ec_gf_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, + dict_t *xdata) { - ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, - oldloc, newloc, xdata); + ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, oldloc, + newloc, xdata); return 0; } -int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc, - int xflags, dict_t * xdata) +int32_t +ec_gf_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, + dict_t *xdata) { ec_rmdir(frame, this, -1, EC_MINIMUM_MIN, default_rmdir_cbk, NULL, loc, xflags, xdata); @@ -749,8 +1303,9 @@ int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc, - struct iatt * stbuf, int32_t valid, dict_t * xdata) +int32_t +ec_gf_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) { ec_setattr(frame, this, -1, EC_MINIMUM_MIN, default_setattr_cbk, NULL, loc, stbuf, valid, xdata); @@ -758,51 +1313,52 @@ int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_fsetattr(call_frame_t * frame, xlator_t * this, fd_t * fd, - struct iatt * stbuf, int32_t valid, dict_t * xdata) +int32_t +ec_gf_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata) { - ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, - fd, stbuf, valid, xdata); + ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, fd, + stbuf, valid, xdata); return 0; } int32_t -ec_gf_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, - dict_t *dict, int32_t flags, dict_t *xdata) +ec_gf_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, + int32_t flags, dict_t *xdata) { - int error = 0; + int error = 0; - EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out); + EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out); - ec_setxattr (frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, - NULL, loc, dict, flags, xdata); + ec_setxattr(frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, NULL, + loc, dict, flags, xdata); - return 0; + return 0; out: - STACK_UNWIND_STRICT (setxattr, frame, -1, error, NULL); - return 0; + STACK_UNWIND_STRICT(setxattr, frame, -1, error, NULL); + return 0; } int32_t -ec_gf_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, - dict_t *dict, int32_t flags, dict_t *xdata) +ec_gf_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict, + int32_t flags, dict_t *xdata) { - int error = 0; + int error = 0; - EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out); + EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out); - ec_fsetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, - NULL, fd, dict, flags, xdata); + ec_fsetxattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, NULL, + fd, dict, flags, xdata); - return 0; + return 0; out: - STACK_UNWIND_STRICT (fsetxattr, frame, -1, error, NULL); - return 0; + STACK_UNWIND_STRICT(fsetxattr, frame, -1, error, NULL); + return 0; } -int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc, - dict_t * xdata) +int32_t +ec_gf_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) { ec_stat(frame, this, -1, EC_MINIMUM_MIN, default_stat_cbk, NULL, loc, xdata); @@ -810,8 +1366,8 @@ int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd, - dict_t * xdata) +int32_t +ec_gf_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) { ec_fstat(frame, this, -1, EC_MINIMUM_MIN, default_fstat_cbk, NULL, fd, xdata); @@ -819,8 +1375,8 @@ int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc, - dict_t * xdata) +int32_t +ec_gf_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) { ec_statfs(frame, this, -1, EC_MINIMUM_MIN, default_statfs_cbk, NULL, loc, xdata); @@ -828,9 +1384,9 @@ int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this, - const char * linkname, loc_t * loc, mode_t umask, - dict_t * xdata) +int32_t +ec_gf_symlink(call_frame_t *frame, xlator_t *this, const char *linkname, + loc_t *loc, mode_t umask, dict_t *xdata) { ec_symlink(frame, this, -1, EC_MINIMUM_MIN, default_symlink_cbk, NULL, linkname, loc, umask, xdata); @@ -838,8 +1394,9 @@ int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this, return 0; } -int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc, - off_t offset, dict_t * xdata) +int32_t +ec_gf_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, + dict_t *xdata) { ec_truncate(frame, this, -1, EC_MINIMUM_MIN, default_truncate_cbk, NULL, loc, offset, xdata); @@ -847,8 +1404,9 @@ int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd, - off_t offset, dict_t * xdata) +int32_t +ec_gf_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + dict_t *xdata) { ec_ftruncate(frame, this, -1, EC_MINIMUM_MIN, default_ftruncate_cbk, NULL, fd, offset, xdata); @@ -856,8 +1414,9 @@ int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc, - int xflags, dict_t * xdata) +int32_t +ec_gf_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, + dict_t *xdata) { ec_unlink(frame, this, -1, EC_MINIMUM_MIN, default_unlink_cbk, NULL, loc, xflags, xdata); @@ -865,9 +1424,10 @@ int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd, - struct iovec * vector, int32_t count, off_t offset, - uint32_t flags, struct iobref * iobref, dict_t * xdata) +int32_t +ec_gf_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata) { ec_writev(frame, this, -1, EC_MINIMUM_MIN, default_writev_cbk, NULL, fd, vector, count, offset, flags, iobref, xdata); @@ -875,9 +1435,9 @@ int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd, return 0; } -int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc, - gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata) +int32_t +ec_gf_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) { ec_xattrop(frame, this, -1, EC_MINIMUM_MIN, default_xattrop_cbk, NULL, loc, optype, xattr, xdata); @@ -885,70 +1445,94 @@ int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc, return 0; } -int32_t ec_gf_fxattrop(call_frame_t * frame, xlator_t * this, fd_t * fd, - gf_xattrop_flags_t optype, dict_t * xattr, - dict_t * xdata) +int32_t +ec_gf_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) { - ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, - fd, optype, xattr, xdata); + ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, fd, + optype, xattr, xdata); return 0; } -int32_t ec_gf_zerofill(call_frame_t * frame, xlator_t * this, fd_t * fd, - off_t offset, off_t len, dict_t * xdata) +int32_t +ec_gf_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + off_t len, dict_t *xdata) { default_zerofill_failure_cbk(frame, ENOTSUP); return 0; } -int32_t ec_gf_forget(xlator_t * this, inode_t * inode) +int32_t +ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + gf_seek_what_t what, dict_t *xdata) +{ + ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd, offset, + what, xdata); + + return 0; +} + +int32_t +ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) +{ + ec_ipc(frame, this, -1, EC_MINIMUM_MIN, default_ipc_cbk, NULL, op, xdata); + return 0; +} + +int32_t +ec_gf_forget(xlator_t *this, inode_t *inode) { uint64_t value = 0; - ec_inode_t * ctx = NULL; + ec_inode_t *ctx = NULL; - if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) - { + if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) { ctx = (ec_inode_t *)(uintptr_t)value; + /* We can only forget an inode if it has been unlocked, so the stripe + * cache should also be empty. */ + GF_ASSERT(list_empty(&ctx->stripe_cache.lru)); GF_FREE(ctx); } return 0; } -void ec_gf_release_fd(xlator_t * this, fd_t * fd) +void +ec_gf_release_fd(xlator_t *this, fd_t *fd) { uint64_t value = 0; - ec_fd_t * ctx = NULL; + ec_fd_t *ctx = NULL; - if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) - { + if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) { ctx = (ec_fd_t *)(uintptr_t)value; loc_wipe(&ctx->loc); GF_FREE(ctx); } } -int32_t ec_gf_release(xlator_t * this, fd_t * fd) +int32_t +ec_gf_release(xlator_t *this, fd_t *fd) { ec_gf_release_fd(this, fd); return 0; } -int32_t ec_gf_releasedir(xlator_t * this, fd_t * fd) +int32_t +ec_gf_releasedir(xlator_t *this, fd_t *fd) { ec_gf_release_fd(this, fd); return 0; } -int32_t ec_dump_private(xlator_t *this) +int32_t +ec_dump_private(xlator_t *this) { ec_t *ec = NULL; - char key_prefix[GF_DUMP_MAX_BUF_LEN]; - char tmp[65]; + char key_prefix[GF_DUMP_MAX_BUF_LEN]; + char tmp[65]; GF_ASSERT(this); @@ -956,7 +1540,8 @@ int32_t ec_dump_private(xlator_t *this) GF_ASSERT(ec); snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name); - gf_proc_dump_add_section(key_prefix); + gf_proc_dump_add_section("%s", key_prefix); + gf_proc_dump_write("up", "%u", ec->up); gf_proc_dump_write("nodes", "%u", ec->nodes); gf_proc_dump_write("redundancy", "%u", ec->redundancy); gf_proc_dump_write("fragment_size", "%u", ec->fragment_size); @@ -964,88 +1549,325 @@ int32_t ec_dump_private(xlator_t *this) gf_proc_dump_write("childs_up", "%u", ec->xl_up_count); gf_proc_dump_write("childs_up_mask", "%s", ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes)); + if (ec->read_mask) { + gf_proc_dump_write("read-mask", "%s", + ec_bin(tmp, sizeof(tmp), ec->read_mask, ec->nodes)); + } + gf_proc_dump_write("background-heals", "%d", ec->background_heals); + gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen); + gf_proc_dump_write("self-heal-window-size", "%" PRIu32, + ec->self_heal_window_size); + gf_proc_dump_write("healers", "%d", ec->healers); + gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters); + gf_proc_dump_write("read-policy", "%s", ec_read_policies[ec->read_policy]); + gf_proc_dump_write("parallel-writes", "%d", ec->parallel_writes); + gf_proc_dump_write("quorum-count", "%u", ec->quorum_count); + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s.stats.stripe_cache", + this->type, this->name); + gf_proc_dump_add_section("%s", key_prefix); + + gf_proc_dump_write("hits", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.hits)); + gf_proc_dump_write("misses", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.misses)); + gf_proc_dump_write("updates", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.updates)); + gf_proc_dump_write("invalidations", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.invals)); + gf_proc_dump_write("evicts", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.evicts)); + gf_proc_dump_write("allocations", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.allocs)); + gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.errors)); + gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.attempted)); + gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.completed)); return 0; } -struct xlator_fops fops = -{ - .lookup = ec_gf_lookup, - .stat = ec_gf_stat, - .fstat = ec_gf_fstat, - .truncate = ec_gf_truncate, - .ftruncate = ec_gf_ftruncate, - .access = ec_gf_access, - .readlink = ec_gf_readlink, - .mknod = ec_gf_mknod, - .mkdir = ec_gf_mkdir, - .unlink = ec_gf_unlink, - .rmdir = ec_gf_rmdir, - .symlink = ec_gf_symlink, - .rename = ec_gf_rename, - .link = ec_gf_link, - .create = ec_gf_create, - .open = ec_gf_open, - .readv = ec_gf_readv, - .writev = ec_gf_writev, - .flush = ec_gf_flush, - .fsync = ec_gf_fsync, - .opendir = ec_gf_opendir, - .readdir = ec_gf_readdir, - .readdirp = ec_gf_readdirp, - .fsyncdir = ec_gf_fsyncdir, - .statfs = ec_gf_statfs, - .setxattr = ec_gf_setxattr, - .getxattr = ec_gf_getxattr, - .fsetxattr = ec_gf_fsetxattr, - .fgetxattr = ec_gf_fgetxattr, - .removexattr = ec_gf_removexattr, - .fremovexattr = ec_gf_fremovexattr, - .lk = ec_gf_lk, - .inodelk = ec_gf_inodelk, - .finodelk = ec_gf_finodelk, - .entrylk = ec_gf_entrylk, - .fentrylk = ec_gf_fentrylk, - .xattrop = ec_gf_xattrop, - .fxattrop = ec_gf_fxattrop, - .setattr = ec_gf_setattr, - .fsetattr = ec_gf_fsetattr, - .fallocate = ec_gf_fallocate, - .discard = ec_gf_discard, - .zerofill = ec_gf_zerofill -}; - -struct xlator_cbks cbks = -{ - .forget = ec_gf_forget, - .release = ec_gf_release, - .releasedir = ec_gf_releasedir -}; - -struct xlator_dumpops dumpops = { - .priv = ec_dump_private -}; - -struct volume_options options[] = -{ +struct xlator_fops fops = {.lookup = ec_gf_lookup, + .stat = ec_gf_stat, + .fstat = ec_gf_fstat, + .truncate = ec_gf_truncate, + .ftruncate = ec_gf_ftruncate, + .access = ec_gf_access, + .readlink = ec_gf_readlink, + .mknod = ec_gf_mknod, + .mkdir = ec_gf_mkdir, + .unlink = ec_gf_unlink, + .rmdir = ec_gf_rmdir, + .symlink = ec_gf_symlink, + .rename = ec_gf_rename, + .link = ec_gf_link, + .create = ec_gf_create, + .open = ec_gf_open, + .readv = ec_gf_readv, + .writev = ec_gf_writev, + .flush = ec_gf_flush, + .fsync = ec_gf_fsync, + .opendir = ec_gf_opendir, + .readdir = ec_gf_readdir, + .readdirp = ec_gf_readdirp, + .fsyncdir = ec_gf_fsyncdir, + .statfs = ec_gf_statfs, + .setxattr = ec_gf_setxattr, + .getxattr = ec_gf_getxattr, + .fsetxattr = ec_gf_fsetxattr, + .fgetxattr = ec_gf_fgetxattr, + .removexattr = ec_gf_removexattr, + .fremovexattr = ec_gf_fremovexattr, + .lk = ec_gf_lk, + .inodelk = ec_gf_inodelk, + .finodelk = ec_gf_finodelk, + .entrylk = ec_gf_entrylk, + .fentrylk = ec_gf_fentrylk, + .xattrop = ec_gf_xattrop, + .fxattrop = ec_gf_fxattrop, + .setattr = ec_gf_setattr, + .fsetattr = ec_gf_fsetattr, + .fallocate = ec_gf_fallocate, + .discard = ec_gf_discard, + .zerofill = ec_gf_zerofill, + .seek = ec_gf_seek, + .ipc = ec_gf_ipc}; + +struct xlator_cbks cbks = {.forget = ec_gf_forget, + .release = ec_gf_release, + .releasedir = ec_gf_releasedir}; + +struct xlator_dumpops dumpops = {.priv = ec_dump_private}; + +struct volume_options options[] = { + {.key = {"redundancy"}, + .type = GF_OPTION_TYPE_INT, + .default_value = "{{ volume.redundancy }}", + .description = "Maximum number of bricks that can fail " + "simultaneously without losing data."}, { - .key = { "redundancy" }, - .type = GF_OPTION_TYPE_INT, - .description = "Maximum number of bricks that can fail " - "simultaneously without losing data." - }, - { - .key = { "self-heal-daemon" }, + .key = {"self-heal-daemon"}, .type = GF_OPTION_TYPE_BOOL, .description = "self-heal daemon enable/disable", .default_value = "enable", + .op_version = {GD_OP_VERSION_3_7_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + }, + {.key = {"iam-self-heal-daemon"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "off", + .description = "This option differentiates if the disperse " + "translator is running as part of self-heal-daemon " + "or not."}, + {.key = {"eager-lock"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_7_10}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Enable/Disable eager lock for regular files on a " + "disperse volume. If a fop takes a lock and completes " + "its operation, it waits for next 1 second before " + "releasing the lock, to see if the lock can be reused " + "for next fop from the same client. If ec finds any lock " + "contention within 1 second it releases the lock " + "immediately before time expires. This improves the " + "performance of file operations. However, as it takes " + "lock on first brick, for few operations like read, " + "discovery of lock contention might take long time and " + "can actually degrade the performance. If eager lock is " + "disabled, lock will be released as soon as fop " + "completes."}, + {.key = {"other-eager-lock"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_13_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "It's equivalent to the eager-lock option but for non " + "regular files."}, + {.key = {"eager-lock-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 60, + .default_value = "1", + .op_version = {GD_OP_VERSION_4_0_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse", "locks", "timeout"}, + .description = "Maximum time (in seconds) that a lock on an inode is " + "kept held if no new operations on the inode are " + "received."}, + {.key = {"other-eager-lock-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 60, + .default_value = "1", + .op_version = {GD_OP_VERSION_4_0_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse", "locks", "timeout"}, + .description = "It's equivalent to eager-lock-timeout option but for " + "non regular files."}, + { + .key = {"background-heals"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, /*Disabling background heals*/ + .max = 256, + .default_value = "8", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of parallel" + " heals", }, - { .key = {"iam-self-heal-daemon"}, - .type = GF_OPTION_TYPE_BOOL, - .default_value = "off", - .description = "This option differentiates if the disperse " - "translator is running as part of self-heal-daemon " - "or not." + { + .key = {"heal-wait-qlength"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, + .max = + 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/ + .default_value = "128", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of heals" + " that can wait", + }, + {.key = {"heal-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 60, + .max = INT_MAX, + .default_value = "600", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE, + .tags = {"disperse"}, + .description = "time interval for checking the need to self-heal " + "in self-heal-daemon"}, + { + .key = {"read-policy"}, + .type = GF_OPTION_TYPE_STR, + .value = {"round-robin", "gfid-hash"}, + .default_value = "gfid-hash", + .op_version = {GD_OP_VERSION_3_7_6}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = + "inode-read fops happen only on 'k' number of bricks in" + " n=k+m disperse subvolume. 'round-robin' selects the read" + " subvolume using round-robin algo. 'gfid-hash' selects read" + " subvolume based on hash of the gfid of that file/directory.", }, - { } + {.key = {"shd-max-threads"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 64, + .default_value = "1", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Maximum number of parallel heals SHD can do per local " + "brick. This can substantially lower heal times, " + "but can also crush your bricks if you don't have " + "the storage hardware to support this."}, + {.key = {"shd-wait-qlength"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 65536, + .default_value = "1024", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of heals" + " that can wait in SHD per subvolume"}, + {.key = {"cpu-extensions"}, + .type = GF_OPTION_TYPE_STR, + .value = {"none", "auto", "x64", "sse", "avx"}, + .default_value = "auto", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "force the cpu extensions to be used to accelerate the " + "galois field computations."}, + {.key = {"self-heal-window-size"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 1024, + .default_value = "1", + .op_version = {GD_OP_VERSION_3_11_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Maximum number blocks(128KB) per file for which " + "self-heal process would be applied simultaneously."}, + {.key = {"optimistic-change-log"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_10_1}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT, + .tags = {"disperse"}, + .description = "Set/Unset dirty flag for every update fop at the start" + "of the fop. If OFF, this option impacts performance of" + "entry operations or metadata operations as it will" + "set dirty flag at the start and unset it at the end of" + "ALL update fop. If ON and all the bricks are good," + "dirty flag will be set at the start only for file fops" + "For metadata and entry fops dirty flag will not be set" + "at the start, if all the bricks are good. This does" + "not impact performance for metadata operations and" + "entry operation but has a very small window to miss" + "marking entry as dirty in case it is required to be" + "healed"}, + {.key = {"parallel-writes"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .description = "This controls if writes can be wound in parallel as long" + "as it doesn't modify same stripes"}, + {.key = {"stripe-cache"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, /*Disabling stripe_cache*/ + .max = EC_STRIPE_CACHE_MAX_SIZE, + .default_value = "4", + .description = "This option will keep the last stripe of write fop" + "in memory. If next write falls in this stripe, we need" + "not to read it again from backend and we can save READ" + "fop going over the network. This will improve performance," + "specially for sequential writes. However, this will also" + "lead to extra memory consumption, maximum " + "(cache size * stripe size) Bytes per open file."}, + { + .key = {"quorum-count"}, + .type = GF_OPTION_TYPE_INT, + .default_value = "0", + .description = + "This option can be used to define how many successes on" + "the bricks constitute a success to the application. This" + " count should be in the range" + "[disperse-data-count, disperse-count] (inclusive)", + }, + { + .key = {"ec-read-mask"}, + .type = GF_OPTION_TYPE_STR, + .default_value = NULL, + .description = "This option can be used to choose which bricks can be" + " used for reading data/metadata of a file/directory", + }, + { + .key = {NULL}, + }, +}; + +xlator_api_t xlator_api = { + .init = init, + .fini = fini, + .notify = notify, + .reconfigure = reconfigure, + .mem_acct_init = mem_acct_init, + .op_version = {1}, + .dumpops = &dumpops, + .fops = &fops, + .cbks = &cbks, + .options = options, + .identifier = "disperse", + .category = GF_MAINTAINED, }; diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h index b6a95a11b18..6f6de6d5981 100644 --- a/xlators/cluster/ec/src/ec.h +++ b/xlators/cluster/ec/src/ec.h @@ -11,43 +11,24 @@ #ifndef __EC_H__ #define __EC_H__ -#include "xlator.h" -#include "timer.h" +#include "ec-method.h" -#define EC_XATTR_PREFIX "trusted.ec." -#define EC_XATTR_CONFIG EC_XATTR_PREFIX"config" -#define EC_XATTR_SIZE EC_XATTR_PREFIX"size" -#define EC_XATTR_VERSION EC_XATTR_PREFIX"version" -#define EC_XATTR_HEAL EC_XATTR_PREFIX"heal" -#define EC_XATTR_DIRTY EC_XATTR_PREFIX"dirty" +#define EC_XATTR_PREFIX "trusted.ec." +#define EC_XATTR_CONFIG EC_XATTR_PREFIX "config" +#define EC_XATTR_SIZE EC_XATTR_PREFIX "size" +#define EC_XATTR_VERSION EC_XATTR_PREFIX "version" +#define EC_XATTR_HEAL EC_XATTR_PREFIX "heal" +#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new" +#define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty" +#define EC_STRIPE_CACHE_MAX_SIZE 10 +#define EC_VERSION_SIZE 2 +#define EC_SHD_INODE_LRU_LIMIT 10 -struct _ec; -typedef struct _ec ec_t; - -struct _ec -{ - xlator_t * xl; - int32_t nodes; - int32_t bits_for_nodes; - int32_t fragments; - int32_t redundancy; - uint32_t fragment_size; - uint32_t stripe_size; - int32_t up; - uint32_t idx; - uint32_t xl_up_count; - uintptr_t xl_up; - uint32_t xl_notify_count; - uintptr_t xl_notify; - uintptr_t node_mask; - xlator_t ** xl_list; - gf_lock_t lock; - gf_timer_t * timer; - struct mem_pool * fop_pool; - struct mem_pool * cbk_pool; - struct mem_pool * lock_pool; - gf_boolean_t shd; - gf_boolean_t iamshd; -}; +#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS +/* The maximum number of nodes is derived from the maximum allowed fragments + * using the rule that redundancy cannot be equal or greater than the number + * of fragments. + */ +#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES) #endif /* __EC_H__ */ |