From 5d25e4a663f30ef221a00a22ed23cb2ed23fefa4 Mon Sep 17 00:00:00 2001 From: aitassou Date: Mon, 7 Jul 2025 19:46:57 +0200 Subject: [PATCH] Add new ISA-L backend for new encoding matrix When parity is higher than 5, the rs_vand decoding matrix is not invertible for some combinations of missing data and parity. Add new backend with modified gen matrix that is suited when parity >= 5. Use rs_cauchy or new modified matrix when parity >= 5. Related-Bug: #1639691 Co-Authored-By: Tim Burke Change-Id: I09abfc619893da7fd3d0740fed3586fdd46791d9 --- include/erasurecode/erasurecode.h | 1 + src/Makefile.am | 1 + src/backends/isa-l/isa_l_rs_vand_inv.c | 240 +++++++++++++++++++++++++ src/erasurecode.c | 2 + test/liberasurecode_test.c | 51 ++++++ 5 files changed, 295 insertions(+) create mode 100644 src/backends/isa-l/isa_l_rs_vand_inv.c diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h index b801206..bdb8562 100644 --- a/include/erasurecode/erasurecode.h +++ b/include/erasurecode/erasurecode.h @@ -50,6 +50,7 @@ typedef enum { EC_BACKEND_LIBERASURECODE_RS_VAND = 6, EC_BACKEND_ISA_L_RS_CAUCHY = 7, EC_BACKEND_LIBPHAZR = 8, + EC_BACKEND_ISA_L_RS_VAND_INV = 9, EC_BACKENDS_MAX, } ec_backend_id_t; diff --git a/src/Makefile.am b/src/Makefile.am index c0731c8..3bcc3e1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -26,6 +26,7 @@ liberasurecode_la_SOURCES = \ backends/jerasure/jerasure_rs_cauchy.c \ backends/isa-l/isa_l_common.c \ backends/isa-l/isa_l_rs_vand.c \ + backends/isa-l/isa_l_rs_vand_inv.c \ backends/isa-l/isa_l_rs_cauchy.c \ backends/rs_vand/liberasurecode_rs_vand.c \ backends/shss/shss.c \ diff --git a/src/backends/isa-l/isa_l_rs_vand_inv.c b/src/backends/isa-l/isa_l_rs_vand_inv.c new file mode 100644 index 0000000..9103083 --- /dev/null +++ b/src/backends/isa-l/isa_l_rs_vand_inv.c @@ -0,0 +1,240 @@ +/* + * Copyright 2014 Kevin M Greenan + * Copyright 2014 Tushar Gohad + * Copyright 2025 Tim Burke + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * isa_l_rs_vand backend implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#include +#include "erasurecode_backend.h" +#include "isa_l_common.h" + +#define ISA_L_RS_VAND_INV_LIB_MAJOR 1 +#define ISA_L_RS_VAND_INV_LIB_MINOR 0 +#define ISA_L_RS_VAND_INV_LIB_REV 0 +#define ISA_L_RS_VAND_INV_LIB_VER_STR "1.0" +#define ISA_L_RS_VAND_INV_LIB_NAME "isa_l_rs_vand" +#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) +#define ISA_L_RS_VAND_INV_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".dylib" +#else +#define ISA_L_RS_VAND_INV_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".so.2" +#endif + +/* Forward declarations */ +struct ec_backend_common backend_isa_l_rs_vand_inv; + +static int gen_encoding_matrix(isa_l_descriptor * desc, int m, int k){ + int i, j, ret = 1, n = m + k; + unsigned char p, gen = 2; + unsigned char *tmp = NULL; + unsigned char *tmp_inv_k = NULL; + + /* Build a (k+m)*k Vandermonde matrix, A */ + tmp = malloc(sizeof(char) * n * k); + if (tmp == NULL) { + goto error_free; + } + for (i = 0; i < n; i++) { + p = 1; + for (j = 0; j < k; j++) { + tmp[k * i + j] = p; + p = desc->gf_mul(p, gen); + } + gen = desc->gf_mul(gen, 2); + } + + /* It starts with a k*k submatrix, A'; calculate inv(A') */ + tmp_inv_k = malloc(sizeof(char) * k * k); + if (tmp_inv_k == NULL) { + goto error_free; + } + int im_ret = desc->gf_invert_matrix(tmp, tmp_inv_k, k); + if (im_ret < 0) { + /** + * Should never happen as it's a proper Vandermonde matrix, + * but belt & bracers... + */ + goto error_free; + } + + /** + * Now we're ready to build the encoding matrix: inv(A') * A. + * Save some multiplies by going straight to I for the start. + */ + memset(desc->matrix, 0, k * n); + for (i = 0; i < k; i++) + desc->matrix[k * i + i] = 1; + + /* Then multiply inv(A') by the rest of A for the parities */ + for (i = k; i < n; i++) { + for (j = 0; j < k; j++) { + p = 0; + for (int u = 0; u < k; u++) { + p ^= desc->gf_mul(tmp[(i*k)+u], tmp_inv_k[(u*k)+j]); + } + desc->matrix[(i*k)+j] = p; + } + } + ret = 0; + +error_free: + free(tmp_inv_k); + free(tmp); + return ret; +} + +static void * isa_l_rs_vand_inv_init(struct ec_backend_args *args, + void *backend_sohandle) +{ + isa_l_descriptor *desc = NULL; + + desc = (isa_l_descriptor *)malloc(sizeof(isa_l_descriptor)); + if (NULL == desc) { + return NULL; + } + /* Set this early so we can have a single error path */ + desc->matrix = NULL; + + desc->k = args->uargs.k; + desc->m = args->uargs.m; + if (args->uargs.w <= 0) + args->uargs.w = ISA_L_W; + desc->w = args->uargs.w; + + /* validate EC arguments */ + { + long long max_symbols = 1LL << desc->w; + if ((desc->k + desc->m) > max_symbols) { + goto error; + } + } + + /* + * ISO C forbids casting a void* to a function pointer. + * Since dlsym return returns a void*, we use this union to + * "transform" the void* to a function pointer. + */ + union { + ec_encode_data_func encodep; + ec_init_tables_func init_tablesp; + gf_gen_encoding_matrix_func gen_matrixp; + gf_invert_matrix_func invert_matrixp; + gf_mul_func gf_mulp; + void *vptr; + } func_handle = {.vptr = NULL}; + + /* fill in function addresses */ + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "ec_encode_data"); + desc->ec_encode_data = func_handle.encodep; + if (NULL == desc->ec_encode_data) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "ec_init_tables"); + desc->ec_init_tables = func_handle.init_tablesp; + if (NULL == desc->ec_init_tables) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "gf_invert_matrix"); + desc->gf_invert_matrix = func_handle.invert_matrixp; + if (NULL == desc->gf_invert_matrix) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "gf_mul"); + desc->gf_mul = func_handle.gf_mulp; + if (NULL == desc->gf_mul) { + goto error; + } + + desc->matrix = malloc(sizeof(char) * desc->k * (desc->k + desc->m)); + if (NULL == desc->matrix) { + goto error; + } + + if (0 != gen_encoding_matrix(desc, desc->m, desc->k)) { + goto error; + } + + /** + * Generate the tables for encoding + */ + desc->encode_tables = malloc(sizeof(unsigned char) * + (desc->k * desc->m * 32)); + if (NULL == desc->encode_tables) { + goto error; + } + + desc->ec_init_tables(desc->k, desc->m, + &desc->matrix[desc->k * desc->k], + desc->encode_tables); + + return desc; + +error: + free(desc->matrix); + free(desc); + + return NULL; +} + +/* + * For the time being, we only claim compatibility with versions that + * match exactly + */ +static bool isa_l_rs_vand_inv_is_compatible_with(uint32_t version) { + return version == backend_isa_l_rs_vand_inv.ec_backend_version; +} + +static struct ec_backend_op_stubs isa_l_rs_vand_inv_op_stubs = { + .INIT = isa_l_rs_vand_inv_init, + .EXIT = isa_l_exit, + .ENCODE = isa_l_encode, + .DECODE = isa_l_decode, + .FRAGSNEEDED = isa_l_min_fragments, + .RECONSTRUCT = isa_l_reconstruct, + .ELEMENTSIZE = isa_l_element_size, + .ISCOMPATIBLEWITH = isa_l_rs_vand_inv_is_compatible_with, + .GETMETADATASIZE = get_backend_metadata_size_zero, + .GETENCODEOFFSET = get_encode_offset_zero, +}; + +__attribute__ ((visibility ("internal"))) +struct ec_backend_common backend_isa_l_rs_vand_inv = { + .id = EC_BACKEND_ISA_L_RS_VAND_INV, + .name = ISA_L_RS_VAND_INV_LIB_NAME, + .soname = ISA_L_RS_VAND_INV_SO_NAME, + .soversion = ISA_L_RS_VAND_INV_LIB_VER_STR, + .ops = &isa_l_rs_vand_inv_op_stubs, + .ec_backend_version = _VERSION(ISA_L_RS_VAND_INV_LIB_MAJOR, + ISA_L_RS_VAND_INV_LIB_MINOR, + ISA_L_RS_VAND_INV_LIB_REV), +}; diff --git a/src/erasurecode.c b/src/erasurecode.c index 0a31889..95643fd 100644 --- a/src/erasurecode.c +++ b/src/erasurecode.c @@ -52,6 +52,7 @@ extern struct ec_backend_common backend_shss; extern struct ec_backend_common backend_liberasurecode_rs_vand; extern struct ec_backend_common backend_isa_l_rs_cauchy; extern struct ec_backend_common backend_libphazr; +extern struct ec_backend_common backend_isa_l_rs_vand_inv; static ec_backend_t ec_backends_supported[] = { (ec_backend_t) &backend_null, @@ -63,6 +64,7 @@ static ec_backend_t ec_backends_supported[] = { (ec_backend_t) &backend_liberasurecode_rs_vand, (ec_backend_t) &backend_isa_l_rs_cauchy, (ec_backend_t) &backend_libphazr, + (ec_backend_t) &backend_isa_l_rs_vand_inv, NULL, }; diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index 81c164e..7200952 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -40,6 +40,7 @@ #define JERASURE_RS_VAND_BACKEND "jerasure_rs_vand" #define JERASURE_RS_CAUCHY_BACKEND "jerasure_rs_cauchy" #define ISA_L_RS_VAND_BACKEND "isa_l_rs_vand" +#define ISA_L_RS_VAND_INV_BACKEND "isa_l_rs_vand_inv" #define ISA_L_RS_CAUCHY_BACKEND "isa_l_rs_cauchy" #define SHSS_BACKEND "shss" #define RS_VAND_BACKEND "liberasurecode_rs_vand" @@ -174,9 +175,24 @@ struct ec_args isa_l_1010_args = { .hd = 11, }; +struct ec_args isa_l_75_args = { + .k = 7, + .m = 5, + .w = 8, + .hd = 6, +}; +struct ec_args isa_l_85_args = { + .k = 8, + .m = 5, + .w = 8, + .hd = 6, +}; + struct ec_args *isa_l_test_args[] = { &isa_l_args, &isa_l_44_args, &isa_l_1010_args, + &isa_l_75_args, + &isa_l_85_args, NULL }; int priv = 128; @@ -296,6 +312,8 @@ char * get_name_from_backend_id(ec_backend_id_t be) { return FLAT_XOR_HD_BACKEND; case EC_BACKEND_ISA_L_RS_VAND: return ISA_L_RS_VAND_BACKEND; + case EC_BACKEND_ISA_L_RS_VAND_INV: + return ISA_L_RS_VAND_INV_BACKEND; case EC_BACKEND_ISA_L_RS_CAUCHY: return ISA_L_RS_CAUCHY_BACKEND; case EC_BACKEND_SHSS: @@ -335,6 +353,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba case EC_BACKEND_ISA_L_RS_VAND: backend_args_array = isa_l_test_args; break; + case EC_BACKEND_ISA_L_RS_VAND_INV: + backend_args_array = isa_l_test_args; + break; case EC_BACKEND_ISA_L_RS_CAUCHY: backend_args_array = isa_l_test_args; break; @@ -1485,6 +1506,32 @@ static void test_decode_with_missing_multi_data_parity( } } +static void test_decode_with_missing_multi_data_parity_fail_with_isal( + const ec_backend_id_t be_id, struct ec_args *args) +{ + int i,j; + struct ec_args specific_75_args = { + .k = 7, + .m = 5, + }; + int bad_positions[4][5] = { + {0, 3, 5, 9, 10 }, + {0, 2, 5, 8, 9 }, + {1, 3, 6, 8, 9}, + {1, 4, 6, 9, 10} + }; + for (i = 0; i < 4; i++) { + int *skips = create_skips_array(&specific_75_args,-1); + assert(skips != NULL); + for (j = 0; j < 5; j++) { + int skipped = bad_positions[i][j]; + skips[skipped]=1; + } + encode_decode_test_impl(be_id, &specific_75_args, skips); + free(skips); + } +} + static void test_isa_l_rs_vand_decode_reconstruct_specific_error_case(void) { struct ec_args specific_1010_args = { @@ -1966,6 +2013,10 @@ struct testcase testcases[] = { TEST({.no_args = test_isa_l_rs_vand_decode_reconstruct_specific_error_case}, EC_BACKENDS_MAX, 0), // ISA-L rs cauchy tests TEST_SUITE(EC_BACKEND_ISA_L_RS_CAUCHY), + TEST({.with_args = test_decode_with_missing_multi_data_parity_fail_with_isal}, EC_BACKEND_ISA_L_RS_CAUCHY, 0), + // ISA-L rs vand inv tests + TEST_SUITE(EC_BACKEND_ISA_L_RS_VAND_INV), + TEST({.with_args = test_decode_with_missing_multi_data_parity_fail_with_isal}, EC_BACKEND_ISA_L_RS_VAND_INV, 0), // shss tests TEST_SUITE(EC_BACKEND_SHSS), // Internal RS Vand backend tests