openstack/liberasurecode
Code Issues Proposed changes

Allow up to 256 total fragments

Browse Source 
... rather than the current limit of 32.

Note that some backends have much lower limits, both before and after:

- jerasure_rs_cauchy caps out at 16 fragments
- the flat xor codes in general are much more prescriptive regarding
  number of data and parity fragments

Those limits are still in place, but the other publicly accessible
backends all seem to support up to the new limit.

Change-Id: Icbc7b2e505442e5b3eb2b844637d5270be6de4d1
Signed-off-by: Tim Burke <tim.burke@gmail.com>
This commit is contained in:
Tim Burke
2026-02-26 08:25:49 -08:00
parent 41cf907b40
commit e1e11e42c7
15 changed files with 179 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.zuul.yaml
View File

@@ -5,6 +5,7 @@
Build and run test for liberasurecode.
pre-run: playbooks/unittests/install-libs-from-source.yaml
run: playbooks/unittests/run.yaml
timeout: 2400
- job:
name: liberasurecode-unittests-memcheck

2
include/erasurecode/erasurecode.h
View File

@@ -32,7 +32,7 @@
#include "erasurecode_stdinc.h"
#include "erasurecode_version.h"
#define EC_MAX_FRAGMENTS 32
#define EC_MAX_FRAGMENTS 256
#ifdef __cplusplus
extern "C" {

75
include/erasurecode/erasurecode_helpers.h
View File

@@ -29,6 +29,7 @@
#ifndef _ERASURECODE_HELPERS_H_
#define _ERASURECODE_HELPERS_H_
#include <assert.h>
#include "erasurecode_stdinc.h"
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
@@ -42,46 +43,50 @@ int is_addr_aligned(unsigned long addr, int align)
return (addr & (align - 1)) == 0;
}
struct ec_bm {
uint64_t v[4];
};
#define NEW_BM {{0, 0, 0, 0}}
// Note that `bit` is 0-indexed
static inline uint64_t bm_get_value(struct ec_bm *bm, int bit) {
assert(bit < EC_MAX_FRAGMENTS);
return bm->v[bit >> 6] & (1LLU << (bit & 0x3f));
}
static inline void bm_set_value(struct ec_bm *bm, int bit, int value) {
assert(bit < EC_MAX_FRAGMENTS);
if (value) {
bm->v[bit >> 6] |= (1LLU << (bit & 0x3f));
} else {
bm->v[bit >> 6] &= ~(1LLU << (bit & 0x3f));
}
}
// Ensure all the bits set in `src` are set in `dst` as well
static inline void bm_combine_or(struct ec_bm *src, struct ec_bm *dst) {
dst->v[0] |= src->v[0];
dst->v[1] |= src->v[1];
dst->v[2] |= src->v[2];
dst->v[3] |= src->v[3];
}
// Ensure only the bits set in `src` may be set in `dst` as well
static inline void bm_combine_and(struct ec_bm *src, struct ec_bm *dst) {
dst->v[0] &= src->v[0];
dst->v[1] &= src->v[1];
dst->v[2] &= src->v[2];
dst->v[3] &= src->v[3];
}
static inline uint64_t bm_any(struct ec_bm *bm) {
return bm->v[0] | bm->v[1] | bm->v[2] | bm->v[3];
}
/*
* Convert an int list into a bitmap
* Assume the list is '-1' terminated.
*/
static inline
unsigned long long convert_list_to_bitmap(int *list)
static inline void convert_list_to_bitmap(int *list, struct ec_bm *bm)
{
int i = 0;
unsigned long long bm = 0;
while (list[i] > -1) {
/*
* TODO: Assert list[i] < 64
*/
bm |= (1 << list[i]);
i++;
}
return bm;
}
/*
* Convert an index list int list into a bitmap
* is_idx_in_erasure[] needs to be allocated by the caller
* @returns number of idxs in error
*/
static inline
int convert_idx_list_to_bitvalues(
int *list_idxs, // input idx_list
int *is_idx_in_erasure, // output idx list as boolean values (1/0)
int num_idxs) // total number of indexes
{
int i = 0, n = 0;
for (i = 0; i < num_idxs; i++)
is_idx_in_erasure[i] = 0;
for (i = 0, n = 0; (list_idxs[i] > -1) && (n < num_idxs); i++, n++)
is_idx_in_erasure[list_idxs[i]] = 1;
return n;
for (int i = 0; list[i] >= 0; i++)
bm_set_value(bm, list[i], 1);
}
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */

2
include/erasurecode/erasurecode_preprocessing.h
View File

@@ -43,7 +43,7 @@ int prepare_fragments_for_decode(
char **data, char **parity,
int *missing_idxs,
int *orig_size, int *fragment_payload_size, int fragment_size,
uint64_t *realloc_bm);
struct ec_bm *realloc_bm);
int get_fragment_partition(
int k, int m,

38
src/backends/isa-l/isa_l_common.c
View File

@@ -57,10 +57,11 @@ static unsigned char* isa_l_get_decode_matrix(int k, int m, unsigned char *encod
int i = 0, j = 0, l = 0;
int n = k + m;
unsigned char *decode_matrix = malloc(sizeof(unsigned char) * k * k);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
while (i < k && l < n) {
if (((1 << l) & missing_bm) == 0) {
if (!bm_get_value(&missing_bm, l)) {
for (j = 0; j < k; j++) {
decode_matrix[(k * i) + j] = encode_matrix[(k * l) + j];
}
@@ -87,7 +88,8 @@ static unsigned char* get_inverse_rows(int k,
int *missing_idxs,
gf_mul_func gf_mul)
{
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int num_missing_elements = get_num_missing_elements(missing_idxs);
unsigned char *inverse_rows = (unsigned char*)malloc(sizeof(unsigned
char*) * k * num_missing_elements);
@@ -105,7 +107,7 @@ static unsigned char* get_inverse_rows(int k,
* Fill in rows for missing data
*/
for (i = 0; i < k; i++) {
if ((1 << i) & missing_bm) {
if (bm_get_value(&missing_bm, i)) {
for (j = 0; j < k; j++) {
inverse_rows[(l * k) + j] = decode_inverse[(i * k) + j];
}
@@ -129,12 +131,12 @@ static unsigned char* get_inverse_rows(int k,
*/
for (i = k; i < n; i++) {
// Parity is missing
if ((1 << i) & missing_bm) {
if (bm_get_value(&missing_bm, i)) {
int d_idx_avail = 0;
int d_idx_unavail = 0;
for (j = 0; j < k; j++) {
// This data is available, so we can use the encode matrix
if (((1 << j) & missing_bm) == 0) {
if (!bm_get_value(&missing_bm, j)) {
inverse_rows[(l * k) + d_idx_avail] ^= encode_matrix[(i * k) + j];
d_idx_avail++;
} else {
@@ -171,7 +173,8 @@ int isa_l_decode(void *desc, char **data, char **parity,
int i, j;
int num_missing_elements = get_num_missing_elements(missing_idxs);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
decode_matrix = isa_l_get_decode_matrix(k, m, isa_l_desc->matrix, missing_idxs);
@@ -210,7 +213,7 @@ int isa_l_decode(void *desc, char **data, char **parity,
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
continue;
}
if (j == k) {
@@ -227,13 +230,13 @@ int isa_l_decode(void *desc, char **data, char **parity,
// Grab pointers to memory needed for missing data fragments
j = 0;
for (i = 0; i < k; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
decoded_elements[j] = (unsigned char*)data[i];
j++;
}
}
for (i = k; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
decoded_elements[j] = (unsigned char*)parity[i - k];
j++;
}
@@ -273,7 +276,8 @@ int isa_l_reconstruct(void *desc, char **data, char **parity,
int n = k + m;
int ret = -1;
int i, j;
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int inverse_row = -1;
/**
@@ -318,7 +322,7 @@ int isa_l_reconstruct(void *desc, char **data, char **parity,
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
continue;
}
if (j == k) {
@@ -337,7 +341,7 @@ int isa_l_reconstruct(void *desc, char **data, char **parity,
*/
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
if (i == destination_idx) {
if (i < k) {
reconstruct_buf = (unsigned char*)data[i];
@@ -376,14 +380,16 @@ int isa_l_min_fragments(void *desc, int *missing_idxs,
{
isa_l_descriptor *isa_l_desc = (isa_l_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm exclude_bm = NEW_BM, missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &exclude_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
bm_combine_or(&exclude_bm, &missing_bm);
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (isa_l_desc->k + isa_l_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

96
src/backends/isa-l/isa_l_rs_lrc.c
View File

@@ -228,19 +228,20 @@ error:
static int isa_l_rs_lrc_check_reconstruct_fragments(void *desc, int *missing_idxs, int destination_idx) {
isa_l_descriptor *d = (isa_l_descriptor*)desc;
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int missing_locals, local_group;
if (destination_idx < d->k) {
local_group = local_group_for_data(d->k, d->l, destination_idx);
int local_parity_index = d->k + d->m - d->l + local_group;
missing_locals = (1 << local_parity_index) & missing_bm;
missing_locals = bm_get_value(&missing_bm, local_parity_index);
for (
int i = local_group_data_lower(d->k, d->l, local_group);
i < local_group_data_upper(d->k, d->l, local_group);
i++
) {
if (i != destination_idx)
missing_locals |= (1 << i) & missing_bm;
missing_locals |= bm_get_value(&missing_bm, i);
}
if (!missing_locals) {
return 0;
@@ -253,7 +254,7 @@ static int isa_l_rs_lrc_check_reconstruct_fragments(void *desc, int *missing_idx
i < local_group_data_upper(d->k, d->l, local_group);
i++
) {
missing_locals |= (1 << i) & missing_bm;
missing_locals |= bm_get_value(&missing_bm, i);
}
if (!missing_locals) {
return 0;
@@ -261,21 +262,22 @@ static int isa_l_rs_lrc_check_reconstruct_fragments(void *desc, int *missing_idx
}
// if we haven't returned yet, we can't do local-only reconstruction
int useful_frags = 0, can_use_local_parity = 0;
int useful_frags = 0;
struct ec_bm can_use_local_parity = NEW_BM;
for (int i = 0; i < d->k + d->m; i++) {
if (i < d->k) {
if ((1 << i) & missing_bm) {
if (bm_get_value(&missing_bm, i)) {
local_group = local_group_for_data(d->k, d->l, i);
can_use_local_parity |= 1 << (d->k + d->m - d->l + local_group);
bm_set_value(&can_use_local_parity, d->k + d->m - d->l + local_group, 1);
} else {
useful_frags++;
}
} else if (i >= d->k + d->m - d->l) {
if ((1 << i) & can_use_local_parity && !((1 << i) & missing_bm)) {
if (bm_get_value(&can_use_local_parity, i) && !bm_get_value(&missing_bm, i)) {
useful_frags++;
}
} else {
if (!((1 << i) & missing_bm)) {
if (!bm_get_value(&missing_bm, i)) {
useful_frags++;
}
}
@@ -303,12 +305,12 @@ static unsigned char* get_lrc_inverse_rows(int k,
int missing_local_parity,
unsigned char *decode_inverse,
unsigned char* encode_matrix,
uint64_t missing_bm,
struct ec_bm *missing_bm,
gf_mul_func gf_mul)
{
int num_missing_elements = 0;
for (int i = 0; i < EC_MAX_FRAGMENTS; i++)
if ((1LLU << i) & missing_bm)
if (bm_get_value(missing_bm, i))
num_missing_elements++;
unsigned char *inverse_rows = (unsigned char*)malloc(sizeof(unsigned
char*) * k * num_missing_elements);
@@ -331,7 +333,7 @@ static unsigned char* get_lrc_inverse_rows(int k,
* Fill in rows for missing data
*/
for (i = 0; i < matrix_size; i++) {
if ((1 << i) & (missing_bm>>min_range)) {
if (bm_get_value(missing_bm, i + min_range)) {
for (j = 0; j < matrix_size; j++) {
inverse_rows[(l * matrix_size) + j] = decode_inverse[(i * matrix_size) + j];
}
@@ -355,12 +357,12 @@ static unsigned char* get_lrc_inverse_rows(int k,
*/
for (i = k; i < n; i++) {
// Parity is missing
if ((1 << i) & (missing_bm)) {
if (bm_get_value(missing_bm, i)) {
int d_idx_avail = 0;
int d_idx_unavail = 0;
for (j = min_range; j < max_range; j++) {
// This data is available, so we can use the encode matrix
if (((1 << j) & (missing_bm)) == 0) {
if (!bm_get_value(missing_bm, j)) {
inverse_rows[(l * matrix_size) + d_idx_avail] ^= encode_matrix[(i * encode_matrix_size) + j];
d_idx_avail++;
} else {
@@ -378,14 +380,14 @@ static unsigned char* get_lrc_inverse_rows(int k,
return inverse_rows;
}
static unsigned char* isa_l_lrc_get_decode_matrix(int k, int m, unsigned local_parity, unsigned char *encode_matrix, uint64_t missing_bm, int *used_idxs, int *use_combined_parity)
static unsigned char* isa_l_lrc_get_decode_matrix(int k, int m, unsigned local_parity, unsigned char *encode_matrix, struct ec_bm *missing_bm, int *used_idxs, int *use_combined_parity)
{
int i = 0, j, locate = 0;
int n = k + m;
int global_parity = m - local_parity;
int group_offset = 0;
uint64_t missing_local_parity = 0;
uint64_t use_parity = 0;
struct ec_bm use_parity = NEW_BM;
int total_missing = 0;
@@ -397,17 +399,17 @@ static unsigned char* isa_l_lrc_get_decode_matrix(int k, int m, unsigned local_p
for (int v = 0; v < local_parity; v++) {
int group_size = local_group_size(k, local_parity, v);
for (int u = group_offset; u < group_offset + group_size; u++) {
if ((1 << u) & missing_bm) {
use_parity |= (1 << (k + global_parity + v));
if (bm_get_value(missing_bm, u)) {
bm_set_value(&use_parity, k + global_parity + v, 1);
}
}
group_offset += group_size;
missing_local_parity |= (1 << (k + global_parity + v)) & missing_bm;
missing_local_parity |= bm_get_value(missing_bm, k + global_parity + v);
}
for (locate = 0; i < k && locate < k + global_parity; locate++) {
if (((1 << locate) & missing_bm)) {
if (bm_get_value(missing_bm, locate)) {
total_missing++;
continue;
}
@@ -437,7 +439,7 @@ static unsigned char* isa_l_lrc_get_decode_matrix(int k, int m, unsigned local_p
// Still not enough? Well, let's add what local parities we have,
// see if we can get lucky
for (locate = k + global_parity; i < k && locate < n; locate++) {
if (use_parity & (1 << locate) && !((1 << locate) & missing_bm)) {
if (bm_get_value(&use_parity, locate) && !bm_get_value(missing_bm, locate)) {
for (j = 0; j < k; j++) {
decode_matrix[(k * i) + j] = encode_matrix[(k * locate) + j];
}
@@ -475,13 +477,14 @@ static int isa_l_lrc_decode(void *desc, char **data, char **parity,
int use_combined_parity = 0;
int num_missing_elements = get_num_missing_elements(missing_idxs);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int *used_idxs = calloc(n, sizeof(int));
if(NULL == used_idxs) {
goto out;
}
decode_matrix = isa_l_lrc_get_decode_matrix(k, m, local_parity, isa_l_desc->matrix, missing_bm, used_idxs, &use_combined_parity);
decode_matrix = isa_l_lrc_get_decode_matrix(k, m, local_parity, isa_l_desc->matrix, &missing_bm, used_idxs, &use_combined_parity);
if (NULL == decode_matrix) {
goto out;
@@ -504,7 +507,7 @@ static int isa_l_lrc_decode(void *desc, char **data, char **parity,
goto out;
}
inverse_rows = get_lrc_inverse_rows(k, m, 0, k, 0, decode_inverse, isa_l_desc->matrix, missing_bm, isa_l_desc->gf_mul);
inverse_rows = get_lrc_inverse_rows(k, m, 0, k, 0, decode_inverse, isa_l_desc->matrix, &missing_bm, isa_l_desc->gf_mul);
decoded_elements = (unsigned char**)malloc(sizeof(unsigned char*)*num_missing_elements);
if (NULL == decoded_elements) {
@@ -518,11 +521,11 @@ static int isa_l_lrc_decode(void *desc, char **data, char **parity,
uint64_t missing_local_parities = 0;
for (j = n - local_parity; j < n; j++) {
missing_local_parities |= (missing_bm & (1 << j));
missing_local_parities |= bm_get_value(&missing_bm, j);
}
for (i = 0, j = 0; i < n - local_parity && j < k; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
continue;
}
if (i < k) {
@@ -557,13 +560,13 @@ static int isa_l_lrc_decode(void *desc, char **data, char **parity,
// Grab pointers to memory needed for missing data fragments
j = 0;
for (i = 0; i < k; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
decoded_elements[j] = (unsigned char*)data[i];
j++;
}
}
for (i = k; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
decoded_elements[j] = (unsigned char*)parity[i - k];
j++;
}
@@ -591,11 +594,11 @@ out:
static unsigned char* isa_l_lrc_get_reconstruct_matrix(
int k, int m, unsigned local_parity, int destination_idx,
unsigned char *encode_matrix, uint64_t *missing_bm, int *used_idxs,
unsigned char *encode_matrix, struct ec_bm *missing_bm, int *used_idxs,
int *min_col, int *max_col, int *mx_size, int *missing_local_parity, int *use_combined_parity)
{
unsigned char *decode_matrix = NULL;
uint64_t useful_mask = 0;
struct ec_bm useful_mask = NEW_BM;
int min_range=0, max_range=0;
if (destination_idx < k) {
@@ -604,19 +607,19 @@ static unsigned char* isa_l_lrc_get_reconstruct_matrix(
min_range = local_group_data_lower(k, local_parity, local_group);
max_range = local_group_data_upper(k, local_parity, local_group);
int local_parity_idx = k + m - local_parity + local_group;
int missing_local = (1 << local_parity_idx) & *missing_bm;
uint64_t missing_local = bm_get_value(missing_bm, local_parity_idx);
for (int i = min_range; !missing_local && i < max_range; i++) {
useful_mask |= 1 << i;
bm_set_value(&useful_mask, i, 1);
if (i == destination_idx) {
// We already knew we were missing *that* one...
continue;
}
missing_local |= (1 << i) & *missing_bm;
missing_local |= bm_get_value(missing_bm, i);
}
if (!missing_local) {
// We have everything we need!
useful_mask |= 1 << local_parity_idx;
*missing_bm &= useful_mask;
bm_set_value(&useful_mask, local_parity_idx, 1);
bm_combine_and(&useful_mask, missing_bm);
*mx_size = max_range - min_range;
decode_matrix = malloc(sizeof(unsigned char) * (*mx_size) * (*mx_size));
if (NULL == decode_matrix) {
@@ -644,15 +647,15 @@ static unsigned char* isa_l_lrc_get_reconstruct_matrix(
int local_group = destination_idx - k - m + local_parity;
min_range = local_group_data_lower(k, local_parity, local_group);
max_range = local_group_data_upper(k, local_parity, local_group);
int missing_local = 0;
uint64_t missing_local = 0;
for (int i = min_range; !missing_local && i < max_range; i++) {
useful_mask |= 1 << i;
missing_local |= (1 << i) & *missing_bm;
bm_set_value(&useful_mask, i, 1);
missing_local |= bm_get_value(missing_bm, i);
}
if (!missing_local) {
// We have everything we need!
useful_mask |= 1 << destination_idx;
*missing_bm &= useful_mask;
bm_set_value(&useful_mask, destination_idx, 1);
bm_combine_and(&useful_mask, missing_bm);
*missing_local_parity = 1;
*mx_size = max_range - min_range;
decode_matrix = malloc(sizeof(unsigned char) * (*mx_size) * (*mx_size));
@@ -670,7 +673,7 @@ static unsigned char* isa_l_lrc_get_reconstruct_matrix(
if (decode_matrix == NULL) {
decode_matrix = isa_l_lrc_get_decode_matrix(k, m, local_parity,
encode_matrix, *missing_bm, used_idxs, use_combined_parity);
encode_matrix, missing_bm, used_idxs, use_combined_parity);
if (decode_matrix) {
*mx_size = k;
}
@@ -699,7 +702,8 @@ static int isa_l_lrc_reconstruct(void *desc, char **data, char **parity,
int n = k + m;
int ret = -1;
int i, j;
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int inverse_row = -1;
int min_range = 0;
int max_range = 0;
@@ -738,7 +742,7 @@ static int isa_l_lrc_reconstruct(void *desc, char **data, char **parity,
/**
* Get the row needed to reconstruct
*/
inverse_rows = get_lrc_inverse_rows(k, m, min_range, max_range, missing_local_parity, decode_inverse, encode, missing_bm, isa_l_desc->gf_mul);
inverse_rows = get_lrc_inverse_rows(k, m, min_range, max_range, missing_local_parity, decode_inverse, encode, &missing_bm, isa_l_desc->gf_mul);
// Generate g_tbls from computed decode matrix (k x k) matrix
g_tbls = malloc(sizeof(unsigned char) * (matrix_size * nb_parity * 32));
@@ -757,7 +761,7 @@ static int isa_l_lrc_reconstruct(void *desc, char **data, char **parity,
j = 0;
if (matrix_size == k) {
for (i = 0; i < n - local_parity && j < k; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
continue;
}
if (i < k) {
@@ -784,7 +788,7 @@ static int isa_l_lrc_reconstruct(void *desc, char **data, char **parity,
j++;
}
for (i = n - local_parity; i < n && j < k; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
continue;
}
if (used_idxs[i]) {
@@ -810,7 +814,7 @@ static int isa_l_lrc_reconstruct(void *desc, char **data, char **parity,
*/
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
if (bm_get_value(&missing_bm, i)) {
if (i == destination_idx) {
if (i < k) {
reconstruct_buf = (unsigned char*)data[i];

8
src/backends/jerasure/jerasure_rs_cauchy.c
View File

@@ -211,14 +211,16 @@ static int jerasure_rs_cauchy_min_fragments(void *desc, int *missing_idxs,
{
struct jerasure_rs_cauchy_descriptor *jerasure_desc =
(struct jerasure_rs_cauchy_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm exclude_bm = NEW_BM, missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &exclude_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
bm_combine_or(&exclude_bm, &missing_bm);
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (jerasure_desc->k + jerasure_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

8
src/backends/jerasure/jerasure_rs_vand.c
View File

@@ -176,14 +176,16 @@ static int jerasure_rs_vand_min_fragments(void *desc, int *missing_idxs,
struct jerasure_rs_vand_descriptor *jerasure_desc =
(struct jerasure_rs_vand_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm exclude_bm = NEW_BM, missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &exclude_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
bm_combine_or(&exclude_bm, &missing_bm);
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (jerasure_desc->k + jerasure_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

7
src/backends/phazrio/libphazr.c
View File

@@ -182,14 +182,15 @@ static int pio_min_fragments(void *desc, int *missing_idxs,
int *fragments_to_exclude, int *fragments_needed)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &missing_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (xdesc->k + xdesc->m); i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

7
src/backends/rs_vand/liberasurecode_rs_vand.c
View File

@@ -122,14 +122,15 @@ static int liberasurecode_rs_vand_min_fragments(void *desc, int *missing_idxs,
struct liberasurecode_rs_vand_descriptor *rs_vand_desc =
(struct liberasurecode_rs_vand_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &missing_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (rs_vand_desc->k + rs_vand_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

8
src/backends/shss/shss.c
View File

@@ -180,14 +180,16 @@ static int shss_fragments_needed(void *desc, int *missing_idxs,
{
struct shss_descriptor *xdesc =
(struct shss_descriptor *) desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
struct ec_bm exclude_bm = NEW_BM, missing_bm = NEW_BM;
convert_list_to_bitmap(fragments_to_exclude, &exclude_bm);
convert_list_to_bitmap(missing_idxs, &missing_bm);
bm_combine_or(&exclude_bm, &missing_bm);
int i;
int j = 0;
int ret = -101;
for (i = 0; i < xdesc->n; i++) {
if (!(missing_bm & (1 << i))) {
if (!bm_get_value(&missing_bm, i)) {
fragments_needed[j] = i;
j++;
}

5
src/backends/xor/flat_xor_hd.c
View File

@@ -105,10 +105,11 @@ static int flat_xor_hd_check_reconstruct_fragments(void *desc,
xor_code_t *xor_desc = (xor_code_t *) xdesc->xor_desc;
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
struct ec_bm missing_bm = NEW_BM;
convert_list_to_bitmap(missing_idxs, &missing_bm);
int num_available = xor_desc->k + xor_desc->m;
for (int i = 0; i < EC_MAX_FRAGMENTS; i++)
if ((1LLU << i) & missing_bm)
if (bm_get_value(&missing_bm, i))
num_available--;
if (xor_desc->hd == 3) {

16
src/erasurecode.c
View File

@@ -546,7 +546,7 @@ int liberasurecode_decode(int desc,
char **parity_segments = NULL;
int *missing_idxs = NULL;
uint64_t realloc_bm = 0;
struct ec_bm realloc_bm = NEW_BM;
int rc = rwlock_rdlock(&active_instances_rwlock);
if (rc) {
@@ -726,15 +726,15 @@ int liberasurecode_decode(int desc,
out:
rwlock_unlock(&active_instances_rwlock);
/* Free the buffers allocated in prepare_fragments_for_decode */
if (realloc_bm != 0) {
if (bm_any(&realloc_bm)) {
for (i = 0; i < k; i++) {
if (realloc_bm & (1 << i)) {
if (bm_get_value(&realloc_bm, i)) {
free(data[i]);
}
}
for (i = 0; i < m; i++) {
if (realloc_bm & (1 << (i + k))) {
if (bm_get_value(&realloc_bm, i + k)) {
free(parity[i]);
}
}
@@ -778,7 +778,7 @@ int liberasurecode_reconstruct_fragment(int desc,
int k = -1;
int m = -1;
int i;
uint64_t realloc_bm = 0;
struct ec_bm realloc_bm = NEW_BM;
char **data_segments = NULL;
char **parity_segments = NULL;
int set_chksum = 1;
@@ -951,15 +951,15 @@ destination_available:
out:
rwlock_unlock(&active_instances_rwlock);
/* Free the buffers allocated in prepare_fragments_for_decode */
if (realloc_bm != 0) {
if (bm_any(&realloc_bm)) {
for (i = 0; i < k; i++) {
if (realloc_bm & (1 << i)) {
if (bm_get_value(&realloc_bm, i)) {
free(data[i]);
}
}
for (i = 0; i < m; i++) {
if (realloc_bm & (1 << (i + k))) {
if (bm_get_value(&realloc_bm, i + k)) {
free(parity[i]);
}
}

20
src/erasurecode_preprocessing.c
View File

@@ -124,14 +124,14 @@ int prepare_fragments_for_decode(
char **data, char **parity,
int *missing_idxs,
int *orig_size, int *fragment_payload_size, int fragment_size,
uint64_t *realloc_bm)
struct ec_bm *realloc_bm)
{
int i; /* a counter */
unsigned long long missing_bm; /* bitmap form of missing indexes list */
int i; /* a counter */
struct ec_bm missing_bm = NEW_BM; /* bitmap form of missing indexes list */
int orig_data_size = -1;
int payload_size = -1;
missing_bm = convert_list_to_bitmap(missing_idxs);
convert_list_to_bitmap(missing_idxs, &missing_bm);
/*
* Determine if each data fragment is:
@@ -152,7 +152,7 @@ int prepare_fragments_for_decode(
log_error("Could not allocate data buffer!");
return -ENOMEM;
}
*realloc_bm = *realloc_bm | (1 << i);
bm_set_value(realloc_bm, i, 1);
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
@@ -161,11 +161,11 @@ int prepare_fragments_for_decode(
}
memcpy(tmp_buf, data[i], fragment_size);
data[i] = tmp_buf;
*realloc_bm = *realloc_bm | (1 << i);
bm_set_value(realloc_bm, i, 1);
}
/* Need to determine the size of the original data */
if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) {
if (!bm_get_value(&missing_bm, i) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(data[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
@@ -191,7 +191,7 @@ int prepare_fragments_for_decode(
log_error("Could not allocate parity buffer!");
return -ENOMEM;
}
*realloc_bm = *realloc_bm | (1 << (k + i));
bm_set_value(realloc_bm, k + i, 1);
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
if (NULL == tmp_buf) {
@@ -200,11 +200,11 @@ int prepare_fragments_for_decode(
}
memcpy(tmp_buf, parity[i], fragment_size);
parity[i] = tmp_buf;
*realloc_bm = *realloc_bm | (1 << (k + i));
bm_set_value(realloc_bm, k + i, 1);
}
/* Need to determine the size of the original data */
if (((missing_bm & (1 << (k + i))) == 0) && orig_data_size < 0) {
if (!bm_get_value(&missing_bm, k + i) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");

29
test/liberasurecode_test.c
View File

@@ -227,8 +227,15 @@ struct ec_args isa_l_lrc_155_args = {
.hd =5,
.priv_args1.lrc_args.l = 2,
};
struct ec_args isa_l_lrc_1208_args = {
.k = 120,
.m = 8,
.w = 8,
.hd =9,
.priv_args1.lrc_args.l = 4,
};
struct ec_args *isa_l_lrc_test_args[] = { &isa_l_lrc_75_args, &isa_l_lrc_84_args, &isa_l_lrc_123_args,
&isa_l_lrc_155_args, NULL };
&isa_l_lrc_155_args, &isa_l_lrc_1208_args, NULL };
int priv = 128;
struct ec_args shss_args = {
.k = 6,
@@ -626,8 +633,8 @@ static void test_create_backend_invalid_args(void)
assert(-EINVALIDPARAMS == desc);
struct ec_args invalid_args = {
.k = 100,
.m = 100,
.k = 1000,
.m = 1000,
};
desc = liberasurecode_instance_create(EC_BACKEND_NULL, &invalid_args);
assert(-EINVALIDPARAMS == desc);
@@ -1910,24 +1917,28 @@ static void test_simple_encode_decode(const ec_backend_id_t be_id,
free(skip);
}
static void test_jerasure_rs_vand_simple_encode_decode_over32(void)
static void test_jerasure_rs_vand_simple_encode_decode_over_max_frags(void)
{
struct ec_args over32_args = {
.k = 30,
struct ec_args over_args = {
.k = 300,
.m = 20,
};
int *skip = create_skips_array(&over32_args, 1);
int *skip = create_skips_array(&over_args, 1);
assert(skip != NULL);
// should return an error
encode_decode_test_impl(EC_BACKEND_JERASURE_RS_VAND,
&over32_args, skip);
&over_args, skip);
free(skip);
}
static void test_simple_reconstruct(const ec_backend_id_t be_id,
struct ec_args *args)
{
if (args->k + args->m > 100) {
fprintf(stderr, "Too many total frags; skipping\n");
return;
}
int i = 0;
for (i = 0; i < args->k + args->m; i++) {
int *skip = create_skips_array(args,i);
@@ -2347,7 +2358,7 @@ struct testcase testcases[] = {
TEST({.no_args = test_flat_xor_can_reconstruct_with_many_failures}, EC_BACKENDS_MAX, 0),
// Jerasure RS Vand backend tests
TEST_SUITE(EC_BACKEND_JERASURE_RS_VAND),
TEST({.no_args = test_jerasure_rs_vand_simple_encode_decode_over32}, EC_BACKENDS_MAX, 0),
TEST({.no_args = test_jerasure_rs_vand_simple_encode_decode_over_max_frags}, EC_BACKENDS_MAX, 0),
// Jerasure RS Cauchy backend tests
TEST_SUITE(EC_BACKEND_JERASURE_RS_CAUCHY),
TEST({.no_args = test_jerasure_rs_cauchy_init_failure}, EC_BACKENDS_MAX, 0),