From 2cdc76aebccba88eb4e906eb6189a7a036e251e3 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Fri, 27 Feb 2015 06:16:30 -0800 Subject: [PATCH 1/4] Add shss id to README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e465057..a04f3b3 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,9 @@ typedef enum { EC_BACKEND_NULL = 0, /* "null" */ EC_BACKEND_JERASURE_RS_VAND = 1, /* "jerasure_rs_vand" */ EC_BACKEND_JERASURE_RS_CAUCHY = 2, /* "jerasure_rs_cauchy" */ - EC_BACKEND_FLAT_XOR_HD = 3, /* "flat_xor_hd */ - EC_BACKEND_ISA_L_RS_VAND = 4, /* "isa_l_rs_vand */ + EC_BACKEND_FLAT_XOR_HD = 3, /* "flat_xor_hd" */ + EC_BACKEND_ISA_L_RS_VAND = 4, /* "isa_l_rs_vand" */ + EC_BACKEND_SHSS = 5, /* "shss" */ EC_BACKENDS_MAX, } ec_backend_id_t; From 9722168b5866769f1bf35bf112bca6d454106e2e Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Fri, 27 Feb 2015 07:55:57 -0800 Subject: [PATCH 2/4] Update documentation about metadata_adder --- README.md | 15 ++++++++++++++- include/erasurecode/erasurecode.h | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a04f3b3..0ced36d 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,8 @@ struct fragment_metadata { uint32_t idx; /* 4 */ - uint32_t size; /* 4 */ + uint32_t size; /* 4 (raw data bytes size of each fragment used for encode/fragment_to_string) */ + uint32_t frag_adder_size; /* 4 (extra metadata_adder bytes size of backend specification) */ uint64_t orig_data_size; /* 8 */ uint8_t chksum_type; /* 1 */ uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 16 */ @@ -339,6 +340,18 @@ int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len); * @return minimum data length length, or -error code on error */ int liberasurecode_get_minimum_encode_size(int desc); + +/** + * This will return the fragment size, which is each fragment data + * length the backend will allocate when encoding. + * + * @param desc - liberasurecode descriptor/handle + * from liberasurecode_instance_create() + * @param data_len - original data length in bytes + * + * @return fragment size - sizeof(fragment_header) + size + frag_adder_size + */ +int liberasurecode_get_fragment_size(int desc, int data_len); ``` ---- diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h index 7dc7ba7..7f78b55 100644 --- a/include/erasurecode/erasurecode.h +++ b/include/erasurecode/erasurecode.h @@ -336,7 +336,7 @@ int liberasurecode_get_minimum_encode_size(int desc); * from liberasurecode_instance_create() * @param data_len - original data length in bytes * - * @return fragment size + * @return fragment size - sizeof(fragment_header) + size + frag_adder_size */ int liberasurecode_get_fragment_size(int desc, int data_len); From dd67555f8425ff5fa3cb9b807224ab5199614394 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Thu, 5 Mar 2015 13:43:19 +0900 Subject: [PATCH 3/4] Rename frag_adder_size This patch renames following variables and functions: - frag_adder_size -> frag_backend_metadata_size - set_fragment_adder_size() -> set_fragment_backend_metadata_size() - get_fragment_adder_size() -> get_fragment_backend_metadata_size() --- README.md | 5 +++-- include/erasurecode/erasurecode.h | 5 +++-- include/erasurecode/erasurecode_helpers.h | 4 ++-- src/erasurecode_helpers.c | 14 +++++++------- src/erasurecode_postprocessing.c | 2 +- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0ced36d..0fd1575 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ fragment_metadata { uint32_t idx; /* 4 */ uint32_t size; /* 4 (raw data bytes size of each fragment used for encode/fragment_to_string) */ - uint32_t frag_adder_size; /* 4 (extra metadata_adder bytes size of backend specification) */ + uint32_t frag_backend_metadata_size; /* 4 (extra metadata bytes size of backend specification) */ uint64_t orig_data_size; /* 8 */ uint8_t chksum_type; /* 1 */ uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 16 */ @@ -349,7 +349,8 @@ int liberasurecode_get_minimum_encode_size(int desc); * from liberasurecode_instance_create() * @param data_len - original data length in bytes * - * @return fragment size - sizeof(fragment_header) + size + frag_adder_size + * @return fragment size - sizeof(fragment_header) + size + * + frag_backend_metadata_size */ int liberasurecode_get_fragment_size(int desc, int data_len); diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h index 7f78b55..4604c2a 100644 --- a/include/erasurecode/erasurecode.h +++ b/include/erasurecode/erasurecode.h @@ -247,7 +247,7 @@ fragment_metadata { uint32_t idx; /* 4 */ uint32_t size; /* 4 */ - uint32_t frag_adder_size; /* 4 */ + uint32_t frag_backend_metadata_size; /* 4 */ uint64_t orig_data_size; /* 8 */ uint8_t chksum_type; /* 1 */ uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */ @@ -336,7 +336,8 @@ int liberasurecode_get_minimum_encode_size(int desc); * from liberasurecode_instance_create() * @param data_len - original data length in bytes * - * @return fragment size - sizeof(fragment_header) + size + frag_adder_size + * @return fragment size - sizeof(fragment_header) + size + * + frag_backend_metadata_size */ int liberasurecode_get_fragment_size(int desc, int data_len); diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h index f7eefd7..33b26bd 100644 --- a/include/erasurecode/erasurecode_helpers.h +++ b/include/erasurecode/erasurecode_helpers.h @@ -135,8 +135,8 @@ int set_fragment_idx(char *buf, int idx); int get_fragment_idx(char *buf); int set_fragment_payload_size(char *buf, int size); int get_fragment_payload_size(char *buf); -int set_fragment_adder_size(char *buf, int size); -int get_fragment_adder_size(char *buf); +int set_fragment_backend_metadata_size(char *buf, int size); +int get_fragment_backend_metadata_size(char *buf); int get_fragment_buffer_size(char *buf); int set_orig_data_size(char *buf, int orig_data_size); int get_orig_data_size(char *buf); diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c index 05bc838..522766e 100644 --- a/src/erasurecode_helpers.c +++ b/src/erasurecode_helpers.c @@ -324,32 +324,32 @@ int get_fragment_payload_size(char *buf) return header->meta.size; } -int set_fragment_adder_size(char *buf, int size) +int set_fragment_backend_metadata_size(char *buf, int size) { fragment_header_t *header = (fragment_header_t *) buf; assert(NULL != header); if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { - log_error("Invalid fragment header (set adder size)!"); + log_error("Invalid fragment header (set fragment backend metadata size)!"); return -1; } - header->meta.frag_adder_size = size; + header->meta.frag_backend_metadata_size = size; return 0; } -int get_fragment_adder_size(char *buf) +int get_fragment_backend_metadata_size(char *buf) { fragment_header_t *header = (fragment_header_t *) buf; assert(NULL != header); if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { - log_error("Invalid fragment header (get adder size)!"); + log_error("Invalid fragment header (get fragment backend metadata size)!"); return -1; } - return header->meta.frag_adder_size; + return header->meta.frag_backend_metadata_size; } int get_fragment_buffer_size(char *buf) @@ -362,7 +362,7 @@ int get_fragment_buffer_size(char *buf) return -1; } - return header->meta.size + header->meta.frag_adder_size; + return header->meta.size + header->meta.frag_backend_metadata_size; } int set_orig_data_size(char *buf, int orig_data_size) diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c index 435325e..b1627f7 100644 --- a/src/erasurecode_postprocessing.c +++ b/src/erasurecode_postprocessing.c @@ -41,7 +41,7 @@ void add_fragment_metadata(ec_backend_t be, char *fragment, set_fragment_payload_size(fragment, blocksize); set_backend_id(fragment, be->common.id); set_backend_version(fragment, be->common.ec_backend_version); - set_fragment_adder_size(fragment, be->common.metadata_adder); + set_fragment_backend_metadata_size(fragment, be->common.metadata_adder); if (add_chksum) { set_checksum(ct, fragment, blocksize); From a63cae119076319e39b53a488a8e3b0735feebc2 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Thu, 5 Mar 2015 13:59:06 +0900 Subject: [PATCH 4/4] Rename metadata_adder on backend_common This patch renames the "metadata_adder" variable to "backend_metadata_size" --- include/erasurecode/erasurecode_backend.h | 4 ++-- src/backends/isa-l/isa_l_rs_vand.c | 2 +- src/backends/jerasure/jerasure_rs_cauchy.c | 2 +- src/backends/jerasure/jerasure_rs_vand.c | 2 +- src/backends/null/null.c | 2 +- src/backends/shss/shss.c | 2 +- src/backends/xor/flat_xor_hd.c | 2 +- src/erasurecode.c | 2 +- src/erasurecode_postprocessing.c | 2 +- src/erasurecode_preprocessing.c | 2 +- test/liberasurecode_test.c | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/erasurecode/erasurecode_backend.h b/include/erasurecode/erasurecode_backend.h index 921a9a1..8b88c1b 100644 --- a/include/erasurecode/erasurecode_backend.h +++ b/include/erasurecode/erasurecode_backend.h @@ -109,8 +109,8 @@ struct ec_backend_common { char soversion[MAX_LEN]; /* EC backend shared library version */ struct ec_backend_op_stubs *ops; /* EC backend stubs */ - size_t metadata_adder; /* EC backend custom metadata adder - - * metadata_adder bytes are added to + size_t backend_metadata_size; /* EC backend custom metadata size - + * backend_metadata_size bytes are added to * the fragment size when allocating * data/parity fragment buffers */ uint32_t ec_backend_version; /* The revision number of this back diff --git a/src/backends/isa-l/isa_l_rs_vand.c b/src/backends/isa-l/isa_l_rs_vand.c index 98a9226..541d308 100644 --- a/src/backends/isa-l/isa_l_rs_vand.c +++ b/src/backends/isa-l/isa_l_rs_vand.c @@ -608,7 +608,7 @@ struct ec_backend_common backend_isa_l_rs_vand = { .soname = ISA_L_RS_VAND_SO_NAME, .soversion = ISA_L_RS_VAND_LIB_VER_STR, .ops = &isa_l_rs_vand_op_stubs, - .metadata_adder = 0, + .backend_metadata_size = 0, .ec_backend_version = _VERSION(ISA_L_RS_VAND_LIB_MAJOR, ISA_L_RS_VAND_LIB_MINOR, ISA_L_RS_VAND_LIB_REV), diff --git a/src/backends/jerasure/jerasure_rs_cauchy.c b/src/backends/jerasure/jerasure_rs_cauchy.c index ee924d2..4728a14 100644 --- a/src/backends/jerasure/jerasure_rs_cauchy.c +++ b/src/backends/jerasure/jerasure_rs_cauchy.c @@ -427,7 +427,7 @@ struct ec_backend_common backend_jerasure_rs_cauchy = { .soname = JERASURE_RS_CAUCHY_SO_NAME, .soversion = JERASURE_RS_CAUCHY_LIB_VER_STR, .ops = &jerasure_rs_cauchy_op_stubs, - .metadata_adder = 0, + .backend_metadata_size = 0, .ec_backend_version = _VERSION(JERASURE_RS_CAUCHY_LIB_MAJOR, JERASURE_RS_CAUCHY_LIB_MINOR, JERASURE_RS_CAUCHY_LIB_REV), diff --git a/src/backends/jerasure/jerasure_rs_vand.c b/src/backends/jerasure/jerasure_rs_vand.c index fee7356..803bd95 100644 --- a/src/backends/jerasure/jerasure_rs_vand.c +++ b/src/backends/jerasure/jerasure_rs_vand.c @@ -342,7 +342,7 @@ struct ec_backend_common backend_jerasure_rs_vand = { .soname = JERASURE_RS_VAND_SO_NAME, .soversion = JERASURE_RS_VAND_LIB_VER_STR, .ops = &jerasure_rs_vand_op_stubs, - .metadata_adder = 0, + .backend_metadata_size = 0, .ec_backend_version = _VERSION(JERASURE_RS_VAND_LIB_MAJOR, JERASURE_RS_VAND_LIB_MINOR, JERASURE_RS_VAND_LIB_REV), diff --git a/src/backends/null/null.c b/src/backends/null/null.c index 294a990..767d9d0 100644 --- a/src/backends/null/null.c +++ b/src/backends/null/null.c @@ -232,7 +232,7 @@ struct ec_backend_common backend_null = { .soname = NULL_SO_NAME, .soversion = NULL_LIB_VER_STR, .ops = &null_op_stubs, - .metadata_adder = 0, + .backend_metadata_size = 0, .ec_backend_version = _VERSION(NULL_LIB_MAJOR, NULL_LIB_MINOR, NULL_LIB_REV), }; diff --git a/src/backends/shss/shss.c b/src/backends/shss/shss.c index 3553456..601fc01 100644 --- a/src/backends/shss/shss.c +++ b/src/backends/shss/shss.c @@ -302,5 +302,5 @@ struct ec_backend_common backend_shss = { .soname = SHSS_SO_NAME, .soversion = SHSS_LIB_VER_STR, .ops = &shss_op_stubs, - .metadata_adder = METADATA, + .backend_metadata_size = METADATA, }; diff --git a/src/backends/xor/flat_xor_hd.c b/src/backends/xor/flat_xor_hd.c index 67a54ae..2af60af 100644 --- a/src/backends/xor/flat_xor_hd.c +++ b/src/backends/xor/flat_xor_hd.c @@ -186,7 +186,7 @@ struct ec_backend_common backend_flat_xor_hd = { .soname = FLAT_XOR_SO_NAME, .soversion = FLAT_XOR_LIB_VER_STR, .ops = &flat_xor_hd_op_stubs, - .metadata_adder = 0, + .backend_metadata_size = 0, .ec_backend_version = _VERSION(FLAT_XOR_LIB_MAJOR, FLAT_XOR_LIB_MINOR, FLAT_XOR_LIB_REV), diff --git a/src/erasurecode.c b/src/erasurecode.c index 66a7e2f..0404e4d 100644 --- a/src/erasurecode.c +++ b/src/erasurecode.c @@ -1123,7 +1123,7 @@ int liberasurecode_get_fragment_size(int desc, int data_len) ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc); // TODO: Create a common function to calculate fragment size also for preprocessing int aligned_data_len = get_aligned_data_size(instance, data_len); - int size = (aligned_data_len / instance->args.uargs.k) + instance->common.metadata_adder; + int size = (aligned_data_len / instance->args.uargs.k) + instance->common.backend_metadata_size; return size; } diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c index b1627f7..af418fa 100644 --- a/src/erasurecode_postprocessing.c +++ b/src/erasurecode_postprocessing.c @@ -41,7 +41,7 @@ void add_fragment_metadata(ec_backend_t be, char *fragment, set_fragment_payload_size(fragment, blocksize); set_backend_id(fragment, be->common.id); set_backend_version(fragment, be->common.ec_backend_version); - set_fragment_backend_metadata_size(fragment, be->common.metadata_adder); + set_fragment_backend_metadata_size(fragment, be->common.backend_metadata_size); if (add_chksum) { set_checksum(ct, fragment, blocksize); diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c index 9666b9d..e8bd510 100644 --- a/src/erasurecode_preprocessing.c +++ b/src/erasurecode_preprocessing.c @@ -46,7 +46,7 @@ int prepare_fragments_for_encode(ec_backend_t instance, data_len = orig_data_size; aligned_data_len = get_aligned_data_size(instance, orig_data_size); *blocksize = payload_size = (aligned_data_len / k); - buffer_size = payload_size + instance->common.metadata_adder; + buffer_size = payload_size + instance->common.backend_metadata_size; for (i = 0; i < k; i++) { int copy_size = data_len > payload_size ? payload_size : data_len; diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index 7fb9557..fe3c006 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -617,7 +617,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, assert(header != NULL); fragment_metadata_t metadata = header->meta; assert(metadata.idx == i); - assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder); + assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size); assert(metadata.orig_data_size == orig_data_size); char *data_ptr = frag + frag_header_size; int cmp_size = remaining >= metadata.size ? metadata.size : remaining;