openstack/liberasurecode
Code Issues Proposed changes

Merged in documentation (pull request #9)

Browse Source 
Documentation Updating
This commit is contained in:
Tushar Gohad
2015-03-04 22:26:17 -07:00
parent 23614e823d a63cae1190
commit 40f4337407
15 changed files with 42 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@@ -55,8 +55,9 @@ typedef enum {
EC_BACKEND_NULL = 0, /* "null" */
EC_BACKEND_JERASURE_RS_VAND = 1, /* "jerasure_rs_vand" */
EC_BACKEND_JERASURE_RS_CAUCHY = 2, /* "jerasure_rs_cauchy" */
EC_BACKEND_FLAT_XOR_HD = 3, /* "flat_xor_hd */
EC_BACKEND_ISA_L_RS_VAND = 4, /* "isa_l_rs_vand */
EC_BACKEND_FLAT_XOR_HD = 3, /* "flat_xor_hd" */
EC_BACKEND_ISA_L_RS_VAND = 4, /* "isa_l_rs_vand" */
EC_BACKEND_SHSS = 5, /* "shss" */
EC_BACKENDS_MAX,
} ec_backend_id_t;
@@ -275,7 +276,8 @@ struct
fragment_metadata
{
uint32_t idx; /* 4 */
uint32_t size; /* 4 */
uint32_t size; /* 4 (raw data bytes size of each fragment used for encode/fragment_to_string) */
uint32_t frag_backend_metadata_size; /* 4 (extra metadata bytes size of backend specification) */
uint64_t orig_data_size; /* 8 */
uint8_t chksum_type; /* 1 */
uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 16 */
@@ -337,6 +339,19 @@ int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len);
* @return minimum data length length, or -error code on error
*/
int liberasurecode_get_minimum_encode_size(int desc);
/**
* This will return the fragment size, which is each fragment data
* length the backend will allocate when encoding.
*
* @param desc - liberasurecode descriptor/handle
* from liberasurecode_instance_create()
* @param data_len - original data length in bytes
*
* @return fragment size - sizeof(fragment_header) + size
* + frag_backend_metadata_size
*/
int liberasurecode_get_fragment_size(int desc, int data_len);
```
----

5
include/erasurecode/erasurecode.h
View File

@@ -247,7 +247,7 @@ fragment_metadata
{
uint32_t idx; /* 4 */
uint32_t size; /* 4 */
uint32_t frag_adder_size; /* 4 */
uint32_t frag_backend_metadata_size; /* 4 */
uint64_t orig_data_size; /* 8 */
uint8_t chksum_type; /* 1 */
uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
@@ -336,7 +336,8 @@ int liberasurecode_get_minimum_encode_size(int desc);
* from liberasurecode_instance_create()
* @param data_len - original data length in bytes
*
* @return fragment size
* @return fragment size - sizeof(fragment_header) + size
* + frag_backend_metadata_size
*/
int liberasurecode_get_fragment_size(int desc, int data_len);

4
include/erasurecode/erasurecode_backend.h
View File

@@ -109,8 +109,8 @@ struct ec_backend_common {
char soversion[MAX_LEN]; /* EC backend shared library version */
struct ec_backend_op_stubs *ops; /* EC backend stubs */
size_t metadata_adder; /* EC backend custom metadata adder -
* metadata_adder bytes are added to
size_t backend_metadata_size; /* EC backend custom metadata size -
* backend_metadata_size bytes are added to
* the fragment size when allocating
* data/parity fragment buffers */
uint32_t ec_backend_version; /* The revision number of this back

4
include/erasurecode/erasurecode_helpers.h
View File

@@ -135,8 +135,8 @@ int set_fragment_idx(char *buf, int idx);
int get_fragment_idx(char *buf);
int set_fragment_payload_size(char *buf, int size);
int get_fragment_payload_size(char *buf);
int set_fragment_adder_size(char *buf, int size);
int get_fragment_adder_size(char *buf);
int set_fragment_backend_metadata_size(char *buf, int size);
int get_fragment_backend_metadata_size(char *buf);
int get_fragment_buffer_size(char *buf);
int set_orig_data_size(char *buf, int orig_data_size);
int get_orig_data_size(char *buf);

2
src/backends/isa-l/isa_l_rs_vand.c
View File

@@ -608,7 +608,7 @@ struct ec_backend_common backend_isa_l_rs_vand = {
.soname = ISA_L_RS_VAND_SO_NAME,
.soversion = ISA_L_RS_VAND_LIB_VER_STR,
.ops = &isa_l_rs_vand_op_stubs,
.metadata_adder = 0,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(ISA_L_RS_VAND_LIB_MAJOR,
ISA_L_RS_VAND_LIB_MINOR,
ISA_L_RS_VAND_LIB_REV),

2
src/backends/jerasure/jerasure_rs_cauchy.c
View File

@@ -427,7 +427,7 @@ struct ec_backend_common backend_jerasure_rs_cauchy = {
.soname = JERASURE_RS_CAUCHY_SO_NAME,
.soversion = JERASURE_RS_CAUCHY_LIB_VER_STR,
.ops = &jerasure_rs_cauchy_op_stubs,
.metadata_adder = 0,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_CAUCHY_LIB_MAJOR,
JERASURE_RS_CAUCHY_LIB_MINOR,
JERASURE_RS_CAUCHY_LIB_REV),

2
src/backends/jerasure/jerasure_rs_vand.c
View File

@@ -342,7 +342,7 @@ struct ec_backend_common backend_jerasure_rs_vand = {
.soname = JERASURE_RS_VAND_SO_NAME,
.soversion = JERASURE_RS_VAND_LIB_VER_STR,
.ops = &jerasure_rs_vand_op_stubs,
.metadata_adder = 0,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_VAND_LIB_MAJOR,
JERASURE_RS_VAND_LIB_MINOR,
JERASURE_RS_VAND_LIB_REV),

2
src/backends/null/null.c
View File

@@ -232,7 +232,7 @@ struct ec_backend_common backend_null = {
.soname = NULL_SO_NAME,
.soversion = NULL_LIB_VER_STR,
.ops = &null_op_stubs,
.metadata_adder = 0,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(NULL_LIB_MAJOR, NULL_LIB_MINOR,
NULL_LIB_REV),
};

2
src/backends/shss/shss.c
View File

@@ -302,5 +302,5 @@ struct ec_backend_common backend_shss = {
.soname = SHSS_SO_NAME,
.soversion = SHSS_LIB_VER_STR,
.ops = &shss_op_stubs,
.metadata_adder = METADATA,
.backend_metadata_size = METADATA,
};

2
src/backends/xor/flat_xor_hd.c
View File

@@ -186,7 +186,7 @@ struct ec_backend_common backend_flat_xor_hd = {
.soname = FLAT_XOR_SO_NAME,
.soversion = FLAT_XOR_LIB_VER_STR,
.ops = &flat_xor_hd_op_stubs,
.metadata_adder = 0,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(FLAT_XOR_LIB_MAJOR,
FLAT_XOR_LIB_MINOR,
FLAT_XOR_LIB_REV),

2
src/erasurecode.c
View File

@@ -1123,7 +1123,7 @@ int liberasurecode_get_fragment_size(int desc, int data_len)
ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
// TODO: Create a common function to calculate fragment size also for preprocessing
int aligned_data_len = get_aligned_data_size(instance, data_len);
int size = (aligned_data_len / instance->args.uargs.k) + instance->common.metadata_adder;
int size = (aligned_data_len / instance->args.uargs.k) + instance->common.backend_metadata_size;
return size;
}

14
src/erasurecode_helpers.c
View File

@@ -324,32 +324,32 @@ int get_fragment_payload_size(char *buf)
return header->meta.size;
}
int set_fragment_adder_size(char *buf, int size)
int set_fragment_backend_metadata_size(char *buf, int size)
{
fragment_header_t *header = (fragment_header_t *) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (set adder size)!");
log_error("Invalid fragment header (set fragment backend metadata size)!");
return -1;
}
header->meta.frag_adder_size = size;
header->meta.frag_backend_metadata_size = size;
return 0;
}
int get_fragment_adder_size(char *buf)
int get_fragment_backend_metadata_size(char *buf)
{
fragment_header_t *header = (fragment_header_t *) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get adder size)!");
log_error("Invalid fragment header (get fragment backend metadata size)!");
return -1;
}
return header->meta.frag_adder_size;
return header->meta.frag_backend_metadata_size;
}
int get_fragment_buffer_size(char *buf)
@@ -362,7 +362,7 @@ int get_fragment_buffer_size(char *buf)
return -1;
}
return header->meta.size + header->meta.frag_adder_size;
return header->meta.size + header->meta.frag_backend_metadata_size;
}
int set_orig_data_size(char *buf, int orig_data_size)

2
src/erasurecode_postprocessing.c
View File

@@ -41,7 +41,7 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
set_fragment_payload_size(fragment, blocksize);
set_backend_id(fragment, be->common.id);
set_backend_version(fragment, be->common.ec_backend_version);
set_fragment_adder_size(fragment, be->common.metadata_adder);
set_fragment_backend_metadata_size(fragment, be->common.backend_metadata_size);
if (add_chksum) {
set_checksum(ct, fragment, blocksize);

2
src/erasurecode_preprocessing.c
View File

@@ -46,7 +46,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = payload_size = (aligned_data_len / k);
buffer_size = payload_size + instance->common.metadata_adder;
buffer_size = payload_size + instance->common.backend_metadata_size;
for (i = 0; i < k; i++) {
int copy_size = data_len > payload_size ? payload_size : data_len;

2
test/liberasurecode_test.c
View File

@@ -617,7 +617,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
assert(header != NULL);
fragment_metadata_t metadata = header->meta;
assert(metadata.idx == i);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size);
assert(metadata.orig_data_size == orig_data_size);
char *data_ptr = frag + frag_header_size;
int cmp_size = remaining >= metadata.size ? metadata.size : remaining;