openstack/liberasurecode
Code Issues Proposed changes

Add encode postprocessing, checksum helpers

Browse Source 
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
This commit is contained in:
Tushar Gohad
2014-07-21 01:19:36 -07:00
parent 4de83981d3
commit 8a07d41e85
10 changed files with 188 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.am
View File

@@ -19,6 +19,8 @@ include_HEADERS = \
include/erasurecode/erasurecode.h \
include/erasurecode/erasurecode_backend.h \
include/erasurecode/erasurecode_helpers.h \
include/erasurecode/erasurecode_preprocessing.h \
include/erasurecode/erasurecode_postprocessing.h \
include/erasurecode/erasurecode_stdinc.h \
include/erasurecode/erasurecode_version.h \
include/erasurecode/list.h \

2
include/erasurecode/erasurecode_helpers.h
View File

@@ -155,6 +155,8 @@ int get_fragment_payload_size(char *buf);
int set_orig_data_size(char *buf, int orig_data_size);
int get_orig_data_size(char *buf);
int validate_fragment(char *buf);
int set_checksum(char *buf, int chksum);
int get_checksum(char *buf);
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */

36
include/erasurecode/erasurecode_postprocessing.h Normal file
View File

@@ -0,0 +1,36 @@
/*
* <Copyright>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this
* list of conditions and the following disclaimer in the documentation and/or
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* liberasurecode API helpers header
*
* vi: set noai tw=79 ts=4 sw=4:
*/
#ifndef _ERASURECODE_POSTPROCESSING_H_
#define _ERASURECODE_POSTPROCESSING_H_
int finalize_fragments_after_encode(ec_backend_t instance,
int k, int m, int blocksize,
char **encoded_data, char **encoded_parity);
#endif

3
src/Makefile.am
View File

@@ -10,7 +10,8 @@ INCLUDES = \
liberasurecode_la_SOURCES = \
erasurecode.c \
erasurecode_helpers.c \
erasurecode_preprocessing.c \
erasurecode_preprocessing.c \
erasurecode_postprocessing.c \
utils/chksum/crc32.c \
utils/chksum/alg_sig.c \
utils/chksum/galois.c \

18
src/backends/jerasure/jerasure_rs_vand.c
View File

@@ -31,6 +31,7 @@
#include "erasurecode.h"
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
/* Forward declarations */
struct ec_backend_op_stubs jerasure_rs_vand_ops;
@@ -58,23 +59,6 @@ struct jerasure_rs_vand_descriptor {
int w;
};
/**
* ToDo (KMG): Move this to a util package, or replace with calloc
*/
static
void * alloc_zeroed_buffer(int size)
{
void * buf = NULL; /* buffer to allocate and return */
/* Allocate and zero the buffer, or set the appropriate error */
buf = malloc((size_t) size);
if (buf) {
buf = memset(buf, 0, (size_t) size);
}
return buf;
}
static int jerasure_rs_vand_encode(void *desc, char **data, char **parity,
int blocksize)
{

9
src/erasurecode.c
View File

@@ -356,9 +356,18 @@ int liberasurecode_encode(int desc,
/* call the backend encode function passing it desc instance */
ret = instance->common.ops->encode(instance->desc.backend_desc,
*encoded_data, *encoded_parity, blocksize);
if (ret < 0) {
goto out;
}
ret = finalize_fragments_after_encode(instance, k, m, blocksize,
*encoded_data, *encoded_parity);
*fragment_len = get_fragment_size((*encoded_data)[0]);
out:
/* FIXME add cleanup API to call when encode() has an error */
if (ret)
log_error("Error in liberasurecode_encode %d", ret);
return ret;
}

32
src/erasurecode_helpers.c
View File

@@ -57,9 +57,7 @@ void *get_aligned_buffer16(int size)
}
/**
* Allocate a zero-ed buffer of a specific size. This method allocates from
* the Python stack in order to comply with the recommendations of the
* Python\C API. On error, return NULL and call PyErr_NoMemory.
* Allocate a zero-ed buffer of a specific size.
*
* @param size integer size in bytes of buffer to allocate
* @return pointer to start of allocated buffer or NULL on error
@@ -307,3 +305,31 @@ int validate_fragment(char *buf)
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
inline int set_chksum(char *buf, int chksum)
{
fragment_header_t* header = (fragment_header_t*) buf;
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (set chksum)!\n");
return -1;
}
header->chksum = chksum;
return 0;
}
inline int get_chksum(char *buf)
{
fragment_header_t* header = (fragment_header_t*) buf;
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get chksum)!");
return -1;
}
return header->chksum;
}
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */

69
src/erasurecode_postprocessing.c Normal file
View File

@@ -0,0 +1,69 @@
/*
* <Copyright>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this
* list of conditions and the following disclaimer in the documentation and/or
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* liberasurecode postprocessing helpers implementation
*
* vi: set noai tw=79 ts=4 sw=4:
*/
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
#include "erasurecode_stdinc.h"
int finalize_fragments_after_encode(ec_backend_t instance,
int k, int m, int blocksize,
char **encoded_data, char **encoded_parity)
{
int i;
// int add_checksum = instance->args.uargs.inline_chksum;
int add_checksum = 1;
char *fragment = get_fragment_ptr_from_data(encoded_parity[0]);
printf ("fragment = %p\n", fragment);
fragment = get_fragment_ptr_from_data(encoded_parity[1]);
printf ("fragment = %p\n", fragment);
/* finalize data fragments */
for (i = 0; i < k; i++) {
char *fragment = get_fragment_ptr_from_data(encoded_data[i]);
set_fragment_idx(fragment, i);
if (add_checksum) {
int chksum = crc32(0, encoded_data[i], blocksize);
set_chksum(fragment, chksum);
}
encoded_data[i] = fragment;
}
/* finalize parity fragments */
for (i = 0; i < m; i++) {
char *fragment = get_fragment_ptr_from_data(encoded_parity[i]);
set_fragment_idx(fragment, i + k);
if (add_checksum) {
int chksum = crc32(0, encoded_parity[i], blocksize);
set_chksum(fragment, chksum);
}
encoded_parity[i] = fragment;
}
return 0;
}

19
src/erasurecode_preprocessing.c
View File

@@ -40,15 +40,16 @@ int prepare_fragments_for_encode(ec_backend_t instance,
int i, ret = 0;
int data_len; /* data len to write to fragment headers */
int aligned_data_len; /* EC algorithm compatible data length */
int bsize = 0;
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = aligned_data_len / k;
*blocksize = bsize = (aligned_data_len / k);
for (i = 0; i < k; i++) {
int payload_size = data_len > *blocksize ? *blocksize : data_len;
char *fragment = alloc_fragment_buffer(*blocksize);
int payload_size = data_len > bsize ? bsize : data_len;
char *fragment = (char *) alloc_fragment_buffer(bsize);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
@@ -56,32 +57,38 @@ int prepare_fragments_for_encode(ec_backend_t instance,
/* Copy existing data into clean, zero'd out buffer */
encoded_data[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(encoded_data[i], orig_data, payload_size);
}
/* Fragment size will always be the same
* (may be able to get rid of this) */
set_fragment_payload_size(fragment, *blocksize);
set_fragment_payload_size(fragment, bsize);
/* Original data length */
set_orig_data_size(fragment, orig_data_size);
orig_data += payload_size;
data_len -= payload_size;
}
for (i = 0; i < m; i++) {
char *fragment = alloc_fragment_buffer(*blocksize);
char *fragment = (char *) alloc_fragment_buffer(bsize);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
}
encoded_parity[i] = get_data_ptr_from_fragment(fragment);
set_fragment_payload_size(fragment, *blocksize);
set_fragment_payload_size(fragment, bsize);
}
out:
return ret;
out_error:
printf ("ERROR in encode\n");
if (encoded_data) {
for (i = 0; i < k; i++) {
if (encoded_data[i])

40
test/liberasurecode_test.c
View File

@@ -42,10 +42,10 @@ struct testcase {
};
//TODO Make this a but more useful
char *create_buffer(int size)
char *create_buffer(int size, int fill)
{
char *buf = malloc(size);
memset(buf, 'x', size);
memset(buf, fill, size);
return buf;
}
@@ -67,32 +67,42 @@ static int test_create_and_destroy_backend(
return 0;
}
static int test_simple_encode(
static int test_simple_encode_decode(
const char *backend,
struct ec_args *args)
{
int rc = 0;
int desc = -1;
int size = 1024 * 1024;
char **encoded_data = NULL, **parity = NULL;
char *data = NULL;
int orig_data_size = 1024 * 1024;
char *orig_data = NULL;
char **encoded_data = NULL, **encoded_parity = NULL;
uint64_t encoded_fragment_len = 0;
uint64_t decoded_data_len = 0;
char *decoded_data = NULL;
desc = liberasurecode_instance_create(backend, args);
assert(desc > 0 || EBACKENDNOTAVAIL == desc);
data = create_buffer(size);
if (NULL == data) {
orig_data = create_buffer(orig_data_size, 'x');
if (NULL == orig_data) {
rc = -ENOMEM;
goto out;
}
rc = liberasurecode_encode(desc, data, size, encoded_data, parity);
rc = liberasurecode_encode(desc, orig_data, orig_data_size,
&encoded_data, &encoded_parity, &encoded_fragment_len);
assert(0 == rc);
rc = liberasurecode_decode(desc, encoded_data,
10, -1, decoded_data, &decoded_data_len);
assert(0 == rc);
if (desc)
liberasurecode_instance_destroy(desc);
free(data);
free(orig_data);
out:
return rc;
@@ -100,7 +110,7 @@ out:
struct ec_args flat_xor_hd_args = {
.k = 10,
.m = 4,
.m = 6,
.priv_args1.flat_xor_hd_args.hd = 3,
};
@@ -118,19 +128,19 @@ struct testcase testcases[] = {
{"create_and_destroy_backend",
test_create_and_destroy_backend,
"flat_xor_hd", &flat_xor_hd_args,
.skip = true},
.skip = false},
{"create_and_destroy_backend",
test_create_and_destroy_backend,
"jerasure_rs_vand", &jerasure_rs_vand_args,
.skip = false},
{"simple_encode_flat_xor_hd",
test_simple_encode,
test_simple_encode_decode,
"flat_xor_hd", &flat_xor_hd_args,
.skip = true},
{"simple_encode_jerasure_rs_vand",
test_simple_encode,
test_simple_encode_decode,
"jerasure_rs_vand", &jerasure_rs_vand_args,
.skip = false},
.skip = true},
{ NULL, NULL, NULL, NULL, false },
};