openstack/liberasurecode
Code Issues Proposed changes

add fragment validation

Browse Source
This commit is contained in:
Eric Lambert
2014-11-09 08:52:11 -08:00
parent 2798a65d81
commit 1b1d18733f
5 changed files with 316 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
include/erasurecode/erasurecode.h
View File

@@ -266,9 +266,23 @@ fragment_metadata
*
* @return 0 on success, non-zero on error
*/
//EDL: This needs to be implemented
int liberasurecode_get_fragment_metadata(char *fragment,
fragment_metadata_t *fragment_metadata);
/**
* Verify that the specified pointer points to a well formed fragment that can
* be processed by both this instance of liberasurecode and the specified
* backend.
*
* @param desc - liberasurecode descriptor/handle
* from liberasurecode_instance_create()
* @param fragment - fragment to verify
*
* @return 0 if fragment validation is successful, 1 otherwise.
*/
int is_valid_fragment(int desc, char *fragment);
/**
* Verify a subset of fragments generated by encode()
*

3
include/erasurecode/erasurecode_helpers.h
View File

@@ -135,9 +135,8 @@ int set_fragment_payload_size(char *buf, int size);
int get_fragment_payload_size(char *buf);
int set_orig_data_size(char *buf, int orig_data_size);
int get_orig_data_size(char *buf);
int validate_fragment(char *buf);
int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
int get_checksum(char *buf);
int get_checksum(char *buf); //TODO implement this
int set_libec_version(char *fragment);
int get_libec_version(char *fragment, uint32_t *ver);
int set_backend_id(char *buf, ec_backend_id_t id);

72
src/erasurecode.c
View File

@@ -33,7 +33,7 @@
#include "erasurecode_preprocessing.h"
#include "erasurecode_postprocessing.h"
#include "erasurecode_stdinc.h"
#include "erasurecode/alg_sig.h"
#include "alg_sig.h"
/* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
@@ -951,22 +951,70 @@ out:
return ret;
}
/**
* Verify a subset of fragments generated by encode()
*
* @param desc - liberasurecode descriptor/handle
* from liberasurecode_instance_create()
* @param fragments - fragments part of the EC stripe to verify
* @num_fragments - number of fragments part of the EC stripe
*
* @ returns 0 if stripe checksum verification is successful
* -1 otherwise
*/
int liberasurecode_verify_fragment_metadata(ec_backend_t be,
fragment_metadata_t *md)
{
if (md->backend_id != be->common.id) {
return 1;
}
if (!be->common.ops->is_compatible_with(md->backend_version)) {
return 1;
}
return 0;
}
int is_valid_fragment(int desc, char *fragment)
{
uint32_t ver = 0;
fragment_metadata_t fragment_metadata;
ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
if (!be) {
log_error("Unable to verify stripe metadata: invalid backend id %d.",
desc);
return 1;
}
if (!fragment) {
log_error("Unable to verify fragment validity: fragments missing.");
return 1;
}
if (get_libec_version(fragment, &ver) != 0 ||
ver != LIBERASURECODE_VERSION) {
return 1;
}
if (liberasurecode_get_fragment_metadata(fragment,
&fragment_metadata) != 0) {
return 1;
}
if (liberasurecode_verify_fragment_metadata(be,
&fragment_metadata) != 0) {
return 1;
}
return 0;
}
int liberasurecode_verify_stripe_metadata(int desc,
char **fragments, int num_fragments)
{
int i = 0;
if (!fragments) {
log_error("Unable to verify stripe metadata: fragments missing.");
return 1;
}
if (num_fragments <= 0) {
log_error("Unable to verify stripe metadata: "
"number of fragments must be greater than 0.");
return 1;
}
for (i = 0; i < num_fragments; i++) {
if (is_valid_fragment(desc, fragments[i]) == 1) {
return 1;
}
}
return 0;
}
/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
/**

16
src/erasurecode_helpers.c
View File

@@ -32,11 +32,11 @@
#include "erasurecode_helpers.h"
#include "erasurecode_stdinc.h"
#include "erasurecode_version.h"
#include "erasurecode/alg_sig.h"
#include "alg_sig.h"
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
static bool is_valid_fragment(char *buf)
static bool is_fragment(char *buf)
{
fragment_header_t *header = (fragment_header_t *) buf;
@@ -343,7 +343,7 @@ int get_orig_data_size(char *buf)
int set_libec_version(char *buf)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;
@@ -353,7 +353,7 @@ int set_libec_version(char *buf)
int get_libec_version(char *buf, uint32_t *ver)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;
@@ -363,7 +363,7 @@ int get_libec_version(char *buf, uint32_t *ver)
int set_backend_id(char *buf, ec_backend_id_t id)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;
@@ -373,7 +373,7 @@ int set_backend_id(char *buf, ec_backend_id_t id)
int get_backend_id(char *buf, ec_backend_id_t *id)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;
@@ -383,7 +383,7 @@ int get_backend_id(char *buf, ec_backend_id_t *id)
int set_backend_version(char *buf, uint32_t version)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;
@@ -393,7 +393,7 @@ int set_backend_version(char *buf, uint32_t version)
int get_backend_version(char *buf, uint32_t *version)
{
if (!is_valid_fragment(buf)) {
if (!is_fragment(buf)) {
return -1;
}
fragment_header_t *header = (fragment_header_t *) buf;

237
test/liberasurecode_test.c
View File

@@ -85,6 +85,13 @@ struct ec_args isa_l_args = {
.hd = 5,
};
typedef enum {
LIBEC_VERSION_MISMATCH,
MAGIC_MISMATCH,
BACKEND_ID_MISMATCH,
BACKEND_VERSION_MISMATCH,
} fragment_mismatch_scenario_t;
char * get_name_from_backend_id(ec_backend_id_t be) {
switch(be) {
case EC_BACKEND_NULL:
@@ -463,16 +470,20 @@ static void test_verify_stripe_metadata_invalid_args() {
char **frags = malloc(sizeof(char *) * num_frags);
rc = liberasurecode_verify_stripe_metadata(desc, frags, num_frags);
assert(rc < 0);
assert(rc == 1);
desc = liberasurecode_instance_create(EC_BACKEND_NULL, &null_args);
assert(desc > 0);
rc = liberasurecode_verify_stripe_metadata(desc, NULL, num_frags);
assert(rc < 0);
assert(rc == 1);
rc = liberasurecode_verify_stripe_metadata(desc, frags, -1);
assert(rc < 0);
assert(rc == 1);
rc = liberasurecode_verify_stripe_metadata(desc, frags, 0);
assert(rc == 1);
}
static void encode_decode_test_impl(const ec_backend_id_t be_id,
@@ -905,6 +916,144 @@ static void test_fragments_needed(const ec_backend_id_t be_id,
test_fragments_needed_impl(be_id, args);
}
static void test_verify_stripe_metadata(const ec_backend_id_t be_id,
struct ec_args *args)
{
int orig_data_size = 1024;
char **encoded_data = NULL, **encoded_parity = NULL;
char **avail_frags = NULL;
uint64_t encoded_fragment_len = 0;
int rc = -1;
int num_avail_frags = -1;
int *skip = create_skips_array(args,-1);
char *orig_data = create_buffer(orig_data_size, 'x');
int desc = liberasurecode_instance_create(be_id, args);
if (-EBACKENDNOTAVAIL == desc) {
fprintf (stderr, "Backend library not available!\n");
return;
}
assert(desc > 0);
assert(orig_data != NULL);
rc = liberasurecode_encode(desc, orig_data, orig_data_size,
&encoded_data, &encoded_parity, &encoded_fragment_len);
assert(0 == rc);
num_avail_frags = create_frags_array(&avail_frags, encoded_data,
encoded_parity, args, skip);
rc = liberasurecode_verify_stripe_metadata(desc, avail_frags,
num_avail_frags);
assert(0 == rc);
liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity);
free(orig_data);
free(skip);
}
static void verify_stripe_metadata_mismatch_impl(const ec_backend_id_t be_id, struct ec_args *args,
fragment_mismatch_scenario_t scenario)
{
int orig_data_size = 1024;
char **encoded_data = NULL, **encoded_parity = NULL;
char **avail_frags = NULL;
uint64_t encoded_fragment_len = 0;
int rc = -1;
int num_avail_frags = -1;
int i = 0;
uint32_t orig_libec_ver = 0;
uint32_t orig_be_ver = 0;
uint8_t orig_be_id = 0;
int *skip = create_skips_array(args,-1);
char *orig_data = create_buffer(orig_data_size, 'x');
int desc = liberasurecode_instance_create(be_id, args);
if (-EBACKENDNOTAVAIL == desc) {
fprintf (stderr, "Backend library not available!\n");
return;
}
assert(desc > 0);
assert(orig_data != NULL);
rc = liberasurecode_encode(desc, orig_data, orig_data_size,
&encoded_data, &encoded_parity, &encoded_fragment_len);
assert(0 == rc);
num_avail_frags = create_frags_array(&avail_frags, encoded_data,
encoded_parity, args, skip);
for (i = 0; i < num_avail_frags; i++) {
char * cur_frag = avail_frags[i];
//corrupt fragment
switch (scenario) {
case LIBEC_VERSION_MISMATCH:
orig_libec_ver = ((fragment_header_t*)cur_frag)->libec_version;
((fragment_header_t*)cur_frag)->libec_version = orig_libec_ver + 1;
break;
case MAGIC_MISMATCH:
((fragment_header_t*)cur_frag)->magic = 0;
break;
case BACKEND_ID_MISMATCH:
orig_be_id = ((fragment_header_t*)cur_frag)->meta.backend_id;
((fragment_header_t*)cur_frag)->meta.backend_id = orig_be_id + 1;
break;
case BACKEND_VERSION_MISMATCH:
orig_be_ver = ((fragment_header_t*)cur_frag)->meta.backend_version;
((fragment_header_t*)cur_frag)->meta.backend_version = orig_be_ver + 1;
break;
default:
assert(false);
}
rc = liberasurecode_verify_stripe_metadata(desc, avail_frags,
num_avail_frags);
assert(rc == 1);
//heal fragment
switch (scenario) {
case LIBEC_VERSION_MISMATCH:
((fragment_header_t*)cur_frag)->libec_version = orig_libec_ver;
break;
case MAGIC_MISMATCH:
((fragment_header_t*)cur_frag)->magic = LIBERASURECODE_FRAG_HEADER_MAGIC;
break;
case BACKEND_ID_MISMATCH:
((fragment_header_t*)cur_frag)->meta.backend_id = orig_be_id;
break;
case BACKEND_VERSION_MISMATCH:
((fragment_header_t*)cur_frag)->meta.backend_version = orig_be_ver;
break;
default:
break;
}
}
liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity);
free(orig_data);
free(skip);
}
static void test_verify_stripe_metadata_libec_mismatch(
const ec_backend_id_t be_id, struct ec_args *args)
{
verify_stripe_metadata_mismatch_impl(be_id, args, LIBEC_VERSION_MISMATCH);
}
static void test_verify_stripe_metadata_magic_mismatch(
const ec_backend_id_t be_id, struct ec_args *args)
{
verify_stripe_metadata_mismatch_impl(be_id, args, MAGIC_MISMATCH);
}
static void test_verify_stripe_metadata_be_id_mismatch(
const ec_backend_id_t be_id, struct ec_args *args)
{
verify_stripe_metadata_mismatch_impl(be_id, args, BACKEND_ID_MISMATCH);
}
static void test_verify_stripe_metadata_be_ver_mismatch(
const ec_backend_id_t be_id, struct ec_args *args)
{
verify_stripe_metadata_mismatch_impl(be_id, args, BACKEND_VERSION_MISMATCH);
}
//static void test_verify_str
struct testcase testcases[] = {
{"test_create_backend_invalid_args",
test_create_backend_invalid_args,
@@ -941,7 +1090,7 @@ struct testcase testcases[] = {
{"test_verify_stripe_metadata_invalid_args",
test_verify_stripe_metadata_invalid_args,
EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
.skip = true}, //EDL, liberasurecode_verify_stripe_metadata is not implemented at the moment
.skip = false},
{"test_fragments_needed_invalid_args",
test_fragments_needed_invalid_args,
EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
@@ -1004,6 +1153,26 @@ struct testcase testcases[] = {
test_get_fragment_metadata,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
.skip = false},
// Jerasure RS Vand backend tests
{"create_and_destroy_backend",
test_create_and_destroy_backend,
@@ -1045,6 +1214,26 @@ struct testcase testcases[] = {
test_get_fragment_metadata,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
.skip = false},
// Jerasure RS Cauchy backend tests
{"create_and_destroy_backend",
test_create_and_destroy_backend,
@@ -1086,6 +1275,26 @@ struct testcase testcases[] = {
test_get_fragment_metadata,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
.skip = false},
// ISA-L tests
{"create_and_destroy_backend",
test_create_and_destroy_backend,
@@ -1123,6 +1332,26 @@ struct testcase testcases[] = {
test_get_fragment_metadata,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_NONE,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
.skip = false},
{ NULL, NULL, 0, 0, false },
};