openstack/pyeclib
Code Issues Proposed changes

Add liberasurecode_rs_vand test cases

Browse Source
This commit is contained in:
Tushar Gohad
2015-07-19 18:56:50 -07:00
parent 04700a4fcf
commit f1524070dd
1 changed files with 128 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
test/test_pyeclib_api.py
View File

@@ -26,6 +26,7 @@ from string import ascii_letters, ascii_uppercase, digits
import sys
import tempfile
import unittest
from pyeclib.ec_iface import ECBackendNotSupported
from pyeclib.ec_iface import ECDriverError
from pyeclib.ec_iface import ECInsufficientFragments
@@ -112,18 +113,31 @@ class TestPyECLibDriver(unittest.TestCase):
print("Skipping test for %s backend" % _type)
continue
try:
if _type is 'shss':
_instance = ECDriver(k=10, m=4, ec_type=_type)
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
if _type is 'shss':
_instance = ECDriver(k=10, m=4, ec_type=_type)
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%s algorithm not supported" % _type)
self.fail("%p: %s algorithm not supported" % _instance, _type)
self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
ec_type="invalid_algo")
def test_small_encode(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=12, m=2,
ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(ECDriver(k=11, m=2,
ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(ECDriver(k=10, m=2,
ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(ECDriver(k=8, m=4,
ec_type="liberasurecode_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@@ -196,85 +210,102 @@ class TestPyECLibDriver(unittest.TestCase):
# pyeclib_driver.verify_stripe_metadata(fragment_metadata_list) == -1)
#
def get_available_backend(self, k, m, ec_type, chksum_type = "inline_crc32"):
if ec_type[:11] == "flat_xor_hd":
return ECDriver(k=k, m=m, ec_type="flat_xor_hd", chksum_type=chksum_type)
elif ec_type in self.available_backends:
return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
else:
return None
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
def test_get_metadata_formatted(self):
pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_str = ''.join(random.choice(ascii_letters)
for i in range(filesize))
file_bytes = file_str.encode('utf-8')
pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
i = 0
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if metadata.has_key('index'):
self.assertEqual(metadata['index'], i)
else:
self.assertTrue(false)
if metadata.has_key('chksum_mismatch'):
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(false)
if metadata.has_key('backend_id'):
self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
else:
self.assertTrue(false)
if metadata.has_key('orig_data_size'):
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(false)
if metadata.has_key('chksum_type'):
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(false)
if not metadata.has_key('backend_version'):
self.assertTrue(false)
if not metadata.has_key('chksum'):
self.assertTrue(false)
metadata = pyeclib_driver.get_metadata(fragment, 1)
if 'index' in metadata:
self.assertEqual(metadata['index'], f)
else:
self.assertTrue(False)
if not metadata.has_key('size'):
self.assertTrue(false)
if 'chksum_mismatch' in metadata:
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(False)
i += 1
if 'backend_id' in metadata:
self.assertEqual(metadata['backend_id'], ec_type)
else:
self.assertTrue(False)
if 'orig_data_size' in metadata:
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(False)
if 'chksum_type' in metadata:
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(False)
if 'backend_version' not in metadata:
self.assertTrue(False)
if 'chksum' not in metadata:
self.assertTrue(False)
if 'size' not in metadata:
self.assertTrue(False)
f += 1
def test_get_metadata_formatted(self):
self.check_metadata_formatted(12, 2, "jerasure_rs_vand",
"inline_crc32")
self.check_metadata_formatted(12, 2, "liberasurecode_rs_vand",
"inline_crc32")
self.check_metadata_formatted(8, 4, "liberasurecode_rs_vand",
"inline_crc32")
def test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=3, ec_type="liberasurecode_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=4, ec_type="liberasurecode_rs_vand",
chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand",
chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
for pyeclib_driver in pyeclib_drivers:
fragments = pyeclib_driver.encode(file_bytes)
fragment_metadata_list = []
first_fragment_to_corrupt = random.randint(0, len(fragments))
num_to_corrupt = 2
fragments_to_corrupt = [
@@ -339,6 +370,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="liberasurecode_rs_vand"))
file_size = 1024 * 1024
segment_size = 3 * 1024
@@ -372,6 +409,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="liberasurecode_rs_vand"))
file_sizes = [
1024 * 1024,
@@ -450,6 +493,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=8, m=6, ec_type="liberasurecode_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=4, ec_type="liberasurecode_rs_vand"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:
@@ -457,7 +506,6 @@ class TestPyECLibDriver(unittest.TestCase):
tmp_file.seek(0)
whole_file_str = tmp_file.read()
whole_file_bytes = whole_file_str.encode('utf-8')
got_exception = False
encode_input = whole_file_bytes
orig_fragments = pyeclib_driver.encode(encode_input)
@@ -511,33 +559,32 @@ class TestPyECLibDriver(unittest.TestCase):
fragments[i] = corrupted_fragment
i += 1
try:
decoded_string = pyeclib_driver.decode(fragments[:], force_metadata_checks=True)
except:
got_exception = True
self.assertTrue(got_exception)
self.assertRaises(ECInsufficientFragments,
pyeclib_driver.decode,
fragments[:], force_metadata_checks=True)
def test_liberasurecode_error(self):
pyeclib_driver = self.get_available_backend(k=10, m=5, ec_type="flat_xor_hd_3")
def get_available_backend(self, k, m, ec_type, chksum_type="inline_crc32"):
if ec_type[:11] == "flat_xor_hd":
return ECDriver(k=k, m=m, ec_type="flat_xor_hd",
chksum_type=chksum_type)
elif ec_type in _available_backends:
return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
else:
return None
def test_liberasurecode_insufficient_frags_error(self):
file_size = self.file_sizes[0]
tmp_file = self.files[file_size]
tmp_file.seek(0)
whole_file_str = tmp_file.read()
whole_file_bytes = whole_file_str.encode('utf-8')
hit_exception = False
fragments = pyeclib_driver.encode(whole_file_bytes)
#
# Test reconstructor with insufficient fragments
#
try:
pyeclib_driver.reconstruct([fragments[0]], [1, 2, 3, 4, 5, 6])
except ECInsufficientFragments as e:
hit_exception = True
self.assertTrue(e.error_str.__str__().find("Insufficient number of fragments") > -1)
self.assertTrue(hit_exception)
for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']:
pyeclib_driver = self.get_available_backend(
k=10, m=5, ec_type=ec_type)
fragments = pyeclib_driver.encode(whole_file_bytes)
self.assertRaises(ECInsufficientFragments,
pyeclib_driver.reconstruct,
[fragments[0]], [1, 2, 3, 4, 5, 6])
def test_min_parity_fragments_needed(self):
pyeclib_drivers = []