openstack/pyeclib
Code Issues Proposed changes

Merge "Add Phazr.IO libphazr backend support"

Browse Source
This commit is contained in:
Jenkins 2017-03-10 06:03:56 +00:00 committed by Gerrit Code Review
commit b526576065
5 changed files with 66 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.rst
View File

@ -51,6 +51,7 @@ Supported ``ec_type`` values:
* ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4]
* ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4]
* ``shss`` => NTT Lab Japan's Erasure Coding Library [5]
* ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6]
The Python API supports the following functions:
@ -240,7 +241,7 @@ Quick Start
An example for ubuntu to install dependency packages::
$ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev
$ sudo pip install -U bindep -r test-requirement.txt
$ sudo pip install -U bindep -r test-requirements.txt
If you want to confirm all dependency packages installed successfully, try::
@ -280,3 +281,5 @@ References
[4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
[5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
[6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"

4
pyeclib/ec_iface.py
View File

@ -124,6 +124,7 @@ class PyECLib_EC_Types(PyECLibEnum):
shss = 5
liberasurecode_rs_vand = 6
isa_l_rs_cauchy = 7
libphazr = 8
# Output of Erasure (en)Coding process are data "fragments". Fragment data
@ -175,6 +176,8 @@ class ECDriver(object):
elif value == "flat_xor_hd_4":
self.hd = 4
value = "flat_xor_hd"
elif value == "libphazr":
self.hd = 1
if PyECLib_EC_Types.has_enum(value):
self.ec_type = PyECLib_EC_Types.get_by_name(value)
else:
@ -521,6 +524,7 @@ ALL_EC_TYPES = [
'shss',
'liberasurecode_rs_vand',
'isa_l_rs_cauchy',
'libphazr',
]

3
src/c/pyeclib_c/pyeclib_c.c
View File

@ -971,6 +971,9 @@ static const char* backend_id_to_str(uint8_t backend_id)
case 7:
backend_id_str = "isa_l_rs_cauchy\0";
break;
case 8:
backend_id_str = "libphazr\0";
break;
default:
backend_id_str = "unknown\0";
}

28
test/test_pyeclib_api.py
View File

@ -134,7 +134,7 @@ class TestPyECLibDriver(unittest.TestCase):
"Invalid Argument: m is required")
with self.assertRaises(ECDriverError) as err_context:
# m is smaller than 1
# k is smaller than 1
ECDriver(ec_type=ec_type, k=-100, m=1)
self.assertEqual(str(err_context.exception),
"Invalid number of data fragments (k)")
@ -151,10 +151,15 @@ class TestPyECLibDriver(unittest.TestCase):
for _type in ALL_EC_TYPES:
try:
if _type is 'shss':
_k = 10
_m = 4
elif _type is 'libphazr':
_k = 4
_m = 4
else:
_k = 10
_m = 5
ECDriver(k=10, m=_m, ec_type=_type, validate=True)
ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
available_ec_types.append(_type)
except Exception:
# ignore any errors, assume backend not available
@ -171,10 +176,12 @@ class TestPyECLibDriver(unittest.TestCase):
try:
if _type is 'shss':
_instance = ECDriver(k=10, m=4, ec_type=_type)
elif _type is 'libphazr':
_instance = ECDriver(k=4, m=4, ec_type=_type)
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%p: %s algorithm not supported" % _instance, _type)
self.fail("%s algorithm not supported" % _type)
self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
ec_type="invalid_algo")
@ -244,6 +251,11 @@ class TestPyECLibDriver(unittest.TestCase):
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6,
chksum_type=csum))
_type7 = 'libphazr'
if _type7 in VALID_EC_TYPES:
pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7,
chksum_type=csum))
return pyeclib_drivers
def test_small_encode(self):
@ -740,7 +752,15 @@ class TestBackendsEnabled(unittest.TestCase):
def dummy(self, ec_type=ec_type):
if ec_type not in VALID_EC_TYPES:
raise unittest.SkipTest
k, m = 10, 4 if ec_type == 'shss' else 5
if ec_type == 'shss':
k = 10
m = 4
elif ec_type == 'libphazr':
k = 4
m = 4
else:
k = 10
m = 5
ECDriver(k=k, m=m, ec_type=ec_type)
dummy.__name__ = 'test_%s_available' % ec_type
cls_dict[dummy.__name__] = dummy

31
test/test_pyeclib_c.py
View File

@ -78,6 +78,7 @@ class TestPyECLib(unittest.TestCase):
(PyECLib_EC_Types.shss, 10, 4),
(PyECLib_EC_Types.shss, 20, 4),
(PyECLib_EC_Types.shss, 11, 7)]
self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@ -357,6 +358,36 @@ class TestPyECLib(unittest.TestCase):
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
@require_backend("libphazr")
def test_libphazr(self):
for (ec_type, k, m) in self.libphazr:
print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
success = self._test_get_required_fragments(k, m, ec_type)
self.assertTrue(success)
for size_str in self.sizes:
avg_time = self.time_encode(k, m, ec_type.value, 0,
size_str,
self.iterations)
print("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_decode(k, m, ec_type.value, 0,
size_str,
self.iterations)
self.assertTrue(success)
print("Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_reconstruct(
k, m, ec_type.value, 0, size_str, self.iterations)
self.assertTrue(success)
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed