openstack/pyeclib
Code Issues Proposed changes

Add shss and its unit tests

Browse Source
This commit is contained in:
Kota Tsuyuzaki 2015-02-27 15:16:58 +09:00
parent e10d5aa330
commit 298826e0a9
4 changed files with 51 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README
View File

@ -42,6 +42,7 @@ Supported ``ec_type`` values:
* ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
* ``flat_xor_hd`` => Flat-XOR based HD combination codes
* ``isa_l_rs_vand`` => SIMD-based Reed-Soloman implementation from ISA-L (Intel(R) Storage Acceleration Library)
* ``shss`` => NTT Lab Japan's Erasure Coding Library
A configuration utility is provided to help compare available EC schemes in
terms of performance and redundancy:: tools/pyeclib_conf_tool.py

4
pyeclib/ec_iface.py
View File

@ -45,7 +45,8 @@ VALID_EC_TYPES = ['jerasure_rs_vand',
'jerasure_rs_cauchy',
'flat_xor_hd_3',
'flat_xor_hd_4',
'isa_l_rs_vand']
'isa_l_rs_vand',
'shss']
@unique
@ -92,6 +93,7 @@ class PyECLib_EC_Types(PyECLibEnum):
jerasure_rs_cauchy = 2
flat_xor_hd = 3
isa_l_rs_vand = 4
shss = 5
# Output of Erasure (en)Coding process are data "fragments". Fragment data

13
test/test_pyeclib_api.py
View File

@ -27,8 +27,9 @@ import sys
import tempfile
import unittest
from pyeclib.ec_iface import ECDriver, VALID_EC_TYPES, ECDriverError
from pyeclib.ec_iface import ECDriver, VALID_EC_TYPES, ECDriverError, \
PyECLib_EC_Types
from test_pyeclib_c import _available_backends
if sys.version < '3':
def b2i(b):
@ -106,7 +107,13 @@ class TestPyECLibDriver(unittest.TestCase):
for _type in VALID_EC_TYPES:
# Check if this algo works
try:
_instance = ECDriver(k=10, m=5, ec_type=_type)
if _type is 'shss':
if PyECLib_EC_Types.shss in _available_backends:
_instance = ECDriver(k=10, m=4, ec_type=_type)
else:
continue
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%s algorithm not supported" % _type)

37
test/test_pyeclib_c.py
View File

@ -26,6 +26,7 @@ from string import ascii_letters
import tempfile
import time
import unittest
from unittest import skipIf
import pyeclib_c
from pyeclib.ec_iface import PyECLib_EC_Types
@ -80,6 +81,10 @@ class TestPyECLib(unittest.TestCase):
self.xor_types = [(PyECLib_EC_Types.flat_xor_hd, 12, 6, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 3)]
self.shss = [(PyECLib_EC_Types.shss, 6, 3),
(PyECLib_EC_Types.shss, 10, 4),
(PyECLib_EC_Types.shss, 20, 4),
(PyECLib_EC_Types.shss, 11, 7)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@ -311,6 +316,38 @@ class TestPyECLib(unittest.TestCase):
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
@skipIf(PyECLib_EC_Types.shss not in _available_backends,
"shss backend is not available in your enviromnet")
def test_shss(self):
for (ec_type, k, m) in self.shss:
print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
success = self._test_get_required_fragments(k, m, ec_type)
self.assertTrue(success)
for size_str in self.sizes:
avg_time = self.time_encode(k, m, ec_type.value, 0,
size_str,
self.iterations)
print("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_decode(k, m, ec_type.value, 0,
size_str,
self.iterations)
self.assertTrue(success)
print("Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_reconstruct(k, m, ec_type.value, 0,
size_str,
self.iterations)
self.assertTrue(success)
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed