openstack/pyeclib
Code Issues Proposed changes

Make library_import_str arg to ECDriver conditional

Browse Source 
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
This commit is contained in:
Tushar Gohad 2014-11-19 09:34:16 -07:00
parent 3254a95f0b
commit 5026e16ab1
8 changed files with 48 additions and 101 deletions

4
README

@ -32,14 +32,14 @@ Examples of using this library are provided in "tools" directory:
PyEClib initialization::
ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver",
k=<num_encoded_data_fragments>,
ec_driver = ECDriver(k=<num_encoded_data_fragments>,
m=<num_encoded_parity_fragments>,
ec_type=<ec_scheme>))
Supported ``ec_type`` values:
* ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding
* ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
* ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes
A configuration utility is provided to help compare available EC schemes in

15
src/python/pyeclib/ec_iface.py

@ -115,12 +115,11 @@ class ECDriverError(Exception):
# Main ECDriver class
class ECDriver(object):
def __init__(self, library_import_str, *args, **kwargs):
def __init__(self, *args, **kwargs):
self.k = -1
self.m = -1
self.ec_type = None
self.chksum_type = None
self.library_import_str = None
for (key, value) in kwargs.items():
if key == "k":
try:
@ -149,17 +148,13 @@ class ECDriver(object):
raise ECDriverError(
"%s is not a valid checksum type for PyECLib!" % value)
if library_import_str is not None:
self.library_import_str = library_import_str
else:
raise ECDriverError(
"Library import string (library_import_str) was not specified "
"and is a required argument!")
self.library_import_str = kwargs.pop('library_import_str',
'pyeclib.core.ECPyECLibDriver')
#
# Instantiate EC backend driver
#
self.ec_lib_reference = create_instance(
library_import_str,
self.library_import_str,
k=self.k,
m=self.m,
ec_type=self.ec_type,
@ -189,7 +184,7 @@ class ECDriver(object):
if len(not_implemented_str) > 0:
raise ECDriverError(
"The following required methods are not implemented "
"in %s: %s" % (library_import_str, not_implemented_str))
"in %s: %s" % (self.library_import_str, not_implemented_str))
def encode(self, data_bytes):
"""

99
test/test_pyeclib_api.py

@ -41,7 +41,8 @@ else:
class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2)
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
@ -54,7 +55,8 @@ class TestNullDriver(unittest.TestCase):
class TestStripeDriver(unittest.TestCase):
def setUp(self):
self.stripe_driver = ECDriver("pyeclib.core.ECStripingDriver", k=8, m=0)
self.stripe_driver = ECDriver(
library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0)
def tearDown(self):
pass
@ -102,12 +104,9 @@ class TestPyECLibDriver(unittest.TestCase):
def test_small_encode(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@ -121,17 +120,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_algsig_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -159,17 +154,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_succeed(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -190,17 +181,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -233,17 +220,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_succeed(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -264,14 +247,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_byterange_info(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=11, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
file_size = 1024 * 1024
segment_size = 3 * 1024
@ -300,14 +280,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_info(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=11, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
file_sizes = [
1024 * 1024,
@ -375,23 +352,17 @@ class TestPyECLibDriver(unittest.TestCase):
def test_rs(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_vand"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=2, ec_type="jerasure_rs_cauchy"))
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_vand"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=3, ec_type="jerasure_rs_cauchy"))
ECDriver(k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd"))
ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd"))
ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:

4
test/test_pyeclib_c.py

@ -184,7 +184,7 @@ class TestPyECLib(unittest.TestCase):
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
@ -367,7 +367,7 @@ class TestPyECLib(unittest.TestCase):
self.num_parities[i],
ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations)
self.assertTrue(success)
print(("Range Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str))))

6
tools/pyeclib_conf_tool.py

@ -58,12 +58,10 @@
# (limit 10)
#
import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse
import time
import math
@ -238,9 +236,7 @@ for scheme in schemes:
string.ascii_uppercase + string.digits) for x in range(args.s))
try:
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue

8
tools/pyeclib_decode.py

@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
@ -44,8 +39,7 @@ print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, ec_type=args.ec_type)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []

9
tools/pyeclib_encode.py

@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse
parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
@ -43,9 +38,7 @@ print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=args.k, m=args.m, ec_type=args.ec_type)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:

4
tools/pyeclib_fragments_needed.py

@ -40,9 +40,7 @@ parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
args = parser.parse_args()
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=args.k, m=args.m, ec_type=args.ec_type)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments)