openstack/pyeclib
Code Issues Proposed changes

Make library_import_str arg to ECDriver conditional

Browse Source 
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
This commit is contained in:
Tushar Gohad
2014-11-19 09:34:16 -07:00
parent 3254a95f0b
commit 5026e16ab1
8 changed files with 48 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README
View File

@@ -32,14 +32,14 @@ Examples of using this library are provided in "tools" directory:
PyEClib initialization:: PyEClib initialization::
ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", ec_driver = ECDriver(k=<num_encoded_data_fragments>,
k=<num_encoded_data_fragments>,
m=<num_encoded_parity_fragments>, m=<num_encoded_parity_fragments>,
ec_type=<ec_scheme>)) ec_type=<ec_scheme>))
Supported ``ec_type`` values: Supported ``ec_type`` values:
* ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding * ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding
* ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
* ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes * ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes
A configuration utility is provided to help compare available EC schemes in A configuration utility is provided to help compare available EC schemes in

15
src/python/pyeclib/ec_iface.py
View File

@@ -115,12 +115,11 @@ class ECDriverError(Exception):
# Main ECDriver class # Main ECDriver class
class ECDriver(object): class ECDriver(object):
def __init__(self, library_import_str, *args, **kwargs): def __init__(self, *args, **kwargs):
self.k = -1 self.k = -1
self.m = -1 self.m = -1
self.ec_type = None self.ec_type = None
self.chksum_type = None self.chksum_type = None
self.library_import_str = None
for (key, value) in kwargs.items(): for (key, value) in kwargs.items():
if key == "k": if key == "k":
try: try:
@@ -149,17 +148,13 @@ class ECDriver(object):
raise ECDriverError( raise ECDriverError(
"%s is not a valid checksum type for PyECLib!" % value) "%s is not a valid checksum type for PyECLib!" % value)
if library_import_str is not None: self.library_import_str = kwargs.pop('library_import_str',
self.library_import_str = library_import_str 'pyeclib.core.ECPyECLibDriver')
else:
raise ECDriverError(
"Library import string (library_import_str) was not specified "
"and is a required argument!")
# #
# Instantiate EC backend driver # Instantiate EC backend driver
# #
self.ec_lib_reference = create_instance( self.ec_lib_reference = create_instance(
library_import_str, self.library_import_str,
k=self.k, k=self.k,
m=self.m, m=self.m,
ec_type=self.ec_type, ec_type=self.ec_type,
@@ -189,7 +184,7 @@ class ECDriver(object):
if len(not_implemented_str) > 0: if len(not_implemented_str) > 0:
raise ECDriverError( raise ECDriverError(
"The following required methods are not implemented " "The following required methods are not implemented "
"in %s: %s" % (library_import_str, not_implemented_str)) "in %s: %s" % (self.library_import_str, not_implemented_str))
def encode(self, data_bytes): def encode(self, data_bytes):
""" """

99
test/test_pyeclib_api.py
View File

@@ -41,7 +41,8 @@ else:
class TestNullDriver(unittest.TestCase): class TestNullDriver(unittest.TestCase):
def setUp(self): def setUp(self):
self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2) self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self): def tearDown(self):
pass pass
@@ -54,7 +55,8 @@ class TestNullDriver(unittest.TestCase):
class TestStripeDriver(unittest.TestCase): class TestStripeDriver(unittest.TestCase):
def setUp(self): def setUp(self):
self.stripe_driver = ECDriver("pyeclib.core.ECStripingDriver", k=8, m=0) self.stripe_driver = ECDriver(
library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0)
def tearDown(self): def tearDown(self):
pass pass
@@ -102,12 +104,9 @@ class TestPyECLibDriver(unittest.TestCase):
def test_small_encode(self): def test_small_encode(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver", pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver", pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=2, ec_type="jerasure_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"] encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@@ -121,17 +120,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_algsig_chksum_fail(self): def disabled_test_verify_fragment_algsig_chksum_fail(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -159,17 +154,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_succeed(self): def disabled_test_verify_fragment_inline_succeed(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -190,17 +181,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_fail(self): def disabled_test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -233,17 +220,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_succeed(self): def disabled_test_verify_fragment_inline_chksum_succeed(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -264,14 +247,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_byterange_info(self): def test_get_segment_byterange_info(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
k=10, m=2, ec_type="jerasure_rs_vand"))
file_size = 1024 * 1024 file_size = 1024 * 1024
segment_size = 3 * 1024 segment_size = 3 * 1024
@@ -300,14 +280,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_info(self): def test_get_segment_info(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
k=10, m=2, ec_type="jerasure_rs_vand"))
file_sizes = [ file_sizes = [
1024 * 1024, 1024 * 1024,
@@ -375,23 +352,17 @@ class TestPyECLibDriver(unittest.TestCase):
def test_rs(self): def test_rs(self):
pyeclib_drivers = [] pyeclib_drivers = []
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy"))
k=12, m=2, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_vand"))
k=12, m=3, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=3, ec_type="jerasure_rs_cauchy"))
k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
k=10, m=5, ec_type="flat_xor_hd"))
for pyeclib_driver in pyeclib_drivers: for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes: for file_size in self.file_sizes:

4
test/test_pyeclib_c.py
View File

@@ -184,7 +184,7 @@ class TestPyECLib(unittest.TestCase):
fragments = pyeclib_c.encode(handle, whole_file_bytes) fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:] orig_fragments = fragments[:]
for i in range(iterations): for i in range(iterations):
missing_idxs = [] missing_idxs = []
num_missing = hd - 1 num_missing = hd - 1
@@ -367,7 +367,7 @@ class TestPyECLib(unittest.TestCase):
self.num_parities[i], self.num_parities[i],
ec_type.value, self.num_parities[i] + 1, ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations) size_str, self.iterations)
self.assertTrue(success) self.assertTrue(success)
print(("Range Decode (%s): %s" % print(("Range Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))) (size_str, self.get_throughput(avg_time, size_str))))

6
tools/pyeclib_conf_tool.py
View File

@@ -58,12 +58,10 @@
# (limit 10) # (limit 10)
# #
import pyeclib
from pyeclib.ec_iface import ECDriver from pyeclib.ec_iface import ECDriver
import random import random
import string import string
import sys import sys
import os
import argparse import argparse
import time import time
import math import math
@@ -238,9 +236,7 @@ for scheme in schemes:
string.ascii_uppercase + string.digits) for x in range(args.s)) string.ascii_uppercase + string.digits) for x in range(args.s))
try: try:
ec_driver = ECDriver( ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
"pyeclib.core.ECPyECLibDriver",
k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e: except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e)) print("Scheme %s is not defined (%s)." % (scheme, e))
continue continue

8
tools/pyeclib_decode.py
View File

@@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse import argparse
parser = argparse.ArgumentParser(description='Decoder for PyECLib.') parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
@@ -44,8 +39,7 @@ print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments) print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename) print("filename = %s" % args.filename)
ec_driver = ECDriver( ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
"pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = [] fragment_list = []

9
tools/pyeclib_encode.py
View File

@@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse import argparse
parser = argparse.ArgumentParser(description='Encoder for PyECLib.') parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
@@ -43,9 +38,7 @@ print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type) print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename) print("filename = %s" % args.filename)
ec_driver = ECDriver( ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
"pyeclib.core.ECPyECLibDriver",
k=args.k, m=args.m, ec_type=args.ec_type)
# read # read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp: with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:

4
tools/pyeclib_fragments_needed.py
View File

@@ -40,9 +40,7 @@ parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
args = parser.parse_args() args = parser.parse_args()
ec_driver = ECDriver( ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
"pyeclib.core.ECPyECLibDriver",
k=args.k, m=args.m, ec_type=args.ec_type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments) fragments_needed = ec_driver.fragments_needed(args.missing_fragments)