use enumerated backend_id to reference a backend

This commit is contained in:
Eric Lambert 2014-10-22 17:12:52 -07:00
parent 3b89f689a7
commit c83be9fef1
6 changed files with 33 additions and 47 deletions

View File

@ -84,7 +84,7 @@ static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
* @param k integer number of data elements * @param k integer number of data elements
* @param m integer number of checksum elements * @param m integer number of checksum elements
* @param w integer word size in bytes * @param w integer word size in bytes
* @param type_str string name of erasure coding algorithm * @param backend_id erasure coding backend
* @return pointer to PyObject or NULL on error * @return pointer to PyObject or NULL on error
*/ */
static PyObject * static PyObject *
@ -94,10 +94,10 @@ pyeclib_c_init(PyObject *self, PyObject *args)
PyObject *pyeclib_obj_handle = NULL; PyObject *pyeclib_obj_handle = NULL;
int k, m, hd=0; int k, m, hd=0;
int use_inline_chksum = 0, use_algsig_chksum = 0; int use_inline_chksum = 0, use_algsig_chksum = 0;
const char *type_str; const ec_backend_id_t backend_id;
/* Obtain and validate the method parameters */ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "iis|iii", &k, &m, &type_str, &hd, &use_inline_chksum, &use_algsig_chksum)) { if (!PyArg_ParseTuple(args, "iii|iii", &k, &m, &backend_id, &hd, &use_inline_chksum, &use_algsig_chksum)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init"); PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init");
return NULL; return NULL;
} }
@ -113,7 +113,7 @@ pyeclib_c_init(PyObject *self, PyObject *args)
pyeclib_handle->ec_args.hd = hd; pyeclib_handle->ec_args.hd = hd;
pyeclib_handle->ec_args.ct = use_inline_chksum ? CHKSUM_CRC32 : CHKSUM_NONE; pyeclib_handle->ec_args.ct = use_inline_chksum ? CHKSUM_CRC32 : CHKSUM_NONE;
pyeclib_handle->ec_desc = liberasurecode_instance_create(type_str, &(pyeclib_handle->ec_args)); pyeclib_handle->ec_desc = liberasurecode_instance_create(backend_id, &(pyeclib_handle->ec_args));
if (pyeclib_handle->ec_desc <= 0) { if (pyeclib_handle->ec_desc <= 0) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create"); PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create");
goto error; goto error;
@ -668,7 +668,7 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args)
return NULL; return NULL;
} }
ret = liberasurecode_get_fragment_metadata(pyeclib_handle->ec_desc, fragment, &c_fragment_metadata); ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata);
if (ret < 0) { if (ret < 0) {
fragment_metadata = NULL; fragment_metadata = NULL;

View File

@ -59,17 +59,13 @@ class ECPyECLibDriver(object):
name = self.ec_type.name name = self.ec_type.name
if name == "flat_xor_hd_3": if name == "flat_xor_hd":
hd = 3
name = "flat_xor_hd"
elif name == "flat_xor_hd_4":
hd = 4 hd = 4
name = "flat_xor_hd"
self.handle = pyeclib_c.init( self.handle = pyeclib_c.init(
self.k, self.k,
self.m, self.m,
name, ec_type.value,
hd, hd,
self.inline_chksum, self.inline_chksum,
self.algsig_chksum) self.algsig_chksum)

View File

@ -84,8 +84,8 @@ class PyECLib_EC_Types(PyECLibEnum):
# 0 is False in the boolean sense but enum members evaluate to True # 0 is False in the boolean sense but enum members evaluate to True
jerasure_rs_vand = 1 jerasure_rs_vand = 1
jerasure_rs_cauchy = 2 jerasure_rs_cauchy = 2
flat_xor_hd_3 = 3 flat_xor_hd = 3
flat_xor_hd_4 = 4 isa_l_rs_vand = 4
# Output of Erasure (en)Coding process are data "fragments". Fragment data # Output of Erasure (en)Coding process are data "fragments". Fragment data

View File

@ -89,4 +89,7 @@ if __name__ == "__main__":
run_under_valgrind = True run_under_valgrind = True
test_cmd_prefix = "valgrind --leak-check=full " test_cmd_prefix = "valgrind --leak-check=full "
log_filename_prefix = "valgrind" log_filename_prefix = "valgrind"
unittest.main(verbosity=2) if sys.version_info<(2,7,0):
unittest.main()
else:
unittest.main(verbosity=2)

View File

@ -128,13 +128,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig")) k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig")) k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -169,13 +166,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig")) k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig")) k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
filesize = 1024 * 1024 * 3 filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -358,16 +352,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_cauchy")) k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4")) k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append( pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver", ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4")) k=10, m=5, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=9, m=5, ec_type="flat_xor_hd_3"))
for pyeclib_driver in pyeclib_drivers: for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes: for file_size in self.file_sizes:

View File

@ -28,6 +28,7 @@ import time
import unittest import unittest
import pyeclib_c import pyeclib_c
from pyeclib.ec_iface import PyECLib_EC_Types
class Timer: class Timer:
@ -58,10 +59,10 @@ class TestPyECLib(unittest.TestCase):
self.iterations = 100 self.iterations = 100
# EC algorithm and config parameters # EC algorithm and config parameters
self.rs_types = [("jerasure_rs_vand"), ("jerasure_rs_cauchy")] self.rs_types = [(PyECLib_EC_Types.jerasure_rs_vand), (PyECLib_EC_Types.jerasure_rs_cauchy)]
self.xor_types = [("flat_xor_hd", 12, 6, 4), self.xor_types = [(PyECLib_EC_Types.flat_xor_hd, 12, 6, 4),
("flat_xor_hd", 10, 5, 4), (PyECLib_EC_Types.flat_xor_hd, 10, 5, 4),
("flat_xor_hd", 10, 5, 3)] (PyECLib_EC_Types.flat_xor_hd, 10, 5, 3)]
# Input temp files for testing # Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"] self.sizes = ["101-K", "202-K", "303-K"]
@ -224,17 +225,15 @@ class TestPyECLib(unittest.TestCase):
for (ec_type, k, m, hd) in self.xor_types: for (ec_type, k, m, hd) in self.xor_types:
print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m))) print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
type_str = "%s" % (ec_type)
for size_str in self.sizes: for size_str in self.sizes:
avg_time = self.time_encode(k, m, type_str, hd, avg_time = self.time_encode(k, m, ec_type.value, hd,
size_str, size_str,
self.iterations) self.iterations)
print("Encode (%s): %s" % print("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str))) (size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes: for size_str in self.sizes:
success, avg_time = self.time_decode(k, m, type_str, hd, success, avg_time = self.time_decode(k, m, ec_type.value, hd,
size_str, size_str,
self.iterations) self.iterations)
self.assertTrue(success) self.assertTrue(success)
@ -242,7 +241,7 @@ class TestPyECLib(unittest.TestCase):
(size_str, self.get_throughput(avg_time, size_str))) (size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes: for size_str in self.sizes:
success, avg_time = self.time_reconstruct(k, m, type_str, hd, success, avg_time = self.time_reconstruct(k, m, ec_type.value, hd,
size_str, size_str,
self.iterations) self.iterations)
self.assertTrue(success) self.assertTrue(success)
@ -253,7 +252,7 @@ class TestPyECLib(unittest.TestCase):
""" """
:return boolean, True if all tests passed :return boolean, True if all tests passed
""" """
handle = pyeclib_c.init(num_data, num_parity, ec_type) handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
success = True success = True
# #
@ -299,9 +298,9 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes: for size_str in self.sizes:
avg_time = self.time_encode(self.num_datas[i], avg_time = self.time_encode(self.num_datas[i],
self.num_parities[i], self.num_parities[i],
ec_type, self.num_parities[i] + 1, ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations) size_str, self.iterations)
print(("Encode (%s): %s" % print(("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))) (size_str, self.get_throughput(avg_time, size_str))))
@ -309,9 +308,9 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes: for size_str in self.sizes:
success, avg_time = self.time_decode(self.num_datas[i], success, avg_time = self.time_decode(self.num_datas[i],
self.num_parities[i], self.num_parities[i],
ec_type, self.num_parities[i] + 1, ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations) size_str, self.iterations)
self.assertTrue(success) self.assertTrue(success)
print(("Decode (%s): %s" % print(("Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))) (size_str, self.get_throughput(avg_time, size_str))))
@ -320,7 +319,7 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes: for size_str in self.sizes:
success, avg_time = self.time_reconstruct(self.num_datas[i], success, avg_time = self.time_reconstruct(self.num_datas[i],
self.num_parities[i], self.num_parities[i],
ec_type, self.num_parities[i] + 1, ec_type.value, self.num_parities[i] + 1,
size_str, size_str,
self.iterations) self.iterations)
self.assertTrue(success) self.assertTrue(success)