openstack/pyeclib
Code Issues Proposed changes

use enumerated backend_id to reference a backend

Browse Source
This commit is contained in:
Eric Lambert 2014-10-22 17:12:52 -07:00
parent 3b89f689a7
commit c83be9fef1
6 changed files with 33 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/c/pyeclib_c/pyeclib_c.c
View File

@ -84,7 +84,7 @@ static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
* @param k integer number of data elements
* @param m integer number of checksum elements
* @param w integer word size in bytes
* @param type_str string name of erasure coding algorithm
* @param backend_id erasure coding backend
* @return pointer to PyObject or NULL on error
*/
static PyObject *
@ -94,10 +94,10 @@ pyeclib_c_init(PyObject *self, PyObject *args)
PyObject *pyeclib_obj_handle = NULL;
int k, m, hd=0;
int use_inline_chksum = 0, use_algsig_chksum = 0;
const char *type_str;
const ec_backend_id_t backend_id;
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "iis|iii", &k, &m, &type_str, &hd, &use_inline_chksum, &use_algsig_chksum)) {
if (!PyArg_ParseTuple(args, "iii|iii", &k, &m, &backend_id, &hd, &use_inline_chksum, &use_algsig_chksum)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init");
return NULL;
}
@ -113,7 +113,7 @@ pyeclib_c_init(PyObject *self, PyObject *args)
pyeclib_handle->ec_args.hd = hd;
pyeclib_handle->ec_args.ct = use_inline_chksum ? CHKSUM_CRC32 : CHKSUM_NONE;
pyeclib_handle->ec_desc = liberasurecode_instance_create(type_str, &(pyeclib_handle->ec_args));
pyeclib_handle->ec_desc = liberasurecode_instance_create(backend_id, &(pyeclib_handle->ec_args));
if (pyeclib_handle->ec_desc <= 0) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create");
goto error;
@ -668,7 +668,7 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args)
return NULL;
}
ret = liberasurecode_get_fragment_metadata(pyeclib_handle->ec_desc, fragment, &c_fragment_metadata);
ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata);
if (ret < 0) {
fragment_metadata = NULL;

8
src/python/pyeclib/core.py
View File

@ -59,17 +59,13 @@ class ECPyECLibDriver(object):
name = self.ec_type.name
if name == "flat_xor_hd_3":
hd = 3
name = "flat_xor_hd"
elif name == "flat_xor_hd_4":
if name == "flat_xor_hd":
hd = 4
name = "flat_xor_hd"
self.handle = pyeclib_c.init(
self.k,
self.m,
name,
ec_type.value,
hd,
self.inline_chksum,
self.algsig_chksum)

4
src/python/pyeclib/ec_iface.py
View File

@ -84,8 +84,8 @@ class PyECLib_EC_Types(PyECLibEnum):
# 0 is False in the boolean sense but enum members evaluate to True
jerasure_rs_vand = 1
jerasure_rs_cauchy = 2
flat_xor_hd_3 = 3
flat_xor_hd_4 = 4
flat_xor_hd = 3
isa_l_rs_vand = 4
# Output of Erasure (en)Coding process are data "fragments". Fragment data

5
test/test_core.py
View File

@ -89,4 +89,7 @@ if __name__ == "__main__":
run_under_valgrind = True
test_cmd_prefix = "valgrind --leak-check=full "
log_filename_prefix = "valgrind"
unittest.main(verbosity=2)
if sys.version_info<(2,7,0):
unittest.main()
else:
unittest.main(verbosity=2)

24
test/test_pyeclib_api.py
View File

@ -128,13 +128,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig"))
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -169,13 +166,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig"))
k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@ -358,16 +352,10 @@ class TestPyECLibDriver(unittest.TestCase):
k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=12, m=6, ec_type="flat_xor_hd_4"))
k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_4"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=10, m=5, ec_type="flat_xor_hd_3"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
k=9, m=5, ec_type="flat_xor_hd_3"))
k=10, m=5, ec_type="flat_xor_hd"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:

29
test/test_pyeclib_c.py
View File

@ -28,6 +28,7 @@ import time
import unittest
import pyeclib_c
from pyeclib.ec_iface import PyECLib_EC_Types
class Timer:
@ -58,10 +59,10 @@ class TestPyECLib(unittest.TestCase):
self.iterations = 100
# EC algorithm and config parameters
self.rs_types = [("jerasure_rs_vand"), ("jerasure_rs_cauchy")]
self.xor_types = [("flat_xor_hd", 12, 6, 4),
("flat_xor_hd", 10, 5, 4),
("flat_xor_hd", 10, 5, 3)]
self.rs_types = [(PyECLib_EC_Types.jerasure_rs_vand), (PyECLib_EC_Types.jerasure_rs_cauchy)]
self.xor_types = [(PyECLib_EC_Types.flat_xor_hd, 12, 6, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 3)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@ -224,17 +225,15 @@ class TestPyECLib(unittest.TestCase):
for (ec_type, k, m, hd) in self.xor_types:
print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
type_str = "%s" % (ec_type)
for size_str in self.sizes:
avg_time = self.time_encode(k, m, type_str, hd,
avg_time = self.time_encode(k, m, ec_type.value, hd,
size_str,
self.iterations)
print("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_decode(k, m, type_str, hd,
success, avg_time = self.time_decode(k, m, ec_type.value, hd,
size_str,
self.iterations)
self.assertTrue(success)
@ -242,7 +241,7 @@ class TestPyECLib(unittest.TestCase):
(size_str, self.get_throughput(avg_time, size_str)))
for size_str in self.sizes:
success, avg_time = self.time_reconstruct(k, m, type_str, hd,
success, avg_time = self.time_reconstruct(k, m, ec_type.value, hd,
size_str,
self.iterations)
self.assertTrue(success)
@ -253,7 +252,7 @@ class TestPyECLib(unittest.TestCase):
"""
:return boolean, True if all tests passed
"""
handle = pyeclib_c.init(num_data, num_parity, ec_type)
handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
success = True
#
@ -299,9 +298,9 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes:
avg_time = self.time_encode(self.num_datas[i],
self.num_parities[i],
ec_type, self.num_parities[i] + 1,
ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations)
print(("Encode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str))))
@ -309,9 +308,9 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes:
success, avg_time = self.time_decode(self.num_datas[i],
self.num_parities[i],
ec_type, self.num_parities[i] + 1,
ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations)
self.assertTrue(success)
print(("Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str))))
@ -320,7 +319,7 @@ class TestPyECLib(unittest.TestCase):
for size_str in self.sizes:
success, avg_time = self.time_reconstruct(self.num_datas[i],
self.num_parities[i],
ec_type, self.num_parities[i] + 1,
ec_type.value, self.num_parities[i] + 1,
size_str,
self.iterations)
self.assertTrue(success)