openstack/deb-python-pyeclib
Code Issues Proposed changes

Merge tag '1.1.1' into debian/unstable

Browse Source 
v1.1.1
This commit is contained in:
Thomas Goirand
2015-11-10 11:07:59 +00:00
parent 97b74eeae6 b9140f9273
commit 93b97c657e
10 changed files with 192 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
AUTHORS
View File

@@ -5,3 +5,5 @@ Tushar Gohad <tusharsg@gmail.com, tushar.gohad@intel.com>
Eric Lambert <eric.d.lambert@gmail.com eric.lambert@seagate.com>
Mark Storer <Mark.Storer@evault.com>
Kota Tsuyuzaki <bloodeagle40123@gmail.com, tsuyuzaki.kota@lab.ntt.co.jp>
Pete Zaitcev <zaitcev@kotori.zaitcev.us>
Victor Stinner <vstinner@redhat.com>

28
ChangeLog
View File

@@ -1,3 +1,31 @@
New in 1.1.0
------------
* Eliminate pyeclib dependency on alloc functions internal to
liberasurecode
* Update include subdirs to be explicit
* Update internal liberasurecode version to 1.1.0
New in 1.0.9
------------
* Eliminate rpath handling in setup.py
* Clean py34 shared libraries created during build
* Fix integer truncation issue with PyBuildValue on Big Endian
systems by explicitly casting the size argument passed in to
Py_ssize_t. Also fix import issue with the API test where
older versions of Python fail to import.
* Add --install-liberasurecode option to setup.py. Requested
by Red Hat/Debian package maintainers.
* Update bundled liberasurecode version to 1.0.9
New in 1.0.8
------------

2
Makefile
View File

@@ -47,6 +47,6 @@ test: build
$(UNITS)
clean:
-rm -f pyeclib_c.so
-rm -f pyeclib_c*.so
-rm -rf build
python setup.py clean

6
README
View File

@@ -1,4 +1,4 @@
This is v1.0 of PyECLib. This library provides a simple Python interface for
This is v1.1 of PyECLib. This library provides a simple Python interface for
implementing erasure codes and is known to work with Python v2.6, 2.7 and 3.x.
To obtain the best possible performance, the library utilizes liberasurecode,
@@ -229,7 +229,7 @@ Quick Start
Install pre-requisites:
* Python 2.6, 2.7 or 3.x (including development packages), argparse, setuptools
* liberasurecode v1.0.8 or greater [3]
* liberasurecode v1.1.0 or greater [3]
* Erasure code backend libraries, gf-complete and Jerasure [1],[2], ISA-L [4] etc
Install PyECLib::
@@ -266,4 +266,4 @@ References
[5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, Ryuta Kon <kon.ryuta@po.ntts.co.jp>, "NTT SHSS Erasure Coding backend"
--
1.0
1.1

17
pyeclib/core.py
View File

@@ -34,13 +34,13 @@ pyver = float('%s.%s' % sys.version_info[:2])
class ECPyECLibDriver(object):
def __init__(self, k, m, ec_type,
def __init__(self, k, m, hd, ec_type,
chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none):
self.k = k
self.m = m
self.hd = hd
self.ec_type = ec_type
self.chksum_type = chksum_type
hd = m
self.inline_chksum = 0
self.algsig_chksum = 0
@@ -50,16 +50,11 @@ class ECPyECLibDriver(object):
name = self.ec_type.name
if name == "flat_xor_hd" or name == "flat_xor_hd_3":
hd = 3
if name == "flat_xor_hd_4":
hd = 4
self.handle = pyeclib_c.init(
self.k,
self.m,
ec_type.value,
hd,
self.hd,
self.inline_chksum,
self.algsig_chksum)
@@ -141,9 +136,10 @@ class ECPyECLibDriver(object):
class ECNullDriver(object):
def __init__(self, k, m, ec_type=None, chksum_type=None):
def __init__(self, k, m, hd, ec_type=None, chksum_type=None):
self.k = k
self.m = m
self.hd = hd
def encode(self, data_bytes):
pass
@@ -177,7 +173,7 @@ class ECNullDriver(object):
#
class ECStripingDriver(object):
def __init__(self, k, m, ec_type=None, chksum_type=None):
def __init__(self, k, m, hd, ec_type=None, chksum_type=None):
"""Stripe an arbitrary-sized string into k fragments
:param k: the number of data fragments to stripe
:param m: the number of parity fragments to stripe
@@ -189,6 +185,7 @@ class ECStripingDriver(object):
raise ECDriverError("This driver only supports m=0")
self.m = m
self.hd = hd
def encode(self, data_bytes):
"""Stripe an arbitrary-sized string into k fragments

14
pyeclib/ec_iface.py
View File

@@ -117,6 +117,7 @@ class ECDriver(object):
def __init__(self, *args, **kwargs):
self.k = -1
self.m = -1
self.hd = -1
self.ec_type = None
self.chksum_type = None
for (key, value) in kwargs.items():
@@ -133,11 +134,14 @@ class ECDriver(object):
raise ECDriverError(
"Invalid number of data fragments (m)")
elif key == "ec_type":
if value in ["flat_xor_hd_3", "flat_xor_hd_4"]:
if value in ["flat_xor_hd", "flat_xor_hd_3", "flat_xor_hd_4"]:
if value == "flat_xor_hd" or value == "flat_xor_hd_3":
self.hd = 3
elif value == "flat_xor_hd_4":
self.hd = 4
value = "flat_xor_hd"
if PyECLib_EC_Types.has_enum(value):
self.ec_type = \
PyECLib_EC_Types.get_by_name(value)
self.ec_type = PyECLib_EC_Types.get_by_name(value)
else:
raise ECBackendNotSupported(
"%s is not a valid EC type for PyECLib!" % value)
@@ -149,6 +153,9 @@ class ECDriver(object):
raise ECDriverError(
"%s is not a valid checksum type for PyECLib!" % value)
if self.hd == -1:
self.hd = self.m
self.library_import_str = kwargs.pop('library_import_str',
'pyeclib.core.ECPyECLibDriver')
#
@@ -158,6 +165,7 @@ class ECDriver(object):
self.library_import_str,
k=self.k,
m=self.m,
hd=self.hd,
ec_type=self.ec_type,
chksum_type=self.chksum_type)
#

121
setup.py
View File

@@ -53,23 +53,6 @@ default_python_libdir = get_python_lib()
default_library_paths = [default_python_libdir]
#
# Prefix directory for ./configure
#
configure_prefix = "/usr"
if platform_str.find("Darwin") > -1:
#
# There appears to be a bug with OS 10.9 and later where
# specifying -L/usr/lib to the linker *will not* search
# /usr/lib, but will resolve to a directory in the Xcode
# tree.
#
mac_major = int(platform.mac_ver()[0].split(".")[0])
mac_minor = int(platform.mac_ver()[0].split(".")[1])
if mac_major == 10 and mac_minor > 9:
configure_prefix = "/usr/local"
# utility routines
def _find_library(name):
target_lib = None
@@ -91,6 +74,7 @@ def _find_library(name):
def _build_default_lib_search_path():
default_library_paths.append('/usr/local/lib')
arch64 = platform_arch[0].startswith('64')
for prefix in ('/', '/usr', '/usr/local', _exec_prefix):
libdir = os.path.join(prefix, 'lib')
@@ -109,20 +93,33 @@ def _read_file_as_str(name):
def _liberasurecode_install_error(library, library_url):
print("**********************************************")
print("** ")
print("*** Error: " + library + " build failed! ")
print("*** Please install " + library + " manually. ")
print("*** ")
print("*** Please install " + library + " manually. ")
print("*** project url: %s" % library_url)
print("** ")
print("**********************************************")
install_libec = True
class build(_build):
boolean_options = _build.boolean_options + ['install-liberasurecode']
user_options = _build.user_options + [
('install-liberasurecode=', None,
'Install liberasurecode dependency (yes/no) [default: yes].')
]
def initialize_options(self):
self.install_liberasurecode = True
_build.initialize_options(self)
def check_liberasure(self):
library_basename = "liberasurecode"
library_version = "1.0.8"
library_version = "1.1.0"
# try using an integrated copy of the library
library = library_basename + "-" + library_version
library_url = "https://bitbucket.org/tsg-/liberasurecode.git"
if platform_str.find("Darwin") > -1:
liberasure_file = \
library_basename + "." + library_version + ".dylib"
@@ -136,7 +133,7 @@ class build(_build):
if found_path:
if found_path.endswith(library_version) or \
found_path.find(library_version + ".") > -1:
# call 1.0.8 the only compatible version for now
# call 1.1.0 the only compatible version for now
notfound = False
if found_path and notfound:
@@ -148,10 +145,16 @@ class build(_build):
notfound = False
break
if notfound:
print("***************************************************")
print("** ")
print("** Can not locate %s" % (liberasure_file))
if not notfound:
return
install_status = 0
print("**************************************************************")
print("*** ")
print("*** Can not locate %s" % (liberasure_file))
if install_libec:
print("** ")
print("** PyECLib requires liberasurecode. Trying to ")
print("** install using bundled tarball. ")
@@ -159,12 +162,6 @@ class build(_build):
print("** If you have liberasurecode already installed, ")
print("** you may need to run 'sudo ldconfig' to update ")
print("** the loader cache. ")
print("** ")
print("***************************************************")
# try using an integrated copy of the library
library = library_basename + "-" + library_version
library_url = "https://bitbucket.org/tsg-/liberasurecode.git"
srcpath = "src/c/"
locallibsrcdir = (srcpath + library)
@@ -174,28 +171,34 @@ class build(_build):
if (os.path.isdir(locallibsrcdir)):
curdir = os.getcwd()
os.chdir(locallibsrcdir)
configure_cmd = ("./configure --prefix=%s" % configure_prefix)
if platform_arch[0].startswith('64'):
if os.path.exists('/usr/lib64'):
configure_cmd = configure_cmd + " --libdir=/usr/lib64"
configure_cmd = ("./configure --prefix=/usr/local")
print(configure_cmd)
retval = os.system(configure_cmd)
if retval != 0:
install_status = os.system(configure_cmd)
if install_status != 0:
_liberasurecode_install_error(library, library_url)
os.chdir(curdir)
sys.exit(retval)
make_cmd = ("make && make install")
retval = os.system(make_cmd)
if retval != 0:
install_status = os.system(make_cmd)
if install_status != 0:
_liberasurecode_install_error(library, library_url)
os.chdir(curdir)
sys.exit(retval)
os.chdir(curdir)
else:
_liberasurecode_install_error(library, library_url)
sys.exit(-1)
else:
_liberasurecode_install_error(library, library_url)
install_status = -1
print("*** ")
print("**************************************************************")
if install_status != 0:
sys.exit(install_status)
def run(self):
global install_libec
if not self.install_liberasurecode:
install_libec = False
self.check_liberasure()
_build.run(self)
@@ -208,7 +211,22 @@ class clean(_clean):
class install(_install):
boolean_options = _install.boolean_options + ['install-liberasurecode']
user_options = _install.user_options + [
('install-liberasurecode=', None,
'Install liberasurecode dependency (yes/no) [default: yes].')
]
def initialize_options(self):
self.install_liberasurecode = True
_install.initialize_options(self)
def run(self):
global install_libec
if not self.install_liberasurecode:
install_libec = False
install_cmd = self.distribution.get_command_obj('install')
install_lib = self.distribution.get_command_obj('install_lib')
for cmd in (install_lib, install_cmd):
@@ -222,8 +240,6 @@ class install(_install):
opts[optname] = os.path.abspath(value)
installroot = install_lib.install_dir
default_library_paths.insert(0, installroot)
_install.run(self)
# Another Mac-ism... If the libraries are installed
@@ -262,17 +278,14 @@ module = Extension('pyeclib_c',
'/usr/include',
'src/c/pyeclib_c',
'/usr/local/include'],
library_dirs=default_library_paths,
runtime_library_dirs=default_library_paths,
libraries=['erasurecode'],
# The extra arguments are for debugging
# extra_compile_args=['-g', '-O0'],
extra_link_args=['-Wl,-rpath,%s' %
l for l in default_library_paths],
sources=['src/c/pyeclib_c/pyeclib_c.c'])
setup(name='PyECLib',
version='1.0.8',
version='1.1.1',
author='Kevin Greenan',
author_email='kmgreen2@gmail.com',
maintainer='Kevin Greenan and Tushar Gohad',

BIN
src/c/liberasurecode-1.1.0.tar.gz Normal file
View File

Binary file not shown.

56
src/c/pyeclib_c/pyeclib_c.c
View File

@@ -25,15 +25,14 @@
*/
#include <Python.h>
#include <math.h>
#include <bytesobject.h>
#include <liberasurecode/erasurecode.h>
/* Compat layer for python <= 2.6 */
#include "capsulethunk.h"
#include <erasurecode.h>
#include <erasurecode_helpers.h>
#include <math.h>
#include <pyeclib_c.h>
#include <bytesobject.h>
/* Python 3 compatibility macros */
#if PY_MAJOR_VERSION >= 3
@@ -46,7 +45,7 @@
PyModuleDef_HEAD_INIT, name, doc, -1, methods, }; \
ob = PyModule_Create(&moduledef);
#define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \
Py_BuildValue("y#", obj, objlen)
Py_BuildValue("y#", obj, (Py_ssize_t)objlen)
#define PyInt_FromLong PyLong_FromLong
#define PyString_FromString PyUnicode_FromString
#define ENCODE_ARGS "Oy#"
@@ -58,7 +57,7 @@
#define MOD_DEF(ob, name, doc, methods) \
ob = Py_InitModule3(name, methods, doc);
#define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \
Py_BuildValue("s#", obj, objlen)
Py_BuildValue("s#", obj, (Py_ssize_t)objlen)
#define ENCODE_ARGS "Os#"
#define GET_METADATA_ARGS "Os#i"
#endif
@@ -87,6 +86,51 @@ static PyObject *import_class(const char *module, const char *cls)
return (PyObject *) PyObject_GetAttrString(s, cls);
}
/**
* Allocate a buffer of a specific size and set its' contents
* to the specified value.
*
* @param size integer size in bytes of buffer to allocate
* @param value
* @return pointer to start of allocated buffer or NULL on error
*/
void * alloc_and_set_buffer(int size, int value) {
void * buf = NULL; /* buffer to allocate and return */
/* Allocate and zero the buffer, or set the appropriate error */
buf = malloc((size_t) size);
if (buf) {
buf = memset(buf, value, (size_t) size);
}
return buf;
}
/**
* Allocate a zero-ed buffer of a specific size.
*
* @param size integer size in bytes of buffer to allocate
* @return pointer to start of allocated buffer or NULL on error
*/
void * alloc_zeroed_buffer(int size)
{
return alloc_and_set_buffer(size, 0);
}
/**
* Deallocate memory buffer if it's not NULL. This methods returns NULL so
* that you can free and reset a buffer using a single line as follows:
*
* my_ptr = check_and_free_buffer(my_ptr);
*
* @return NULL
*/
void * check_and_free_buffer(void * buf)
{
if (buf)
free(buf);
return NULL;
}
void
pyeclib_c_seterr(int ret, const char * prefix)
{

28
test/test_pyeclib_api.py
View File

@@ -31,7 +31,10 @@ from pyeclib.ec_iface import ECDriverError
from pyeclib.ec_iface import ECInsufficientFragments
from pyeclib.ec_iface import ECDriver, PyECLib_EC_Types
from .test_pyeclib_c import _available_backends
if sys.version < '3':
from test_pyeclib_c import _available_backends
else:
from .test_pyeclib_c import _available_backends
if sys.version < '3':
def b2i(b):
@@ -147,11 +150,25 @@ class TestPyECLibDriver(unittest.TestCase):
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type2,
chksum_type=csum))
_type3 = 'flat_xor_hd'
if _type3 in _available_backends:
pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3,
_type3_1 = 'flat_xor_hd'
if _type3_1 in _available_backends:
pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_1,
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3,
pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_1,
chksum_type=csum))
_type3_2 = 'flat_xor_hd_4'
if _type3_2 in _available_backends:
pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_2,
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_2,
chksum_type=csum))
_type4 = 'shss'
if _type4 in _available_backends:
pyeclib_drivers.append(ECDriver(k=10, m=4, ec_type=_type4,
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=20, m=4, ec_type=_type4,
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type4,
chksum_type=csum))
return pyeclib_drivers
@@ -536,5 +553,6 @@ class TestPyECLibDriver(unittest.TestCase):
self.assertTrue(
pyeclib_drivers[0].min_parity_fragments_needed() == 1)
if __name__ == '__main__':
unittest.main()