Merge tag '1.1.1' into debian/unstable

v1.1.1

AUTHORS

@@ -5,3 +5,5 @@ Tushar Gohad <tusharsg@gmail.com, tushar.gohad@intel.com>
 Eric Lambert <eric.d.lambert@gmail.com eric.lambert@seagate.com>
 Mark Storer <Mark.Storer@evault.com>
 Kota Tsuyuzaki <bloodeagle40123@gmail.com, tsuyuzaki.kota@lab.ntt.co.jp>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us>
+Victor Stinner <vstinner@redhat.com>

ChangeLog

@@ -1,3 +1,31 @@
+New in 1.1.0
+------------
+
+ * Eliminate pyeclib dependency on alloc functions internal to
+   liberasurecode
+
+ * Update include subdirs to be explicit
+
+ * Update internal liberasurecode version to 1.1.0
+
+New in 1.0.9
+------------
+
+ * Eliminate rpath handling in setup.py
+
+ * Clean py34 shared libraries created during build
+
+ * Fix integer truncation issue with PyBuildValue on Big Endian
+   systems by explicitly casting the size argument passed in to
+   Py_ssize_t.  Also fix import issue with the API test where
+   older versions of Python fail to import.
+
+ * Add --install-liberasurecode option to setup.py.  Requested
+   by Red Hat/Debian package maintainers.
+
+ * Update bundled liberasurecode version to 1.0.9
+
+
 New in 1.0.8
 ------------
 

Makefile

@@ -47,6 +47,6 @@ test:		build
 		$(UNITS)
 
 clean:
-	-rm -f pyeclib_c.so
+	-rm -f pyeclib_c*.so
 	-rm -rf build
 	python setup.py clean

README

@@ -1,4 +1,4 @@
-This is v1.0 of PyECLib.  This library provides a simple Python interface for
+This is v1.1 of PyECLib.  This library provides a simple Python interface for
 implementing erasure codes and is known to work with Python v2.6, 2.7 and 3.x.
 
 To obtain the best possible performance, the library utilizes liberasurecode,
@@ -229,7 +229,7 @@ Quick Start
   Install pre-requisites:
   
     * Python 2.6, 2.7 or 3.x (including development packages), argparse, setuptools
-    * liberasurecode v1.0.8 or greater [3]
+    * liberasurecode v1.1.0 or greater [3]
     * Erasure code backend libraries, gf-complete and Jerasure [1],[2], ISA-L [4] etc
 
   Install PyECLib::
@@ -266,4 +266,4 @@ References
  [5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, Ryuta Kon <kon.ryuta@po.ntts.co.jp>, "NTT SHSS Erasure Coding backend"
 
 --
-1.0
+1.1

pyeclib/core.py

@@ -34,13 +34,13 @@ pyver = float('%s.%s' % sys.version_info[:2])
 
 class ECPyECLibDriver(object):
 
-    def __init__(self, k, m, ec_type,
+    def __init__(self, k, m, hd, ec_type,
                  chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none):
         self.k = k
         self.m = m
+        self.hd = hd
         self.ec_type = ec_type
         self.chksum_type = chksum_type
-        hd = m
 
         self.inline_chksum = 0
         self.algsig_chksum = 0
@@ -50,16 +50,11 @@ class ECPyECLibDriver(object):
 
         name = self.ec_type.name
 
-        if name == "flat_xor_hd" or name == "flat_xor_hd_3":
-            hd = 3
-        if name == "flat_xor_hd_4":
-            hd = 4
-
         self.handle = pyeclib_c.init(
             self.k,
             self.m,
             ec_type.value,
-            hd,
+            self.hd,
             self.inline_chksum,
             self.algsig_chksum)
 
@@ -141,9 +136,10 @@ class ECPyECLibDriver(object):
 
 class ECNullDriver(object):
 
-    def __init__(self, k, m, ec_type=None, chksum_type=None):
+    def __init__(self, k, m, hd, ec_type=None, chksum_type=None):
         self.k = k
         self.m = m
+        self.hd = hd
 
     def encode(self, data_bytes):
         pass
@@ -177,7 +173,7 @@ class ECNullDriver(object):
 #
 class ECStripingDriver(object):
 
-    def __init__(self, k, m, ec_type=None, chksum_type=None):
+    def __init__(self, k, m, hd, ec_type=None, chksum_type=None):
         """Stripe an arbitrary-sized string into k fragments
         :param k: the number of data fragments to stripe
         :param m: the number of parity fragments to stripe
@@ -189,6 +185,7 @@ class ECStripingDriver(object):
             raise ECDriverError("This driver only supports m=0")
 
         self.m = m
+        self.hd = hd
 
     def encode(self, data_bytes):
         """Stripe an arbitrary-sized string into k fragments

pyeclib/ec_iface.py

@@ -117,6 +117,7 @@ class ECDriver(object):
     def __init__(self, *args, **kwargs):
         self.k = -1
         self.m = -1
+        self.hd = -1
         self.ec_type = None
         self.chksum_type = None
         for (key, value) in kwargs.items():
@@ -133,11 +134,14 @@ class ECDriver(object):
                     raise ECDriverError(
                         "Invalid number of data fragments (m)")
             elif key == "ec_type":
-                if value in ["flat_xor_hd_3", "flat_xor_hd_4"]:
+                if value in ["flat_xor_hd", "flat_xor_hd_3", "flat_xor_hd_4"]:
+                    if value == "flat_xor_hd" or value == "flat_xor_hd_3":
+                        self.hd = 3
+                    elif value == "flat_xor_hd_4":
+                        self.hd = 4
                     value = "flat_xor_hd"
                 if PyECLib_EC_Types.has_enum(value):
-                    self.ec_type = \
+                    self.ec_type = PyECLib_EC_Types.get_by_name(value)
-                        PyECLib_EC_Types.get_by_name(value)
                 else:
                     raise ECBackendNotSupported(
                         "%s is not a valid EC type for PyECLib!" % value)
@@ -149,6 +153,9 @@ class ECDriver(object):
                     raise ECDriverError(
                         "%s is not a valid checksum type for PyECLib!" % value)
 
+        if self.hd == -1:
+            self.hd = self.m
+
         self.library_import_str = kwargs.pop('library_import_str',
                                              'pyeclib.core.ECPyECLibDriver')
         #
@@ -158,6 +165,7 @@ class ECDriver(object):
             self.library_import_str,
             k=self.k,
             m=self.m,
+            hd=self.hd,
             ec_type=self.ec_type,
             chksum_type=self.chksum_type)
         #

setup.py

@@ -53,23 +53,6 @@ default_python_libdir = get_python_lib()
 default_library_paths = [default_python_libdir]
 
 
-#
-# Prefix directory for ./configure
-#
-configure_prefix = "/usr"
-if platform_str.find("Darwin") > -1:
-    #
-    # There appears to be a bug with OS 10.9 and later where
-    # specifying -L/usr/lib to the linker *will not* search
-    # /usr/lib, but will resolve to a directory in the Xcode
-    # tree.
-    #
-    mac_major = int(platform.mac_ver()[0].split(".")[0])
-    mac_minor = int(platform.mac_ver()[0].split(".")[1])
-    if mac_major == 10 and mac_minor > 9:
-        configure_prefix = "/usr/local"
-
-
 # utility routines
 def _find_library(name):
     target_lib = None
@@ -91,6 +74,7 @@ def _find_library(name):
 
 
 def _build_default_lib_search_path():
+    default_library_paths.append('/usr/local/lib')
     arch64 = platform_arch[0].startswith('64')
     for prefix in ('/', '/usr', '/usr/local', _exec_prefix):
         libdir = os.path.join(prefix, 'lib')
@@ -109,20 +93,33 @@ def _read_file_as_str(name):
 
 
 def _liberasurecode_install_error(library, library_url):
-    print("**********************************************")
+    print("***                                                ")
-    print("**                                            ")
-    print("*** Error: " + library + " build failed!      ")
     print("*** Please install " + library + " manually.       ")
     print("*** project url: %s" % library_url)
-    print("**                                            ")
+
-    print("**********************************************")
+
+install_libec = True
 
 
 class build(_build):
 
+    boolean_options = _build.boolean_options + ['install-liberasurecode']
+    user_options = _build.user_options + [
+        ('install-liberasurecode=', None,
+         'Install liberasurecode dependency (yes/no) [default: yes].')
+    ]
+
+    def initialize_options(self):
+        self.install_liberasurecode = True
+        _build.initialize_options(self)
+
     def check_liberasure(self):
         library_basename = "liberasurecode"
-        library_version = "1.0.8"
+        library_version = "1.1.0"
+        # try using an integrated copy of the library
+        library = library_basename + "-" + library_version
+        library_url = "https://bitbucket.org/tsg-/liberasurecode.git"
+
         if platform_str.find("Darwin") > -1:
             liberasure_file = \
                 library_basename + "." + library_version + ".dylib"
@@ -136,7 +133,7 @@ class build(_build):
         if found_path:
             if found_path.endswith(library_version) or \
                     found_path.find(library_version + ".") > -1:
-                # call 1.0.8 the only compatible version for now
+                # call 1.1.0 the only compatible version for now
                 notfound = False
 
         if found_path and notfound:
@@ -148,10 +145,16 @@ class build(_build):
                     notfound = False
                     break
 
-        if notfound:
+        if not notfound:
-            print("***************************************************")
+            return
-            print("**                                                 ")
+
-            print("** Can not locate %s" % (liberasure_file))
+        install_status = 0
+
+        print("**************************************************************")
+        print("***                                                           ")
+        print("*** Can not locate %s" % (liberasure_file))
+
+        if install_libec:
             print("**                                                 ")
             print("** PyECLib requires liberasurecode.  Trying to     ")
             print("** install using bundled tarball.                  ")
@@ -159,12 +162,6 @@ class build(_build):
             print("** If you have liberasurecode already installed,   ")
             print("** you may need to run 'sudo ldconfig' to update   ")
             print("** the loader cache.                               ")
-            print("**                                                 ")
-            print("***************************************************")
-
-            # try using an integrated copy of the library
-            library = library_basename + "-" + library_version
-            library_url = "https://bitbucket.org/tsg-/liberasurecode.git"
 
             srcpath = "src/c/"
             locallibsrcdir = (srcpath + library)
@@ -174,28 +171,34 @@ class build(_build):
             if (os.path.isdir(locallibsrcdir)):
                 curdir = os.getcwd()
                 os.chdir(locallibsrcdir)
-                configure_cmd = ("./configure --prefix=%s" % configure_prefix)
+                configure_cmd = ("./configure --prefix=/usr/local")
-                if platform_arch[0].startswith('64'):
-                    if os.path.exists('/usr/lib64'):
-                        configure_cmd = configure_cmd + " --libdir=/usr/lib64"
                 print(configure_cmd)
-                retval = os.system(configure_cmd)
+                install_status = os.system(configure_cmd)
-                if retval != 0:
+                if install_status != 0:
                     _liberasurecode_install_error(library, library_url)
                     os.chdir(curdir)
-                    sys.exit(retval)
                 make_cmd = ("make && make install")
-                retval = os.system(make_cmd)
+                install_status = os.system(make_cmd)
-                if retval != 0:
+                if install_status != 0:
                     _liberasurecode_install_error(library, library_url)
                     os.chdir(curdir)
-                    sys.exit(retval)
                 os.chdir(curdir)
         else:
             _liberasurecode_install_error(library, library_url)
-                sys.exit(-1)
+            install_status = -1
+
+        print("***                                                           ")
+        print("**************************************************************")
+
+        if install_status != 0:
+            sys.exit(install_status)
 
     def run(self):
+        global install_libec
+
+        if not self.install_liberasurecode:
+            install_libec = False
+
         self.check_liberasure()
         _build.run(self)
 
@@ -208,7 +211,22 @@ class clean(_clean):
 
 class install(_install):
 
+    boolean_options = _install.boolean_options + ['install-liberasurecode']
+    user_options = _install.user_options + [
+        ('install-liberasurecode=', None,
+         'Install liberasurecode dependency (yes/no) [default: yes].')
+    ]
+
+    def initialize_options(self):
+        self.install_liberasurecode = True
+        _install.initialize_options(self)
+
     def run(self):
+        global install_libec
+
+        if not self.install_liberasurecode:
+            install_libec = False
+
         install_cmd = self.distribution.get_command_obj('install')
         install_lib = self.distribution.get_command_obj('install_lib')
         for cmd in (install_lib, install_cmd):
@@ -222,8 +240,6 @@ class install(_install):
                 opts[optname] = os.path.abspath(value)
 
         installroot = install_lib.install_dir
-
-        default_library_paths.insert(0, installroot)
         _install.run(self)
 
         # Another Mac-ism...  If the libraries are installed
@@ -262,17 +278,14 @@ module = Extension('pyeclib_c',
                                  '/usr/include',
                                  'src/c/pyeclib_c',
                                  '/usr/local/include'],
-                   library_dirs=default_library_paths,
                    runtime_library_dirs=default_library_paths,
                    libraries=['erasurecode'],
                    # The extra arguments are for debugging
                    # extra_compile_args=['-g', '-O0'],
-                   extra_link_args=['-Wl,-rpath,%s' %
-                                    l for l in default_library_paths],
                    sources=['src/c/pyeclib_c/pyeclib_c.c'])
 
 setup(name='PyECLib',
-      version='1.0.8',
+      version='1.1.1',
       author='Kevin Greenan',
       author_email='kmgreen2@gmail.com',
       maintainer='Kevin Greenan and Tushar Gohad',

src/c/pyeclib_c/pyeclib_c.c

@@ -25,15 +25,14 @@
  */
 
 #include <Python.h>
+#include <math.h>
+#include <bytesobject.h>
+#include <liberasurecode/erasurecode.h>
 
 /* Compat layer for python <= 2.6 */
 #include "capsulethunk.h"
 
-#include <erasurecode.h>
-#include <erasurecode_helpers.h>
-#include <math.h>
 #include <pyeclib_c.h>
-#include <bytesobject.h>
 
 /* Python 3 compatibility macros */
 #if PY_MAJOR_VERSION >= 3
@@ -46,7 +45,7 @@
               PyModuleDef_HEAD_INIT, name, doc, -1, methods, }; \
           ob = PyModule_Create(&moduledef);
   #define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \
-          Py_BuildValue("y#", obj, objlen)
+          Py_BuildValue("y#", obj, (Py_ssize_t)objlen)
   #define PyInt_FromLong PyLong_FromLong
   #define PyString_FromString PyUnicode_FromString
   #define ENCODE_ARGS "Oy#"
@@ -58,7 +57,7 @@
   #define MOD_DEF(ob, name, doc, methods) \
           ob = Py_InitModule3(name, methods, doc);
   #define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \
-          Py_BuildValue("s#", obj, objlen)
+          Py_BuildValue("s#", obj, (Py_ssize_t)objlen)
   #define ENCODE_ARGS "Os#"
   #define GET_METADATA_ARGS "Os#i"
 #endif
@@ -87,6 +86,51 @@ static PyObject *import_class(const char *module, const char *cls)
     return (PyObject *) PyObject_GetAttrString(s, cls);
 }
 
+/**
+ * Allocate a buffer of a specific size and set its' contents
+ * to the specified value.
+ *
+ * @param size integer size in bytes of buffer to allocate
+ * @param value
+ * @return pointer to start of allocated buffer or NULL on error
+ */
+void * alloc_and_set_buffer(int size, int value) {
+    void * buf = NULL;  /* buffer to allocate and return */
+  
+    /* Allocate and zero the buffer, or set the appropriate error */
+    buf = malloc((size_t) size);
+    if (buf) {
+        buf = memset(buf, value, (size_t) size);
+    }
+    return buf;
+}
+
+/**
+ * Allocate a zero-ed buffer of a specific size.
+ *
+ * @param size integer size in bytes of buffer to allocate
+ * @return pointer to start of allocated buffer or NULL on error
+ */
+void * alloc_zeroed_buffer(int size)
+{
+    return alloc_and_set_buffer(size, 0);
+}
+
+/**
+ * Deallocate memory buffer if it's not NULL.  This methods returns NULL so 
+ * that you can free and reset a buffer using a single line as follows:
+ *
+ * my_ptr = check_and_free_buffer(my_ptr);
+ *
+ * @return NULL
+ */
+void * check_and_free_buffer(void * buf)
+{
+    if (buf)
+        free(buf);
+    return NULL;
+}
+
 void
 pyeclib_c_seterr(int ret, const char * prefix)
 {

test/test_pyeclib_api.py

@@ -31,7 +31,10 @@ from pyeclib.ec_iface import ECDriverError
 from pyeclib.ec_iface import ECInsufficientFragments
 
 from pyeclib.ec_iface import ECDriver, PyECLib_EC_Types
-from .test_pyeclib_c import _available_backends
+if sys.version < '3':
+    from test_pyeclib_c import _available_backends
+else:
+    from .test_pyeclib_c import _available_backends
 
 if sys.version < '3':
     def b2i(b):
@@ -147,11 +150,25 @@ class TestPyECLibDriver(unittest.TestCase):
                                    chksum_type=csum))
             pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type2,
                                    chksum_type=csum))
-        _type3 = 'flat_xor_hd'
+        _type3_1 = 'flat_xor_hd'
-        if _type3 in _available_backends:
+        if _type3_1 in _available_backends:
-            pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3,
+            pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_1,
                                    chksum_type=csum))
-            pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3,
+            pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_1,
+                                   chksum_type=csum))
+        _type3_2 = 'flat_xor_hd_4'
+        if _type3_2 in _available_backends:
+            pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_2,
+                                   chksum_type=csum))
+            pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_2,
+                                   chksum_type=csum))
+        _type4 = 'shss'
+        if _type4 in _available_backends:
+            pyeclib_drivers.append(ECDriver(k=10, m=4, ec_type=_type4,
+                                   chksum_type=csum))
+            pyeclib_drivers.append(ECDriver(k=20, m=4, ec_type=_type4,
+                                   chksum_type=csum))
+            pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type4,
                                    chksum_type=csum))
         return pyeclib_drivers
 
@@ -536,5 +553,6 @@ class TestPyECLibDriver(unittest.TestCase):
         self.assertTrue(
             pyeclib_drivers[0].min_parity_fragments_needed() == 1)
 
+
 if __name__ == '__main__':
     unittest.main()