Add Phazr.IO libphazr backend support

Currently, there are several implementations of erasure codes that are
available within OpenStack Swift.  Most, if not all, of which are based
on the Reed Solomon coding algorithm.

Phazr.IO’s Erasure Coding technology uses a patented algorithm which are
significantly more efficient and improves the speed of coding, decoding
and reconstruction.  In addition, Phazr.IO Erasure Code use a non-systematic
algorithm which provides data protection at rest and in transport without
the need to use encryption.

Please contact support@phazr.io for more info on our technology.

Change-Id: I9377fa32426a190efd0a7f0675ecf13d7e90367d

README.rst

@@ -51,6 +51,7 @@ Supported ``ec_type`` values:
   * ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4]
   * ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4]
   * ``shss`` => NTT Lab Japan's Erasure Coding Library [5]
+  * ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6]
 
 
 The Python API supports the following functions:
@@ -240,7 +241,7 @@ Quick Start
   An example for ubuntu to install dependency packages::
 
       $ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev
-      $ sudo pip install -U bindep -r test-requirement.txt
+      $ sudo pip install -U bindep -r test-requirements.txt
 
   If you want to confirm all dependency packages installed successfully, try::
 
@@ -280,3 +281,5 @@ References
  [4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
 
  [5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
+
+ [6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"

pyeclib/ec_iface.py

@@ -124,6 +124,7 @@ class PyECLib_EC_Types(PyECLibEnum):
     shss = 5
     liberasurecode_rs_vand = 6
     isa_l_rs_cauchy = 7
+    libphazr = 8
 
 
 # Output of Erasure (en)Coding process are data "fragments".  Fragment data
@@ -175,6 +176,8 @@ class ECDriver(object):
                     elif value == "flat_xor_hd_4":
                         self.hd = 4
                     value = "flat_xor_hd"
+                elif value == "libphazr":
+                    self.hd = 1
                 if PyECLib_EC_Types.has_enum(value):
                     self.ec_type = PyECLib_EC_Types.get_by_name(value)
                 else:
@@ -521,6 +524,7 @@ ALL_EC_TYPES = [
     'shss',
     'liberasurecode_rs_vand',
     'isa_l_rs_cauchy',
+    'libphazr',
 ]
 
 

src/c/pyeclib_c/pyeclib_c.c

@@ -971,6 +971,9 @@ static const char* backend_id_to_str(uint8_t backend_id)
     case 7:
       backend_id_str = "isa_l_rs_cauchy\0";
       break;
+    case 8:
+      backend_id_str = "libphazr\0";
+      break;
     default:
       backend_id_str = "unknown\0";
   }

test/test_pyeclib_api.py

@@ -134,7 +134,7 @@ class TestPyECLibDriver(unittest.TestCase):
                              "Invalid Argument: m is required")
 
             with self.assertRaises(ECDriverError) as err_context:
-                # m is smaller than 1
+                # k is smaller than 1
                 ECDriver(ec_type=ec_type, k=-100, m=1)
             self.assertEqual(str(err_context.exception),
                              "Invalid number of data fragments (k)")
@@ -151,10 +151,15 @@ class TestPyECLibDriver(unittest.TestCase):
         for _type in ALL_EC_TYPES:
             try:
                 if _type is 'shss':
+                    _k = 10
+                    _m = 4
+                elif _type is 'libphazr':
+                    _k = 4
                     _m = 4
                 else:
+                    _k = 10
                     _m = 5
-                ECDriver(k=10, m=_m, ec_type=_type, validate=True)
+                ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
                 available_ec_types.append(_type)
             except Exception:
                 # ignore any errors, assume backend not available
@@ -171,10 +176,12 @@ class TestPyECLibDriver(unittest.TestCase):
             try:
                 if _type is 'shss':
                     _instance = ECDriver(k=10, m=4, ec_type=_type)
+                elif _type is 'libphazr':
+                    _instance = ECDriver(k=4, m=4, ec_type=_type)
                 else:
                     _instance = ECDriver(k=10, m=5, ec_type=_type)
             except ECDriverError:
-                self.fail("%p: %s algorithm not supported" % _instance, _type)
+                self.fail("%s algorithm not supported" % _type)
 
         self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
                           ec_type="invalid_algo")
@@ -244,6 +251,11 @@ class TestPyECLibDriver(unittest.TestCase):
                                    chksum_type=csum))
             pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6,
                                    chksum_type=csum))
+
+        _type7 = 'libphazr'
+        if _type7 in VALID_EC_TYPES:
+            pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7,
+                                   chksum_type=csum))
         return pyeclib_drivers
 
     def test_small_encode(self):
@@ -740,7 +752,15 @@ class TestBackendsEnabled(unittest.TestCase):
                 def dummy(self, ec_type=ec_type):
                     if ec_type not in VALID_EC_TYPES:
                         raise unittest.SkipTest
-                    k, m = 10, 4 if ec_type == 'shss' else 5
+                    if ec_type == 'shss':
+                        k = 10
+                        m = 4
+                    elif ec_type == 'libphazr':
+                        k = 4
+                        m = 4
+                    else:
+                        k = 10
+                        m = 5
                     ECDriver(k=k, m=m, ec_type=ec_type)
                 dummy.__name__ = 'test_%s_available' % ec_type
                 cls_dict[dummy.__name__] = dummy

test/test_pyeclib_c.py

@@ -78,6 +78,7 @@ class TestPyECLib(unittest.TestCase):
                      (PyECLib_EC_Types.shss, 10, 4),
                      (PyECLib_EC_Types.shss, 20, 4),
                      (PyECLib_EC_Types.shss, 11, 7)]
+        self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)]
 
         # Input temp files for testing
         self.sizes = ["101-K", "202-K", "303-K"]
@@ -357,6 +358,36 @@ class TestPyECLib(unittest.TestCase):
                 print("Reconstruct (%s): %s" %
                       (size_str, self.get_throughput(avg_time, size_str)))
 
+    @require_backend("libphazr")
+    def test_libphazr(self):
+        for (ec_type, k, m) in self.libphazr:
+            print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
+
+            success = self._test_get_required_fragments(k, m, ec_type)
+            self.assertTrue(success)
+
+            for size_str in self.sizes:
+                avg_time = self.time_encode(k, m, ec_type.value, 0,
+                                            size_str,
+                                            self.iterations)
+                print("Encode (%s): %s" %
+                      (size_str, self.get_throughput(avg_time, size_str)))
+
+            for size_str in self.sizes:
+                success, avg_time = self.time_decode(k, m, ec_type.value, 0,
+                                                     size_str,
+                                                     self.iterations)
+                self.assertTrue(success)
+                print("Decode (%s): %s" %
+                      (size_str, self.get_throughput(avg_time, size_str)))
+
+            for size_str in self.sizes:
+                success, avg_time = self.time_reconstruct(
+                    k, m, ec_type.value, 0, size_str, self.iterations)
+                self.assertTrue(success)
+                print("Reconstruct (%s): %s" %
+                      (size_str, self.get_throughput(avg_time, size_str)))
+
     def _test_get_required_fragments(self, num_data, num_parity, ec_type):
         """
         :return boolean, True if all tests passed