EC: Use best available ec_type in unittests

To minimize external library dependencies for Swift unit
tests and SAIO, PyECLib 1.1.1 introduces a native backend
'liberasurecode_rs_vand.'  This patch is to migrate over
the unit tests to the new ec_type when available.

This change will work with current pyeclib requirements
(==1.0.7) and also future requirements (>=1.0.7).

When we're able to raise *our* requirements to >=1.1.1 we
should remove jerasure from the list of preferred backends.
Related SAIO doc and example config changes should be
included with that patch.

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: Idf657f0acf0479bc8158972e568a29dbc08eaf3b

test/unit/__init__.py

@@ -41,7 +41,8 @@ import logging.handlers
 
 from six.moves.http_client import HTTPException
 from swift.common import storage_policy
-from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
+from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
+                                         VALID_EC_TYPES)
 import functools
 import six.moves.cPickle as pickle
 from gzip import GzipFile
@@ -56,6 +57,22 @@ if not os.path.basename(sys.argv[0]).startswith('swift'):
     utils.HASH_PATH_SUFFIX = 'endcap'
 
 
+EC_TYPE_PREFERENCE = [
+    'liberasurecode_rs_vand',
+    'jerasure_rs_vand',
+]
+for eclib_name in EC_TYPE_PREFERENCE:
+    if eclib_name in VALID_EC_TYPES:
+        break
+else:
+    raise SystemExit('ERROR: unable to find suitable PyECLib type'
+                     ' (none of %r found in %r)' % (
+                         EC_TYPE_PREFERENCE,
+                         VALID_EC_TYPES,
+                     ))
+DEFAULT_TEST_EC_TYPE = eclib_name
+
+
 def patch_policies(thing_or_policies=None, legacy_only=False,
                    with_ec_default=False, fake_ring_args=None):
     if isinstance(thing_or_policies, (
@@ -70,7 +87,7 @@ def patch_policies(thing_or_policies=None, legacy_only=False,
     elif with_ec_default:
         default_policies = [
             ECStoragePolicy(0, name='ec', is_default=True,
-                            ec_type='jerasure_rs_vand', ec_ndata=10,
+                            ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10,
                             ec_nparity=4, ec_segment_size=4096),
             StoragePolicy(1, name='unu'),
         ]

test/unit/common/test_storage_policy.py

@@ -20,7 +20,7 @@ from functools import partial
 
 from six.moves.configparser import ConfigParser
 from tempfile import NamedTemporaryFile
-from test.unit import patch_policies, FakeRing, temptree
+from test.unit import patch_policies, FakeRing, temptree, DEFAULT_TEST_EC_TYPE
 from swift.common.storage_policy import (
     StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies,
     reload_storage_policies, get_policy_string, split_policy_string,
@@ -70,7 +70,7 @@ class TestStoragePolicies(unittest.TestCase):
         StoragePolicy(1, 'one'),
         StoragePolicy(2, 'two'),
         StoragePolicy(3, 'three', is_deprecated=True),
-        ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
+        ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE,
                         ec_ndata=10, ec_nparity=4),
     ])
     def test_swift_info(self):
@@ -144,7 +144,8 @@ class TestStoragePolicies(unittest.TestCase):
         test_policies = [StoragePolicy(0, 'aay', True),
                          StoragePolicy(1, 'bee', False),
                          StoragePolicy(2, 'cee', False),
-                         ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
+                         ECStoragePolicy(10, 'ten',
+                                         ec_type=DEFAULT_TEST_EC_TYPE,
                                          ec_ndata=10, ec_nparity=3)]
         policies = StoragePolicyCollection(test_policies)
         for policy in policies:
@@ -295,7 +296,7 @@ class TestStoragePolicies(unittest.TestCase):
             StoragePolicy(1, 'one'),
             StoragePolicy(2, 'two'),
             StoragePolicy(3, 'three', is_deprecated=True),
-            ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
+            ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=10, ec_nparity=3),
         ]
         policies = StoragePolicyCollection(test_policies)
@@ -561,9 +562,9 @@ class TestStoragePolicies(unittest.TestCase):
         [storage-policy:1]
         name = ec10-4
         policy_type = erasure_coding
-        ec_type = jerasure_rs_vand
+        ec_type = %(ec_type)s
         ec_num_data_fragments = 10
-        """)
+        """ % {'ec_type': DEFAULT_TEST_EC_TYPE})
 
         self.assertRaisesWithMessage(PolicyError,
                                      'Invalid ec_num_parity_fragments',
@@ -576,10 +577,11 @@ class TestStoragePolicies(unittest.TestCase):
             [storage-policy:1]
             name = ec10-4
             policy_type = erasure_coding
-            ec_type = jerasure_rs_vand
+            ec_type = %(ec_type)s
             ec_num_data_fragments = 10
-            ec_num_parity_fragments = %s
-            """ % num_parity)
+            ec_num_parity_fragments = %(num_parity)s
+            """ % {'ec_type': DEFAULT_TEST_EC_TYPE,
+                   'num_parity': num_parity})
 
             self.assertRaisesWithMessage(PolicyError,
                                          'Invalid ec_num_parity_fragments',
@@ -592,9 +594,9 @@ class TestStoragePolicies(unittest.TestCase):
         [storage-policy:1]
         name = ec10-4
         policy_type = erasure_coding
-        ec_type = jerasure_rs_vand
+        ec_type = %(ec_type)s
         ec_num_parity_fragments = 4
-        """)
+        """ % {'ec_type': DEFAULT_TEST_EC_TYPE})
 
         self.assertRaisesWithMessage(PolicyError,
                                      'Invalid ec_num_data_fragments',
@@ -607,10 +609,10 @@ class TestStoragePolicies(unittest.TestCase):
             [storage-policy:1]
             name = ec10-4
             policy_type = erasure_coding
-            ec_type = jerasure_rs_vand
-            ec_num_data_fragments = %s
+            ec_type = %(ec_type)s
+            ec_num_data_fragments = %(num_data)s
             ec_num_parity_fragments = 4
-            """ % num_data)
+            """ % {'num_data': num_data, 'ec_type': DEFAULT_TEST_EC_TYPE})
 
             self.assertRaisesWithMessage(PolicyError,
                                          'Invalid ec_num_data_fragments',
@@ -624,11 +626,12 @@ class TestStoragePolicies(unittest.TestCase):
             [storage-policy:1]
             name = ec10-4
             policy_type = erasure_coding
-            ec_object_segment_size = %s
-            ec_type = jerasure_rs_vand
+            ec_object_segment_size = %(segment_size)s
+            ec_type = %(ec_type)s
             ec_num_data_fragments = 10
             ec_num_parity_fragments = 4
-            """ % segment_size)
+            """ % {'segment_size': segment_size,
+                   'ec_type': DEFAULT_TEST_EC_TYPE})
 
             self.assertRaisesWithMessage(PolicyError,
                                          'Invalid ec_object_segment_size',
@@ -900,7 +903,7 @@ class TestStoragePolicies(unittest.TestCase):
 
     def test_quorum_size_erasure_coding(self):
         test_ec_policies = [
-            ECStoragePolicy(10, 'ec8-2', ec_type='jerasure_rs_vand',
+            ECStoragePolicy(10, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=8, ec_nparity=2),
             ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4',
                             ec_ndata=10, ec_nparity=6),
@@ -913,14 +916,14 @@ class TestStoragePolicies(unittest.TestCase):
 
     def test_validate_ring(self):
         test_policies = [
-            ECStoragePolicy(0, 'ec8-2', ec_type='jerasure_rs_vand',
+            ECStoragePolicy(0, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=8, ec_nparity=2,
                             object_ring=FakeRing(replicas=8),
                             is_default=True),
-            ECStoragePolicy(1, 'ec10-4', ec_type='jerasure_rs_vand',
+            ECStoragePolicy(1, 'ec10-4', ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=10, ec_nparity=4,
                             object_ring=FakeRing(replicas=10)),
-            ECStoragePolicy(2, 'ec4-2', ec_type='jerasure_rs_vand',
+            ECStoragePolicy(2, 'ec4-2', ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=4, ec_nparity=2,
                             object_ring=FakeRing(replicas=7)),
         ]
@@ -939,10 +942,10 @@ class TestStoragePolicies(unittest.TestCase):
             StoragePolicy(0, 'zero', is_default=True),
             StoragePolicy(1, 'one', is_deprecated=True),
             ECStoragePolicy(10, 'ten',
-                            ec_type='jerasure_rs_vand',
+                            ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=10, ec_nparity=3),
             ECStoragePolicy(11, 'done', is_deprecated=True,
-                            ec_type='jerasure_rs_vand',
+                            ec_type=DEFAULT_TEST_EC_TYPE,
                             ec_ndata=10, ec_nparity=3),
         ]
         policies = StoragePolicyCollection(test_policies)
@@ -975,7 +978,7 @@ class TestStoragePolicies(unittest.TestCase):
                 'default': False,
                 'deprecated': False,
                 'policy_type': EC_POLICY,
-                'ec_type': 'jerasure_rs_vand',
+                'ec_type': DEFAULT_TEST_EC_TYPE,
                 'ec_num_data_fragments': 10,
                 'ec_num_parity_fragments': 3,
                 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,
@@ -989,7 +992,7 @@ class TestStoragePolicies(unittest.TestCase):
                 'default': False,
                 'deprecated': True,
                 'policy_type': EC_POLICY,
-                'ec_type': 'jerasure_rs_vand',
+                'ec_type': DEFAULT_TEST_EC_TYPE,
                 'ec_num_data_fragments': 10,
                 'ec_num_parity_fragments': 3,
                 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,

test/unit/obj/test_diskfile.py

@@ -39,7 +39,7 @@ from gzip import GzipFile
 from eventlet import hubs, timeout, tpool
 from test.unit import (FakeLogger, mock as unit_mock, temptree,
                        patch_policies, debug_logger, EMPTY_ETAG,
-                       make_timestamp_iter)
+                       make_timestamp_iter, DEFAULT_TEST_EC_TYPE)
 
 from nose import SkipTest
 from swift.obj import diskfile
@@ -59,7 +59,7 @@ from swift.common.storage_policy import (
 test_policies = [
     StoragePolicy(0, name='zero', is_default=True),
     ECStoragePolicy(1, name='one', is_default=False,
-                    ec_type='jerasure_rs_vand',
+                    ec_type=DEFAULT_TEST_EC_TYPE,
                     ec_ndata=10, ec_nparity=4),
 ]
 

test/unit/obj/test_reconstructor.py

@@ -39,7 +39,8 @@ from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
 from swift.obj.reconstructor import REVERT
 
 from test.unit import (patch_policies, debug_logger, mocked_http_conn,
-                       FabricatedRing, make_timestamp_iter)
+                       FabricatedRing, make_timestamp_iter,
+                       DEFAULT_TEST_EC_TYPE)
 
 
 @contextmanager
@@ -131,7 +132,8 @@ def get_header_frag_index(self, body):
 
 
 @patch_policies([StoragePolicy(0, name='zero', is_default=True),
-                 ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand',
+                 ECStoragePolicy(1, name='one',
+                                 ec_type=DEFAULT_TEST_EC_TYPE,
                                  ec_ndata=2, ec_nparity=1)])
 class TestGlobalSetupObjectReconstructor(unittest.TestCase):
 

test/unit/obj/test_server.py

@@ -45,7 +45,7 @@ from nose import SkipTest
 from swift import __version__ as swift_version
 from swift.common.http import is_success
 from test.unit import FakeLogger, debug_logger, mocked_http_conn, \
-    make_timestamp_iter
+    make_timestamp_iter, DEFAULT_TEST_EC_TYPE
 from test.unit import connect_tcp, readuntil2crlfs, patch_policies
 from swift.obj import server as object_server
 from swift.obj import diskfile
@@ -66,7 +66,7 @@ def mock_time(*args, **kwargs):
 
 test_policies = [
     StoragePolicy(0, name='zero', is_default=True),
-    ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand',
+    ECStoragePolicy(1, name='one', ec_type=DEFAULT_TEST_EC_TYPE,
                     ec_ndata=10, ec_nparity=4),
 ]
 

test/unit/proxy/test_server.py

@@ -54,7 +54,7 @@ from swift.common.utils import hash_path, json, storage_directory, \
 from test.unit import (
     connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing,
     FakeMemcache, debug_logger, patch_policies, write_fake_ring,
-    mocked_http_conn)
+    mocked_http_conn, DEFAULT_TEST_EC_TYPE)
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers.obj import ReplicatedObjectController
 from swift.account import server as account_server
@@ -139,7 +139,7 @@ def do_setup(the_object_server):
         StoragePolicy(0, 'zero', True),
         StoragePolicy(1, 'one', False),
         StoragePolicy(2, 'two', False),
-        ECStoragePolicy(3, 'ec', ec_type='jerasure_rs_vand',
+        ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE,
                         ec_ndata=2, ec_nparity=1, ec_segment_size=4096)])
     obj_rings = {
         0: ('sda1', 'sdb1'),
@@ -1831,7 +1831,7 @@ class TestObjectController(unittest.TestCase):
                     '4096')
                 self.assertEqual(
                     lmeta['x-object-sysmeta-ec-scheme'],
-                    'jerasure_rs_vand 2+1')
+                    '%s 2+1' % DEFAULT_TEST_EC_TYPE)
                 self.assertEqual(
                     lmeta['etag'],
                     md5(contents).hexdigest())