From 2d85a3f6997fbadac210207f841c4ec25ff50cc4 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 23 Oct 2015 13:16:33 +0000 Subject: [PATCH] EC: Use best available ec_type in unittests To minimize external library dependencies for Swift unit tests and SAIO, PyECLib 1.1.1 introduces a native backend 'liberasurecode_rs_vand.' This patch is to migrate over the unit tests to the new ec_type when available. This change will work with current pyeclib requirements (==1.0.7) and also future requirements (>=1.0.7). When we're able to raise *our* requirements to >=1.1.1 we should remove jerasure from the list of preferred backends. Related SAIO doc and example config changes should be included with that patch. Co-Authored-By: Clay Gerrard Change-Id: Idf657f0acf0479bc8158972e568a29dbc08eaf3b --- test/unit/__init__.py | 21 +++++++++- test/unit/common/test_storage_policy.py | 53 +++++++++++++------------ test/unit/obj/test_diskfile.py | 4 +- test/unit/obj/test_reconstructor.py | 6 ++- test/unit/obj/test_server.py | 4 +- test/unit/proxy/test_server.py | 6 +-- 6 files changed, 58 insertions(+), 36 deletions(-) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 4a07e176d7..6ab3618780 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -41,7 +41,8 @@ import logging.handlers from six.moves.http_client import HTTPException from swift.common import storage_policy -from swift.common.storage_policy import StoragePolicy, ECStoragePolicy +from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy, + VALID_EC_TYPES) import functools import six.moves.cPickle as pickle from gzip import GzipFile @@ -56,6 +57,22 @@ if not os.path.basename(sys.argv[0]).startswith('swift'): utils.HASH_PATH_SUFFIX = 'endcap' +EC_TYPE_PREFERENCE = [ + 'liberasurecode_rs_vand', + 'jerasure_rs_vand', +] +for eclib_name in EC_TYPE_PREFERENCE: + if eclib_name in VALID_EC_TYPES: + break +else: + raise SystemExit('ERROR: unable to find suitable PyECLib type' + ' (none of %r found in %r)' % ( + EC_TYPE_PREFERENCE, + VALID_EC_TYPES, + )) +DEFAULT_TEST_EC_TYPE = eclib_name + + def patch_policies(thing_or_policies=None, legacy_only=False, with_ec_default=False, fake_ring_args=None): if isinstance(thing_or_policies, ( @@ -70,7 +87,7 @@ def patch_policies(thing_or_policies=None, legacy_only=False, elif with_ec_default: default_policies = [ ECStoragePolicy(0, name='ec', is_default=True, - ec_type='jerasure_rs_vand', ec_ndata=10, + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, ec_segment_size=4096), StoragePolicy(1, name='unu'), ] diff --git a/test/unit/common/test_storage_policy.py b/test/unit/common/test_storage_policy.py index 2d473f7dae..5a2e332ba5 100644 --- a/test/unit/common/test_storage_policy.py +++ b/test/unit/common/test_storage_policy.py @@ -20,7 +20,7 @@ from functools import partial from six.moves.configparser import ConfigParser from tempfile import NamedTemporaryFile -from test.unit import patch_policies, FakeRing, temptree +from test.unit import patch_policies, FakeRing, temptree, DEFAULT_TEST_EC_TYPE from swift.common.storage_policy import ( StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies, reload_storage_policies, get_policy_string, split_policy_string, @@ -70,7 +70,7 @@ class TestStoragePolicies(unittest.TestCase): StoragePolicy(1, 'one'), StoragePolicy(2, 'two'), StoragePolicy(3, 'three', is_deprecated=True), - ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4), ]) def test_swift_info(self): @@ -144,7 +144,8 @@ class TestStoragePolicies(unittest.TestCase): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False), - ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ECStoragePolicy(10, 'ten', + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3)] policies = StoragePolicyCollection(test_policies) for policy in policies: @@ -295,7 +296,7 @@ class TestStoragePolicies(unittest.TestCase): StoragePolicy(1, 'one'), StoragePolicy(2, 'two'), StoragePolicy(3, 'three', is_deprecated=True), - ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', + ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) @@ -561,9 +562,9 @@ class TestStoragePolicies(unittest.TestCase): [storage-policy:1] name = ec10-4 policy_type = erasure_coding - ec_type = jerasure_rs_vand + ec_type = %(ec_type)s ec_num_data_fragments = 10 - """) + """ % {'ec_type': DEFAULT_TEST_EC_TYPE}) self.assertRaisesWithMessage(PolicyError, 'Invalid ec_num_parity_fragments', @@ -576,10 +577,11 @@ class TestStoragePolicies(unittest.TestCase): [storage-policy:1] name = ec10-4 policy_type = erasure_coding - ec_type = jerasure_rs_vand + ec_type = %(ec_type)s ec_num_data_fragments = 10 - ec_num_parity_fragments = %s - """ % num_parity) + ec_num_parity_fragments = %(num_parity)s + """ % {'ec_type': DEFAULT_TEST_EC_TYPE, + 'num_parity': num_parity}) self.assertRaisesWithMessage(PolicyError, 'Invalid ec_num_parity_fragments', @@ -592,9 +594,9 @@ class TestStoragePolicies(unittest.TestCase): [storage-policy:1] name = ec10-4 policy_type = erasure_coding - ec_type = jerasure_rs_vand + ec_type = %(ec_type)s ec_num_parity_fragments = 4 - """) + """ % {'ec_type': DEFAULT_TEST_EC_TYPE}) self.assertRaisesWithMessage(PolicyError, 'Invalid ec_num_data_fragments', @@ -607,10 +609,10 @@ class TestStoragePolicies(unittest.TestCase): [storage-policy:1] name = ec10-4 policy_type = erasure_coding - ec_type = jerasure_rs_vand - ec_num_data_fragments = %s + ec_type = %(ec_type)s + ec_num_data_fragments = %(num_data)s ec_num_parity_fragments = 4 - """ % num_data) + """ % {'num_data': num_data, 'ec_type': DEFAULT_TEST_EC_TYPE}) self.assertRaisesWithMessage(PolicyError, 'Invalid ec_num_data_fragments', @@ -624,11 +626,12 @@ class TestStoragePolicies(unittest.TestCase): [storage-policy:1] name = ec10-4 policy_type = erasure_coding - ec_object_segment_size = %s - ec_type = jerasure_rs_vand + ec_object_segment_size = %(segment_size)s + ec_type = %(ec_type)s ec_num_data_fragments = 10 ec_num_parity_fragments = 4 - """ % segment_size) + """ % {'segment_size': segment_size, + 'ec_type': DEFAULT_TEST_EC_TYPE}) self.assertRaisesWithMessage(PolicyError, 'Invalid ec_object_segment_size', @@ -900,7 +903,7 @@ class TestStoragePolicies(unittest.TestCase): def test_quorum_size_erasure_coding(self): test_ec_policies = [ - ECStoragePolicy(10, 'ec8-2', ec_type='jerasure_rs_vand', + ECStoragePolicy(10, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=8, ec_nparity=2), ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4', ec_ndata=10, ec_nparity=6), @@ -913,14 +916,14 @@ class TestStoragePolicies(unittest.TestCase): def test_validate_ring(self): test_policies = [ - ECStoragePolicy(0, 'ec8-2', ec_type='jerasure_rs_vand', + ECStoragePolicy(0, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=8, ec_nparity=2, object_ring=FakeRing(replicas=8), is_default=True), - ECStoragePolicy(1, 'ec10-4', ec_type='jerasure_rs_vand', + ECStoragePolicy(1, 'ec10-4', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, object_ring=FakeRing(replicas=10)), - ECStoragePolicy(2, 'ec4-2', ec_type='jerasure_rs_vand', + ECStoragePolicy(2, 'ec4-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=4, ec_nparity=2, object_ring=FakeRing(replicas=7)), ] @@ -939,10 +942,10 @@ class TestStoragePolicies(unittest.TestCase): StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one', is_deprecated=True), ECStoragePolicy(10, 'ten', - ec_type='jerasure_rs_vand', + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3), ECStoragePolicy(11, 'done', is_deprecated=True, - ec_type='jerasure_rs_vand', + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) @@ -975,7 +978,7 @@ class TestStoragePolicies(unittest.TestCase): 'default': False, 'deprecated': False, 'policy_type': EC_POLICY, - 'ec_type': 'jerasure_rs_vand', + 'ec_type': DEFAULT_TEST_EC_TYPE, 'ec_num_data_fragments': 10, 'ec_num_parity_fragments': 3, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, @@ -989,7 +992,7 @@ class TestStoragePolicies(unittest.TestCase): 'default': False, 'deprecated': True, 'policy_type': EC_POLICY, - 'ec_type': 'jerasure_rs_vand', + 'ec_type': DEFAULT_TEST_EC_TYPE, 'ec_num_data_fragments': 10, 'ec_num_parity_fragments': 3, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 39b4d299ec..991f38a496 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -39,7 +39,7 @@ from gzip import GzipFile from eventlet import hubs, timeout, tpool from test.unit import (FakeLogger, mock as unit_mock, temptree, patch_policies, debug_logger, EMPTY_ETAG, - make_timestamp_iter) + make_timestamp_iter, DEFAULT_TEST_EC_TYPE) from nose import SkipTest from swift.obj import diskfile @@ -59,7 +59,7 @@ from swift.common.storage_policy import ( test_policies = [ StoragePolicy(0, name='zero', is_default=True), ECStoragePolicy(1, name='one', is_default=False, - ec_type='jerasure_rs_vand', + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4), ] diff --git a/test/unit/obj/test_reconstructor.py b/test/unit/obj/test_reconstructor.py index 891b2a69cd..7baa1342af 100755 --- a/test/unit/obj/test_reconstructor.py +++ b/test/unit/obj/test_reconstructor.py @@ -39,7 +39,8 @@ from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy, from swift.obj.reconstructor import REVERT from test.unit import (patch_policies, debug_logger, mocked_http_conn, - FabricatedRing, make_timestamp_iter) + FabricatedRing, make_timestamp_iter, + DEFAULT_TEST_EC_TYPE) @contextmanager @@ -131,7 +132,8 @@ def get_header_frag_index(self, body): @patch_policies([StoragePolicy(0, name='zero', is_default=True), - ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', + ECStoragePolicy(1, name='one', + ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1)]) class TestGlobalSetupObjectReconstructor(unittest.TestCase): diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 52ea670e98..232f0091b2 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -45,7 +45,7 @@ from nose import SkipTest from swift import __version__ as swift_version from swift.common.http import is_success from test.unit import FakeLogger, debug_logger, mocked_http_conn, \ - make_timestamp_iter + make_timestamp_iter, DEFAULT_TEST_EC_TYPE from test.unit import connect_tcp, readuntil2crlfs, patch_policies from swift.obj import server as object_server from swift.obj import diskfile @@ -66,7 +66,7 @@ def mock_time(*args, **kwargs): test_policies = [ StoragePolicy(0, name='zero', is_default=True), - ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', + ECStoragePolicy(1, name='one', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4), ] diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index cc2eef727f..7f3e201def 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -54,7 +54,7 @@ from swift.common.utils import hash_path, json, storage_directory, \ from test.unit import ( connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing, FakeMemcache, debug_logger, patch_policies, write_fake_ring, - mocked_http_conn) + mocked_http_conn, DEFAULT_TEST_EC_TYPE) from swift.proxy import server as proxy_server from swift.proxy.controllers.obj import ReplicatedObjectController from swift.account import server as account_server @@ -139,7 +139,7 @@ def do_setup(the_object_server): StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), - ECStoragePolicy(3, 'ec', ec_type='jerasure_rs_vand', + ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096)]) obj_rings = { 0: ('sda1', 'sdb1'), @@ -1831,7 +1831,7 @@ class TestObjectController(unittest.TestCase): '4096') self.assertEqual( lmeta['x-object-sysmeta-ec-scheme'], - 'jerasure_rs_vand 2+1') + '%s 2+1' % DEFAULT_TEST_EC_TYPE) self.assertEqual( lmeta['etag'], md5(contents).hexdigest())