openstack/swift
Code Issues Proposed changes

EC: Use best available ec_type in unittests

Browse Source 
To minimize external library dependencies for Swift unit
tests and SAIO, PyECLib 1.1.1 introduces a native backend
'liberasurecode_rs_vand.'  This patch is to migrate over
the unit tests to the new ec_type when available.

This change will work with current pyeclib requirements
(==1.0.7) and also future requirements (>=1.0.7).

When we're able to raise *our* requirements to >=1.1.1 we
should remove jerasure from the list of preferred backends.
Related SAIO doc and example config changes should be
included with that patch.

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: Idf657f0acf0479bc8158972e568a29dbc08eaf3b
This commit is contained in:
Tushar Gohad 2015-10-23 13:16:33 +00:00 committed by Clay Gerrard
parent 705642db4c
commit 2d85a3f699
6 changed files with 58 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
test/unit/__init__.py
View File

@ -41,7 +41,8 @@ import logging.handlers
from six.moves.http_client import HTTPException from six.moves.http_client import HTTPException
from swift.common import storage_policy from swift.common import storage_policy
from swift.common.storage_policy import StoragePolicy, ECStoragePolicy from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
VALID_EC_TYPES)
import functools import functools
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
from gzip import GzipFile from gzip import GzipFile
@ -56,6 +57,22 @@ if not os.path.basename(sys.argv[0]).startswith('swift'):
utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_SUFFIX = 'endcap'
EC_TYPE_PREFERENCE = [
'liberasurecode_rs_vand',
'jerasure_rs_vand',
]
for eclib_name in EC_TYPE_PREFERENCE:
if eclib_name in VALID_EC_TYPES:
break
else:
raise SystemExit('ERROR: unable to find suitable PyECLib type'
' (none of %r found in %r)' % (
EC_TYPE_PREFERENCE,
VALID_EC_TYPES,
))
DEFAULT_TEST_EC_TYPE = eclib_name
def patch_policies(thing_or_policies=None, legacy_only=False, def patch_policies(thing_or_policies=None, legacy_only=False,
with_ec_default=False, fake_ring_args=None): with_ec_default=False, fake_ring_args=None):
if isinstance(thing_or_policies, ( if isinstance(thing_or_policies, (
@ -70,7 +87,7 @@ def patch_policies(thing_or_policies=None, legacy_only=False,
elif with_ec_default: elif with_ec_default:
default_policies = [ default_policies = [
ECStoragePolicy(0, name='ec', is_default=True, ECStoragePolicy(0, name='ec', is_default=True,
ec_type='jerasure_rs_vand', ec_ndata=10, ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10,
ec_nparity=4, ec_segment_size=4096), ec_nparity=4, ec_segment_size=4096),
StoragePolicy(1, name='unu'), StoragePolicy(1, name='unu'),
] ]

53
test/unit/common/test_storage_policy.py
View File

@ -20,7 +20,7 @@ from functools import partial
from six.moves.configparser import ConfigParser from six.moves.configparser import ConfigParser
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from test.unit import patch_policies, FakeRing, temptree from test.unit import patch_policies, FakeRing, temptree, DEFAULT_TEST_EC_TYPE
from swift.common.storage_policy import ( from swift.common.storage_policy import (
StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies, StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies,
reload_storage_policies, get_policy_string, split_policy_string, reload_storage_policies, get_policy_string, split_policy_string,
@ -70,7 +70,7 @@ class TestStoragePolicies(unittest.TestCase):
StoragePolicy(1, 'one'), StoragePolicy(1, 'one'),
StoragePolicy(2, 'two'), StoragePolicy(2, 'two'),
StoragePolicy(3, 'three', is_deprecated=True), StoragePolicy(3, 'three', is_deprecated=True),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=4), ec_ndata=10, ec_nparity=4),
]) ])
def test_swift_info(self): def test_swift_info(self):
@ -144,7 +144,8 @@ class TestStoragePolicies(unittest.TestCase):
test_policies = [StoragePolicy(0, 'aay', True), test_policies = [StoragePolicy(0, 'aay', True),
StoragePolicy(1, 'bee', False), StoragePolicy(1, 'bee', False),
StoragePolicy(2, 'cee', False), StoragePolicy(2, 'cee', False),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ECStoragePolicy(10, 'ten',
ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=3)] ec_ndata=10, ec_nparity=3)]
policies = StoragePolicyCollection(test_policies) policies = StoragePolicyCollection(test_policies)
for policy in policies: for policy in policies:
@ -295,7 +296,7 @@ class TestStoragePolicies(unittest.TestCase):
StoragePolicy(1, 'one'), StoragePolicy(1, 'one'),
StoragePolicy(2, 'two'), StoragePolicy(2, 'two'),
StoragePolicy(3, 'three', is_deprecated=True), StoragePolicy(3, 'three', is_deprecated=True),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=3), ec_ndata=10, ec_nparity=3),
] ]
policies = StoragePolicyCollection(test_policies) policies = StoragePolicyCollection(test_policies)
@ -561,9 +562,9 @@ class TestStoragePolicies(unittest.TestCase):
[storage-policy:1] [storage-policy:1]
name = ec10-4 name = ec10-4
policy_type = erasure_coding policy_type = erasure_coding
ec_type = jerasure_rs_vand ec_type = %(ec_type)s
ec_num_data_fragments = 10 ec_num_data_fragments = 10
""") """ % {'ec_type': DEFAULT_TEST_EC_TYPE})
self.assertRaisesWithMessage(PolicyError, self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_parity_fragments', 'Invalid ec_num_parity_fragments',
@ -576,10 +577,11 @@ class TestStoragePolicies(unittest.TestCase):
[storage-policy:1] [storage-policy:1]
name = ec10-4 name = ec10-4
policy_type = erasure_coding policy_type = erasure_coding
ec_type = jerasure_rs_vand ec_type = %(ec_type)s
ec_num_data_fragments = 10 ec_num_data_fragments = 10
ec_num_parity_fragments = %s ec_num_parity_fragments = %(num_parity)s
""" % num_parity) """ % {'ec_type': DEFAULT_TEST_EC_TYPE,
'num_parity': num_parity})
self.assertRaisesWithMessage(PolicyError, self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_parity_fragments', 'Invalid ec_num_parity_fragments',
@ -592,9 +594,9 @@ class TestStoragePolicies(unittest.TestCase):
[storage-policy:1] [storage-policy:1]
name = ec10-4 name = ec10-4
policy_type = erasure_coding policy_type = erasure_coding
ec_type = jerasure_rs_vand ec_type = %(ec_type)s
ec_num_parity_fragments = 4 ec_num_parity_fragments = 4
""") """ % {'ec_type': DEFAULT_TEST_EC_TYPE})
self.assertRaisesWithMessage(PolicyError, self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_data_fragments', 'Invalid ec_num_data_fragments',
@ -607,10 +609,10 @@ class TestStoragePolicies(unittest.TestCase):
[storage-policy:1] [storage-policy:1]
name = ec10-4 name = ec10-4
policy_type = erasure_coding policy_type = erasure_coding
ec_type = jerasure_rs_vand ec_type = %(ec_type)s
ec_num_data_fragments = %s ec_num_data_fragments = %(num_data)s
ec_num_parity_fragments = 4 ec_num_parity_fragments = 4
""" % num_data) """ % {'num_data': num_data, 'ec_type': DEFAULT_TEST_EC_TYPE})
self.assertRaisesWithMessage(PolicyError, self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_data_fragments', 'Invalid ec_num_data_fragments',
@ -624,11 +626,12 @@ class TestStoragePolicies(unittest.TestCase):
[storage-policy:1] [storage-policy:1]
name = ec10-4 name = ec10-4
policy_type = erasure_coding policy_type = erasure_coding
ec_object_segment_size = %s ec_object_segment_size = %(segment_size)s
ec_type = jerasure_rs_vand ec_type = %(ec_type)s
ec_num_data_fragments = 10 ec_num_data_fragments = 10
ec_num_parity_fragments = 4 ec_num_parity_fragments = 4
""" % segment_size) """ % {'segment_size': segment_size,
'ec_type': DEFAULT_TEST_EC_TYPE})
self.assertRaisesWithMessage(PolicyError, self.assertRaisesWithMessage(PolicyError,
'Invalid ec_object_segment_size', 'Invalid ec_object_segment_size',
@ -900,7 +903,7 @@ class TestStoragePolicies(unittest.TestCase):
def test_quorum_size_erasure_coding(self): def test_quorum_size_erasure_coding(self):
test_ec_policies = [ test_ec_policies = [
ECStoragePolicy(10, 'ec8-2', ec_type='jerasure_rs_vand', ECStoragePolicy(10, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=8, ec_nparity=2), ec_ndata=8, ec_nparity=2),
ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4', ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4',
ec_ndata=10, ec_nparity=6), ec_ndata=10, ec_nparity=6),
@ -913,14 +916,14 @@ class TestStoragePolicies(unittest.TestCase):
def test_validate_ring(self): def test_validate_ring(self):
test_policies = [ test_policies = [
ECStoragePolicy(0, 'ec8-2', ec_type='jerasure_rs_vand', ECStoragePolicy(0, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=8, ec_nparity=2, ec_ndata=8, ec_nparity=2,
object_ring=FakeRing(replicas=8), object_ring=FakeRing(replicas=8),
is_default=True), is_default=True),
ECStoragePolicy(1, 'ec10-4', ec_type='jerasure_rs_vand', ECStoragePolicy(1, 'ec10-4', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=4, ec_ndata=10, ec_nparity=4,
object_ring=FakeRing(replicas=10)), object_ring=FakeRing(replicas=10)),
ECStoragePolicy(2, 'ec4-2', ec_type='jerasure_rs_vand', ECStoragePolicy(2, 'ec4-2', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=4, ec_nparity=2, ec_ndata=4, ec_nparity=2,
object_ring=FakeRing(replicas=7)), object_ring=FakeRing(replicas=7)),
] ]
@ -939,10 +942,10 @@ class TestStoragePolicies(unittest.TestCase):
StoragePolicy(0, 'zero', is_default=True), StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_deprecated=True), StoragePolicy(1, 'one', is_deprecated=True),
ECStoragePolicy(10, 'ten', ECStoragePolicy(10, 'ten',
ec_type='jerasure_rs_vand', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=3), ec_ndata=10, ec_nparity=3),
ECStoragePolicy(11, 'done', is_deprecated=True, ECStoragePolicy(11, 'done', is_deprecated=True,
ec_type='jerasure_rs_vand', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=3), ec_ndata=10, ec_nparity=3),
] ]
policies = StoragePolicyCollection(test_policies) policies = StoragePolicyCollection(test_policies)
@ -975,7 +978,7 @@ class TestStoragePolicies(unittest.TestCase):
'default': False, 'default': False,
'deprecated': False, 'deprecated': False,
'policy_type': EC_POLICY, 'policy_type': EC_POLICY,
'ec_type': 'jerasure_rs_vand', 'ec_type': DEFAULT_TEST_EC_TYPE,
'ec_num_data_fragments': 10, 'ec_num_data_fragments': 10,
'ec_num_parity_fragments': 3, 'ec_num_parity_fragments': 3,
'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,
@ -989,7 +992,7 @@ class TestStoragePolicies(unittest.TestCase):
'default': False, 'default': False,
'deprecated': True, 'deprecated': True,
'policy_type': EC_POLICY, 'policy_type': EC_POLICY,
'ec_type': 'jerasure_rs_vand', 'ec_type': DEFAULT_TEST_EC_TYPE,
'ec_num_data_fragments': 10, 'ec_num_data_fragments': 10,
'ec_num_parity_fragments': 3, 'ec_num_parity_fragments': 3,
'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,

4
test/unit/obj/test_diskfile.py
View File

@ -39,7 +39,7 @@ from gzip import GzipFile
from eventlet import hubs, timeout, tpool from eventlet import hubs, timeout, tpool
from test.unit import (FakeLogger, mock as unit_mock, temptree, from test.unit import (FakeLogger, mock as unit_mock, temptree,
patch_policies, debug_logger, EMPTY_ETAG, patch_policies, debug_logger, EMPTY_ETAG,
make_timestamp_iter) make_timestamp_iter, DEFAULT_TEST_EC_TYPE)
from nose import SkipTest from nose import SkipTest
from swift.obj import diskfile from swift.obj import diskfile
@ -59,7 +59,7 @@ from swift.common.storage_policy import (
test_policies = [ test_policies = [
StoragePolicy(0, name='zero', is_default=True), StoragePolicy(0, name='zero', is_default=True),
ECStoragePolicy(1, name='one', is_default=False, ECStoragePolicy(1, name='one', is_default=False,
ec_type='jerasure_rs_vand', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=4), ec_ndata=10, ec_nparity=4),
] ]

6
test/unit/obj/test_reconstructor.py
View File

@ -39,7 +39,8 @@ from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
from swift.obj.reconstructor import REVERT from swift.obj.reconstructor import REVERT
from test.unit import (patch_policies, debug_logger, mocked_http_conn, from test.unit import (patch_policies, debug_logger, mocked_http_conn,
FabricatedRing, make_timestamp_iter) FabricatedRing, make_timestamp_iter,
DEFAULT_TEST_EC_TYPE)
@contextmanager @contextmanager
@ -131,7 +132,8 @@ def get_header_frag_index(self, body):
@patch_policies([StoragePolicy(0, name='zero', is_default=True), @patch_policies([StoragePolicy(0, name='zero', is_default=True),
ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', ECStoragePolicy(1, name='one',
ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1)]) ec_ndata=2, ec_nparity=1)])
class TestGlobalSetupObjectReconstructor(unittest.TestCase): class TestGlobalSetupObjectReconstructor(unittest.TestCase):

4
test/unit/obj/test_server.py
View File

@ -45,7 +45,7 @@ from nose import SkipTest
from swift import __version__ as swift_version from swift import __version__ as swift_version
from swift.common.http import is_success from swift.common.http import is_success
from test.unit import FakeLogger, debug_logger, mocked_http_conn, \ from test.unit import FakeLogger, debug_logger, mocked_http_conn, \
make_timestamp_iter make_timestamp_iter, DEFAULT_TEST_EC_TYPE
from test.unit import connect_tcp, readuntil2crlfs, patch_policies from test.unit import connect_tcp, readuntil2crlfs, patch_policies
from swift.obj import server as object_server from swift.obj import server as object_server
from swift.obj import diskfile from swift.obj import diskfile
@ -66,7 +66,7 @@ def mock_time(*args, **kwargs):
test_policies = [ test_policies = [
StoragePolicy(0, name='zero', is_default=True), StoragePolicy(0, name='zero', is_default=True),
ECStoragePolicy(1, name='one', ec_type='jerasure_rs_vand', ECStoragePolicy(1, name='one', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=10, ec_nparity=4), ec_ndata=10, ec_nparity=4),
] ]

6
test/unit/proxy/test_server.py
View File

@ -54,7 +54,7 @@ from swift.common.utils import hash_path, json, storage_directory, \
from test.unit import ( from test.unit import (
connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing, connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing,
FakeMemcache, debug_logger, patch_policies, write_fake_ring, FakeMemcache, debug_logger, patch_policies, write_fake_ring,
mocked_http_conn) mocked_http_conn, DEFAULT_TEST_EC_TYPE)
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.proxy.controllers.obj import ReplicatedObjectController from swift.proxy.controllers.obj import ReplicatedObjectController
from swift.account import server as account_server from swift.account import server as account_server
@ -139,7 +139,7 @@ def do_setup(the_object_server):
StoragePolicy(0, 'zero', True), StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'one', False), StoragePolicy(1, 'one', False),
StoragePolicy(2, 'two', False), StoragePolicy(2, 'two', False),
ECStoragePolicy(3, 'ec', ec_type='jerasure_rs_vand', ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1, ec_segment_size=4096)]) ec_ndata=2, ec_nparity=1, ec_segment_size=4096)])
obj_rings = { obj_rings = {
0: ('sda1', 'sdb1'), 0: ('sda1', 'sdb1'),
@ -1831,7 +1831,7 @@ class TestObjectController(unittest.TestCase):
'4096') '4096')
self.assertEqual( self.assertEqual(
lmeta['x-object-sysmeta-ec-scheme'], lmeta['x-object-sysmeta-ec-scheme'],
'jerasure_rs_vand 2+1') '%s 2+1' % DEFAULT_TEST_EC_TYPE)
self.assertEqual( self.assertEqual(
lmeta['etag'], lmeta['etag'],
md5(contents).hexdigest()) md5(contents).hexdigest())