openstack
/
sahara
Code Issues Proposed changes

Adapt to the additional rules from pycodestyle 2.5.0 

It is wreaking havoc on the openstack gates, let's fix the issues.

Fix:
- E117: Over-indented code blocks;
- E305: Expected 2 blank lines after class or function definition

Ignore for now:
- W504: Line break after binary operator
- W605: Invalid escape sequence

W504 should be fixed together with its sibling W503.

Change-Id: I1483fae77153ffb24bfc51de2b83ea9db6737bfd
This commit is contained in:
Luigi Toscano 2019-01-30 11:41:54 +01:00
parent 2e09de9c82
commit b21e696926
18 changed files with 255 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sahara/conductor/__init__.py
View File

@ -34,4 +34,5 @@ def Api(use_local=True, **kwargs):
return api(**kwargs)
API = Api()

1
sahara/db/migration/alembic_migrations/env.py
View File

@ -89,6 +89,7 @@ def run_migrations_online():
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:

1
sahara/db/sqlalchemy/model_base.py
View File

@ -49,4 +49,5 @@ def datetime_to_str(dct, attr_name):
value = value[:ms_delimiter]
dct[attr_name] = value
SaharaBase = declarative.declarative_base(cls=_SaharaBase)

1
sahara/db/templates/api.py
View File

@ -43,6 +43,7 @@ def set_conf(conf):
global CONF
CONF = conf
ng_validator = api_validator.ApiValidator(ngt.NODE_GROUP_TEMPLATE_SCHEMA)
ct_validator = api_validator.ApiValidator(clt.CLUSTER_TEMPLATE_SCHEMA)

1
sahara/plugins/provisioning.py
View File

@ -319,6 +319,7 @@ class ValidationError(object):
def __repr__(self):
return "<ValidationError %s>" % self.config.name
# COMMON FOR ALL PLUGINS CONFIGS
XFS_ENABLED = Config(

1
sahara/service/edp/resources/launch_command.py
View File

@ -54,6 +54,7 @@ def parse_env_vars():
return env_vars, sys.argv[index:]
log.info("Running %s" % ' '.join(sys.argv[1:]))
try:

8
sahara/service/validations/edp/job_execution.py
View File

@ -156,10 +156,10 @@ def check_job_execution_cancel(job_id, **kwargs):
je = conductor.job_execution_get(ctx, job_id)
if je.tenant_id != ctx.tenant_id:
raise ex.CancelingFailed(
_("Job execution with id '%s' cannot be canceled "
"because it wasn't created in this tenant")
% job_id)
raise ex.CancelingFailed(
_("Job execution with id '%s' cannot be canceled "
"because it wasn't created in this tenant")
% job_id)
if je.is_protected:
raise ex.CancelingFailed(

128
sahara/tests/unit/service/edp/job_binaries/internal_db/test_internal_db_type.py
View File

@ -23,76 +23,76 @@ from sahara.service.edp.job_binaries.internal_db import implementation
class TestInternalDBType(testtools.TestCase):
def setUp(self):
super(TestInternalDBType, self).setUp()
self.internal_db = implementation.InternalDBType()
def setUp(self):
super(TestInternalDBType, self).setUp()
self.internal_db = implementation.InternalDBType()
def test_validate_job_location_format(self):
def test_validate_job_location_format(self):
self.assertFalse(self.internal_db.
validate_job_location_format(''))
self.assertFalse(self.internal_db.
validate_job_location_format('invalid-scheme://'))
self.assertFalse(self.internal_db.
validate_job_location_format('internal-db://abc'))
self.assertTrue(self.internal_db.
validate_job_location_format(
'internal-db://' + uuidutils.generate_uuid()))
self.assertFalse(self.internal_db.
validate_job_location_format(''))
self.assertFalse(self.internal_db.
validate_job_location_format('invalid-scheme://'))
self.assertFalse(self.internal_db.
validate_job_location_format('internal-db://abc'))
self.assertTrue(self.internal_db.
validate_job_location_format(
'internal-db://' + uuidutils.generate_uuid()))
@mock.patch('sahara.conductor.API.job_binary_internal_get_raw_data')
def test_copy_binary_to_cluster(self, conductor_get_raw_data):
remote = mock.Mock()
context = mock.Mock()
conductor_get_raw_data.return_value = 'ok'
job_binary = mock.Mock()
job_binary.name = 'test'
job_binary.url = 'internal-db://somebinary'
@mock.patch('sahara.conductor.API.job_binary_internal_get_raw_data')
def test_copy_binary_to_cluster(self, conductor_get_raw_data):
remote = mock.Mock()
context = mock.Mock()
conductor_get_raw_data.return_value = 'ok'
job_binary = mock.Mock()
job_binary.name = 'test'
job_binary.url = 'internal-db://somebinary'
res = self.internal_db.copy_binary_to_cluster(job_binary,
context=context,
remote=remote)
res = self.internal_db.copy_binary_to_cluster(job_binary,
context=context,
remote=remote)
self.assertEqual('/tmp/test', res)
remote.write_file_to.assert_called_with(
'/tmp/test',
'ok')
self.assertEqual('/tmp/test', res)
remote.write_file_to.assert_called_with(
'/tmp/test',
'ok')
@mock.patch('sahara.conductor.API.job_binary_internal_get_raw_data')
def test_get_raw_data(self, conductor_get_raw_data):
context = mock.Mock()
conductor_get_raw_data.return_value = 'ok'
job_binary = mock.Mock()
job_binary.url = 'internal-db://somebinary'
@mock.patch('sahara.conductor.API.job_binary_internal_get_raw_data')
def test_get_raw_data(self, conductor_get_raw_data):
context = mock.Mock()
conductor_get_raw_data.return_value = 'ok'
job_binary = mock.Mock()
job_binary.url = 'internal-db://somebinary'
self.internal_db.get_raw_data(job_binary,
context=context)
self.internal_db.get_raw_data(job_binary,
context=context)
@mock.patch('sahara.service.validations.edp.base.'
'check_job_binary_internal_exists')
def test_data_validation(self, check_exists):
data = {
'url': '',
'description': 'empty url'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'invalid-url://',
'description': 'not empty, but invalid url'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'internal-db://must-be-uuid',
'description': 'correct scheme, but not netloc is not uuid'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'internal-db://' + uuidutils.generate_uuid(),
'description': 'correct scheme and netloc'
}
@mock.patch('sahara.service.validations.edp.base.'
'check_job_binary_internal_exists')
def test_data_validation(self, check_exists):
data = {
'url': '',
'description': 'empty url'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'invalid-url://',
'description': 'not empty, but invalid url'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'internal-db://must-be-uuid',
'description': 'correct scheme, but not netloc is not uuid'
}
with testtools.ExpectedException(ex.InvalidDataException):
self.internal_db.validate(data)
data = {
'url': 'internal-db://' + uuidutils.generate_uuid(),
'description': 'correct scheme and netloc'
}
self.internal_db.validate(data)

62
sahara/tests/unit/service/edp/job_binaries/job_binary_manager_support.py
View File

@ -22,48 +22,48 @@ from sahara.tests.unit import base
class JobBinaryManagerSupportTest(base.SaharaTestCase):
def setUp(self):
super(JobBinaryManagerSupportTest, self).setUp()
jb_manager.setup_job_binaries()
def setUp(self):
super(JobBinaryManagerSupportTest, self).setUp()
jb_manager.setup_job_binaries()
def test_job_binaries_loaded(self):
jb_types = [jb.name for jb in
jb_manager.JOB_BINARIES.get_job_binaries()]
def test_job_binaries_loaded(self):
jb_types = [jb.name for jb in
jb_manager.JOB_BINARIES.get_job_binaries()]
self.assertIn('internal-db', jb_types)
self.assertIn('manila', jb_types)
self.assertIn('swift', jb_types)
self.assertIn('internal-db', jb_types)
self.assertIn('manila', jb_types)
self.assertIn('swift', jb_types)
def test_get_job_binary_by_url(self):
def test_get_job_binary_by_url(self):
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary_by_url('')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary_by_url('')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary_by_url('internal-db')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary_by_url('internal-db')
self.assertEqual('internal-db', jb_manager.JOB_BINARIES
.get_job_binary_by_url('internal-db://').name)
self.assertEqual('internal-db', jb_manager.JOB_BINARIES
.get_job_binary_by_url('internal-db://').name)
self.assertEqual('manila', jb_manager.JOB_BINARIES
.get_job_binary_by_url('manila://').name)
self.assertEqual('manila', jb_manager.JOB_BINARIES
.get_job_binary_by_url('manila://').name)
self.assertEqual('swift', jb_manager.JOB_BINARIES
.get_job_binary_by_url('swift://').name)
self.assertEqual('swift', jb_manager.JOB_BINARIES
.get_job_binary_by_url('swift://').name)
def test_get_job_binary(self):
def test_get_job_binary(self):
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary('')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary('')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary('internaldb')
with testtools.ExpectedException(ex.InvalidDataException):
jb_manager.JOB_BINARIES.get_job_binary('internaldb')
self.assertEqual('internal-db', jb_manager.JOB_BINARIES
.get_job_binary('internal-db').name)
self.assertEqual('internal-db', jb_manager.JOB_BINARIES
.get_job_binary('internal-db').name)
self.assertEqual('manila', jb_manager.JOB_BINARIES
.get_job_binary('manila').name)
self.assertEqual('manila', jb_manager.JOB_BINARIES
.get_job_binary('manila').name)
self.assertEqual('swift', jb_manager.JOB_BINARIES
.get_job_binary('swift').name)
self.assertEqual('swift', jb_manager.JOB_BINARIES
.get_job_binary('swift').name)

238
sahara/tests/unit/service/edp/job_binaries/manila/test_manila_type.py
View File

@ -30,147 +30,147 @@ class _FakeShare(object):
class TestManilaType(base.SaharaTestCase):
def setUp(self):
super(TestManilaType, self).setUp()
self.manila_type = implementation.ManilaType()
def setUp(self):
super(TestManilaType, self).setUp()
self.manila_type = implementation.ManilaType()
def test_validate_job_location_format(self):
def test_validate_job_location_format(self):
invalid_url_1 = 'manila://abc'
invalid_url_2 = 'manila://' + uuidutils.generate_uuid()
valid_url = 'manila://' + uuidutils.generate_uuid() + '/path'
invalid_url_1 = 'manila://abc'
invalid_url_2 = 'manila://' + uuidutils.generate_uuid()
valid_url = 'manila://' + uuidutils.generate_uuid() + '/path'
self.assertFalse(self.manila_type.validate_job_location_format(''))
self.assertFalse(self.manila_type.
validate_job_location_format(invalid_url_1))
self.assertFalse(self.manila_type.
validate_job_location_format(invalid_url_2))
self.assertFalse(self.manila_type.validate_job_location_format(''))
self.assertFalse(self.manila_type.
validate_job_location_format(invalid_url_1))
self.assertFalse(self.manila_type.
validate_job_location_format(invalid_url_2))
self.assertTrue(self.manila_type.
validate_job_location_format(valid_url))
self.assertTrue(self.manila_type.
validate_job_location_format(valid_url))
@mock.patch('sahara.service.edp.utils.shares.default_mount')
@mock.patch('sahara.utils.openstack.manila.client')
def test_copy_binary_to_cluster(self, f_manilaclient, default_mount):
cluster_shares = [
{'id': 'the_share_id',
'path': '/mnt/mymountpoint'}
]
@mock.patch('sahara.service.edp.utils.shares.default_mount')
@mock.patch('sahara.utils.openstack.manila.client')
def test_copy_binary_to_cluster(self, f_manilaclient, default_mount):
cluster_shares = [
{'id': 'the_share_id',
'path': '/mnt/mymountpoint'}
]
ng_shares = [
{'id': 'the_share_id',
'path': '/mnt/othermountpoint'},
{'id': '123456',
'path': '/mnt/themountpoint'}
]
ng_shares = [
{'id': 'the_share_id',
'path': '/mnt/othermountpoint'},
{'id': '123456',
'path': '/mnt/themountpoint'}
]
job_binary = mock.Mock()
job_binary.url = 'manila://the_share_id/the_path'
job_binary = mock.Mock()
job_binary.url = 'manila://the_share_id/the_path'
remote = mock.Mock()
remote.instance.node_group.cluster.shares = cluster_shares
remote.instance.node_group.shares = ng_shares
remote = mock.Mock()
remote.instance.node_group.cluster.shares = cluster_shares
remote.instance.node_group.shares = ng_shares
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
self.assertItemsEqual('/mnt/mymountpoint/the_path', info)
self.assertItemsEqual('/mnt/mymountpoint/the_path', info)
job_binary.url = 'manila://123456/the_path'
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
self.assertItemsEqual('/mnt/themountpoint/the_path', info)
job_binary.url = 'manila://123456/the_path'
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
self.assertItemsEqual('/mnt/themountpoint/the_path', info)
# missing id
default_mount.return_value = '/mnt/missing_id'
# missing id
default_mount.return_value = '/mnt/missing_id'
job_binary.url = 'manila://missing_id/the_path'
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
job_binary.url = 'manila://missing_id/the_path'
info = self.manila_type.copy_binary_to_cluster(job_binary,
remote=remote)
self.assertItemsEqual('/mnt/missing_id/the_path', info)
self.assertItemsEqual('/mnt/missing_id/the_path', info)
@mock.patch('sahara.utils.openstack.manila.client')
@mock.patch('sahara.conductor.API.cluster_update')
@mock.patch('sahara.service.edp.utils.shares.mount_shares')
def test_prepare_cluster(self, mount_shares, cluster_update,
f_manilaclient):
@mock.patch('sahara.utils.openstack.manila.client')
@mock.patch('sahara.conductor.API.cluster_update')
@mock.patch('sahara.service.edp.utils.shares.mount_shares')
def test_prepare_cluster(self, mount_shares, cluster_update,
f_manilaclient):
cluster_shares = [
{'id': 'the_share_id',
'path': '/mnt/mymountpoint'}
]
cluster_shares = [
{'id': 'the_share_id',
'path': '/mnt/mymountpoint'}
]
ng_shares = [
{'id': 'the_share_id',
'path': '/mnt/othermountpoint'},
{'id': '123456',
'path': '/mnt/themountpoint'}
]
ng_shares = [
{'id': 'the_share_id',
'path': '/mnt/othermountpoint'},
{'id': '123456',
'path': '/mnt/themountpoint'}
]
job_binary = mock.Mock()
job_binary = mock.Mock()
remote = mock.Mock()
remote.instance.node_group.cluster.shares = cluster_shares
remote.instance.node_group.shares = ng_shares
remote = mock.Mock()
remote.instance.node_group.cluster.shares = cluster_shares
remote.instance.node_group.shares = ng_shares
# This should return a default path, and should cause
# a mount at the default location
share = _FakeShare("missing_id")
f_manilaclient.return_value = mock.Mock(shares=mock.Mock(
get=mock.Mock(return_value=share)))
# This should return a default path, and should cause
# a mount at the default location
share = _FakeShare("missing_id")
f_manilaclient.return_value = mock.Mock(shares=mock.Mock(
get=mock.Mock(return_value=share)))
job_binary.url = 'manila://missing_id/the_path'
job_binary.url = 'manila://missing_id/the_path'
self.manila_type.prepare_cluster(job_binary, remote=remote)
self.assertEqual(1, mount_shares.call_count)
self.assertEqual(1, cluster_update.call_count)
self.manila_type.prepare_cluster(job_binary, remote=remote)
self.assertEqual(1, mount_shares.call_count)
self.assertEqual(1, cluster_update.call_count)
def test_get_raw_data(self):
with testtools.ExpectedException(ex.NotImplementedException):
self.manila_type.get_raw_data({})
def test_get_raw_data(self):
with testtools.ExpectedException(ex.NotImplementedException):
self.manila_type.get_raw_data({})
def test_data_validation(self):
data = {
"name": "test",
"url": "man://%s" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("incorrect url schema for")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "",
"type": "manila",
"description": ("empty url")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://bob",
"type": "manila",
"description": ("netloc is not a uuid")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://%s" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("netloc is not a uuid")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://%s/foo" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("correct url")
}
def test_data_validation(self):
data = {
"name": "test",
"url": "man://%s" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("incorrect url schema for")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "",
"type": "manila",
"description": ("empty url")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://bob",
"type": "manila",
"description": ("netloc is not a uuid")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://%s" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("netloc is not a uuid")
}
with testtools.ExpectedException(ex.InvalidDataException):
self.manila_type.validate(data)
data = {
"name": "test",
"url": "manila://%s/foo" % uuidutils.generate_uuid(),
"type": "manila",
"description": ("correct url")
}
self.manila_type.validate(data)

16
sahara/tests/unit/service/edp/job_binaries/test_base.py
View File

@ -26,12 +26,12 @@ class _FakeJobBinary(jb_base.JobBinaryType):
class JobBinaryManagerSupportTest(base.SaharaTestCase):
def setUp(self):
super(JobBinaryManagerSupportTest, self).setUp()
self.job_binary = _FakeJobBinary()
def setUp(self):
super(JobBinaryManagerSupportTest, self).setUp()
self.job_binary = _FakeJobBinary()
def test_generate_valid_path(self):
jb = mock.Mock()
jb.name = 'jb_name.jar'
res = self.job_binary._generate_valid_path(jb)
self.assertEqual('/tmp/jb_name.jar', res)
def test_generate_valid_path(self):
jb = mock.Mock()
jb.name = 'jb_name.jar'
res = self.job_binary._generate_valid_path(jb)
self.assertEqual('/tmp/jb_name.jar', res)

1
sahara/tests/unit/service/test_quotas.py
View File

@ -110,6 +110,7 @@ class FakeNodeGroup(object):
self.open_ports = ports
self.id = uuidutils.generate_uuid()
nova_limits = {
'absolute': {
'maxTotalRAMSize': 10,

1
sahara/tests/unit/service/validation/edp/test_job_interface.py
View File

@ -202,6 +202,7 @@ class TestJobInterfaceValidation(u.ValidationTestCase):
self._assert_create_object_validation(data=job)
assert param['value_type'] == j_i.DEFAULT_DATA_TYPE
int_arg = collections.namedtuple("int_arg",
["name", "mapping_type", "location",
"value_type", "required", "default"])

2
sahara/tests/unit/service/validation/test_cluster_template_create_validation.py
View File

@ -289,4 +289,4 @@ class TestClusterTemplateCreateValidation(u.ValidationTestCase):
)
def test_cluster_create_v_default_image_required_tags(self):
self._assert_cluster_default_image_tags_validation()
self._assert_cluster_default_image_tags_validation()

22
sahara/tests/unit/utils/openstack/test_images.py
View File

@ -79,14 +79,14 @@ class TestImages(base.SaharaTestCase):
@mock.patch('sahara.utils.openstack.images.SaharaImageManager.get')
@mock.patch('sahara.utils.openstack.images.SaharaImageManager.delete_meta')
def test_unset_image_info(self, delete_meta, get_image):
manager = sahara_images.image_manager()
image = mock.MagicMock()
image.tags = ['fake', 'fake_2.0']
image.username = 'ubuntu'
image.description = 'some description'
get_image.return_value = image
manager.unset_image_info('id')
self.assertEqual(
('id', ['_sahara_tag_fake', '_sahara_tag_fake_2.0',
'_sahara_description', '_sahara_username']),
delete_meta.call_args[0])
manager = sahara_images.image_manager()
image = mock.MagicMock()
image.tags = ['fake', 'fake_2.0']
image.username = 'ubuntu'
image.description = 'some description'
get_image.return_value = image
manager.unset_image_info('id')
self.assertEqual(
('id', ['_sahara_tag_fake', '_sahara_tag_fake_2.0',
'_sahara_description', '_sahara_username']),
delete_meta.call_args[0])

9
sahara/utils/hacking/import_checks.py
View File

@ -21,10 +21,10 @@ def _find_module(module, path=None):
mod_base = module
parent_path = None
while '.' in mod_base:
first, _, mod_base = mod_base.partition('.')
parent_path = path
_, path, _ = imp.find_module(first, path)
path = [path]
first, _, mod_base = mod_base.partition('.')
parent_path = path
_, path, _ = imp.find_module(first, path)
path = [path]
try:
_, path, _ = imp.find_module(mod_base, path)
except ImportError:
@ -46,6 +46,7 @@ def _find_module(module, path=None):
raise
return path
module_cache = dict()
# List of all Python 2 stdlib modules - anything not in this list will be

1
sahara/utils/openstack/keystone.py
View File

@ -71,6 +71,7 @@ deprecated. Please add the trustee credentials you need to the [trustee]
section of your sahara.conf file.
""")
opts = [
# TODO(alazarev) Move to [keystone] section
cfg.BoolOpt('use_identity_api_v3',

4
tox.ini
View File

@ -132,7 +132,9 @@ enable-extensions=H904,H106,H203,H204,H205
# [E402] Module level import not at top of file
# [E731] Do not assign a lambda expression, use a def
# [W503] Line break occurred before a binary operator
ignore=E123,E226,E402,E731,W503
# [W504] Line break occurred after a binary operator
# [W605] Invalid escape sequence 'x'
ignore=E123,E226,E402,E731,W503,W504,W605
[hacking]
import_exceptions = sahara.i18n