From ceb5420cf4708ff870fdca5f8c07d02561e2405e Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Tue, 18 Jun 2019 08:25:14 +0200 Subject: [PATCH] Add support for dynamic parameters allocation in Heat stacks - dynamically allocate subnet CIDR from configed pool Change-Id: I8a166cb348ca46b1267b1507e9b2f9a8b95eb2fe --- tobiko/openstack/heat/_stack.py | 192 ++++++++++-------- tobiko/openstack/heat/_template.py | 22 +- tobiko/openstack/neutron/_cidr.py | 23 +-- tobiko/openstack/stacks/_neutron.py | 50 +++-- .../tests/functional/openstack/test_stacks.py | 19 +- .../tests/unit/openstack/heat/test_stack.py | 62 +++--- 6 files changed, 212 insertions(+), 156 deletions(-) diff --git a/tobiko/openstack/heat/_stack.py b/tobiko/openstack/heat/_stack.py index 25de57c68..f94f49d73 100644 --- a/tobiko/openstack/heat/_stack.py +++ b/tobiko/openstack/heat/_stack.py @@ -41,17 +41,28 @@ DELETE_FAILED = 'DELETE_FAILED' TEMPLATE_FILE_SUFFIX = '.yaml' +def _stack_parameters(obj, stack=None): + if obj is None or isinstance(obj, collections.Mapping): + parameters = HeatStackParametersFixture(stack, obj) + else: + parameters = tobiko.get_fixture(obj) + if not isinstance(parameters, HeatStackParametersFixture): + msg = "Object {!r} is not an HeatStackParametersFixture".format( + parameters) + raise TypeError(msg) + return parameters + + class HeatStackFixture(tobiko.SharedFixture): """Manages Heat stacks.""" client = None - client_fixture = None retry_create_stack = 1 wait_interval = 5 stack_name = None template = None - parameters = None stack = None + parameters = None def __init__(self, stack_name=None, template=None, parameters=None, wait_interval=None, client=None): @@ -60,62 +71,24 @@ class HeatStackFixture(tobiko.SharedFixture): self.stack_name or self.fixture_name) - template = template or self.template - if template: - if isinstance(template, collections.Mapping): - template = _template.heat_template(template) - else: - template = tobiko.get_fixture(template) - if not isinstance(template, _template.HeatTemplateFixture): - msg = "Object {!r} is not an HeatTemplateFixture".format( - template) - raise TypeError(msg) - self.template = template - - self._parameters = parameters - - if tobiko.is_fixture(client): - self.client_fixture = client - elif client: - self.client = client + self.template = _template.heat_template(template or self.template) + self.parameters = _stack_parameters( + stack=self, obj=(parameters or self.parameters)) + self.client = client or self.client if wait_interval: self.wait_interval = wait_interval def setup_fixture(self): self.setup_template() - self.setup_parameters() self.setup_client() self.setup_stack() def setup_template(self): tobiko.setup_fixture(self.template) - def setup_parameters(self): - self.parameters = {} - # Merge all parameters dictionaries in the class hierarchy - for cls in reversed(type(self).__mro__): - parameters = cls.__dict__.get('parameters') - if parameters: - self.parameters.update(parameters) - - if self._parameters: - self.parameters.update(self._parameters) - - # Add template's missing stack parameters - template_parameters = set(self.template.template.get('parameters', [])) - missing_parameters = sorted(template_parameters - set(self.parameters)) - for name in missing_parameters: - value = getattr(self, name, None) - if value is not None: - self.parameters[name] = value - def setup_client(self): - client_fixture = self.client_fixture - if client_fixture: - self.client = tobiko.setup_fixture(client_fixture).client - elif not self.client: - self.client = _client.get_heat_client() + self.client = _client.heat_client(self.client) def setup_stack(self): self.create_stack() @@ -159,13 +132,15 @@ class HeatStackFixture(tobiko.SharedFixture): expected_status={DELETE_COMPLETE}) self.stack = self._outputs = None + # Compile template parameters + parameters = tobiko.reset_fixture(self.parameters).values try: LOG.debug('Creating stack %r (re-tries left %d)...', self.stack_name, retry) stack_id = self.client.stacks.create( stack_name=self.stack_name, template=self.template.template_yaml, - parameters=self.parameters)['stack']['id'] + parameters=parameters)['stack']['id'] except exc.HTTPConflict: LOG.debug('Stack %r already exists.', self.stack_name) else: @@ -252,80 +227,141 @@ class HeatStackFixture(tobiko.SharedFixture): def __getattr__(self, name): try: - return self.get_stack_outputs().get_output(name) - except InvalidHeatStackOutputKey: + return self.get_stack_outputs().get_value(name) + except HeatStackOutputKeyError: pass message = "Object {!r} has no attribute {!r}".format(self, name) raise AttributeError(message) -class HeatStackOutputsFixture(tobiko.SharedFixture): +class HeatStackKeyError(tobiko.TobikoException): + message = "key {key!r} not found in stack {name!r}" + +class HeatStackParameterKeyError(HeatStackKeyError): + message = "parameter key {key!r} not found in stack {name!r}" + + +class HeatStackOutputKeyError(HeatStackKeyError): + message = "output key {key!r} not found in stack {name!r}" + + +class HeatStackNamespaceFixture(tobiko.SharedFixture): + + key_error = HeatStackKeyError _keys = None _values = None def __init__(self, stack): - super(HeatStackOutputsFixture, self).__init__() + super(HeatStackNamespaceFixture, self).__init__() if not isinstance(stack, HeatStackFixture): message = "Object {!r} is not an HeatStackFixture".format(stack) raise TypeError(message) self.stack = stack def setup_fixture(self): - self.get_output_keys() - self.get_output_values() + self.setup_keys() + self.setup_values() - def get_output_keys(self): + def setup_keys(self): keys = self._keys if keys is None: - self._keys = keys = frozenset(tobiko.setup_fixture( - self.stack.template).outputs.keys()) + self._keys = keys = self.get_keys() self.addCleanup(self.cleanup_keys) return keys - keys = property(get_output_keys) + keys = tobiko.fixture_property(setup_keys) + + def get_keys(self): + raise NotImplementedError def cleanup_keys(self): del self._keys - def get_output_values(self): + def setup_values(self): values = self._values if values is None: - # Can't get output values before stack creation is complete - self.stack.wait_for_create_complete() - outputs = self.stack.get_stack(resolve_outputs=True).outputs - self._values = values = {o['output_key']: o['output_value'] - for o in outputs} - self.addCleanup(self.cleanup_output_values) + self._values = values = self.get_values() + self.addCleanup(self.cleanup_values) return values - def cleanup_output_values(self): + values = tobiko.fixture_property(setup_values) + + def get_values(self): + raise NotImplementedError + + def cleanup_values(self): del self._values - values = property(get_output_values) - - def get_output(self, key): - # Match template outputs definition before fetching getting values - if key not in self.keys: - LOG.error('Output key %r not found in stack %r template', key, - self.stack.stack_name) - else: + def get_value(self, key): + # Match template outputs definition before getting value + if key in self.keys: try: return self.values[key] except KeyError: - LOG.error('Output key %r not found in stack %r outputs', key, + LOG.error('Key %r not found in stack %r', key, self.stack.stack_name) - raise InvalidHeatStackOutputKey(name=self.stack.stack_name, key=key) + else: + LOG.error('Key %r not found in template for stack %r', key, + self.stack.stack_name) + raise self.key_error(name=self.stack.stack_name, key=key) + + def set_value(self, key, value): + # Match template outputs definition before setting value + if key in self.keys: + self.values[key] = value + else: + LOG.error('Key %r not found in template for stack %r', key, + self.stack.stack_name) + raise self.key_error(name=self.stack.stack_name, key=key) def __getattr__(self, name): try: - return self.get_output(name) - except InvalidHeatStackOutputKey: + return self.get_value(name) + except self.key_error: pass message = "Object {!r} has no attribute {!r}".format(self, name) raise AttributeError(message) +class HeatStackParametersFixture(HeatStackNamespaceFixture): + + key_error = HeatStackParameterKeyError + + def __init__(self, stack, parameters=None): + super(HeatStackParametersFixture, self).__init__(stack) + self.parameters = parameters and dict(parameters) or {} + + def get_keys(self): + template = tobiko.setup_fixture(self.stack.template) + return frozenset(template.parameters or []) + + def get_values(self): + values = dict(self.parameters) + missing_keys = sorted(self.keys - set(values)) + for key in missing_keys: + value = getattr(self.stack, key, None) + if value is not None: + values[key] = value + return values + + +class HeatStackOutputsFixture(HeatStackNamespaceFixture): + + key_error = HeatStackOutputKeyError + + def get_keys(self): + template = tobiko.setup_fixture(self.stack.template) + return frozenset(template.outputs or []) + + def get_values(self): + # Can't get output values before stack creation is complete + self.stack.wait_for_create_complete() + outputs = self.stack.get_stack(resolve_outputs=True).outputs + return {o['output_key']: o['output_value'] + for o in outputs} + + def check_stack_status(stack, expected): observed = stack.stack_status if observed not in expected: @@ -341,10 +377,6 @@ def check_stack_status(stack, expected): status_reason=stack.stack_status_reason) -class InvalidHeatStackOutputKey(tobiko.TobikoException, AttributeError): - message = "output key {key!r} not found in stack {name!r}" - - class HeatStackNotFound(tobiko.TobikoException): message = "stack {name!r} not found" diff --git a/tobiko/openstack/heat/_template.py b/tobiko/openstack/heat/_template.py index 2922f4e45..1b2e2a018 100644 --- a/tobiko/openstack/heat/_template.py +++ b/tobiko/openstack/heat/_template.py @@ -13,6 +13,7 @@ # under the License. from __future__ import absolute_import +import collections import os import sys @@ -49,7 +50,12 @@ class HeatTemplateFixture(tobiko.SharedFixture): @property def outputs(self): template = self.template - return template and template.get('outputs') or {} + return template and template.get('outputs') or None + + @property + def parameters(self): + template = self.template + return template and template.get('parameters') or None class HeatTemplateFileFixture(HeatTemplateFixture): @@ -80,9 +86,17 @@ class HeatTemplateFileFixture(HeatTemplateFixture): super(HeatTemplateFileFixture, self).setup_template() -def heat_template(template, template_files=None): - return HeatTemplateFixture(template=template, - template_files=template_files) +def heat_template(obj, template_files=None): + if isinstance(obj, collections.Mapping): + template = HeatTemplateFixture(template=obj, + template_files=template_files) + else: + template = tobiko.get_fixture(obj) + + if not isinstance(template, HeatTemplateFixture): + msg = "Object {!r} is not an HeatTemplateFixture".format(template) + raise TypeError(msg) + return template def heat_template_file(template_file, template_dirs=None): diff --git a/tobiko/openstack/neutron/_cidr.py b/tobiko/openstack/neutron/_cidr.py index 2fd453ccd..211681e90 100644 --- a/tobiko/openstack/neutron/_cidr.py +++ b/tobiko/openstack/neutron/_cidr.py @@ -35,7 +35,6 @@ class CIDRGeneratorFixture(tobiko.SharedFixture): client = None config = None cidr_generator = None - subnet_cidrs = None def __init__(self, cidr=None, prefixlen=None, client=None): super(CIDRGeneratorFixture, self).__init__() @@ -48,7 +47,7 @@ class CIDRGeneratorFixture(tobiko.SharedFixture): def setup_fixture(self): self.setup_config() - self.setup_subnet_cidrs() + self.setup_client() self.setup_cidr_generator() def setup_config(self): @@ -56,18 +55,16 @@ class CIDRGeneratorFixture(tobiko.SharedFixture): CONF = config.CONF self.config = CONF.tobiko.neutron - def setup_subnet_cidrs(self): - client = _client.neutron_client(self.client) - self.subnet_cidrs = set(_client.list_subnet_cidrs(client=client)) + def setup_client(self): + self.client = _client.neutron_client(self.client) def setup_cidr_generator(self): - cidr = netaddr.IPNetwork(self.cidr) - prefixlen = int(self.prefixlen) - self.cidr_generator = cidr.subnet(prefixlen) + self.cidr_generator = self.cidr.subnet(self.prefixlen) def new_cidr(self): + used_cidrs = set(_client.list_subnet_cidrs(client=self.client)) for cidr in self.cidr_generator: - if cidr not in self.subnet_cidrs: + if cidr not in used_cidrs: return cidr raise NoSuchCIDRLeft(cidr=self.cidr, prefixlen=self.prefixlen) @@ -76,22 +73,22 @@ class IPv4CIDRGeneratorFixture(CIDRGeneratorFixture): @property def cidr(self): - return self.config.ipv4_cidr + return netaddr.IPNetwork(self.config.ipv4_cidr) @property def prefixlen(self): - return self.config.ipv4_prefixlen + return int(self.config.ipv4_prefixlen) class IPv6CIDRGeneratorFixture(CIDRGeneratorFixture): @property def cidr(self): - return self.config.ipv6_cidr + return netaddr.IPNetwork(self.config.ipv6_cidr) @property def prefixlen(self): - return self.config.ipv6_prefixlen + return int(self.config.ipv6_prefixlen) class NoSuchCIDRLeft(tobiko.TobikoException): diff --git a/tobiko/openstack/stacks/_neutron.py b/tobiko/openstack/stacks/_neutron.py index ec509b18a..effbb660d 100644 --- a/tobiko/openstack/stacks/_neutron.py +++ b/tobiko/openstack/stacks/_neutron.py @@ -15,6 +15,7 @@ # under the License. from __future__ import absolute_import +import netaddr import tobiko from tobiko import config @@ -38,21 +39,33 @@ class NetworkStackFixture(heat.HeatStackFixture): #: Disable port security by default for new network ports port_security_enabled = False - #: Default IPv4 sub-net CIDR - ipv4_cidr = '190.40.2.0/24' - @property def has_ipv4(self): """Whenever to setup IPv4 subnet""" - return bool(self.ipv4_cidr) + return bool(CONF.tobiko.neutron.ipv4_cidr) - #: IPv6 sub-net CIDR - ipv6_cidr = '2001:db8:1:2::/64' + @property + def ipv4_cidr(self): + if self.has_ipv4: + return neutron.new_ipv4_cidr() + else: + return None @property def has_ipv6(self): """Whenever to setup IPv6 subnet""" - return bool(self.ipv6_cidr) + return bool(CONF.tobiko.neutron.ipv4_cidr) + + @property + def ipv6_cidr(self): + if self.has_ipv6: + return neutron.new_ipv6_cidr() + else: + return None + + @property + def value_specs(self): + return {} #: Floating IP network where the Neutron floating IPs are created gateway_network = CONF.tobiko.neutron.floating_network @@ -73,10 +86,18 @@ class NetworkStackFixture(heat.HeatStackFixture): def ipv4_subnet_details(self): return neutron.show_subnet(self.ipv4_subnet_id) + @property + def ipv4_subnet_cidr(self): + return netaddr.IPNetwork(self.ipv4_subnet_details['cidr']) + @property def ipv6_subnet_details(self): return neutron.show_subnet(self.ipv6_subnet_id) + @property + def ipv6_subnet_cidr(self): + return netaddr.IPNetwork(self.ipv6_subnet_details['cidr']) + @property def gateway_details(self): return neutron.show_router(self.gateway_id) @@ -95,16 +116,11 @@ class NetworkWithNetMtuWriteStackFixture(NetworkStackFixture): #: Value for maximum transfer unit on the internal network mtu = 1000 - def setup_parameters(self): - """Setup Heat template parameters""" - super(NetworkWithNetMtuWriteStackFixture, self).setup_parameters() - if self.mtu: - self.setup_net_mtu_writable() - - @neutron.skip_if_missing_networking_extensions('net-mtu-writable') - def setup_net_mtu_writable(self): - """Setup maximum transfer unit size for the network""" - self.parameters.setdefault('value_specs', {}).update(mtu=self.mtu) + @property + def value_specs(self): + return dict( + super(NetworkWithNetMtuWriteStackFixture, self).value_specs, + mtu=int(self.mtu)) @neutron.skip_if_missing_networking_extensions('security-group') diff --git a/tobiko/tests/functional/openstack/test_stacks.py b/tobiko/tests/functional/openstack/test_stacks.py index e244ef6e3..3d2f58f87 100644 --- a/tobiko/tests/functional/openstack/test_stacks.py +++ b/tobiko/tests/functional/openstack/test_stacks.py @@ -15,9 +15,6 @@ # under the License. from __future__ import absolute_import -import unittest - -import netaddr import testtools import tobiko @@ -45,27 +42,19 @@ class NetworkTestCase(testtools.TestCase): self.assertEqual(self.stack.network_details['mtu'], self.stack.outputs.mtu) - @unittest.skip('Feature not implemented') def test_ipv4_subnet_cidr(self): if not self.stack.has_ipv4: tobiko.skip('Stack {!s} has no ipv4 subnet', self.stack.stack_name) - self.assertEqual(str(self.stack.ipv4_cidr), - self.stack.ipv4_subnet_details['cidr']) - - subnet = neutron.find_subnet(str(self.stack.ipv4_cidr), + subnet = neutron.find_subnet(str(self.stack.ipv4_subnet_cidr), properties=['cidr']) self.assertEqual(neutron.show_subnet(self.stack.ipv4_subnet_id), subnet) - @unittest.skip('Feature not implemented') def test_ipv6_subnet_cidr(self): if not self.stack.has_ipv6: tobiko.skip('Stack {!s} has no ipv4 subnet', self.stack.stack_name) - self.assertEqual(str(self.stack.ipv6_cidr), - self.stack.ipv6_subnet_details['cidr']) - - subnet = neutron.find_subnet(str(self.stack.ipv6_cidr), + subnet = neutron.find_subnet(str(self.stack.ipv6_subnet_cidr), properties=['cidr']) self.assertEqual(neutron.show_subnet(self.stack.ipv6_subnet_id), subnet) @@ -74,14 +63,14 @@ class NetworkTestCase(testtools.TestCase): if not self.stack.has_ipv4 or self.stack.has_gateway: tobiko.skip('Stack {!s} has no IPv4 gateway', self.stack.stack_name) - self.assertEqual(str(netaddr.IPNetwork(self.stack.ipv4_cidr).ip + 1), + self.assertEqual(str(self.stack.ipv4_cidr.ip + 1), self.stack.ipv4_subnet_details['gateway_ip']) def test_ipv6_subnet_gateway_ip(self): if not self.stack.has_ipv6 or self.stack.has_gateway: tobiko.skip('Stack {!s} has no IPv6 gateway', self.stack.stack_name) - self.assertEqual(str(netaddr.IPNetwork(self.stack.ipv6_cidr).ip + 1), + self.assertEqual(str(self.stack.ipv6_cidr.ip + 1), self.stack.ipv6_subnet_details['gateway_ip']) def test_gateway_network(self): diff --git a/tobiko/tests/unit/openstack/heat/test_stack.py b/tobiko/tests/unit/openstack/heat/test_stack.py index bd364bdea..4b094956b 100644 --- a/tobiko/tests/unit/openstack/heat/test_stack.py +++ b/tobiko/tests/unit/openstack/heat/test_stack.py @@ -23,6 +23,7 @@ import yaml import tobiko from tobiko.openstack import heat +from tobiko.openstack.heat import _stack from tobiko.openstack import keystone from tobiko.tests.unit import openstack @@ -47,8 +48,16 @@ class MyTemplateFixture(heat.HeatTemplateFixture): template = {'template': 'from-class'} +class MockClient(mock.NonCallableMagicMock): + pass + + class HeatStackFixtureTest(openstack.OpenstackTest): + def setUp(self): + super(HeatStackFixtureTest, self).setUp() + self.patch(heatclient, 'Client', MockClient) + def test_init(self, fixture_class=MyStack, stack_name=None, template=None, parameters=None, wait_interval=None, client=None): @@ -62,20 +71,13 @@ class HeatStackFixtureTest(openstack.OpenstackTest): self.check_stack_template(stack=stack, template=template) - self.assertIs(fixture_class.parameters, stack.parameters) - + self.assertIsInstance(stack.parameters, + _stack.HeatStackParametersFixture) + self.assertEqual(parameters or fixture_class.parameters or {}, + stack.parameters.parameters) self.assertEqual(wait_interval or fixture_class.wait_interval, stack.wait_interval) - - if tobiko.is_fixture(client): - self.assertIsNone(stack.client) - self.assertIs(client, stack.client_fixture) - elif client: - self.assertIs(client, stack.client) - self.assertIsNone(stack.client_fixture) - else: - self.assertIsNone(stack.client) - self.assertIsNone(stack.client_fixture) + self.assertIs(client or fixture_class.client, stack.client) def test_init_with_stack_name(self): self.test_init(stack_name='my-stack-name') @@ -112,12 +114,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest): stack_name=None, parameters=None, wait_interval=None, stacks=None, create_conflict=False, call_create=True, call_delete=False, call_sleep=False): - from tobiko.openstack.heat import _client - - client = mock.MagicMock(specs=heatclient.Client) - get_heat_client = self.patch(_client, 'get_heat_client', - return_value=client) - + client = MockClient() stacks = stacks or [ exc.HTTPNotFound, mock_stack('CREATE_IN_PROGRESS')] @@ -131,14 +128,14 @@ class HeatStackFixtureTest(openstack.OpenstackTest): sleep = self.patch(time, 'sleep') stack = fixture_class(stack_name=stack_name, parameters=parameters, - template=template, wait_interval=wait_interval) + template=template, wait_interval=wait_interval, + client=client) stack.setUp() self.assertIs(client, stack.client) self.assertEqual(wait_interval or fixture_class.wait_interval, stack.wait_interval) - get_heat_client.assert_called_once_with() client.stacks.get.assert_has_calls([mock.call(stack.stack_name, resolve_outputs=False)]) @@ -147,11 +144,11 @@ class HeatStackFixtureTest(openstack.OpenstackTest): else: client.stacks.delete.assert_not_called() - self.assertEqual(parameters or fixture_class.parameters or {}, - stack.parameters) + parameters = parameters or fixture_class.parameters or {} + self.assertEqual(parameters, stack.parameters.values) if call_create: client.stacks.create.assert_called_once_with( - parameters=stack.parameters, stack_name=stack.stack_name, + parameters=parameters, stack_name=stack.stack_name, template=yaml.safe_dump(stack.template.template)) else: client.stacks.create.assert_not_called() @@ -169,8 +166,8 @@ class HeatStackFixtureTest(openstack.OpenstackTest): self.test_setup(template={'other': 'template'}) def test_setup_with_template_fixture(self): - self.test_setup(template=heat.heat_template(template={'template': - 'from-fixture'})) + self.test_setup(template=heat.heat_template({'template': + 'from-fixture'})) def test_setup_with_template_fixture_type(self): self.test_setup(template=MyTemplateFixture) @@ -242,7 +239,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest): self.test_setup(create_conflict=True) def test_cleanup(self): - client = mock.MagicMock(specs=heatclient.Client) + client = MockClient() stack = MyStack(client=client) stack.cleanUp() client.stacks.delete.assert_called_once_with(stack.stack_name) @@ -253,7 +250,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest): 'output_value': 'value1'}, {'output_key': 'key2', 'output_value': 'value2'}]) - client = mock.MagicMock(specs=heatclient.Client) + client = MockClient() client.stacks.get.return_value = stack stack_fixture = MyStack( template={'outputs': {'key1': {}, 'key2': {}}}, @@ -264,6 +261,17 @@ class HeatStackFixtureTest(openstack.OpenstackTest): self.assertEqual('value1', outputs.key1) self.assertEqual('value2', outputs.key2) + def test_parameters(self): + stack_fixture = MyStack( + template={'parameters': {'key1': {}, 'key2': {}}}, + parameters={'key1': 'value1', + 'key2': 'value2'}) + + parameters = stack_fixture.parameters + + self.assertEqual('value1', parameters.key1) + self.assertEqual('value2', parameters.key2) + def check_stack_template(self, stack, template): expected_template = template or type(stack).template if tobiko.is_fixture(expected_template):