x/tobiko
Code Issues Proposed changes

Add support for dynamic parameters allocation in Heat stacks

Browse Source 
- dynamically allocate subnet CIDR from configed pool

Change-Id: I8a166cb348ca46b1267b1507e9b2f9a8b95eb2fe
This commit is contained in:
Federico Ressi 2019-06-18 08:25:14 +02:00
parent d9e9bcab86
commit ceb5420cf4
6 changed files with 212 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
tobiko/openstack/heat/_stack.py
View File

@ -41,17 +41,28 @@ DELETE_FAILED = 'DELETE_FAILED'
TEMPLATE_FILE_SUFFIX = '.yaml'
def _stack_parameters(obj, stack=None):
if obj is None or isinstance(obj, collections.Mapping):
parameters = HeatStackParametersFixture(stack, obj)
else:
parameters = tobiko.get_fixture(obj)
if not isinstance(parameters, HeatStackParametersFixture):
msg = "Object {!r} is not an HeatStackParametersFixture".format(
parameters)
raise TypeError(msg)
return parameters
class HeatStackFixture(tobiko.SharedFixture):
"""Manages Heat stacks."""
client = None
client_fixture = None
retry_create_stack = 1
wait_interval = 5
stack_name = None
template = None
parameters = None
stack = None
parameters = None
def __init__(self, stack_name=None, template=None, parameters=None,
wait_interval=None, client=None):
@ -60,62 +71,24 @@ class HeatStackFixture(tobiko.SharedFixture):
self.stack_name or
self.fixture_name)
template = template or self.template
if template:
if isinstance(template, collections.Mapping):
template = _template.heat_template(template)
else:
template = tobiko.get_fixture(template)
if not isinstance(template, _template.HeatTemplateFixture):
msg = "Object {!r} is not an HeatTemplateFixture".format(
template)
raise TypeError(msg)
self.template = template
self._parameters = parameters
if tobiko.is_fixture(client):
self.client_fixture = client
elif client:
self.client = client
self.template = _template.heat_template(template or self.template)
self.parameters = _stack_parameters(
stack=self, obj=(parameters or self.parameters))
self.client = client or self.client
if wait_interval:
self.wait_interval = wait_interval
def setup_fixture(self):
self.setup_template()
self.setup_parameters()
self.setup_client()
self.setup_stack()
def setup_template(self):
tobiko.setup_fixture(self.template)
def setup_parameters(self):
self.parameters = {}
# Merge all parameters dictionaries in the class hierarchy
for cls in reversed(type(self).__mro__):
parameters = cls.__dict__.get('parameters')
if parameters:
self.parameters.update(parameters)
if self._parameters:
self.parameters.update(self._parameters)
# Add template's missing stack parameters
template_parameters = set(self.template.template.get('parameters', []))
missing_parameters = sorted(template_parameters - set(self.parameters))
for name in missing_parameters:
value = getattr(self, name, None)
if value is not None:
self.parameters[name] = value
def setup_client(self):
client_fixture = self.client_fixture
if client_fixture:
self.client = tobiko.setup_fixture(client_fixture).client
elif not self.client:
self.client = _client.get_heat_client()
self.client = _client.heat_client(self.client)
def setup_stack(self):
self.create_stack()
@ -159,13 +132,15 @@ class HeatStackFixture(tobiko.SharedFixture):
expected_status={DELETE_COMPLETE})
self.stack = self._outputs = None
# Compile template parameters
parameters = tobiko.reset_fixture(self.parameters).values
try:
LOG.debug('Creating stack %r (re-tries left %d)...',
self.stack_name, retry)
stack_id = self.client.stacks.create(
stack_name=self.stack_name,
template=self.template.template_yaml,
parameters=self.parameters)['stack']['id']
parameters=parameters)['stack']['id']
except exc.HTTPConflict:
LOG.debug('Stack %r already exists.', self.stack_name)
else:
@ -252,80 +227,141 @@ class HeatStackFixture(tobiko.SharedFixture):
def __getattr__(self, name):
try:
return self.get_stack_outputs().get_output(name)
except InvalidHeatStackOutputKey:
return self.get_stack_outputs().get_value(name)
except HeatStackOutputKeyError:
pass
message = "Object {!r} has no attribute {!r}".format(self, name)
raise AttributeError(message)
class HeatStackOutputsFixture(tobiko.SharedFixture):
class HeatStackKeyError(tobiko.TobikoException):
message = "key {key!r} not found in stack {name!r}"
class HeatStackParameterKeyError(HeatStackKeyError):
message = "parameter key {key!r} not found in stack {name!r}"
class HeatStackOutputKeyError(HeatStackKeyError):
message = "output key {key!r} not found in stack {name!r}"
class HeatStackNamespaceFixture(tobiko.SharedFixture):
key_error = HeatStackKeyError
_keys = None
_values = None
def __init__(self, stack):
super(HeatStackOutputsFixture, self).__init__()
super(HeatStackNamespaceFixture, self).__init__()
if not isinstance(stack, HeatStackFixture):
message = "Object {!r} is not an HeatStackFixture".format(stack)
raise TypeError(message)
self.stack = stack
def setup_fixture(self):
self.get_output_keys()
self.get_output_values()
self.setup_keys()
self.setup_values()
def get_output_keys(self):
def setup_keys(self):
keys = self._keys
if keys is None:
self._keys = keys = frozenset(tobiko.setup_fixture(
self.stack.template).outputs.keys())
self._keys = keys = self.get_keys()
self.addCleanup(self.cleanup_keys)
return keys
keys = property(get_output_keys)
keys = tobiko.fixture_property(setup_keys)
def get_keys(self):
raise NotImplementedError
def cleanup_keys(self):
del self._keys
def get_output_values(self):
def setup_values(self):
values = self._values
if values is None:
# Can't get output values before stack creation is complete
self.stack.wait_for_create_complete()
outputs = self.stack.get_stack(resolve_outputs=True).outputs
self._values = values = {o['output_key']: o['output_value']
for o in outputs}
self.addCleanup(self.cleanup_output_values)
self._values = values = self.get_values()
self.addCleanup(self.cleanup_values)
return values
def cleanup_output_values(self):
values = tobiko.fixture_property(setup_values)
def get_values(self):
raise NotImplementedError
def cleanup_values(self):
del self._values
values = property(get_output_values)
def get_output(self, key):
# Match template outputs definition before fetching getting values
if key not in self.keys:
LOG.error('Output key %r not found in stack %r template', key,
self.stack.stack_name)
else:
def get_value(self, key):
# Match template outputs definition before getting value
if key in self.keys:
try:
return self.values[key]
except KeyError:
LOG.error('Output key %r not found in stack %r outputs', key,
LOG.error('Key %r not found in stack %r', key,
self.stack.stack_name)
raise InvalidHeatStackOutputKey(name=self.stack.stack_name, key=key)
else:
LOG.error('Key %r not found in template for stack %r', key,
self.stack.stack_name)
raise self.key_error(name=self.stack.stack_name, key=key)
def set_value(self, key, value):
# Match template outputs definition before setting value
if key in self.keys:
self.values[key] = value
else:
LOG.error('Key %r not found in template for stack %r', key,
self.stack.stack_name)
raise self.key_error(name=self.stack.stack_name, key=key)
def __getattr__(self, name):
try:
return self.get_output(name)
except InvalidHeatStackOutputKey:
return self.get_value(name)
except self.key_error:
pass
message = "Object {!r} has no attribute {!r}".format(self, name)
raise AttributeError(message)
class HeatStackParametersFixture(HeatStackNamespaceFixture):
key_error = HeatStackParameterKeyError
def __init__(self, stack, parameters=None):
super(HeatStackParametersFixture, self).__init__(stack)
self.parameters = parameters and dict(parameters) or {}
def get_keys(self):
template = tobiko.setup_fixture(self.stack.template)
return frozenset(template.parameters or [])
def get_values(self):
values = dict(self.parameters)
missing_keys = sorted(self.keys - set(values))
for key in missing_keys:
value = getattr(self.stack, key, None)
if value is not None:
values[key] = value
return values
class HeatStackOutputsFixture(HeatStackNamespaceFixture):
key_error = HeatStackOutputKeyError
def get_keys(self):
template = tobiko.setup_fixture(self.stack.template)
return frozenset(template.outputs or [])
def get_values(self):
# Can't get output values before stack creation is complete
self.stack.wait_for_create_complete()
outputs = self.stack.get_stack(resolve_outputs=True).outputs
return {o['output_key']: o['output_value']
for o in outputs}
def check_stack_status(stack, expected):
observed = stack.stack_status
if observed not in expected:
@ -341,10 +377,6 @@ def check_stack_status(stack, expected):
status_reason=stack.stack_status_reason)
class InvalidHeatStackOutputKey(tobiko.TobikoException, AttributeError):
message = "output key {key!r} not found in stack {name!r}"
class HeatStackNotFound(tobiko.TobikoException):
message = "stack {name!r} not found"

22
tobiko/openstack/heat/_template.py
View File

@ -13,6 +13,7 @@
# under the License.
from __future__ import absolute_import
import collections
import os
import sys
@ -49,7 +50,12 @@ class HeatTemplateFixture(tobiko.SharedFixture):
@property
def outputs(self):
template = self.template
return template and template.get('outputs') or {}
return template and template.get('outputs') or None
@property
def parameters(self):
template = self.template
return template and template.get('parameters') or None
class HeatTemplateFileFixture(HeatTemplateFixture):
@ -80,9 +86,17 @@ class HeatTemplateFileFixture(HeatTemplateFixture):
super(HeatTemplateFileFixture, self).setup_template()
def heat_template(template, template_files=None):
return HeatTemplateFixture(template=template,
template_files=template_files)
def heat_template(obj, template_files=None):
if isinstance(obj, collections.Mapping):
template = HeatTemplateFixture(template=obj,
template_files=template_files)
else:
template = tobiko.get_fixture(obj)
if not isinstance(template, HeatTemplateFixture):
msg = "Object {!r} is not an HeatTemplateFixture".format(template)
raise TypeError(msg)
return template
def heat_template_file(template_file, template_dirs=None):

23
tobiko/openstack/neutron/_cidr.py
View File

@ -35,7 +35,6 @@ class CIDRGeneratorFixture(tobiko.SharedFixture):
client = None
config = None
cidr_generator = None
subnet_cidrs = None
def __init__(self, cidr=None, prefixlen=None, client=None):
super(CIDRGeneratorFixture, self).__init__()
@ -48,7 +47,7 @@ class CIDRGeneratorFixture(tobiko.SharedFixture):
def setup_fixture(self):
self.setup_config()
self.setup_subnet_cidrs()
self.setup_client()
self.setup_cidr_generator()
def setup_config(self):
@ -56,18 +55,16 @@ class CIDRGeneratorFixture(tobiko.SharedFixture):
CONF = config.CONF
self.config = CONF.tobiko.neutron
def setup_subnet_cidrs(self):
client = _client.neutron_client(self.client)
self.subnet_cidrs = set(_client.list_subnet_cidrs(client=client))
def setup_client(self):
self.client = _client.neutron_client(self.client)
def setup_cidr_generator(self):
cidr = netaddr.IPNetwork(self.cidr)
prefixlen = int(self.prefixlen)
self.cidr_generator = cidr.subnet(prefixlen)
self.cidr_generator = self.cidr.subnet(self.prefixlen)
def new_cidr(self):
used_cidrs = set(_client.list_subnet_cidrs(client=self.client))
for cidr in self.cidr_generator:
if cidr not in self.subnet_cidrs:
if cidr not in used_cidrs:
return cidr
raise NoSuchCIDRLeft(cidr=self.cidr, prefixlen=self.prefixlen)
@ -76,22 +73,22 @@ class IPv4CIDRGeneratorFixture(CIDRGeneratorFixture):
@property
def cidr(self):
return self.config.ipv4_cidr
return netaddr.IPNetwork(self.config.ipv4_cidr)
@property
def prefixlen(self):
return self.config.ipv4_prefixlen
return int(self.config.ipv4_prefixlen)
class IPv6CIDRGeneratorFixture(CIDRGeneratorFixture):
@property
def cidr(self):
return self.config.ipv6_cidr
return netaddr.IPNetwork(self.config.ipv6_cidr)
@property
def prefixlen(self):
return self.config.ipv6_prefixlen
return int(self.config.ipv6_prefixlen)
class NoSuchCIDRLeft(tobiko.TobikoException):

50
tobiko/openstack/stacks/_neutron.py
View File

@ -15,6 +15,7 @@
# under the License.
from __future__ import absolute_import
import netaddr
import tobiko
from tobiko import config
@ -38,21 +39,33 @@ class NetworkStackFixture(heat.HeatStackFixture):
#: Disable port security by default for new network ports
port_security_enabled = False
#: Default IPv4 sub-net CIDR
ipv4_cidr = '190.40.2.0/24'
@property
def has_ipv4(self):
"""Whenever to setup IPv4 subnet"""
return bool(self.ipv4_cidr)
return bool(CONF.tobiko.neutron.ipv4_cidr)
#: IPv6 sub-net CIDR
ipv6_cidr = '2001:db8:1:2::/64'
@property
def ipv4_cidr(self):
if self.has_ipv4:
return neutron.new_ipv4_cidr()
else:
return None
@property
def has_ipv6(self):
"""Whenever to setup IPv6 subnet"""
return bool(self.ipv6_cidr)
return bool(CONF.tobiko.neutron.ipv4_cidr)
@property
def ipv6_cidr(self):
if self.has_ipv6:
return neutron.new_ipv6_cidr()
else:
return None
@property
def value_specs(self):
return {}
#: Floating IP network where the Neutron floating IPs are created
gateway_network = CONF.tobiko.neutron.floating_network
@ -73,10 +86,18 @@ class NetworkStackFixture(heat.HeatStackFixture):
def ipv4_subnet_details(self):
return neutron.show_subnet(self.ipv4_subnet_id)
@property
def ipv4_subnet_cidr(self):
return netaddr.IPNetwork(self.ipv4_subnet_details['cidr'])
@property
def ipv6_subnet_details(self):
return neutron.show_subnet(self.ipv6_subnet_id)
@property
def ipv6_subnet_cidr(self):
return netaddr.IPNetwork(self.ipv6_subnet_details['cidr'])
@property
def gateway_details(self):
return neutron.show_router(self.gateway_id)
@ -95,16 +116,11 @@ class NetworkWithNetMtuWriteStackFixture(NetworkStackFixture):
#: Value for maximum transfer unit on the internal network
mtu = 1000
def setup_parameters(self):
"""Setup Heat template parameters"""
super(NetworkWithNetMtuWriteStackFixture, self).setup_parameters()
if self.mtu:
self.setup_net_mtu_writable()
@neutron.skip_if_missing_networking_extensions('net-mtu-writable')
def setup_net_mtu_writable(self):
"""Setup maximum transfer unit size for the network"""
self.parameters.setdefault('value_specs', {}).update(mtu=self.mtu)
@property
def value_specs(self):
return dict(
super(NetworkWithNetMtuWriteStackFixture, self).value_specs,
mtu=int(self.mtu))
@neutron.skip_if_missing_networking_extensions('security-group')

19
tobiko/tests/functional/openstack/test_stacks.py
View File

@ -15,9 +15,6 @@
# under the License.
from __future__ import absolute_import
import unittest
import netaddr
import testtools
import tobiko
@ -45,27 +42,19 @@ class NetworkTestCase(testtools.TestCase):
self.assertEqual(self.stack.network_details['mtu'],
self.stack.outputs.mtu)
@unittest.skip('Feature not implemented')
def test_ipv4_subnet_cidr(self):
if not self.stack.has_ipv4:
tobiko.skip('Stack {!s} has no ipv4 subnet', self.stack.stack_name)
self.assertEqual(str(self.stack.ipv4_cidr),
self.stack.ipv4_subnet_details['cidr'])
subnet = neutron.find_subnet(str(self.stack.ipv4_cidr),
subnet = neutron.find_subnet(str(self.stack.ipv4_subnet_cidr),
properties=['cidr'])
self.assertEqual(neutron.show_subnet(self.stack.ipv4_subnet_id),
subnet)
@unittest.skip('Feature not implemented')
def test_ipv6_subnet_cidr(self):
if not self.stack.has_ipv6:
tobiko.skip('Stack {!s} has no ipv4 subnet', self.stack.stack_name)
self.assertEqual(str(self.stack.ipv6_cidr),
self.stack.ipv6_subnet_details['cidr'])
subnet = neutron.find_subnet(str(self.stack.ipv6_cidr),
subnet = neutron.find_subnet(str(self.stack.ipv6_subnet_cidr),
properties=['cidr'])
self.assertEqual(neutron.show_subnet(self.stack.ipv6_subnet_id),
subnet)
@ -74,14 +63,14 @@ class NetworkTestCase(testtools.TestCase):
if not self.stack.has_ipv4 or self.stack.has_gateway:
tobiko.skip('Stack {!s} has no IPv4 gateway',
self.stack.stack_name)
self.assertEqual(str(netaddr.IPNetwork(self.stack.ipv4_cidr).ip + 1),
self.assertEqual(str(self.stack.ipv4_cidr.ip + 1),
self.stack.ipv4_subnet_details['gateway_ip'])
def test_ipv6_subnet_gateway_ip(self):
if not self.stack.has_ipv6 or self.stack.has_gateway:
tobiko.skip('Stack {!s} has no IPv6 gateway',
self.stack.stack_name)
self.assertEqual(str(netaddr.IPNetwork(self.stack.ipv6_cidr).ip + 1),
self.assertEqual(str(self.stack.ipv6_cidr.ip + 1),
self.stack.ipv6_subnet_details['gateway_ip'])
def test_gateway_network(self):

62
tobiko/tests/unit/openstack/heat/test_stack.py
View File

@ -23,6 +23,7 @@ import yaml
import tobiko
from tobiko.openstack import heat
from tobiko.openstack.heat import _stack
from tobiko.openstack import keystone
from tobiko.tests.unit import openstack
@ -47,8 +48,16 @@ class MyTemplateFixture(heat.HeatTemplateFixture):
template = {'template': 'from-class'}
class MockClient(mock.NonCallableMagicMock):
pass
class HeatStackFixtureTest(openstack.OpenstackTest):
def setUp(self):
super(HeatStackFixtureTest, self).setUp()
self.patch(heatclient, 'Client', MockClient)
def test_init(self, fixture_class=MyStack, stack_name=None,
template=None, parameters=None, wait_interval=None,
client=None):
@ -62,20 +71,13 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
self.check_stack_template(stack=stack, template=template)
self.assertIs(fixture_class.parameters, stack.parameters)
self.assertIsInstance(stack.parameters,
_stack.HeatStackParametersFixture)
self.assertEqual(parameters or fixture_class.parameters or {},
stack.parameters.parameters)
self.assertEqual(wait_interval or fixture_class.wait_interval,
stack.wait_interval)
if tobiko.is_fixture(client):
self.assertIsNone(stack.client)
self.assertIs(client, stack.client_fixture)
elif client:
self.assertIs(client, stack.client)
self.assertIsNone(stack.client_fixture)
else:
self.assertIsNone(stack.client)
self.assertIsNone(stack.client_fixture)
self.assertIs(client or fixture_class.client, stack.client)
def test_init_with_stack_name(self):
self.test_init(stack_name='my-stack-name')
@ -112,12 +114,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
stack_name=None, parameters=None, wait_interval=None,
stacks=None, create_conflict=False, call_create=True,
call_delete=False, call_sleep=False):
from tobiko.openstack.heat import _client
client = mock.MagicMock(specs=heatclient.Client)
get_heat_client = self.patch(_client, 'get_heat_client',
return_value=client)
client = MockClient()
stacks = stacks or [
exc.HTTPNotFound,
mock_stack('CREATE_IN_PROGRESS')]
@ -131,14 +128,14 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
sleep = self.patch(time, 'sleep')
stack = fixture_class(stack_name=stack_name, parameters=parameters,
template=template, wait_interval=wait_interval)
template=template, wait_interval=wait_interval,
client=client)
stack.setUp()
self.assertIs(client, stack.client)
self.assertEqual(wait_interval or fixture_class.wait_interval,
stack.wait_interval)
get_heat_client.assert_called_once_with()
client.stacks.get.assert_has_calls([mock.call(stack.stack_name,
resolve_outputs=False)])
@ -147,11 +144,11 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
else:
client.stacks.delete.assert_not_called()
self.assertEqual(parameters or fixture_class.parameters or {},
stack.parameters)
parameters = parameters or fixture_class.parameters or {}
self.assertEqual(parameters, stack.parameters.values)
if call_create:
client.stacks.create.assert_called_once_with(
parameters=stack.parameters, stack_name=stack.stack_name,
parameters=parameters, stack_name=stack.stack_name,
template=yaml.safe_dump(stack.template.template))
else:
client.stacks.create.assert_not_called()
@ -169,8 +166,8 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
self.test_setup(template={'other': 'template'})
def test_setup_with_template_fixture(self):
self.test_setup(template=heat.heat_template(template={'template':
'from-fixture'}))
self.test_setup(template=heat.heat_template({'template':
'from-fixture'}))
def test_setup_with_template_fixture_type(self):
self.test_setup(template=MyTemplateFixture)
@ -242,7 +239,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
self.test_setup(create_conflict=True)
def test_cleanup(self):
client = mock.MagicMock(specs=heatclient.Client)
client = MockClient()
stack = MyStack(client=client)
stack.cleanUp()
client.stacks.delete.assert_called_once_with(stack.stack_name)
@ -253,7 +250,7 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
'output_value': 'value1'},
{'output_key': 'key2',
'output_value': 'value2'}])
client = mock.MagicMock(specs=heatclient.Client)
client = MockClient()
client.stacks.get.return_value = stack
stack_fixture = MyStack(
template={'outputs': {'key1': {}, 'key2': {}}},
@ -264,6 +261,17 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
self.assertEqual('value1', outputs.key1)
self.assertEqual('value2', outputs.key2)
def test_parameters(self):
stack_fixture = MyStack(
template={'parameters': {'key1': {}, 'key2': {}}},
parameters={'key1': 'value1',
'key2': 'value2'})
parameters = stack_fixture.parameters
self.assertEqual('value1', parameters.key1)
self.assertEqual('value2', parameters.key2)
def check_stack_template(self, stack, template):
expected_template = template or type(stack).template
if tobiko.is_fixture(expected_template):