Use TripleO Heat Merge to manage the stack

Generate the YAML file describing Tuskar compute nodes and invoke Heat
Merge. This review adds the master branch of tripleo-heat-templates as
a requirement.

At the moment this patch hardcodes the parameters which define the
different Overcloud Roles. On a create over the Overcloud it creates one
controller and one compute Overcloud role. When the Overcloud is updated
it created a second compute node.

This can be manually tested with these commands.

Create the overcloud:
curl -H "Content-Type:application/json"\
 -XPOST http://localhost:8585/v1/overclouds/ -d '{}'

Update the overcloud:
curl -H "Content-Type:application/json"\
 -XPUT http://localhost:8585/v1/overclouds/1 -d '{}'

Delete the overcloud:
curl -H "Content-Type:application/json"\
 -XDELETE http://localhost:8585/v1/overclouds/1 -d '{}'

Change-Id: I578b4e9f238590ea245b827bc75d252568d194fe
This commit is contained in:
marios 2013-10-16 12:31:39 +03:00 committed by Dougal Matthews
parent dad9a3e0df
commit 64f0310042
10 changed files with 413 additions and 21 deletions

View File

@ -20,3 +20,5 @@ WSME>=0.5b6
python-novaclient>=2.15.0
PyYAML>=3.1.0
python-heatclient>=0.2.3
-e git+http://git.openstack.org/cgit/openstack/tripleo-heat-templates#egg=tripleo_heat_templates-master

View File

@ -34,7 +34,7 @@ build-dir = doc/build
source-dir = doc/source
[egg_info]
tag_build =
tag_build =
tag_date = 0
tag_svn_revision = 0

View File

@ -1,5 +1,3 @@
# -*- encoding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -14,19 +12,70 @@
import logging
import pecan
from pecan import rest
import wsme
from pecan import rest
from wsmeext import pecan as wsme_pecan
from tuskar.api.controllers.v1 import models
from tuskar.common import exception
from tuskar.heat.client import HeatClient
import tuskar.heat.template_tools as template_tools
LOG = logging.getLogger(__name__)
# FIXME(lsmola) mocked params for POC, remove later by real ones
POC_PARAMS = {'controller': 1, 'compute': 1}
POC_PARAMS_UPDATE = {'controller': 1, 'compute': 2}
def process_stack(params, create=False):
"""Helper function for processing the stack. Given a params dict containing
the Overcloud Roles and initialization parameters create or update the
stack.
:param params: Dictionary of initialization params and overcloud roles for
heat template and initialization of stack/
:type params: dict
:param create: A flag to designate if we are creating or updating the stack
:type create: bool
"""
overcloud = template_tools.merge_templates(params)
heat_client = HeatClient()
stack_exists = heat_client.exists_stack()
if not heat_client.validate_template(overcloud):
raise exception.InvalidHeatTemplate()
if stack_exists and create:
raise exception.StackAlreadyCreated()
elif not stack_exists and not create:
raise exception.StackNotFound()
res = heat_client.create_stack(overcloud, params)
if not res:
if create:
raise exception.HeatTemplateCreateFailed()
raise exception.HeatTemplateUpdateFailed()
class OvercloudsController(rest.RestController):
"""REST controller for the Overcloud class."""
_custom_actions = {'template_get': ['GET']}
# FIXME(lsmola) this is for debugging purposes only, remove before I3
@pecan.expose()
def template_get(self):
overcloud = template_tools.merge_templates(POC_PARAMS)
return overcloud
@wsme.validate(models.Overcloud)
@wsme_pecan.wsexpose(models.Overcloud,
body=models.Overcloud,
@ -55,6 +104,14 @@ class OvercloudsController(rest.RestController):
saved_overcloud =\
models.Overcloud.from_db_model(result)
# FIXME(lsmola) This is just POC of creating a stack
# this has to be done properly with proper Work-flow abstraction of:
# step one- build template and start stack-create
# step 2- put the right stack_id to the overcloud
# step 3- initialize the stack
# step 4- set the correct overcloud status
process_stack(POC_PARAMS, create=True)
return saved_overcloud
@wsme.validate(models.Overcloud)
@ -90,6 +147,12 @@ class OvercloudsController(rest.RestController):
updated = models.Overcloud.from_db_model(result)
# FIXME(lsmola) This is just POC of updating a stack
# this probably should also have workflow
# step one- build template and stack-update
# step 2- set the correct overcloud status
process_stack(POC_PARAMS_UPDATE)
return updated
@wsme_pecan.wsexpose(None, int, status_code=204)
@ -103,9 +166,23 @@ class OvercloudsController(rest.RestController):
is no overcloud with the given ID
"""
# FIXME(lsmola) this should always try to delete both overcloud
# and stack. So it requires some exception catch over below.
LOG.debug('Deleting overcloud with ID: %s' % overcloud_id)
pecan.request.dbapi.delete_overcloud_by_id(overcloud_id)
heat_client = HeatClient()
if not heat_client.exists_stack():
# If the stack doesn't exist, we have nothing else to do here.
return
result = heat_client.delete_stack()
if not result:
raise wsme.exc.ClientSideError(_(
"Failed to delete the Heat overcloud."
))
@wsme_pecan.wsexpose(models.Overcloud, int)
def get_one(self, overcloud_id):
"""Returns a specific overcloud.

View File

@ -170,3 +170,23 @@ class DuplicateAttribute(DuplicateEntry):
class ConfigNotFound(TuskarException):
message = _("Could not find config at %(path)s")
class InvalidHeatTemplate(TuskarException):
message = _("Validation of the Heat Template failed.")
class StackNotFound(NotFound):
message = _("The Stack for this Overcloud can't be found.")
class StackAlreadyCreated(DuplicateEntry):
message = _("The Stack for this Overcloud already exists.")
class HeatTemplateUpdateFailed(TuskarException):
message = _("The Heat template failed to update.")
class HeatTemplateCreateFailed(TuskarException):
message = _("The Heat template failed to create.")

View File

@ -80,15 +80,16 @@ class HeatClient(object):
try:
keystone = ksclient.Client(**CONF.heat_keystone)
endpoint = keystone.service_catalog.url_for(
service_type=CONF.heat['service_type'],
endpoint_type=CONF.heat['endpoint_type'])
service_type=CONF.heat['service_type'],
endpoint_type=CONF.heat['endpoint_type']
)
self.connection = heatclient(
endpoint=endpoint,
token=keystone.auth_token,
username=CONF.heat_keystone['username'],
password=CONF.heat_keystone['password'])
except Exception as e:
LOG.exception(e)
except Exception:
LOG.exception("An error occurred initialising the HeatClient")
self.connection = None
def validate_template(self, template_body):
@ -96,8 +97,8 @@ class HeatClient(object):
try:
self.connection.stacks.validate(template=template_body)
return True
except Exception as e:
LOG.exception(e)
except Exception:
LOG.exception("Validation of the Heat template failed.")
return False
def get_stack(self, name=None):
@ -110,8 +111,8 @@ class HeatClient(object):
def get_template(self):
"""Get JSON representation of the Heat overcloud template."""
return self.connection.stacks.template(
stack_id=CONF.heat['stack_name']
)
stack_id=CONF.heat['stack_name']
)
def update_stack(self, template_body, params):
"""Update the Heat overcloud stack."""
@ -120,8 +121,17 @@ class HeatClient(object):
template=template_body,
parameters=params)
return True
except Exception as e:
LOG.exception(e)
except Exception:
LOG.exception("An error occurred updating the stack.")
return False
def delete_stack(self):
"""Delete the Heat overcloud stack."""
try:
self.connection.stacks.delete(stack_id=CONF.heat['stack_name'])
return True
except Exception:
LOG.exception("An error occurred deleting the stack.")
return False
def create_stack(self, template_body, params):
@ -131,8 +141,8 @@ class HeatClient(object):
template=template_body,
parameters=params)
return True
except Exception as e:
LOG.exception(e)
except Exception:
LOG.exception("An error occurred creating the stack.")
return False
def exists_stack(self, name=None):

View File

@ -0,0 +1,78 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Utilities for using merge.py to generate overcloud.yaml to hand over to Heat.
Translates Tuskar resources into the overcloud heat template, using merge.py
from upstream tripleo-heat-templates.
"""
import os
from oslo.config import cfg
from tripleo_heat_merge import merge
# The name of the compute Overcloud role - defined for special case handling
OVERCLOUD_COMPUTE_ROLE = 'compute'
def generate_scaling_params(overcloud_roles):
"""Given a dictionary containing a key value mapping of Overcloud Role name
to a count of the nodes return the scaling parameters to be used by
tripleo_heat_merge
:param overcloud_roles: Dictionary with role names and a count of the nodes
:type overcloud_roles: dict
:return: scaling parameters dict
:rtype: dict
"""
scaling = {}
for overcloud_role, count in overcloud_roles.items():
overcloud_role = overcloud_role.lower()
if overcloud_role == OVERCLOUD_COMPUTE_ROLE:
scaling = dict(scaling.items() +
merge.parse_scaling(["NovaCompute=%s" % (count)]).items())
return scaling
def _join_template_path(file_name):
return os.path.abspath(
os.path.join(os.path.dirname(cfg.CONF.tht_local_dir), file_name)
)
def merge_templates(overcloud_roles):
"""Merge the Overcloud Roles with overcloud.yaml using merge from
tripleo_heat_merge
See tripleo-heat-templates for further details.
"""
# TODO(dmatthews): Add exception handling to catch merge errors
scale_params = generate_scaling_params(overcloud_roles)
overcloud_src_path = _join_template_path("overcloud-source.yaml")
ssl_src_path = _join_template_path("ssl-source.yaml")
swift_src_path = _join_template_path("swift-source.yaml")
template = merge.merge(
[overcloud_src_path, ssl_src_path, swift_src_path], None, None,
included_template_dir=cfg.CONF.tht_local_dir, scaling=scale_params
)
return template

View File

@ -12,11 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os
import mock
from pecan.testing import load_test_app
from tuskar.api.controllers.v1 import overcloud
from tuskar.common import exception
from tuskar.db.sqlalchemy import models as db_models
from tuskar.tests import base
@ -69,13 +71,85 @@ class OvercloudTests(base.TestCase):
mock_db_get.assert_called_once_with(12345)
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': False,
'create_stack.return_value': True,
})
)
def test_create_stack(self, mock_heat_client, mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test
response = overcloud.process_stack({}, create=True)
# Verify
self.assertEqual(response, None)
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': False,
'create_stack.return_value': False,
})
)
def test_create_stack_heat_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.HeatTemplateCreateFailed,
overcloud.process_stack, {}, True)
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': True,
'create_stack.return_value': True,
})
)
def test_create_stack_existing_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.StackAlreadyCreated, overcloud.process_stack, {}, True)
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': False,
'exists_stack.return_value': False,
'create_stack.return_value': True,
})
)
def test_create_stack_not_valid_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.InvalidHeatTemplate, overcloud.process_stack, {}, True)
@mock.patch('tuskar.api.controllers.v1.overcloud.process_stack')
@mock.patch('tuskar.db.sqlalchemy.api.Connection.create_overcloud')
def test_post(self, mock_db_create):
def test_post(self, mock_db_create, mock_process_stack):
# Setup
create_me = {'name': 'new'}
fake_created = db_models.Overcloud(name='created')
mock_db_create.return_value = fake_created
mock_process_stack.return_value = None
# Test
response = self.app.post_json(URL_OVERCLOUDS, params=create_me)
@ -91,8 +165,78 @@ class OvercloudTests(base.TestCase):
db_models.Overcloud))
self.assertEqual(db_create_model.name, create_me['name'])
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': True,
'create_stack.return_value': True,
})
)
def test_update_stack(self, mock_heat_client, mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test
response = overcloud.process_stack({})
# Verify
self.assertEqual(response, None)
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': True,
'create_stack.return_value': False,
})
)
def test_update_stack_heat_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.HeatTemplateUpdateFailed, overcloud.process_stack, {})
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': True,
'exists_stack.return_value': False,
'create_stack.return_value': True,
})
)
def test_update_stack_not_existing_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.StackNotFound, overcloud.process_stack, {})
@mock.patch('tuskar.heat.template_tools.merge_templates')
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'validate_template.return_value': False,
'exists_stack.return_value': True,
'create_stack.return_value': True,
})
)
def test_update_stack_not_valid_exception(self, mock_heat_client,
mock_heat_merge_templates):
# Setup
mock_heat_merge_templates.return_value = None
# Test and Verify
self.assertRaises(
exception.InvalidHeatTemplate, overcloud.process_stack, {})
@mock.patch('tuskar.api.controllers.v1.overcloud.process_stack')
@mock.patch('tuskar.db.sqlalchemy.api.Connection.update_overcloud')
def test_put(self, mock_db_update):
def test_put(self, mock_db_update, mock_process_stack):
# Setup
changes = {'name': 'updated'}
@ -100,6 +244,7 @@ class OvercloudTests(base.TestCase):
attributes=[],
counts=[])
mock_db_update.return_value = fake_updated
mock_process_stack.return_value = None
# Test
url = URL_OVERCLOUDS + '/' + '12345'
@ -119,7 +264,12 @@ class OvercloudTests(base.TestCase):
@mock.patch('tuskar.db.sqlalchemy.api.'
'Connection.delete_overcloud_by_id')
def test_delete(self, mock_db_delete):
@mock.patch(
'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
'delete_stack.return_value': True,
})
)
def test_delete(self, mock_heat_client, mock_db_delete):
# Test
url = URL_OVERCLOUDS + '/' + '12345'
response = self.app.delete(url)

View File

@ -42,5 +42,7 @@ class ConfFixture(fixtures.Fixture):
self.conf.set_default('sqlite_synchronous', False)
self.conf.set_default('use_ipv6', True)
self.conf.set_default('verbose', True)
self.conf.set_default('tht_local_dir',
'/etc/tuskar/tripleo-heat-templates/')
config.parse_args([], default_config_files=[])
self.addCleanup(self.conf.reset)

View File

View File

@ -0,0 +1,53 @@
# -*- encoding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import unittest
from tuskar.heat import template_tools
class TemplateToolsTests(unittest.TestCase):
@mock.patch('tripleo_heat_merge.merge.parse_scaling')
def test_generate_scaling_params(self, mock_parse_scaling):
# Setup
overcloud_roles = {'controller': 1, 'compute': 12}
# Test
template_tools.generate_scaling_params(overcloud_roles)
# Verify
mock_parse_scaling.assert_called_once_with(['NovaCompute=12'])
@mock.patch('tripleo_heat_merge.merge.merge')
def test_merge_templates(self, mock_merge):
# Setup
overcloud_roles = {'controller': 1, 'compute': 12}
# Test
template_tools.merge_templates(overcloud_roles)
# Verify
mock_merge.assert_called_once_with([
'/etc/tuskar/tripleo-heat-templates/overcloud-source.yaml',
'/etc/tuskar/tripleo-heat-templates/ssl-source.yaml',
'/etc/tuskar/tripleo-heat-templates/swift-source.yaml'],
None,
None,
scaling={
'NovaCompute0': 12
},
included_template_dir='/etc/tuskar/tripleo-heat-templates/'
)