Series Upgrade

Implement the series-upgrade feature allowing to move between Ubuntu
series.

Change-Id: I4377125f537c555e4b0b63dc08a3d9c0fc9d5251
This commit is contained in:
David Ames 2018-10-11 15:37:25 -07:00
parent 7d015f64d3
commit 0bcdc44c01
13 changed files with 395 additions and 37 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ tests/cirros-*-disk.img
func-results.json func-results.json
__pycache__ __pycache__
.stestr/ .stestr/
tests/cirros*

View File

@ -5,3 +5,14 @@ openstack-upgrade:
domain-setup: domain-setup:
description: description:
Setup the keystone domains, roles and user required for Heat to operate. Only required for OpenStack >= Kilo. Setup the keystone domains, roles and user required for Heat to operate. Only required for OpenStack >= Kilo.
pause:
description: |
Pause heat services.
If the heat deployment is clustered using the hacluster charm, the
corresponding hacluster unit on the node must first be paused as well.
Not doing so may lead to an interruption of service.
resume:
description: |
Resume heat services.
If the heat deployment is clustered using the hacluster charm, the
corresponding hacluster unit on the node must be resumed as well.

80
actions/actions.py Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
_path = os.path.dirname(os.path.realpath(__file__))
_parent = os.path.abspath(os.path.join(_path, ".."))
_hooks = os.path.abspath(os.path.join(_parent, "hooks"))
def _add_path(path):
if path not in sys.path:
sys.path.insert(1, path)
_add_path(_parent)
_add_path(_hooks)
from charmhelpers.core.hookenv import action_fail
sys.path.append('hooks/')
from heat_utils import (
pause_unit_helper,
resume_unit_helper,
register_configs,
)
def pause(args):
"""Pause all the Glance services.
@raises Exception if any services fail to stop
"""
pause_unit_helper(register_configs())
def resume(args):
"""Resume all the Glance services.
@raises Exception if any services fail to start
"""
resume_unit_helper(register_configs())
# A dictionary of all the defined actions to callables (which take
# parsed arguments).
ACTIONS = {"pause": pause, "resume": resume}
def main(args):
action_name = os.path.basename(args[0])
try:
action = ACTIONS[action_name]
except KeyError:
return "Action %s undefined" % action_name
else:
try:
action(args)
except Exception as e:
action_fail(str(e))
if __name__ == "__main__":
sys.exit(main(sys.argv))

1
actions/pause Symbolic link
View File

@ -0,0 +1 @@
actions.py

1
actions/resume Symbolic link
View File

@ -0,0 +1 @@
actions.py

View File

@ -64,9 +64,9 @@ from charmhelpers.contrib.network.ip import (
from charmhelpers.contrib.openstack.utils import ( from charmhelpers.contrib.openstack.utils import (
configure_installation_source, configure_installation_source,
openstack_upgrade_available, openstack_upgrade_available,
set_os_workload_status,
sync_db_with_multi_ipv6_addresses, sync_db_with_multi_ipv6_addresses,
os_application_version_set, series_upgrade_prepare,
series_upgrade_complete,
) )
from charmhelpers.contrib.openstack.ha.utils import ( from charmhelpers.contrib.openstack.ha.utils import (
@ -88,9 +88,10 @@ from heat_utils import (
register_configs, register_configs,
CLUSTER_RES, CLUSTER_RES,
HEAT_CONF, HEAT_CONF,
REQUIRED_INTERFACES,
setup_ipv6, setup_ipv6,
VERSION_PACKAGE, pause_unit_helper,
resume_unit_helper,
assess_status,
) )
from heat_context import ( from heat_context import (
@ -456,13 +457,26 @@ def certs_changed(relation_id=None, unit=None):
configure_https() configure_https()
@hooks.hook('pre-series-upgrade')
def pre_series_upgrade():
log("Running prepare series upgrade hook", "INFO")
series_upgrade_prepare(
pause_unit_helper, CONFIGS)
@hooks.hook('post-series-upgrade')
def post_series_upgrade():
log("Running complete series upgrade hook", "INFO")
series_upgrade_complete(
resume_unit_helper, CONFIGS)
def main(): def main():
try: try:
hooks.execute(sys.argv) hooks.execute(sys.argv)
except UnregisteredHookError as e: except UnregisteredHookError as e:
log('Unknown hook {} - skipping.'.format(e)) log('Unknown hook {} - skipping.'.format(e))
set_os_workload_status(CONFIGS, REQUIRED_INTERFACES) assess_status(CONFIGS)
os_application_version_set(VERSION_PACKAGE)
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -14,6 +14,7 @@
import os import os
from copy import deepcopy
from collections import OrderedDict from collections import OrderedDict
from subprocess import check_call from subprocess import check_call
@ -26,8 +27,17 @@ from charmhelpers.contrib.openstack.utils import (
token_cache_pkgs, token_cache_pkgs,
enable_memcache, enable_memcache,
CompareOpenStackReleases, CompareOpenStackReleases,
os_application_version_set,
make_assess_status_func,
pause_unit,
resume_unit,
) )
from charmhelpers.contrib.hahelpers.cluster import (
get_hacluster_config,
)
from charmhelpers.fetch import ( from charmhelpers.fetch import (
add_source, add_source,
apt_install, apt_install,
@ -41,6 +51,7 @@ from charmhelpers.fetch import (
from charmhelpers.core.hookenv import ( from charmhelpers.core.hookenv import (
log, log,
config, config,
relation_ids,
) )
from charmhelpers.core.host import ( from charmhelpers.core.host import (
@ -151,31 +162,41 @@ CONFIG_FILES = OrderedDict([
'services': [] 'services': []
}), }),
(MEMCACHED_CONF, { (MEMCACHED_CONF, {
'hook_contexts': [context.MemcacheContext()], 'contexts': [context.MemcacheContext()],
'services': ['memcached'], 'services': ['memcached'],
}), }),
]) ])
def register_configs(): def resource_map(release=None):
release = os_release('heat-common') """
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, Dynamically generate a map of resources that will be managed for a single
openstack_release=release) hook execution.
"""
confs = [HEAT_CONF, HEAT_API_PASTE, HAPROXY_CONF, ADMIN_OPENRC] _release = release or os_release('heat-common', base='icehouse')
for conf in confs: _resource_map = deepcopy(CONFIG_FILES)
configs.register(conf, CONFIG_FILES[conf]['contexts'])
if os.path.exists('/etc/apache2/conf-available'): if os.path.exists('/etc/apache2/conf-available'):
configs.register(HTTPS_APACHE_24_CONF, _resource_map.pop(HTTPS_APACHE_CONF)
CONFIG_FILES[HTTPS_APACHE_24_CONF]['contexts'])
else: else:
configs.register(HTTPS_APACHE_CONF, _resource_map.pop(HTTPS_APACHE_24_CONF)
CONFIG_FILES[HTTPS_APACHE_CONF]['contexts'])
if enable_memcache(release=release): if not enable_memcache(release=_release):
configs.register(MEMCACHED_CONF, _resource_map.pop(MEMCACHED_CONF)
CONFIG_FILES[MEMCACHED_CONF]['hook_contexts'])
return _resource_map
def register_configs(release=None):
"""Register config files with their respective contexts.
Regstration of some configs may not be required depending on
existing of certain relations.
"""
release = release or os_release('heat-common', base='icehouse')
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
openstack_release=release)
for cfg, rscs in resource_map(release).items():
configs.register(cfg, rscs['contexts'])
return configs return configs
@ -249,22 +270,15 @@ def do_openstack_upgrade(configs):
def restart_map(): def restart_map():
"""Restarts on config change. '''Determine the correct resource map to be passed to
Determine the correct resource map to be passed to
charmhelpers.core.restart_on_change() based on the services configured. charmhelpers.core.restart_on_change() based on the services configured.
:returns: dict: A dictionary mapping config file to lists of services :returns: dict: A dictionary mapping config file to lists of services
that should be restarted when file changes. that should be restarted when file changes.
""" '''
_map = [] return OrderedDict([(cfg, v['services'])
for f, ctxt in CONFIG_FILES.items(): for cfg, v in resource_map().items()
svcs = [] if v['services']])
for svc in ctxt['services']:
svcs.append(svc)
if svcs:
_map.append((f, svcs))
return OrderedDict(_map)
def services(): def services():
@ -297,3 +311,113 @@ def setup_ipv6():
'main') 'main')
apt_update() apt_update()
apt_install('haproxy/trusty-backports', fatal=True) apt_install('haproxy/trusty-backports', fatal=True)
def check_optional_relations(configs):
"""Check that if we have a relation_id for high availability that we can
get the hacluster config. If we can't then we are blocked.
This function is called from assess_status/set_os_workload_status as the
charm_func and needs to return either None, None if there is no problem or
the status, message if there is a problem.
:param configs: an OSConfigRender() instance.
:return 2-tuple: (string, string) = (status, message)
"""
if relation_ids('ha'):
try:
get_hacluster_config()
except:
return ('blocked',
'hacluster missing configuration: '
'vip, vip_iface, vip_cidr')
# return 'unknown' as the lowest priority to not clobber an existing
# status.
return "unknown", ""
def get_optional_interfaces():
"""Return the optional interfaces that should be checked if the relavent
relations have appeared.
:returns: {general_interface: [specific_int1, specific_int2, ...], ...}
"""
optional_interfaces = {}
if relation_ids('ha'):
optional_interfaces['ha'] = ['cluster']
return optional_interfaces
def assess_status(configs):
"""Assess status of current unit
Decides what the state of the unit should be based on the current
configuration.
SIDE EFFECT: calls set_os_workload_status(...) which sets the workload
status of the unit.
Also calls status_set(...) directly if paused state isn't complete.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
assess_status_func(configs)()
os_application_version_set(VERSION_PACKAGE)
def assess_status_func(configs):
"""Helper function to create the function that will assess_status() for
the unit.
Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to
create the appropriate status function and then returns it.
Used directly by assess_status() and also for pausing and resuming
the unit.
NOTE: REQUIRED_INTERFACES is augmented with the optional interfaces
depending on the current config before being passed to the
make_assess_status_func() function.
NOTE(ajkavanagh) ports are not checked due to race hazards with services
that don't behave sychronously w.r.t their service scripts. e.g.
apache2.
@param configs: a templating.OSConfigRenderer() object
@return f() -> None : a function that assesses the unit's workload status
"""
required_interfaces = REQUIRED_INTERFACES.copy()
required_interfaces.update(get_optional_interfaces())
return make_assess_status_func(
configs, required_interfaces,
charm_func=check_optional_relations,
services=services(), ports=None)
def pause_unit_helper(configs):
"""Helper function to pause a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(pause_unit, configs)
def resume_unit_helper(configs):
"""Helper function to resume a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(resume_unit, configs)
def _pause_resume_helper(f, configs):
"""Helper function that uses the make_assess_status_func(...) from
charmhelpers.contrib.openstack.utils to create an assess_status(...)
function that can be used with the pause/resume of the unit
@param f: the function to be used with the assess_status(...) function
@returns None - this function is executed for its side-effect
"""
# TODO(ajkavanagh) - ports= has been left off because of the race hazard
# that exists due to service_start()
f(assess_status_func(configs),
services=services(),
ports=None)

1
hooks/post-series-upgrade Symbolic link
View File

@ -0,0 +1 @@
heat_relations.py

1
hooks/pre-series-upgrade Symbolic link
View File

@ -0,0 +1 @@
heat_relations.py

View File

@ -719,3 +719,27 @@ class HeatBasicDeployment(OpenStackAmuletDeployment):
sleep_time = 0 sleep_time = 0
self.d.configure(juju_service, set_default) self.d.configure(juju_service, set_default)
def test_901_pause_resume(self):
"""Test pause and resume actions."""
u.log.debug('Checking pause and resume actions...')
unit = self.d.sentry['heat'][0]
unit_name = unit.info['unit_name']
u.log.debug('Checking for active status on {}'.format(unit_name))
assert u.status_get(unit)[0] == "active"
u.log.debug('Running pause action on {}'.format(unit_name))
action_id = u.run_action(unit, "pause")
u.log.debug('Waiting on action {}'.format(action_id))
assert u.wait_on_action(action_id), "Pause action failed."
u.log.debug('Checking for maintenance status on {}'.format(unit_name))
assert u.status_get(unit)[0] == "maintenance"
u.log.debug('Running resume action on {}'.format(unit_name))
action_id = u.run_action(unit, "resume")
u.log.debug('Waiting on action {}'.format(action_id))
assert u.wait_on_action(action_id), "Resume action failed."
u.log.debug('Checking for active status on {}'.format(unit_name))
assert u.status_get(unit)[0] == "active"
u.log.debug('OK')

View File

@ -0,0 +1,80 @@
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import mock
from test_utils import CharmTestCase
os.environ['JUJU_UNIT_NAME'] = 'heat'
with mock.patch('heat_utils.register_configs') as configs:
configs.return_value = 'test-config'
import actions
class PauseTestCase(CharmTestCase):
def setUp(self):
super(PauseTestCase, self).setUp(
actions, ["pause_unit_helper"])
def test_pauses_services(self):
actions.pause([])
self.pause_unit_helper.assert_called_once_with('test-config')
class ResumeTestCase(CharmTestCase):
def setUp(self):
super(ResumeTestCase, self).setUp(
actions, ["resume_unit_helper"])
def test_pauses_services(self):
actions.resume([])
self.resume_unit_helper.assert_called_once_with('test-config')
class MainTestCase(CharmTestCase):
def setUp(self):
super(MainTestCase, self).setUp(actions, ["action_fail"])
def test_invokes_action(self):
dummy_calls = []
def dummy_action(args):
dummy_calls.append(True)
with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}):
actions.main(["foo"])
self.assertEqual(dummy_calls, [True])
def test_unknown_action(self):
"""Unknown actions aren't a traceback."""
exit_string = actions.main(["foo"])
self.assertEqual("Action foo undefined", exit_string)
def test_failing_action(self):
"""Actions which traceback trigger action_fail() calls."""
dummy_calls = []
self.action_fail.side_effect = dummy_calls.append
def dummy_action(args):
raise ValueError("uh oh")
with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}):
actions.main(["foo"])
self.assertEqual(dummy_calls, ["uh oh"])

View File

@ -28,7 +28,8 @@ mock_apt.apt_pkg = MagicMock()
with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
lambda *args, **kwargs: f(*args, **kwargs)) lambda *args, **kwargs: f(*args, **kwargs))
with patch('heat_utils.register_configs') as register_configs: with patch('heat_utils.register_configs') as register_configs, \
patch('heat_utils.resource_map') as resource_map:
import openstack_upgrade import openstack_upgrade
from test_utils import ( from test_utils import (

View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from copy import deepcopy
from collections import OrderedDict from collections import OrderedDict
from mock import patch, MagicMock, call from mock import patch, MagicMock, call
from test_utils import CharmTestCase from test_utils import CharmTestCase
@ -39,6 +40,7 @@ TO_PATCH = [
'service_stop', 'service_stop',
'token_cache_pkgs', 'token_cache_pkgs',
'enable_memcache', 'enable_memcache',
'os'
] ]
@ -101,7 +103,24 @@ class HeatUtilsTests(CharmTestCase):
['python-heat', 'python-memcache']) ['python-heat', 'python-memcache'])
def test_restart_map(self): def test_restart_map(self):
self.assertEqual(RESTART_MAP, utils.restart_map()) # Icehouse
self.os_release.return_value = "icehouse"
self.enable_memcache.return_value = False
self.os.path.exists.return_value = False
_restart_map = deepcopy(RESTART_MAP)
_restart_map.pop(
"/etc/apache2/sites-available/openstack_https_frontend.conf")
_restart_map.pop("/etc/memcached.conf")
self.assertEqual(_restart_map, utils.restart_map())
# Mitaka
self.os_release.return_value = "mitaka"
self.enable_memcache.return_value = True
self.os.path.exists.return_value = True
_restart_map = deepcopy(RESTART_MAP)
_restart_map.pop(
"/etc/apache2/sites-available/openstack_https_frontend")
self.assertEqual(_restart_map, utils.restart_map())
def test_openstack_upgrade(self): def test_openstack_upgrade(self):
self.config.side_effect = None self.config.side_effect = None