Series Upgrade

Implement the series-upgrade feature allowing to move between Ubuntu
series.

Change-Id: I4377125f537c555e4b0b63dc08a3d9c0fc9d5251

.gitignore

@@ -13,3 +13,4 @@ tests/cirros-*-disk.img
 func-results.json
 __pycache__
 .stestr/
+tests/cirros*

actions.yaml

@@ -5,3 +5,14 @@ openstack-upgrade:
 domain-setup:
   description:
     Setup the keystone domains, roles and user required for Heat to operate. Only required for OpenStack >= Kilo.
+pause:
+  description: |
+    Pause heat services.
+    If the heat deployment is clustered using the hacluster charm, the
+    corresponding hacluster unit on the node must first be paused as well.
+    Not doing so may lead to an interruption of service.
+resume:
+  description: |
+    Resume heat services.
+    If the heat deployment is clustered using the hacluster charm, the
+    corresponding hacluster unit on the node must be resumed as well.

actions/actions.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+
+_path = os.path.dirname(os.path.realpath(__file__))
+_parent = os.path.abspath(os.path.join(_path, ".."))
+_hooks = os.path.abspath(os.path.join(_parent, "hooks"))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+
+_add_path(_parent)
+_add_path(_hooks)
+
+
+from charmhelpers.core.hookenv import action_fail
+
+sys.path.append('hooks/')
+
+from heat_utils import (
+    pause_unit_helper,
+    resume_unit_helper,
+    register_configs,
+)
+
+
+def pause(args):
+    """Pause all the Glance services.
+
+    @raises Exception if any services fail to stop
+    """
+    pause_unit_helper(register_configs())
+
+
+def resume(args):
+    """Resume all the Glance services.
+
+    @raises Exception if any services fail to start
+    """
+    resume_unit_helper(register_configs())
+
+
+# A dictionary of all the defined actions to callables (which take
+# parsed arguments).
+ACTIONS = {"pause": pause, "resume": resume}
+
+
+def main(args):
+    action_name = os.path.basename(args[0])
+    try:
+        action = ACTIONS[action_name]
+    except KeyError:
+        return "Action %s undefined" % action_name
+    else:
+        try:
+            action(args)
+        except Exception as e:
+            action_fail(str(e))
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

actions/pause

@@ -0,0 +1 @@
+actions.py

actions/resume

@@ -0,0 +1 @@
+actions.py

hooks/heat_relations.py

@@ -64,9 +64,9 @@ from charmhelpers.contrib.network.ip import (
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
     openstack_upgrade_available,
-    set_os_workload_status,
     sync_db_with_multi_ipv6_addresses,
-    os_application_version_set,
+    series_upgrade_prepare,
+    series_upgrade_complete,
 )
 
 from charmhelpers.contrib.openstack.ha.utils import (
@@ -88,9 +88,10 @@ from heat_utils import (
     register_configs,
     CLUSTER_RES,
     HEAT_CONF,
-    REQUIRED_INTERFACES,
     setup_ipv6,
-    VERSION_PACKAGE,
+    pause_unit_helper,
+    resume_unit_helper,
+    assess_status,
 )
 
 from heat_context import (
@@ -456,13 +457,26 @@ def certs_changed(relation_id=None, unit=None):
     configure_https()
 
 
+@hooks.hook('pre-series-upgrade')
+def pre_series_upgrade():
+    log("Running prepare series upgrade hook", "INFO")
+    series_upgrade_prepare(
+        pause_unit_helper, CONFIGS)
+
+
+@hooks.hook('post-series-upgrade')
+def post_series_upgrade():
+    log("Running complete series upgrade hook", "INFO")
+    series_upgrade_complete(
+        resume_unit_helper, CONFIGS)
+
+
 def main():
     try:
         hooks.execute(sys.argv)
     except UnregisteredHookError as e:
         log('Unknown hook {} - skipping.'.format(e))
-    set_os_workload_status(CONFIGS, REQUIRED_INTERFACES)
+    assess_status(CONFIGS)
-    os_application_version_set(VERSION_PACKAGE)
 
 
 if __name__ == '__main__':

hooks/heat_utils.py

@@ -14,6 +14,7 @@
 
 import os
 
+from copy import deepcopy
 from collections import OrderedDict
 from subprocess import check_call
 
@@ -26,8 +27,17 @@ from charmhelpers.contrib.openstack.utils import (
     token_cache_pkgs,
     enable_memcache,
     CompareOpenStackReleases,
+    os_application_version_set,
+    make_assess_status_func,
+    pause_unit,
+    resume_unit,
 )
 
+from charmhelpers.contrib.hahelpers.cluster import (
+    get_hacluster_config,
+)
+
+
 from charmhelpers.fetch import (
     add_source,
     apt_install,
@@ -41,6 +51,7 @@ from charmhelpers.fetch import (
 from charmhelpers.core.hookenv import (
     log,
     config,
+    relation_ids,
 )
 
 from charmhelpers.core.host import (
@@ -151,31 +162,41 @@ CONFIG_FILES = OrderedDict([
         'services': []
     }),
     (MEMCACHED_CONF, {
-        'hook_contexts': [context.MemcacheContext()],
+        'contexts': [context.MemcacheContext()],
         'services': ['memcached'],
     }),
 ])
 
 
-def register_configs():
+def resource_map(release=None):
-    release = os_release('heat-common')
+    """
-    configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
+    Dynamically generate a map of resources that will be managed for a single
-                                          openstack_release=release)
+    hook execution.
-
+    """
-    confs = [HEAT_CONF, HEAT_API_PASTE, HAPROXY_CONF, ADMIN_OPENRC]
+    _release = release or os_release('heat-common', base='icehouse')
-    for conf in confs:
+    _resource_map = deepcopy(CONFIG_FILES)
-        configs.register(conf, CONFIG_FILES[conf]['contexts'])
 
     if os.path.exists('/etc/apache2/conf-available'):
-        configs.register(HTTPS_APACHE_24_CONF,
+        _resource_map.pop(HTTPS_APACHE_CONF)
-                         CONFIG_FILES[HTTPS_APACHE_24_CONF]['contexts'])
     else:
-        configs.register(HTTPS_APACHE_CONF,
+        _resource_map.pop(HTTPS_APACHE_24_CONF)
-                         CONFIG_FILES[HTTPS_APACHE_CONF]['contexts'])
 
-    if enable_memcache(release=release):
+    if not enable_memcache(release=_release):
-        configs.register(MEMCACHED_CONF,
+        _resource_map.pop(MEMCACHED_CONF)
-                         CONFIG_FILES[MEMCACHED_CONF]['hook_contexts'])
+
+    return _resource_map
+
+
+def register_configs(release=None):
+    """Register config files with their respective contexts.
+    Regstration of some configs may not be required depending on
+    existing of certain relations.
+    """
+    release = release or os_release('heat-common', base='icehouse')
+    configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
+                                          openstack_release=release)
+    for cfg, rscs in resource_map(release).items():
+        configs.register(cfg, rscs['contexts'])
     return configs
 
 
@@ -249,22 +270,15 @@ def do_openstack_upgrade(configs):
 
 
 def restart_map():
-    """Restarts on config change.
+    '''Determine the correct resource map to be passed to
-
-    Determine the correct resource map to be passed to
     charmhelpers.core.restart_on_change() based on the services configured.
 
     :returns: dict: A dictionary mapping config file to lists of services
                     that should be restarted when file changes.
-    """
+    '''
-    _map = []
+    return OrderedDict([(cfg, v['services'])
-    for f, ctxt in CONFIG_FILES.items():
+                        for cfg, v in resource_map().items()
-        svcs = []
+                        if v['services']])
-        for svc in ctxt['services']:
-            svcs.append(svc)
-        if svcs:
-            _map.append((f, svcs))
-    return OrderedDict(_map)
 
 
 def services():
@@ -297,3 +311,113 @@ def setup_ipv6():
                    'main')
         apt_update()
         apt_install('haproxy/trusty-backports', fatal=True)
+
+
+def check_optional_relations(configs):
+    """Check that if we have a relation_id for high availability that we can
+    get the hacluster config.  If we can't then we are blocked.
+
+    This function is called from assess_status/set_os_workload_status as the
+    charm_func and needs to return either None, None if there is no problem or
+    the status, message if there is a problem.
+
+    :param configs: an OSConfigRender() instance.
+    :return 2-tuple: (string, string) = (status, message)
+    """
+    if relation_ids('ha'):
+        try:
+            get_hacluster_config()
+        except:
+            return ('blocked',
+                    'hacluster missing configuration: '
+                    'vip, vip_iface, vip_cidr')
+    # return 'unknown' as the lowest priority to not clobber an existing
+    # status.
+    return "unknown", ""
+
+
+def get_optional_interfaces():
+    """Return the optional interfaces that should be checked if the relavent
+    relations have appeared.
+
+    :returns: {general_interface: [specific_int1, specific_int2, ...], ...}
+    """
+    optional_interfaces = {}
+    if relation_ids('ha'):
+        optional_interfaces['ha'] = ['cluster']
+
+    return optional_interfaces
+
+
+def assess_status(configs):
+    """Assess status of current unit
+    Decides what the state of the unit should be based on the current
+    configuration.
+    SIDE EFFECT: calls set_os_workload_status(...) which sets the workload
+    status of the unit.
+    Also calls status_set(...) directly if paused state isn't complete.
+    @param configs: a templating.OSConfigRenderer() object
+    @returns None - this function is executed for its side-effect
+    """
+    assess_status_func(configs)()
+    os_application_version_set(VERSION_PACKAGE)
+
+
+def assess_status_func(configs):
+    """Helper function to create the function that will assess_status() for
+    the unit.
+    Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to
+    create the appropriate status function and then returns it.
+    Used directly by assess_status() and also for pausing and resuming
+    the unit.
+
+    NOTE: REQUIRED_INTERFACES is augmented with the optional interfaces
+    depending on the current config before being passed to the
+    make_assess_status_func() function.
+
+    NOTE(ajkavanagh) ports are not checked due to race hazards with services
+    that don't behave sychronously w.r.t their service scripts.  e.g.
+    apache2.
+    @param configs: a templating.OSConfigRenderer() object
+    @return f() -> None : a function that assesses the unit's workload status
+    """
+    required_interfaces = REQUIRED_INTERFACES.copy()
+    required_interfaces.update(get_optional_interfaces())
+    return make_assess_status_func(
+        configs, required_interfaces,
+        charm_func=check_optional_relations,
+        services=services(), ports=None)
+
+
+def pause_unit_helper(configs):
+    """Helper function to pause a unit, and then call assess_status(...) in
+    effect, so that the status is correctly updated.
+    Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work.
+    @param configs: a templating.OSConfigRenderer() object
+    @returns None - this function is executed for its side-effect
+    """
+    _pause_resume_helper(pause_unit, configs)
+
+
+def resume_unit_helper(configs):
+    """Helper function to resume a unit, and then call assess_status(...) in
+    effect, so that the status is correctly updated.
+    Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work.
+    @param configs: a templating.OSConfigRenderer() object
+    @returns None - this function is executed for its side-effect
+    """
+    _pause_resume_helper(resume_unit, configs)
+
+
+def _pause_resume_helper(f, configs):
+    """Helper function that uses the make_assess_status_func(...) from
+    charmhelpers.contrib.openstack.utils to create an assess_status(...)
+    function that can be used with the pause/resume of the unit
+    @param f: the function to be used with the assess_status(...) function
+    @returns None - this function is executed for its side-effect
+    """
+    # TODO(ajkavanagh) - ports= has been left off because of the race hazard
+    # that exists due to service_start()
+    f(assess_status_func(configs),
+      services=services(),
+      ports=None)

hooks/post-series-upgrade

@@ -0,0 +1 @@
+heat_relations.py

hooks/pre-series-upgrade

@@ -0,0 +1 @@
+heat_relations.py

tests/basic_deployment.py

@@ -719,3 +719,27 @@ class HeatBasicDeployment(OpenStackAmuletDeployment):
             sleep_time = 0
 
         self.d.configure(juju_service, set_default)
+
+    def test_901_pause_resume(self):
+        """Test pause and resume actions."""
+        u.log.debug('Checking pause and resume actions...')
+        unit = self.d.sentry['heat'][0]
+        unit_name = unit.info['unit_name']
+
+        u.log.debug('Checking for active status on {}'.format(unit_name))
+        assert u.status_get(unit)[0] == "active"
+
+        u.log.debug('Running pause action on {}'.format(unit_name))
+        action_id = u.run_action(unit, "pause")
+        u.log.debug('Waiting on action {}'.format(action_id))
+        assert u.wait_on_action(action_id), "Pause action failed."
+        u.log.debug('Checking for maintenance status on {}'.format(unit_name))
+        assert u.status_get(unit)[0] == "maintenance"
+
+        u.log.debug('Running resume action on {}'.format(unit_name))
+        action_id = u.run_action(unit, "resume")
+        u.log.debug('Waiting on action {}'.format(action_id))
+        assert u.wait_on_action(action_id), "Resume action failed."
+        u.log.debug('Checking for active status on {}'.format(unit_name))
+        assert u.status_get(unit)[0] == "active"
+        u.log.debug('OK')

unit_tests/test_actions.py

@@ -0,0 +1,80 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import mock
+
+from test_utils import CharmTestCase
+
+os.environ['JUJU_UNIT_NAME'] = 'heat'
+with mock.patch('heat_utils.register_configs') as configs:
+    configs.return_value = 'test-config'
+    import actions
+
+
+class PauseTestCase(CharmTestCase):
+
+    def setUp(self):
+        super(PauseTestCase, self).setUp(
+            actions, ["pause_unit_helper"])
+
+    def test_pauses_services(self):
+        actions.pause([])
+        self.pause_unit_helper.assert_called_once_with('test-config')
+
+
+class ResumeTestCase(CharmTestCase):
+
+    def setUp(self):
+        super(ResumeTestCase, self).setUp(
+            actions, ["resume_unit_helper"])
+
+    def test_pauses_services(self):
+        actions.resume([])
+        self.resume_unit_helper.assert_called_once_with('test-config')
+
+
+class MainTestCase(CharmTestCase):
+
+    def setUp(self):
+        super(MainTestCase, self).setUp(actions, ["action_fail"])
+
+    def test_invokes_action(self):
+        dummy_calls = []
+
+        def dummy_action(args):
+            dummy_calls.append(True)
+
+        with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}):
+            actions.main(["foo"])
+        self.assertEqual(dummy_calls, [True])
+
+    def test_unknown_action(self):
+        """Unknown actions aren't a traceback."""
+        exit_string = actions.main(["foo"])
+        self.assertEqual("Action foo undefined", exit_string)
+
+    def test_failing_action(self):
+        """Actions which traceback trigger action_fail() calls."""
+        dummy_calls = []
+
+        self.action_fail.side_effect = dummy_calls.append
+
+        def dummy_action(args):
+            raise ValueError("uh oh")
+
+        with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}):
+            actions.main(["foo"])
+        self.assertEqual(dummy_calls, ["uh oh"])

unit_tests/test_actions_openstack_upgrade.py

@@ -28,7 +28,8 @@ mock_apt.apt_pkg = MagicMock()
 with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
     mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
                             lambda *args, **kwargs: f(*args, **kwargs))
-    with patch('heat_utils.register_configs') as register_configs:
+    with patch('heat_utils.register_configs') as register_configs, \
+            patch('heat_utils.resource_map') as resource_map:
         import openstack_upgrade
 
 from test_utils import (

unit_tests/test_heat_utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from copy import deepcopy
 from collections import OrderedDict
 from mock import patch, MagicMock, call
 from test_utils import CharmTestCase
@@ -39,6 +40,7 @@ TO_PATCH = [
     'service_stop',
     'token_cache_pkgs',
     'enable_memcache',
+    'os'
 ]
 
 
@@ -101,7 +103,24 @@ class HeatUtilsTests(CharmTestCase):
                          ['python-heat', 'python-memcache'])
 
     def test_restart_map(self):
-        self.assertEqual(RESTART_MAP, utils.restart_map())
+        # Icehouse
+        self.os_release.return_value = "icehouse"
+        self.enable_memcache.return_value = False
+        self.os.path.exists.return_value = False
+        _restart_map = deepcopy(RESTART_MAP)
+        _restart_map.pop(
+            "/etc/apache2/sites-available/openstack_https_frontend.conf")
+        _restart_map.pop("/etc/memcached.conf")
+        self.assertEqual(_restart_map, utils.restart_map())
+
+        # Mitaka
+        self.os_release.return_value = "mitaka"
+        self.enable_memcache.return_value = True
+        self.os.path.exists.return_value = True
+        _restart_map = deepcopy(RESTART_MAP)
+        _restart_map.pop(
+            "/etc/apache2/sites-available/openstack_https_frontend")
+        self.assertEqual(_restart_map, utils.restart_map())
 
     def test_openstack_upgrade(self):
         self.config.side_effect = None