From b3f6993f2031233c8a90a300396fe7fef75334ec Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 06:59:21 +0000 Subject: [PATCH] Implementation of deferred restarts Add deferred restart actions and config. Change-Id: I6aeceae10b7a25ba5bfd6f2cb07f510a9481d0ba --- actions.yaml | 27 +++++++++++++++ actions/actions.py | 57 ++++++++++++++++++++++++++++++-- actions/restart-services | 1 + actions/run-deferred-hooks | 1 + actions/show-deferred-events | 1 + config.yaml | 6 ++++ hooks/neutron_hooks.py | 15 ++++++--- hooks/neutron_utils.py | 20 +++++++++++ tests/tests.yaml | 2 ++ unit_tests/test_neutron_hooks.py | 18 +++++++--- 10 files changed, 137 insertions(+), 11 deletions(-) create mode 120000 actions/restart-services create mode 120000 actions/run-deferred-hooks create mode 120000 actions/show-deferred-events diff --git a/actions.yaml b/actions.yaml index 787cc697..da514587 100644 --- a/actions.yaml +++ b/actions.yaml @@ -24,3 +24,30 @@ resume: descrpition: Resume the neutron-gateway unit. security-checklist: description: Validate the running configuration against the OpenStack security guides checklist +restart-services: + description: | + Restarts services this charm manages. + params: + deferred-only: + type: boolean + default: false + description: | + Restart all deferred services. + services: + type: string + default: "" + description: | + List of services to restart. + run-hooks: + type: boolean + default: true + description: | + Run any hooks which have been deferred. +run-deferred-hooks: + description: | + Run deferable hooks and restart services. + . + NOTE: Service will be restarted as needed irrespective of enable-auto-restarts +show-deferred-events: + descrpition: | + Show the outstanding restarts diff --git a/actions/actions.py b/actions/actions.py index cff79c8e..78d81f99 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -15,8 +15,15 @@ def _add_path(path): _add_path(_hooks_dir) -from charmhelpers.core.hookenv import action_fail +import charmhelpers.contrib.openstack.utils as os_utils +from charmhelpers.core.hookenv import ( + DEBUG, + action_get, + action_fail, + log, +) from neutron_utils import ( + assess_status, pause_unit_helper, resume_unit_helper, register_configs, @@ -36,9 +43,55 @@ def resume(args): resume_unit_helper(register_configs()) +def restart(args): + """Restart services. + + :param args: Unused + :type args: List[str] + """ + deferred_only = action_get("deferred-only") + services = action_get("services").split() + # Check input + if deferred_only and services: + action_fail("Cannot set deferred-only and services") + return + if not (deferred_only or services): + action_fail("Please specify deferred-only or services") + return + if action_get('run-hooks'): + log("Charm does not defer any hooks at present", DEBUG) + if deferred_only: + os_utils.restart_services_action(deferred_only=True) + else: + os_utils.restart_services_action(services=services) + assess_status(register_configs()) + + +def run_deferred_hooks(args): + """Run deferred hooks. + + :param args: Unused + :type args: List[str] + """ + # Charm defers restarts on a case-by-case basis so no full + # hook deferalls are needed. + action_fail("Charm does not defer any hooks at present") + + +def show_deferred_events(args): + """Show the deferred events. + + :param args: Unused + :type args: List[str] + """ + os_utils.show_deferred_events_action_helper() + + # A dictionary of all the defined actions to callables (which take # parsed arguments). -ACTIONS = {"pause": pause, "resume": resume} +ACTIONS = {"pause": pause, "resume": resume, "restart-services": restart, + "show-deferred-events": show_deferred_events, + "run-deferred-hooks": run_deferred_hooks} def main(args): diff --git a/actions/restart-services b/actions/restart-services new file mode 120000 index 00000000..405a394e --- /dev/null +++ b/actions/restart-services @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/actions/run-deferred-hooks b/actions/run-deferred-hooks new file mode 120000 index 00000000..405a394e --- /dev/null +++ b/actions/run-deferred-hooks @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/actions/show-deferred-events b/actions/show-deferred-events new file mode 120000 index 00000000..405a394e --- /dev/null +++ b/actions/show-deferred-events @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/config.yaml b/config.yaml index 592f69dc..e454c984 100644 --- a/config.yaml +++ b/config.yaml @@ -382,3 +382,9 @@ options: by sending a ping and if that fails they trigger a vrrp transition. This option defines how frequently this check is performed. Setting this value to 0 will disable the healthchecks. + enable-auto-restarts: + type: boolean + default: True + description: | + Allow the charm and packages to restart services automatically when + required. diff --git a/hooks/neutron_hooks.py b/hooks/neutron_hooks.py index 2644b314..5a83c2ca 100755 --- a/hooks/neutron_hooks.py +++ b/hooks/neutron_hooks.py @@ -12,8 +12,11 @@ from charmhelpers.core.hookenv import ( UnregisteredHookError, status_set, ) -from charmhelpers.core.host import service_restart from charmhelpers.core.unitdata import kv +from charmhelpers.contrib.openstack.deferred_events import ( + configure_deferred_restarts, + deferrable_svc_restart, +) from charmhelpers.fetch import ( apt_update, apt_install, @@ -33,8 +36,8 @@ from charmhelpers.contrib.hahelpers.apache import ( from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, - pausable_restart_on_change as restart_on_change, is_unit_paused_set, + os_restart_on_change as restart_on_change, series_upgrade_prepare, series_upgrade_complete, ) @@ -76,6 +79,7 @@ from neutron_utils import ( disable_neutron_lbaas, remove_old_packages, deprecated_services, + deferrable_services, ) hooks = Hooks() @@ -140,6 +144,7 @@ def install(): @restart_on_change(restart_map) @harden() def config_changed(): + configure_deferred_restarts(deferrable_services()) if not config('action-managed-upgrade'): if openstack_upgrade_available(NEUTRON_COMMON): status_set('maintenance', 'Running openstack upgrade') @@ -205,7 +210,7 @@ def upgrade_charm(): if packages_removed and not is_unit_paused_set(): log("Package purge detected, restarting services", "INFO") for s in services(): - service_restart(s) + deferrable_svc_restart(s, 'Package purge detected') config_changed() update_legacy_ha_files(force=True) @@ -288,7 +293,9 @@ def nm_changed(): previous_nonce = db.get('restart_nonce') if previous_nonce != restart_nonce: if not is_unit_paused_set(): - service_restart('nova-api-metadata') + deferrable_svc_restart( + 'nova-api-metadata', + 'Restart trigger received') db.set('restart_nonce', restart_nonce) db.flush() # LP: #1812813 diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index b426de4f..c2ad916e 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -803,6 +803,26 @@ def services(): return list(set(_services)) +def deferrable_services(): + """Services which should be stopped from restarting. + + All services from services() are deferable. But the charm may + install a package which install a service that the charm does not add + to its restart_map. In that case it will be missing from + self.services. However one of the jobs of deferred events is to ensure + that packages updates outside of charms also do not restart services. + To ensure there is a complete list take the services from services{} + and also add in a known list of networking services. + + NOTE: It does not matter if one of the services in the list is not + installed on the system. + """ + _svcs = services() + _svcs.extend(['ovs-vswitchd', 'ovsdb-server', + 'openvswitch-switch']) + return list(set(_svcs)) + + def do_openstack_upgrade(configs): """ Perform an upgrade. Takes care of upgrading packages, rewriting diff --git a/tests/tests.yaml b/tests/tests.yaml index cd67102c..ea1f7011 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -60,12 +60,14 @@ configure_options: configure_gateway_ext_port_use_juju_wait: false tests: + - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest - zaza.openstack.charm_tests.neutron.tests.SecurityTest - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest - zaza.openstack.charm_tests.neutron.tests.NeutronBridgePortMappingTest - migrate-ovn: + - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest - zaza.openstack.charm_tests.neutron.tests.SecurityTest - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest diff --git a/unit_tests/test_neutron_hooks.py b/unit_tests/test_neutron_hooks.py index b7f3a190..f37cf049 100644 --- a/unit_tests/test_neutron_hooks.py +++ b/unit_tests/test_neutron_hooks.py @@ -50,7 +50,7 @@ TO_PATCH = [ 'stop_neutron_ha_monitor_daemon', 'use_l3ha', 'kv', - 'service_restart', + 'deferrable_svc_restart', 'install_systemd_override', 'configure_apparmor', 'disable_nova_metadata', @@ -60,6 +60,8 @@ TO_PATCH = [ 'services', 'remove_old_packages', 'is_container', + 'configure_deferred_restarts', + 'deferrable_services', 'charmhelpers.contrib.openstack.utils.is_unit_paused_set', ] @@ -236,7 +238,9 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(_install.called) self.assertTrue(_config_changed.called) self.assertTrue(self.install_systemd_override.called) - self.service_restart.assert_called_once_with('neutron-metadata-agent') + self.deferrable_svc_restart.assert_called_once_with( + 'neutron-metadata-agent', + 'Package purge detected') def test_amqp_joined(self): self._call_hook('amqp-relation-joined') @@ -326,7 +330,9 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('quantum-network-service-relation-changed') self.assertTrue(self.CONFIGS.write_all.called) self.install_ca_cert.assert_called_with('cert') - self.service_restart.assert_called_with('nova-api-metadata') + self.deferrable_svc_restart.assert_called_with( + 'nova-api-metadata', + 'Restart trigger received') kv_mock.get.assert_called_with('restart_nonce') kv_mock.set.assert_called_with('restart_nonce', '1111111222222333333') @@ -365,7 +371,9 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('quantum-network-service-relation-changed') self.assertTrue(self.CONFIGS.write_all.called) self.install_ca_cert.assert_called_with('cert') - self.service_restart.assert_called_with('nova-api-metadata') + self.deferrable_svc_restart.assert_called_with( + 'nova-api-metadata', + 'Restart trigger received') kv_mock.get.assert_called_with('restart_nonce') kv_mock.set.assert_called_with('restart_nonce', '1111111222222333333') @@ -405,7 +413,7 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('quantum-network-service-relation-changed') self.assertTrue(self.CONFIGS.write_all.called) self.install_ca_cert.assert_called_with('cert') - self.assertFalse(self.service_restart.called) + self.assertFalse(self.deferrable_svc_restart.called) kv_mock.get.assert_called_with('restart_nonce') self.assertFalse(kv_mock.set.called) self.assertFalse(kv_mock.flush.called)