From b2a11a58301330c29c6bd9d010ed74cd8398301d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 29 Jan 2019 11:14:03 +0000 Subject: [PATCH] Prune unused Galaxy roles during upgrade The 'kayobe control host upgrade' command updates the installed Ansible Galaxy roles based on requirements.yml. Sometimes roles are removed from that file, but there is currently no way of removing them from the local system. Normally this causes no problems, but due to the upstream role containing symlinks with whitespace, we are switching out yatesr.timezone to a stackhpc.timezone fork. In order to make upgrades work, we need to ensure the old role is removed. It also makes sense to clean up old roles generally. This change adds support for removing stale roles during control host upgrade, currently including the following roles: stackhpc.os-flavors stackhpc.os-projects stackhpc.parted-1.1 yatesr.timezone Change-Id: I174c7e6f19cbefda56777229a2441bf6469c0982 Story: 2004252 Task: 29166 --- kayobe/ansible.py | 16 ++++++++++++++++ kayobe/cli/commands.py | 3 +++ kayobe/tests/unit/cli/test_commands.py | 4 +++- kayobe/tests/unit/test_ansible.py | 17 +++++++++++++++++ kayobe/tests/unit/test_utils.py | 14 ++++++++++++++ kayobe/utils.py | 15 +++++++++++++++ 6 files changed, 68 insertions(+), 1 deletion(-) diff --git a/kayobe/ansible.py b/kayobe/ansible.py index 48dfcdd7b..7a92b2787 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -254,3 +254,19 @@ def install_galaxy_roles(parsed_args, force=False): # Install roles from kayobe-config. utils.galaxy_install(kc_reqs_path, kc_roles_path, force=force) + + +def prune_galaxy_roles(parsed_args): + """Prune galaxy roles that are no longer necessary. + + :param parsed_args: Parsed command line arguments. + """ + LOG.info("Removing unnecessary galaxy roles from kayobe") + roles_to_remove = [ + 'stackhpc.os-flavors', + 'stackhpc.os-projects', + 'stackhpc.parted-1-1', + 'yatesr.timezone', + ] + LOG.debug("Removing roles: %s", ",".join(roles_to_remove)) + utils.galaxy_remove(roles_to_remove, "ansible/roles") diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 1b1dabe3f..8ef4b1110 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -136,6 +136,9 @@ class ControlHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command): def take_action(self, parsed_args): self.app.LOG.debug("Upgrading Kayobe Ansible control host") + # Remove roles that are no longer used. Do this before installing new + # ones, just in case a custom role dependency includes any. + ansible.prune_galaxy_roles(parsed_args) # Use force to upgrade roles. ansible.install_galaxy_roles(parsed_args, force=True) playbooks = _build_playbook_list("bootstrap") diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 67681c221..29372364b 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -51,15 +51,17 @@ class TestCase(unittest.TestCase): self.assertEqual(expected_calls, mock_run.call_args_list) @mock.patch.object(ansible, "install_galaxy_roles", autospec=True) + @mock.patch.object(ansible, "prune_galaxy_roles", autospec=True) @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") - def test_control_host_upgrade(self, mock_run, mock_install): + def test_control_host_upgrade(self, mock_run, mock_prune, mock_install): command = commands.ControlHostUpgrade(TestApp(), []) parser = command.get_parser("test") parsed_args = parser.parse_args([]) result = command.run(parsed_args) self.assertEqual(0, result) mock_install.assert_called_once_with(parsed_args, force=True) + mock_prune.assert_called_once_with(parsed_args) expected_calls = [ mock.call(mock.ANY, ["ansible/bootstrap.yml"]), mock.call(mock.ANY, ["ansible/kolla-ansible.yml"], diff --git a/kayobe/tests/unit/test_ansible.py b/kayobe/tests/unit/test_ansible.py index 56765f272..35cd0ab3d 100644 --- a/kayobe/tests/unit/test_ansible.py +++ b/kayobe/tests/unit/test_ansible.py @@ -389,3 +389,20 @@ class TestCase(unittest.TestCase): mock_is_readable.assert_called_once_with( "/etc/kayobe/ansible/requirements.yml") mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles") + + @mock.patch.object(utils, 'galaxy_remove', autospec=True) + def test_prune_galaxy_roles(self, mock_remove): + parser = argparse.ArgumentParser() + ansible.add_args(parser) + parsed_args = parser.parse_args([]) + + ansible.prune_galaxy_roles(parsed_args) + + expected_roles = [ + 'stackhpc.os-flavors', + 'stackhpc.os-projects', + 'stackhpc.parted-1-1', + 'yatesr.timezone', + ] + mock_remove.assert_called_once_with(expected_roles, + "ansible/roles") diff --git a/kayobe/tests/unit/test_utils.py b/kayobe/tests/unit/test_utils.py index 7c9efee3b..4f5029ba1 100644 --- a/kayobe/tests/unit/test_utils.py +++ b/kayobe/tests/unit/test_utils.py @@ -48,6 +48,20 @@ class TestCase(unittest.TestCase): utils.galaxy_install, "/path/to/role/file", "/path/to/roles") + @mock.patch.object(utils, "run_command") + def test_galaxy_remove(self, mock_run): + utils.galaxy_remove(["role1", "role2"], "/path/to/roles") + mock_run.assert_called_once_with(["ansible-galaxy", "remove", + "--roles-path", "/path/to/roles", + "role1", "role2"]) + + @mock.patch.object(utils, "run_command") + def test_galaxy_remove_failure(self, mock_run): + mock_run.side_effect = subprocess.CalledProcessError(1, "command") + self.assertRaises(SystemExit, + utils.galaxy_install, ["role1", "role2"], + "/path/to/roles") + @mock.patch.object(utils, "read_file") def test_read_yaml_file(self, mock_read): mock_read.return_value = """--- diff --git a/kayobe/utils.py b/kayobe/utils.py index 2a666d166..b96179502 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -51,6 +51,21 @@ def galaxy_install(role_file, roles_path, force=False): sys.exit(e.returncode) +def galaxy_remove(roles_to_remove, roles_path): + + """Remove Ansible roles via Ansible Galaxy.""" + cmd = ["ansible-galaxy", "remove"] + cmd += ["--roles-path", roles_path] + cmd += roles_to_remove + try: + run_command(cmd) + except subprocess.CalledProcessError as e: + LOG.error("Failed to remove Ansible roles %s via Ansible " + "Galaxy: returncode %d", + ",".join(roles_to_remove), e.returncode) + sys.exit(e.returncode) + + def read_file(path, mode="r"): """Read the content of a file.""" with open(path, mode) as f: