From bcc9c667471e832739c20ae06b8325ef95a5e56b Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 19 Jul 2019 21:45:52 -0500 Subject: [PATCH] Replace ansible shell with python runner This change replaces all of the ansible shell commands with the python library, ansible-runner. This library is supported by upstream ansible, is approved by the openstack foundation, is supported in global requirements, and provides a better, more programatic interface into running ansible playbooks. All tests that interacted with the old shell commands have been updated to now test using the library. Change-Id: I8db50da826e2fbc074f4e7986d6fd00f6d488648 Signed-off-by: Kevin Carter --- lower-constraints.txt | 1 + requirements.txt | 1 + tripleoclient/constants.py | 35 +- tripleoclient/tests/fakes.py | 5 + tripleoclient/tests/test_utils.py | 429 +++--------- .../overcloud_deploy/test_overcloud_deploy.py | 12 +- .../tests/v1/test_container_image.py | 26 +- .../tests/v1/tripleo/test_tripleo_deploy.py | 81 +-- .../tests/v1/tripleo/test_tripleo_upgrade.py | 54 +- .../v1/tripleo/test_tripleo_validator.py | 54 -- .../tests/v1/undercloud/minion/test_config.py | 10 +- .../v1/undercloud/test_install_upgrade.py | 45 +- .../tests/workflows/test_deployment.py | 15 +- tripleoclient/tests/workflows/test_support.py | 6 +- tripleoclient/utils.py | 631 ++++++++++++------ tripleoclient/v1/container_image.py | 80 +-- tripleoclient/v1/minion_config.py | 7 +- tripleoclient/v1/overcloud_config.py | 4 +- tripleoclient/v1/overcloud_deploy.py | 25 +- tripleoclient/v1/tripleo_deploy.py | 83 +-- tripleoclient/v1/tripleo_validator.py | 58 +- tripleoclient/v1/undercloud_config.py | 7 +- tripleoclient/v1/undercloud_preflight.py | 22 +- tripleoclient/workflows/deployment.py | 16 +- tripleoclient/workflows/support.py | 8 +- 25 files changed, 789 insertions(+), 926 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index c4a3d8f76..169309bdb 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -1,5 +1,6 @@ alembic==0.8.10 amqp==2.1.1 +ansible-runner===1.4.4 aodhclient==0.9.0 appdirs==1.3.0 asn1crypto==0.23.0 diff --git a/requirements.txt b/requirements.txt index 739e8e1dc..3392053b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ websocket-client>=0.44.0 # LGPLv2+ tripleo-common>=11.3.1 # Apache-2.0 cryptography>=2.1 # BSD/Apache-2.0 futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +ansible-runner>=1.4.4 # Apache 2.0 diff --git a/tripleoclient/constants.py b/tripleoclient/constants.py index 0f7ec6f1a..1798e1a4b 100644 --- a/tripleoclient/constants.py +++ b/tripleoclient/constants.py @@ -102,16 +102,33 @@ CTLPLANE_INSPECTION_IPRANGE_DEFAULT = '192.168.24.100,192.168.24.120' CTLPLANE_GATEWAY_DEFAULT = '192.168.24.1' CTLPLANE_DNS_NAMESERVERS_DEFAULT = [] -# Ansible parameters used for the actions being -# executed during tripleo deploy/upgrade. +# Ansible parameters used for the actions being executed during tripleo +# deploy/upgrade. Used as kwargs in the `utils.run_ansible_playbook` +# function. A playbook entry is either a string representing the name of +# one the playbook or a list of playbooks to execute. The lookup +# will search for the playbook in the work directory path. DEPLOY_ANSIBLE_ACTIONS = { - 'deploy': 'deploy_steps_playbook.yaml', - 'upgrade': 'upgrade_steps_playbook.yaml --skip-tags ' - 'validation', - 'post-upgrade': 'post_upgrade_steps_playbook.yaml ' - '--skip-tags validation', - 'online-upgrade': 'external_upgrade_steps_playbook.yaml ' - '--tags online_upgrade', + 'deploy': { + 'playbook': 'deploy_steps_playbook.yaml' + }, + 'upgrade': { + 'playbook': 'upgrade_steps_playbook.yaml', + 'skip_tags': 'validation' + }, + 'post-upgrade': { + 'playbook': 'post_upgrade_steps_playbook.yaml', + 'skip_tags': 'validation' + }, + 'online-upgrade': { + 'playbook': 'external_upgrade_steps_playbook.yaml', + 'tags': 'online_upgrade' + }, + 'preflight-deploy': { + 'playbook': 'undercloud-disk-space.yaml' + }, + 'preflight-upgrade': { + 'playbook': 'undercloud-disk-space-pre-upgrade.yaml' + }, } # Key-value pair of deprecated service and its warning message diff --git a/tripleoclient/tests/fakes.py b/tripleoclient/tests/fakes.py index 6ce95be18..e17f55878 100644 --- a/tripleoclient/tests/fakes.py +++ b/tripleoclient/tests/fakes.py @@ -80,3 +80,8 @@ class FakeClientWrapper(object): def messaging_websocket(self): return self.ws + + +def fake_ansible_runner_run_return(rc=0): + + return 'Test Status', rc diff --git a/tripleoclient/tests/test_utils.py b/tripleoclient/tests/test_utils.py index 5ae0aa6cc..fed15f1f8 100644 --- a/tripleoclient/tests/test_utils.py +++ b/tripleoclient/tests/test_utils.py @@ -38,9 +38,13 @@ import yaml from tripleoclient import exceptions from tripleoclient import utils +from tripleoclient.tests import fakes + from six.moves.configparser import ConfigParser from six.moves.urllib import error as url_error +from ansible_runner import Runner + class TestRunAnsiblePlaybook(TestCase): def setUp(self): @@ -48,329 +52,130 @@ class TestRunAnsiblePlaybook(TestCase): self.addCleanup(self.unlink_patch.stop) self.unlink_patch.start() self.mock_log = mock.Mock('logging.getLogger') - python_version = sys.version_info[0] - self.ansible_playbook_cmd = "ansible-playbook-%s" % (python_version) + self.ansible_playbook_cmd = "ansible-playbook" @mock.patch('os.path.exists', return_value=False) @mock.patch('tripleoclient.utils.run_command_and_log') def test_no_playbook(self, mock_run, mock_exists): - self.assertRaises(RuntimeError, - utils.run_ansible_playbook, - self.mock_log, - '/tmp', - 'non-existing.yaml', - 'localhost,' - ) - mock_exists.assert_called_once_with('/tmp/non-existing.yaml') + self.assertRaises( + RuntimeError, + utils.run_ansible_playbook, + 'non-existing.yaml', + 'localhost,', + '/tmp' + ) + mock_exists.assert_called_with('/tmp/non-existing.yaml') mock_run.assert_not_called() @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_subprocess_error(self, mock_run, mock_exists, mock_mkstemp): - mock_process = mock.Mock() - mock_process.returncode = 1 - mock_process.stdout.read.side_effect = ["Error\n"] - mock_run.return_value = mock_process - - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - - self.assertRaises(RuntimeError, - utils.run_ansible_playbook, - self.mock_log, - '/tmp', - 'existing.yaml', - 'localhost,' - ) - mock_run.assert_called_once_with(self.mock_log, - [self.ansible_playbook_cmd, - '-u', 'root', - '-i', 'localhost,', '-v', - '-c', 'smart', - '/tmp/existing.yaml'], - env=env, retcode_only=False) - - @mock.patch('os.path.isabs') - @mock.patch('os.path.exists', return_value=False) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_non_existing_config(self, mock_run, mock_exists, mock_isabs): - self.assertRaises(RuntimeError, - utils.run_ansible_playbook, self.mock_log, - '/tmp', 'existing.yaml', 'localhost,', - '/home/foo', '/tmp/foo.cfg' - ) - mock_exists.assert_called_with('/tmp/foo.cfg') - mock_isabs.assert_called_with('/tmp/foo.cfg') - mock_run.assert_not_called() - - @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) - @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_run_success_default(self, mock_run, mock_exists, mock_mkstemp): - mock_process = mock.Mock() - mock_process.returncode = 0 - mock_run.return_value = mock_process - - retcode, output = utils.run_ansible_playbook( - self.mock_log, '/tmp', 'existing.yaml', 'localhost,') - self.assertEqual(retcode, 0) - mock_exists.assert_called_once_with('/tmp/existing.yaml') - - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - - mock_run.assert_called_once_with(self.mock_log, - [self.ansible_playbook_cmd, - '-u', 'root', - '-i', 'localhost,', '-v', - '-c', 'smart', - '/tmp/existing.yaml'], - env=env, retcode_only=False) - - @mock.patch('os.path.isabs') - @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_run_success_ansible_cfg(self, mock_run, mock_exists, mock_isabs): - mock_process = mock.Mock() - mock_process.returncode = 0 - mock_run.return_value = mock_process - - retcode, output = utils.run_ansible_playbook( - self.mock_log, - '/tmp', + @mock.patch('os.makedirs') + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return(rc=1) + ) + def test_subprocess_error(self, mock_run, mock_mkdirs, mock_exists, + mock_mkstemp): + self.assertRaises( + RuntimeError, + utils.run_ansible_playbook, 'existing.yaml', 'localhost,', - ansible_config='/tmp/foo.cfg') - self.assertEqual(retcode, 0) - - mock_isabs.assert_called_once_with('/tmp/foo.cfg') - - exist_calls = [mock.call('/tmp/foo.cfg'), - mock.call('/tmp/existing.yaml')] - mock_exists.assert_has_calls(exist_calls, any_order=False) - - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/foo.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - - mock_run.assert_called_once_with(self.mock_log, - [self.ansible_playbook_cmd, - '-u', 'root', - '-i', 'localhost,', '-v', - '-c', 'smart', - '/tmp/existing.yaml'], - env=env, retcode_only=False) + '/tmp' + ) @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_run_success_connection_local(self, mock_run, mock_exists, - mok_mkstemp): - mock_process = mock.Mock() - mock_process.returncode = 0 - mock_run.return_value = mock_process - + @mock.patch('os.makedirs') + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) + def test_run_success_default(self, mock_run, mock_mkdirs, mock_exists, + mock_mkstemp): retcode, output = utils.run_ansible_playbook( - self.mock_log, - '/tmp', - 'existing.yaml', - 'localhost,', - connection='local') + playbook='existing.yaml', + inventory='localhost,', + workdir='/tmp' + ) self.assertEqual(retcode, 0) - mock_exists.assert_called_once_with('/tmp/existing.yaml') - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - mock_run.assert_called_once_with(self.mock_log, - [self.ansible_playbook_cmd, - '-u', 'root', - '-i', 'localhost,', '-v', - '-c', 'local', - '/tmp/existing.yaml'], - env=env, retcode_only=False) + @mock.patch('os.path.exists', return_value=True) + @mock.patch('os.makedirs') + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) + def test_run_success_ansible_cfg(self, mock_run, mock_mkdirs, mock_exists): + retcode, output = utils.run_ansible_playbook( + playbook='existing.yaml', + inventory='localhost,', + workdir='/tmp' + ) + self.assertEqual(retcode, 0) @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') + @mock.patch('os.makedirs') + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) + def test_run_success_connection_local(self, mock_run, mock_mkdirs, + mock_exists, mock_mkstemp): + retcode, output = utils.run_ansible_playbook( + playbook='existing.yaml', + inventory='localhost,', + workdir='/tmp', + connection='local' + ) + self.assertEqual(retcode, 0) + + @mock.patch('os.makedirs', return_value=None) + @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) + @mock.patch('os.path.exists', return_value=True) + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) def test_run_success_gathering_policy(self, mock_run, mock_exists, - mok_mkstemp): - mock_process = mock.Mock() - mock_process.returncode = 0 - mock_run.return_value = mock_process - + mock_mkstemp, mock_makedirs): retcode, output = utils.run_ansible_playbook( - self.mock_log, - '/tmp', - 'existing.yaml', - 'localhost,', - gathering_policy='explicit') + playbook='existing.yaml', + inventory='localhost,', + workdir='/tmp', + connection='local', + gathering_policy='smart' + ) self.assertEqual(retcode, 0) - mock_exists.assert_called_once_with('/tmp/existing.yaml') - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - env['ANSIBLE_GATHERING'] = 'explicit' - - mock_run.assert_called_once_with(self.mock_log, - [self.ansible_playbook_cmd, - '-u', 'root', - '-i', 'localhost,', '-v', - '-c', 'smart', - '/tmp/existing.yaml'], - env=env, retcode_only=False) + @mock.patch('os.makedirs', return_value=None) @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg')) @mock.patch('os.path.exists', return_value=True) - @mock.patch('tripleoclient.utils.run_command_and_log') - def test_run_success_extra_vars(self, mock_run, mock_exists, mock_mkstemp): - mock_process = mock.Mock() - mock_process.returncode = 0 - mock_run.return_value = mock_process - + @mock.patch.object( + Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) + def test_run_success_extra_vars(self, mock_run, mock_exists, mock_mkstemp, + mock_makedirs): arglist = { 'var_one': 'val_one', } - retcode, output = utils.run_ansible_playbook( - self.mock_log, - '/tmp', - 'existing.yaml', - 'localhost,', - extra_vars=arglist) - + playbook='existing.yaml', + inventory='localhost,', + workdir='/tmp', + connection='local', + gathering_policy='smart', + extra_vars=arglist + ) self.assertEqual(retcode, 0) - mock_exists.assert_called_once_with('/tmp/existing.yaml') - env = os.environ.copy() - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '/usr/share/openstack-tripleo-validations/library') - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '/usr/share/openstack-tripleo-validations/lookup_plugins') - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '/usr/share/openstack-tripleo-validations/callback_plugins') - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '/usr/share/openstack-tripleo-validations/roles') - env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg' - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' - env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log' - env['TRIPLEO_PLAN_NAME'] = 'overcloud' - - mock_run.assert_called_once_with( - self.mock_log, [ - self.ansible_playbook_cmd, '-u', 'root', - '-i', 'localhost,', '-v', - '--extra-vars', '%s' % arglist, - '-c', 'smart', '/tmp/existing.yaml' - ], - env=env, - retcode_only=False) class TestRunCommandAndLog(TestCase): @@ -1170,14 +975,6 @@ class TestStoreCliParam(TestCase): def setUp(self): self.args = argparse.ArgumentParser() - @mock.patch('os.mkdir') - @mock.patch('os.path.exists') - def test_fail_to_create_file(self, mock_exists, mock_mkdir): - mock_exists.return_value = False - mock_mkdir.side_effect = OSError() - command = "undercloud install" - self.assertRaises(OSError, utils.store_cli_param, command, self.args) - @mock.patch('os.path.isdir') @mock.patch('os.path.exists') def test_exists_but_not_dir(self, mock_exists, mock_isdir): @@ -1741,36 +1538,6 @@ class TestGetLocalTimezone(TestCase): self.assertEqual('UTC', utils.get_local_timezone()) -class TestAnsibleSymlink(TestCase): - @mock.patch('tripleoclient.utils.run_command') - @mock.patch('os.path.exists', side_effect=[False, True]) - def test_ansible_symlink_needed(self, mock_path, mock_cmd): - utils.ansible_symlink() - python_version = sys.version_info[0] - ansible_playbook_cmd = "ansible-playbook-{}".format(python_version) - mock_cmd.assert_called_once_with(['sudo', 'ln', '-s', - '/usr/bin/' + ansible_playbook_cmd, - '/usr/bin/ansible-playbook'], - name='ansible-playbook-symlink') - - @mock.patch('tripleoclient.utils.run_command') - @mock.patch('os.path.exists', side_effect=[True, False]) - def test_ansible3_symlink_needed(self, mock_path, mock_cmd): - utils.ansible_symlink() - python_version = sys.version_info[0] - ansible_playbook_cmd = "ansible-playbook-{}".format(python_version) - mock_cmd.assert_called_once_with(['sudo', 'ln', '-s', - '/usr/bin/ansible-playbook', - '/usr/bin/' + ansible_playbook_cmd], - name='ansible-playbook-3-symlink') - - @mock.patch('tripleoclient.utils.run_command') - @mock.patch('os.path.exists', side_effect=[False, False]) - def test_ansible_symlink_not_needed(self, mock_path, mock_cmd): - utils.ansible_symlink() - mock_cmd.assert_not_called() - - class TestGetParamFieldName(TestCase): def test_with_empty_val_data(self): input_parameter = {} diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py index ba011d09d..ca002f887 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py +++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py @@ -262,9 +262,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): 'UndercloudHostsEntries': ['192.168.0.1 uc.ctlplane.localhost uc.ctlplane']}} - mock_rm = shutil.rmtree = mock.MagicMock() self.cmd.take_action(parsed_args) - mock_rm.assert_not_called() self.assertFalse(orchestration_client.stacks.create.called) @@ -309,6 +307,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertEqual(env_map.get('parameter_defaults'), parameters_env.get('parameter_defaults')) + @mock.patch('os.chdir') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_get_undercloud_host_entry', autospec=True, return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane') @@ -339,7 +338,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_postconfig, mock_shutil_rmtree, mock_invoke_plan_env_wf, mock_stack_network_check, - mock_get_undercloud_host_entry): + mock_get_undercloud_host_entry, + mock_chdir): fixture = deployment.DeploymentWorkflowFixture() self.useFixture(fixture) plane_management_fixture = deployment.PlanManagementFixture() @@ -446,6 +446,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): clients.tripleoclient.object_store.put_object.assert_called() self.assertTrue(mock_invoke_plan_env_wf.called) + @mock.patch('os.chdir') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_get_undercloud_host_entry', autospec=True, return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane') @@ -470,7 +471,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_parameters_env, mock_validate_args, mock_breakpoints_cleanup, mock_postconfig, mock_deprecated_params, mock_stack_network_check, - mock_get_undercloud_host_entry): + mock_get_undercloud_host_entry, + mock_chdir): fixture = deployment.DeploymentWorkflowFixture() self.useFixture(fixture) plane_management_fixture = deployment.PlanManagementFixture() @@ -526,9 +528,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_parameters_env.side_effect = _custom_create_params_env - mock_rm = shutil.rmtree = mock.MagicMock() self.cmd.take_action(parsed_args) - mock_rm.assert_called_once() execution_calls = workflow_client.executions.create.call_args_list deploy_plan_call = execution_calls[1] deploy_plan_call_input = deploy_plan_call[1]['workflow_input'] diff --git a/tripleoclient/tests/v1/test_container_image.py b/tripleoclient/tests/v1/test_container_image.py index f25cebfee..593874257 100644 --- a/tripleoclient/tests/v1/test_container_image.py +++ b/tripleoclient/tests/v1/test_container_image.py @@ -968,17 +968,21 @@ class TestContainerImageBuild(TestPluginV1): parsed_args = self.check_parser(self.cmd, arglist, verifylist) f, path = tempfile.mkstemp(dir=self.temp_dir) - with mock.patch('tempfile.mkstemp') as mock_mkstemp: - mock_mkstemp.return_value = f, path - self.cmd.take_action(parsed_args) + with mock.patch('tempfile.mkdtemp') as mock_mkd: + mock_mkd.return_value = '/tmp/testing' + with mock.patch('tempfile.mkstemp') as mock_mkstemp: + with mock.patch('os.chdir'): + mock_mkstemp.return_value = f, path + self.cmd.take_action(parsed_args) mock_builder.assert_called_once_with([ '/tmp/foo.yaml', '/tmp/bar.yaml']) mock_builder.return_value.build_images.assert_called_once_with([ self.default_kolla_conf, '/tmp/kolla.conf', path - ], [], False, None) + ], [], False, '/tmp/testing') + @mock.patch('os.chdir') @mock.patch('os.fdopen', autospec=True) @mock.patch('tempfile.mkdtemp') @mock.patch('tempfile.mkstemp') @@ -997,7 +1001,8 @@ class TestContainerImageBuild(TestPluginV1): mock_builder, mock_buildah, mock_kolla_boolean_cfg, mock_kolla_cfg, mock_mkstemp, - mock_mkdtemp, mock_fdopen): + mock_mkdtemp, mock_fdopen, + mock_chdir): arglist = [ '--config-file', '/tmp/bar.yaml', @@ -1056,16 +1061,19 @@ class TestContainerImageBuild(TestPluginV1): parsed_args = self.check_parser(self.cmd, arglist, verifylist) f, path = tempfile.mkstemp(dir=self.temp_dir) - with mock.patch('tempfile.mkstemp') as mock_mkstemp: - mock_mkstemp.return_value = f, path - self.cmd.take_action(parsed_args) + with mock.patch('tempfile.mkdtemp') as mock_mkd: + mock_mkd.return_value = '/tmp/testing' + with mock.patch('tempfile.mkstemp') as mock_mkstemp: + with mock.patch('os.chdir'): + mock_mkstemp.return_value = f, path + self.cmd.take_action(parsed_args) mock_builder.assert_called_once_with([ '/tmp/foo.yaml', '/tmp/bar.yaml']) mock_builder.return_value.build_images.assert_called_once_with([ self.default_kolla_conf, '/tmp/kolla.conf', path - ], ['foo', 'bar'], False, None) + ], ['foo', 'bar'], False, '/tmp/testing') @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder', autospec=True) diff --git a/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py b/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py index 470e3b263..60f54085b 100644 --- a/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py +++ b/tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py @@ -24,11 +24,15 @@ from heatclient import exc as hc_exc from tripleo_common.image import kolla_builder from tripleoclient import exceptions +from tripleoclient.tests import fakes from tripleoclient.tests.v1.test_plugin import TestPluginV1 # Load the plugin init module for the plugin list and show commands from tripleoclient.v1 import tripleo_deploy +import ansible_runner + + # TODO(sbaker) Remove after a tripleo-common release contains this new function if not hasattr(kolla_builder, 'container_images_prepare_multi'): setattr(kolla_builder, 'container_images_prepare_multi', mock.Mock()) @@ -47,6 +51,7 @@ class TestDeployUndercloud(TestPluginV1): # Get the command object to test self.cmd = tripleo_deploy.Deploy(self.app, None) + self.cmd.ansible_dir = '/tmp' tripleo_deploy.Deploy.heat_pid = mock.MagicMock( return_value=False) @@ -59,8 +64,7 @@ class TestDeployUndercloud(TestPluginV1): self.orc.stacks.create = mock.MagicMock( return_value={'stack': {'id': 'foo'}}) - python_version = sys.version_info[0] - self.ansible_playbook_cmd = "ansible-playbook-%s" % (python_version) + self.ansible_playbook_cmd = "ansible-playbook" @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy._is_undercloud_deploy') @mock.patch('tripleoclient.utils.check_hostname') @@ -833,37 +837,6 @@ class TestDeployUndercloud(TestPluginV1): mock_inventory.write_static_inventory.assert_called_once_with( fake_output_dir + '/inventory.yaml', extra_vars) - @mock.patch('tripleoclient.utils.' - 'run_command_and_log', autospec=True) - @mock.patch('os.chdir') - @mock.patch('os.execvp') - def test_launch_ansible_deploy(self, mock_execvp, mock_chdir, mock_run): - - self.cmd._launch_ansible('/tmp') - mock_chdir.assert_called_once() - mock_run.assert_called_once_with(self.cmd.log, [ - self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml', - 'deploy_steps_playbook.yaml']) - - @mock.patch('tripleoclient.utils.' - 'run_command_and_log', autospec=True) - @mock.patch('os.chdir') - @mock.patch('os.execvp') - def test_launch_ansible_with_args(self, mock_execvp, mock_chdir, mock_run): - - args = ['--skip-tags', 'validation'] - self.cmd._launch_ansible('/tmp', args, operation='deploy') - mock_chdir.assert_called_once() - mock_run.assert_called_once_with(self.cmd.log, [ - self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml', - 'deploy_steps_playbook.yaml', '--skip-tags', 'validation']) - - @mock.patch('os.execvp') - def test_launch_ansible_invalid_op(self, mock_execvp): - - self.assertRaises(exceptions.DeploymentError, self.cmd._launch_ansible, - '/tmp', operation='unploy') - @mock.patch('tripleo_common.image.kolla_builder.' 'container_images_prepare_multi') def test_prepare_container_images(self, mock_cipm): @@ -886,6 +859,18 @@ class TestDeployUndercloud(TestPluginV1): env ) + @mock.patch.object( + ansible_runner.runner_config.RunnerConfig, + 'prepare', + return_value=fakes.fake_ansible_runner_run_return() + ) + @mock.patch.object( + ansible_runner.Runner, + 'run', + return_value=fakes.fake_ansible_runner_run_return() + ) + @mock.patch('os.path.exists') + @mock.patch('os.chdir') @mock.patch('tripleoclient.utils.reset_cmdline') @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' '_download_stack_outputs') @@ -903,8 +888,6 @@ class TestDeployUndercloud(TestPluginV1): '_populate_templates_dir') @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' '_create_install_artifact', return_value='/tmp/foo.tar.bzip2') - @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' - '_launch_ansible', return_value=0) @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' '_cleanup_working_dirs') @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' @@ -928,18 +911,17 @@ class TestDeployUndercloud(TestPluginV1): @mock.patch('tripleoclient.utils.wait_for_stack_ready', return_value=True) @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.' '_set_default_plan') - @mock.patch('tripleoclient.utils.ansible_symlink') - def test_take_action_standalone(self, mock_slink, mock_def_plan, mock_poll, + def test_take_action_standalone(self, mock_def_plan, mock_poll, mock_environ, mock_geteuid, mock_puppet, mock_killheat, mock_launchheat, mock_download, mock_tht, mock_wait_for_port, mock_createdirs, - mock_cleanupdirs, mock_launchansible, - mock_tarball, mock_templates_dir, - mock_open, mock_os, mock_user, mock_cc, - mock_chmod, mock_ac, mock_outputs, - mock_cmdline): - mock_slink.side_effect = 'fake-cmd' + mock_cleanupdirs, mock_tarball, + mock_templates_dir, mock_open, mock_os, + mock_user, mock_cc, mock_chmod, mock_ac, + mock_outputs, mock_cmdline, mock_chdir, + mock_file_exists, mock_run, + mock_run_prepare): parsed_args = self.check_parser(self.cmd, ['--local-ip', '127.0.0.1', '--templates', '/tmp/thtroot', @@ -957,6 +939,7 @@ class TestDeployUndercloud(TestPluginV1): '-e', '../../../outside.yaml', '--standalone'], []) + mock_file_exists.return_value = True fake_orchestration = mock_launchheat(parsed_args) self.cmd.take_action(parsed_args) mock_createdirs.assert_called_once() @@ -967,16 +950,12 @@ class TestDeployUndercloud(TestPluginV1): mock_download.assert_called_with(self.cmd, fake_orchestration, 'undercloud', 'Undercloud', sys.executable) - mock_launchansible.assert_called_once() mock_tarball.assert_called_once() mock_cleanupdirs.assert_called_once() self.assertEqual(mock_killheat.call_count, 2) - mock_cmdline.assert_called_once() @mock.patch('tripleoclient.utils.reset_cmdline') - @mock.patch('tripleoclient.utils.ansible_symlink') - def test_take_action(self, mock_slink, mock_cmdline): - mock_slink.side_effect = 'fake-cmd' + def test_take_action(self, mock_cmdline): parsed_args = self.check_parser(self.cmd, ['--local-ip', '127.0.0.1', '--templates', '/tmp/thtroot', @@ -984,14 +963,11 @@ class TestDeployUndercloud(TestPluginV1): '--output-dir', '/my'], []) self.assertRaises(exceptions.DeploymentError, self.cmd.take_action, parsed_args) - mock_cmdline.assert_called_once() @mock.patch('tripleoclient.utils.reset_cmdline') @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy._standalone_deploy', return_value=1) - @mock.patch('tripleoclient.utils.ansible_symlink') - def test_take_action_failure(self, mock_slink, mock_deploy, mock_cmdline): - mock_slink.side_effect = 'fake-cmd' + def test_take_action_failure(self, mock_deploy, mock_cmdline): parsed_args = self.check_parser(self.cmd, ['--local-ip', '127.0.0.1', '--templates', '/tmp/thtroot', @@ -1000,7 +976,6 @@ class TestDeployUndercloud(TestPluginV1): '--standalone'], []) self.assertRaises(exceptions.DeploymentError, self.cmd.take_action, parsed_args) - mock_cmdline.assert_called_once() @mock.patch('os.path.isfile', return_value=False) def test_set_stack_action_default_create(self, mock_isfile): diff --git a/tripleoclient/tests/v1/tripleo/test_tripleo_upgrade.py b/tripleoclient/tests/v1/tripleo/test_tripleo_upgrade.py index d86fd16bf..388fa6bf4 100644 --- a/tripleoclient/tests/v1/tripleo/test_tripleo_upgrade.py +++ b/tripleoclient/tests/v1/tripleo/test_tripleo_upgrade.py @@ -14,7 +14,6 @@ # import mock -import sys from osc_lib.tests import utils import six @@ -31,48 +30,8 @@ class TestUpgrade(utils.TestCommand): # Get the command object to test self.cmd = tripleo_upgrade.Upgrade(self.app, None) - - python_version = sys.version_info[0] - self.ansible_playbook_cmd = "ansible-playbook-%s" % (python_version) - - @mock.patch('tripleoclient.utils.' - 'run_command_and_log', autospec=True) - @mock.patch('os.chdir') - @mock.patch('os.execvp') - def test_launch_ansible_upgrade(self, mock_execvp, mock_chdir, mock_run): - - self.cmd._launch_ansible('/tmp', operation='upgrade') - mock_chdir.assert_called_once() - mock_run.assert_called_once_with(self.cmd.log, [ - self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml', - 'upgrade_steps_playbook.yaml', - '--skip-tags', 'validation']) - - @mock.patch('tripleoclient.utils.' - 'run_command_and_log', autospec=True) - @mock.patch('os.chdir') - @mock.patch('os.execvp') - def test_launch_ansible_post_upgrade(self, mock_execvp, mock_chdir, - mock_run): - self.cmd._launch_ansible('/tmp', operation='post-upgrade') - mock_chdir.assert_called_once() - mock_run.assert_called_once_with(self.cmd.log, [ - self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml', - 'post_upgrade_steps_playbook.yaml', - '--skip-tags', 'validation']) - - @mock.patch('tripleoclient.utils.' - 'run_command_and_log', autospec=True) - @mock.patch('os.chdir') - @mock.patch('os.execvp') - def test_launch_ansible_online_upgrade(self, mock_execvp, mock_chdir, - mock_run): - self.cmd._launch_ansible('/tmp', operation='online-upgrade') - mock_chdir.assert_called_once() - mock_run.assert_called_once_with(self.cmd.log, [ - self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml', - 'external_upgrade_steps_playbook.yaml', - '--tags', 'online_upgrade']) + self.cmd.ansible_dir = '/tmp' + self.ansible_playbook_cmd = "ansible-playbook" @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.take_action', autospec=True) @@ -127,9 +86,7 @@ class TestUpgrade(utils.TestCommand): @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy', autospec=True) @mock.patch('sys.stdin', spec=six.StringIO) - @mock.patch('tripleoclient.utils.ansible_symlink') - def test_take_action_prompt_no(self, mock_slink, mock_stdin, mock_deploy): - mock_slink.side_effect = 'fake-cmd' + def test_take_action_prompt_no(self, mock_stdin, mock_deploy): mock_stdin.isatty.return_value = True mock_stdin.readline.return_value = 'n' parsed_args = self.check_parser(self.cmd, @@ -153,10 +110,7 @@ class TestUpgrade(utils.TestCommand): @mock.patch('tripleoclient.v1.tripleo_deploy.Deploy', autospec=True) @mock.patch('sys.stdin', spec=six.StringIO) - @mock.patch('tripleoclient.utils.ansible_symlink') - def test_take_action_prompt_invalid_option(self, mock_slink, mock_stdin, - mock_deploy): - mock_slink.side_effect = 'fake-cmd' + def test_take_action_prompt_invalid_option(self, mock_stdin, mock_deploy): mock_stdin.isatty.return_value = True mock_stdin.readline.return_value = 'Dontwant' parsed_args = self.check_parser(self.cmd, diff --git a/tripleoclient/tests/v1/tripleo/test_tripleo_validator.py b/tripleoclient/tests/v1/tripleo/test_tripleo_validator.py index 36637b315..6cbb61c45 100644 --- a/tripleoclient/tests/v1/tripleo/test_tripleo_validator.py +++ b/tripleoclient/tests/v1/tripleo/test_tripleo_validator.py @@ -14,7 +14,6 @@ # import mock -import sys from osc_lib.tests import utils from tripleoclient.v1 import tripleo_validator @@ -115,56 +114,3 @@ class TestValidatorShowParameter(utils.TestCommand): parsed_args = self.check_parser(self.cmd, arglist, verifylist) self.cmd.take_action(parsed_args) - - -class TestValidatorRun(utils.TestCommand): - - def setUp(self): - super(TestValidatorRun, self).setUp() - - # Get the command object to test - self.cmd = tripleo_validator.TripleOValidatorRun(self.app, None) - - @mock.patch('sys.exit') - @mock.patch('logging.getLogger') - @mock.patch('pwd.getpwuid') - @mock.patch('os.getuid') - @mock.patch('tripleoclient.utils.get_tripleo_ansible_inventory', - return_value='/home/stack/inventory.yaml') - @mock.patch('tripleoclient.utils.run_ansible_playbook', - autospec=True) - def test_validation_run_with_ansible(self, plan_mock, mock_inventory, - mock_getuid, mock_getpwuid, - mock_logger, mock_sysexit): - mock_pwuid = mock.Mock() - mock_pwuid.pw_dir = '/home/stack' - mock_getpwuid.return_value = mock_pwuid - - mock_log = mock.Mock() - mock_logger.return_value = mock_log - - playbooks_dir = '/usr/share/openstack-tripleo-validations/playbooks' - arglist = [ - '--validation', - 'check-ftype' - ] - verifylist = [('validation_name', ['check-ftype'])] - - parsed_args = self.check_parser(self.cmd, arglist, verifylist) - self.cmd.take_action(parsed_args) - - plan_mock.assert_called_once_with( - logger=mock_log, - plan='overcloud', - inventory='/home/stack/inventory.yaml', - workdir=playbooks_dir, - log_path_dir='/home/stack', - playbook='check-ftype.yaml', - retries=False, - output_callback='validation_output', - extra_vars={}, - python_interpreter='/usr/bin/python{}'.format(sys.version_info[0]), - gathering_policy='explicit' - ) - - assert mock_sysexit.called diff --git a/tripleoclient/tests/v1/undercloud/minion/test_config.py b/tripleoclient/tests/v1/undercloud/minion/test_config.py index 3667e3acc..6788b94c3 100644 --- a/tripleoclient/tests/v1/undercloud/minion/test_config.py +++ b/tripleoclient/tests/v1/undercloud/minion/test_config.py @@ -36,8 +36,7 @@ class TestMinionDeploy(base.TestCase): @mock.patch('tripleoclient.v1.minion_config._process_undercloud_passwords') @mock.patch('tripleoclient.v1.undercloud_preflight.minion_check') - @mock.patch('tripleoclient.utils.ansible_symlink') - @mock.patch('os.path.isdir', return_value=True) + @mock.patch('os.makedirs', return_value=None) @mock.patch('tripleoclient.v1.minion_config._process_undercloud_output', return_value='output.yaml') @mock.patch('tripleoclient.v1.minion_config._container_images_config') @@ -46,7 +45,7 @@ class TestMinionDeploy(base.TestCase): @mock.patch('tripleoclient.utils.load_config') def test_basic_deploy(self, mock_load_config, mock_get_user, mock_write_env, mock_undercloud_output, - mock_images_config, mock_isdir, mock_ans_symlink, + mock_images_config, mock_isdir, mock_check, mock_pass): mock_get_user.return_value = 'foo' cmd = minion_config.prepare_minion_deploy() @@ -102,9 +101,8 @@ class TestMinionDeploy(base.TestCase): @mock.patch('tripleoclient.v1.minion_config._process_undercloud_passwords') @mock.patch('tripleoclient.v1.undercloud_preflight.minion_check') - @mock.patch('tripleoclient.utils.ansible_symlink') @mock.patch('os.path.exists', return_value=True) - @mock.patch('os.path.isdir', return_value=True) + @mock.patch('os.makedirs', return_value=None) @mock.patch('tripleoclient.v1.minion_config._process_undercloud_output', return_value='output.yaml') @mock.patch('tripleoclient.v1.minion_config._container_images_config') @@ -113,7 +111,7 @@ class TestMinionDeploy(base.TestCase): def test_configured_deploy(self, mock_load_config, mock_write_env, mock_undercloud_output, mock_images_config, mock_isdir, mock_exists, - mock_ans_symlink, mock_check, mock_pass): + mock_check, mock_pass): self.conf.set_default('deployment_user', 'bar') self.conf.set_default('enable_heat_engine', False) self.conf.set_default('enable_ironic_conductor', True) diff --git a/tripleoclient/tests/v1/undercloud/test_install_upgrade.py b/tripleoclient/tests/v1/undercloud/test_install_upgrade.py index 1f8d82cef..c08e64497 100644 --- a/tripleoclient/tests/v1/undercloud/test_install_upgrade.py +++ b/tripleoclient/tests/v1/undercloud/test_install_upgrade.py @@ -54,6 +54,7 @@ class TestUndercloudInstall(TestPluginV1): self.cmd = undercloud.InstallUndercloud(self.app, app_args) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('six.moves.builtins.open') @mock.patch('shutil.copy') @@ -63,7 +64,7 @@ class TestUndercloudInstall(TestPluginV1): def test_undercloud_install_default(self, mock_subprocess, mock_wr, mock_os, mock_copy, - mock_open, mock_user): + mock_open, mock_user, mock_getuid): arglist = ['--no-validations'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -118,14 +119,16 @@ class TestUndercloudInstall(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') - @mock.patch('os.mkdir') + @mock.patch('os.makedirs', return_value=None) @mock.patch('tripleoclient.utils.write_env_file', autospec=True) @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_install_with_heat_customized(self, mock_subprocess, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): self.conf.config(output_dir='/foo') self.conf.config(templates='/usertht') self.conf.config(heat_native='false') @@ -178,6 +181,7 @@ class TestUndercloudInstall(TestPluginV1): '--force-stack-update']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -200,7 +204,8 @@ class TestUndercloudInstall(TestPluginV1): mock_sroutes, mock_masq, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): self.conf.config(net_config_override='/foo/net-config.json') self.conf.config(local_interface='ethX') self.conf.config(undercloud_public_host='4.3.2.1') @@ -348,6 +353,7 @@ class TestUndercloudInstall(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('six.moves.builtins.open') @mock.patch('shutil.copy') @@ -357,7 +363,8 @@ class TestUndercloudInstall(TestPluginV1): def test_undercloud_install_with_heat_and_debug(self, mock_subprocess, mock_wr, mock_os, mock_copy, - mock_open, mock_user): + mock_open, mock_user, + mock_getuid): self.conf.config(undercloud_log_file='/foo/bar') arglist = ['--no-validations'] verifylist = [] @@ -416,6 +423,7 @@ class TestUndercloudInstall(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('six.moves.builtins.open') @mock.patch('shutil.copy') @@ -425,7 +433,8 @@ class TestUndercloudInstall(TestPluginV1): def test_undercloud_install_with_heat_true(self, mock_subprocess, mock_wr, mock_os, mock_copy, - mock_open, mock_user): + mock_open, mock_user, + mock_getuid): self.conf.config(undercloud_log_file='/foo/bar') arglist = ['--no-validations'] verifylist = [] @@ -480,6 +489,7 @@ class TestUndercloudInstall(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -487,7 +497,8 @@ class TestUndercloudInstall(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_install_with_swift_encryption(self, mock_subprocess, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations'] verifylist = [] self.conf.set_default('enable_swift_encryption', True) @@ -564,6 +575,7 @@ class TestUndercloudUpgrade(TestPluginV1): self.cmd = undercloud.UpgradeUndercloud(self.app, app_args) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -571,7 +583,8 @@ class TestUndercloudUpgrade(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_upgrade_default(self, mock_subprocess, mock_wr, - mock_os, mock_copy, mock_user): + mock_os, mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -629,6 +642,7 @@ class TestUndercloudUpgrade(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -636,7 +650,8 @@ class TestUndercloudUpgrade(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_upgrade_with_heat_enabled(self, mock_subprocess, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -693,6 +708,7 @@ class TestUndercloudUpgrade(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -700,7 +716,8 @@ class TestUndercloudUpgrade(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_upgrade_with_heat_true(self, mock_subprocess, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -757,6 +774,7 @@ class TestUndercloudUpgrade(TestPluginV1): '/usr/share/openstack-tripleo-heat-templates/' 'undercloud-stack-vstate-dropin.yaml']) + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -764,7 +782,8 @@ class TestUndercloudUpgrade(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_upgrade_with_heat_and_yes(self, mock_subprocess, mock_wr, mock_os, - mock_user, mock_copy): + mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations', '-y'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) @@ -822,6 +841,7 @@ class TestUndercloudUpgrade(TestPluginV1): 'undercloud-stack-vstate-dropin.yaml']) # TODO(cjeanner) drop once we have proper oslo.privsep + @mock.patch('os.geteuid', return_value=1001) @mock.patch('getpass.getuser', return_value='stack') @mock.patch('shutil.copy') @mock.patch('os.mkdir') @@ -829,7 +849,8 @@ class TestUndercloudUpgrade(TestPluginV1): @mock.patch('subprocess.check_call', autospec=True) def test_undercloud_upgrade_with_heat_and_debug(self, mock_subprocess, mock_wr, mock_os, - mock_copy, mock_user): + mock_copy, mock_user, + mock_getuid): arglist = ['--no-validations'] verifylist = [] parsed_args = self.check_parser(self.cmd, arglist, verifylist) diff --git a/tripleoclient/tests/workflows/test_deployment.py b/tripleoclient/tests/workflows/test_deployment.py index 57b1c888f..90727a6cc 100644 --- a/tripleoclient/tests/workflows/test_deployment.py +++ b/tripleoclient/tests/workflows/test_deployment.py @@ -46,14 +46,16 @@ class TestDeploymentWorkflows(utils.TestCommand): "message": "Fail.", }]) + @mock.patch('os.chdir') @mock.patch('tripleoclient.workflows.deployment.wait_for_ssh_port') @mock.patch('tripleoclient.workflows.deployment.time.sleep') - @mock.patch('tripleoclient.workflows.deployment.shutil.rmtree') + @mock.patch('tripleoclient.utils.shutil.rmtree') @mock.patch('tripleoclient.workflows.deployment.open') - @mock.patch('tripleoclient.workflows.deployment.tempfile') + @mock.patch('tripleoclient.utils.tempfile') @mock.patch('tripleoclient.workflows.deployment.subprocess.check_call') def test_enable_ssh_admin(self, mock_check_call, mock_tempfile, mock_open, - mock_rmtree, mock_sleep, mock_wait_for_ssh_port): + mock_rmtree, mock_sleep, mock_wait_for_ssh_port, + mock_chdir): log = mock.Mock() hosts = 'a', 'b', 'c' ssh_user = 'test-user' @@ -86,15 +88,16 @@ class TestDeploymentWorkflows(utils.TestCommand): self.assertEqual(1, mock_rmtree.call_count) self.assertEqual('/foo', mock_rmtree.call_args[0][0]) + @mock.patch('os.chdir') @mock.patch('tripleoclient.workflows.deployment.wait_for_ssh_port') @mock.patch('tripleoclient.workflows.deployment.time.sleep') - @mock.patch('tripleoclient.workflows.deployment.shutil.rmtree') + @mock.patch('tripleoclient.utils.shutil.rmtree') @mock.patch('tripleoclient.workflows.deployment.open') - @mock.patch('tripleoclient.workflows.deployment.tempfile') + @mock.patch('tripleoclient.utils.tempfile') @mock.patch('tripleoclient.workflows.deployment.subprocess.check_call') def test_enable_ssh_admin_error(self, mock_check_call, mock_tempfile, mock_open, mock_rmtree, mock_sleep, - mock_wait_for_ssh_port): + mock_wait_for_ssh_port, mock_chdir): log = mock.Mock() hosts = 'a', 'b', 'c' ssh_user = 'test-user' diff --git a/tripleoclient/tests/workflows/test_support.py b/tripleoclient/tests/workflows/test_support.py index 2b2c20afa..429b61313 100644 --- a/tripleoclient/tests/workflows/test_support.py +++ b/tripleoclient/tests/workflows/test_support.py @@ -155,11 +155,11 @@ class TestDownloadContainer(fakes.TestDeployOvercloud): 'test', 'test') - @mock.patch('os.path.exists') - def test_download_files(self, exists_mock): + @mock.patch('os.makedirs') + def test_download_files(self, makedirs_mock): support.check_local_space = mock.MagicMock() support.check_local_space.return_value = True - exists_mock.return_value = True + makedirs_mock.return_value = None oc = self.app.client_manager.object_store oc.object_list.return_value = [ {'name': 'test1'} diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index fc192fec3..3115a56e1 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -28,13 +28,14 @@ import getpass import glob import hashlib import logging -import shutil from six.moves.configparser import ConfigParser import json import netaddr import os import os.path +import pwd +import shutil import simplejson import six import socket @@ -44,6 +45,8 @@ import tempfile import time import yaml +import ansible_runner + from heatclient.common import event_utils from heatclient.common import template_utils from heatclient.common import utils as heat_utils @@ -63,204 +66,450 @@ from tripleoclient import exceptions LOG = logging.getLogger(__name__ + ".utils") -def run_ansible_playbook(logger, - workdir, - playbook, - inventory, - log_path_dir=None, - ansible_config=None, - retries=True, - connection='smart', - output_callback='json', - python_interpreter=None, - ssh_user='root', - key=None, - module_path=None, - limit_hosts=None, - tags=None, - skip_tags=None, - verbosity=1, - extra_vars=None, - plan='overcloud', - gathering_policy=None): - """Simple wrapper for ansible-playbook +# NOTE(cloudnull): This is setting the FileExistsError for py2 environments. +# When we no longer support py2 (centos7) this should be +# removed. +try: + FileExistsError = FileExistsError +except NameError: + FileExistsError = OSError - :param logger: logger instance - :type logger: Logger - :param plan: plan name (Defaults to "overcloud") - :type plan: String +class Pushd(object): + """Simple context manager to change directories and then return.""" - :param workdir: location of the playbook - :type workdir: String + def __init__(self, directory): + """This context manager will enter and exit directories. - :param playbook: playbook filename + >>> with Pushd(directory='/tmp'): + ... with open('file', 'w') as f: + ... f.write('test') + + :param directory: path to change directory to + :type directory: `string` + """ + self.dir = directory + self.pwd = self.cwd = os.getcwd() + + def __enter__(self): + os.chdir(self.dir) + self.cwd = os.getcwd() + return self + + def __exit__(self, *args): + if self.pwd != self.cwd: + os.chdir(self.pwd) + + +class TempDirs(object): + """Simple context manager to manage temp directories.""" + + def __init__(self, dir_path=None, dir_prefix='tripleo', cleanup=True, + chdir=True): + """This context manager will create, push, and cleanup temp directories. + + >>> with TempDirs() as t: + ... with open('file', 'w') as f: + ... f.write('test') + ... print(t) + ... os.mkdir('testing') + ... with open(os.path.join(t, 'file')) as w: + ... print(w.read()) + ... with open('testing/file', 'w') as f: + ... f.write('things') + ... with open(os.path.join(t, 'testing/file')) as w: + ... print(w.read()) + + :param dir_path: path to create the temp directory + :type dir_path: `string` + :param dir_prefix: prefix to add to a temp directory + :type dir_prefix: `string` + :paramm cleanup: when enabled the temp directory will be + removed on exit. + :type cleanup: `boolean` + :param chdir: Change to/from the created temporary dir on enter/exit. + :type chdir: `boolean` + """ + + # NOTE(cloudnull): kwargs for tempfile.mkdtemp are created + # because args are not processed correctly + # in py2. When we drop py2 support (cent7) + # these args can be removed and used directly + # in the `tempfile.mkdtemp` function. + tempdir_kwargs = dict() + if dir_path: + tempdir_kwargs['dir'] = dir_path + + if dir_prefix: + tempdir_kwargs['prefix'] = dir_prefix + + self.dir = tempfile.mkdtemp(**tempdir_kwargs) + self.pushd = Pushd(directory=self.dir) + self.cleanup = cleanup + self.chdir = chdir + + def __enter__(self): + if self.chdir: + self.pushd.__enter__() + return self.dir + + def __exit__(self, *args): + if self.chdir: + self.pushd.__exit__() + if self.cleanup: + self.clean() + else: + LOG.warning("Not cleaning temporary directory [ %s ]" % self.dir) + + def clean(self): + shutil.rmtree(self.dir, ignore_errors=True) + LOG.info("Temporary directory [ %s ] cleaned up" % self.dir) + + +def _encode_envvars(env): + """Encode a hash of values. + + :param env: A hash of key=value items. + :type env: `dict`. + """ + for key, value in env.items(): + env[key] = six.text_type(value) + else: + return env + + +def makedirs(dir_path): + """Recursively make directories and log the interaction. + + :param dir_path: full path of the directories to make. + :type dir_path: `string` + :returns: `boolean` + """ + + try: + os.makedirs(dir_path) + except FileExistsError: + LOG.debug( + 'Directory "{}" was not created because it' + ' already exists.'.format( + dir_path + ) + ) + return False + else: + LOG.debug('Directory "{}" was created.'.format(dir_path)) + return True + + +def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None, + connection='smart', output_callback='yaml', + ssh_user='root', key=None, module_path=None, + limit_hosts=None, tags=None, skip_tags=None, + verbosity=0, quiet=False, extra_vars=None, + plan='overcloud', gathering_policy='smart', + extra_env_variables=None): + """Simple wrapper for ansible-playbook. + + :param playbook: Playbook filename. :type playbook: String - :param inventory: either proper inventory file, or a coma-separated list + :param inventory: Either proper inventory file, or a coma-separated list. :type inventory: String - :param ansible_config: Pass either Absolute Path, or None to generate a - temporary file, or False to not manage configuration at all - :type ansible_config: String + :param workdir: Location of the working directory. + :type workdir: String - :param log_path_dir: Dir path location for ansible log file. - Defaults to "None" - :type retries: String + :param playbook_dir: Location of the playbook directory. + (defaults to workdir). + :type playbook_dir: String - :param retries: do you want to get a retry_file? - :type retries: Boolean - - :param connection: connection type (local, smart, etc) + :param connection: Connection type (local, smart, etc). :type connection: String - :param output_callback: Callback for output format. Defaults to "json" + :param output_callback: Callback for output format. Defaults to "json". :type output_callback: String - :param python_interpreter: Absolute path for the Python interpreter - on the host where Ansible is run. - :type python_interpreter: String - - :param ssh_user: user for the ssh connection + :param ssh_user: User for the ssh connection. :type ssh_user: String - :param key: private key to use for the ssh connection + :param key: Private key to use for the ssh connection. :type key: String - :param module_path: location of the ansible module and library + :param module_path: Location of the ansible module and library. :type module_path: String - :param limit_hosts: limit the execution to the hosts + :param limit_hosts: Limit the execution to the hosts. :type limit_hosts: String - :param tags: run specific tags + :param tags: Run specific tags. :type tags: String - :param skip_tags: skip specific tags + :param skip_tags: Skip specific tags. :type skip_tags: String - :param verbosity: verbosity level for Ansible execution + :param verbosity: Verbosity level for Ansible execution. :type verbosity: Integer - :param extra_vars: set additional variables as a Dict - or the absolute path of a JSON or YAML file type + :param quiet: Disable all output (Defaults to False) + :type quiet: Boolean + + :param extra_vars: Set additional variables as a Dict or the absolute + path of a JSON or YAML file type. :type extra_vars: Either a Dict or the absolute path of JSON or YAML - :param gathering_policy: This setting controls the default policy of - fact gathering ('smart', 'implicit', 'explicit'). Defaults to None. - When not specified, the policy will be the default Ansible one, ie. - 'implicit'. - :type gathering_facts: String - """ - env = os.environ.copy() + :param plan: Plan name (Defaults to "overcloud"). + :type plan: String - env['ANSIBLE_LIBRARY'] = \ - ('/root/.ansible/plugins/modules:' - '/usr/share/ansible/plugins/modules:' - '%s/library' % constants.DEFAULT_VALIDATIONS_BASEDIR) - env['ANSIBLE_LOOKUP_PLUGINS'] = \ - ('root/.ansible/plugins/lookup:' - '/usr/share/ansible/plugins/lookup:' - '%s/lookup_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR) - env['ANSIBLE_CALLBACK_PLUGINS'] = \ - ('~/.ansible/plugins/callback:' - '/usr/share/ansible/plugins/callback:' - '%s/callback_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR) - env['ANSIBLE_ROLES_PATH'] = \ - ('/root/.ansible/roles:' - '/usr/share/ansible/roles:' - '/etc/ansible/roles:' - '%s/roles' % constants.DEFAULT_VALIDATIONS_BASEDIR) + :param gathering_policy: This setting controls the default policy of + fact gathering ('smart', 'implicit', 'explicit'). + :type gathering_facts: String + + :param extra_env_variables: Dict option to extend or override any of the + default environment variables. + :type extra_env_variables: Dict + """ + + def _playbook_check(play): + if not os.path.exists(play): + play = os.path.join(playbook_dir, play) + if not os.path.exists(play): + raise RuntimeError('No such playbook: {}'.format(play)) + LOG.debug('Ansible playbook {} found'.format(play)) + return play + + if not playbook_dir: + playbook_dir = workdir + + if isinstance(playbook, (list, set)): + playbook = [_playbook_check(play=i) for i in playbook] + LOG.info( + 'Running Ansible playbooks: {},' + ' Working directory: {},' + ' Playbook directory: {}'.format( + playbook, + workdir, + playbook_dir + ) + ) + else: + playbook = _playbook_check(play=playbook) + LOG.info( + 'Running Ansible playbook: {},' + ' Working directory: {},' + ' Playbook directory: {}'.format( + playbook, + workdir, + playbook_dir + ) + ) + + cwd = os.getcwd() + ansible_fact_path = os.path.join( + os.path.join( + tempfile.gettempdir(), + 'tripleo-ansible' + ), + 'fact_cache' + ) + makedirs(ansible_fact_path) + extravars = { + 'ansible_ssh_extra_args': ( + '-o UserKnownHostsFile={} ' + '-o StrictHostKeyChecking=no ' + '-o ControlMaster=auto ' + '-o ControlPersist=30m ' + '-o ServerAliveInterval=64 ' + '-o ServerAliveCountMax=1024 ' + '-o Compression=no ' + '-o TCPKeepAlive=yes ' + '-o VerifyHostKeyDNS=no ' + '-o ForwardX11=no ' + '-o ForwardAgent=yes ' + '-o PreferredAuthentications=publickey ' + '-T' + ).format(os.devnull) + } + if extra_vars: + if isinstance(extra_vars, dict): + extravars.update(extra_vars) + elif os.path.exists(extra_vars) and os.path.isfile(extra_vars): + with open(extra_vars) as f: + extravars.update(yaml.safe_load(f.read())) + + env = os.environ.copy() + env['ANSIBLE_DISPLAY_FAILED_STDERR'] = True + env['ANSIBLE_FORKS'] = 36 + env['ANSIBLE_TIMEOUT'] = 30 + env['ANSIBLE_GATHER_TIMEOUT'] = 45 + env['ANSIBLE_SSH_RETRIES'] = 3 + env['ANSIBLE_PIPELINING'] = True + env['ANSIBLE_REMOTE_USER'] = ssh_user + env['ANSIBLE_STDOUT_CALLBACK'] = output_callback + env['ANSIBLE_LIBRARY'] = os.path.expanduser( + '~/.ansible/plugins/modules:' + '{}:{}:' + '/usr/share/ansible/tripleo-plugins/modules:' + '/usr/share/ansible/plugins/modules:' + '/usr/share/ceph-ansible/library:' + '{}/library'.format( + os.path.join(workdir, 'modules'), + os.path.join(cwd, 'modules'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_LOOKUP_PLUGINS'] = os.path.expanduser( + '~/.ansible/plugins/lookup:' + '{}:{}:' + '/usr/share/ansible/tripleo-plugins/lookup:' + '/usr/share/ansible/plugins/lookup:' + '/usr/share/ceph-ansible/plugins/lookup:' + '{}/lookup_plugins'.format( + os.path.join(workdir, 'lookup'), + os.path.join(cwd, 'lookup'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_CALLBACK_PLUGINS'] = os.path.expanduser( + '~/.ansible/plugins/callback:' + '{}:{}:' + '/usr/share/ansible/tripleo-plugins/callback:' + '/usr/share/ansible/plugins/callback:' + '/usr/share/ceph-ansible/plugins/callback:' + '{}/callback_plugins'.format( + os.path.join(workdir, 'callback'), + os.path.join(cwd, 'callback'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_ACTION_PLUGINS'] = os.path.expanduser( + '~/.ansible/plugins/action:' + '{}:{}:' + '/usr/share/ansible/tripleo-plugins/action:' + '/usr/share/ansible/plugins/action:' + '/usr/share/ceph-ansible/plugins/actions:' + '{}/action_plugins'.format( + os.path.join(workdir, 'action'), + os.path.join(cwd, 'action'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_FILTER_PLUGINS'] = os.path.expanduser( + '~/.ansible/plugins/filter:' + '{}:{}:' + '/usr/share/ansible/tripleo-plugins/filter:' + '/usr/share/ansible/plugins/filter:' + '/usr/share/ceph-ansible/plugins/filter:' + '{}/filter_plugins'.format( + os.path.join(workdir, 'filter'), + os.path.join(cwd, 'filter'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_ROLES_PATH'] = os.path.expanduser( + '~/.ansible/roles:' + '{}:{}:' + '/usr/share/ansible/tripleo-roles:' + '/usr/share/ansible/roles:' + '/usr/share/ceph-ansible/roles:' + '/etc/ansible/roles:' + '{}/roles'.format( + os.path.join(workdir, 'roles'), + os.path.join(cwd, 'roles'), + constants.DEFAULT_VALIDATIONS_BASEDIR + ) + ) + env['ANSIBLE_CALLBACK_WHITELIST'] = 'profile_tasks,validation_output' + env['ANSIBLE_RETRY_FILES_ENABLED'] = False + env['ANSIBLE_HOST_KEY_CHECKING'] = False + env['ANSIBLE_TRANSPORT'] = connection + env['ANSIBLE_CACHE_PLUGIN_TIMEOUT'] = 7200 + + if connection == 'local': + env['ANSIBLE_PYTHON_INTERPRETER'] = sys.executable + + if gathering_policy in ('smart', 'explicit', 'implicit'): + env['ANSIBLE_GATHERING'] = gathering_policy + + if module_path: + env['ANSIBLE_LIBRARY'] = ':'.join( + [env['ANSIBLE_LIBRARY'], module_path] + ) env['TRIPLEO_PLAN_NAME'] = plan - if not log_path_dir or not os.path.exists(log_path_dir): - env['ANSIBLE_LOG_PATH'] = os.path.join(workdir, 'ansible.log') + try: + user_pwd = pwd.getpwuid(int(os.getenv('SUDO_UID', os.getuid()))) + except TypeError: + home = os.path.expanduser('~') else: - env['ANSIBLE_LOG_PATH'] = os.path.join(log_path_dir, 'ansible.log') + home = user_pwd.pw_dir - env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' + env['ANSIBLE_LOG_PATH'] = os.path.join(home, 'ansible.log') - if gathering_policy in ['smart', 'explicit', 'implicit']: - env['ANSIBLE_GATHERING'] = gathering_policy + if key: + env['ANSIBLE_PRIVATE_KEY_FILE'] = key - if extra_vars is None: - extra_vars = {} - - cleanup = False - if ansible_config is None: - _, tmp_config = tempfile.mkstemp(prefix=playbook, suffix='ansible.cfg') - with open(tmp_config, 'w+') as f: - f.write("[defaults]\nstdout_callback = %s\n" % output_callback) - if not retries: - f.write("retry_files_enabled = False\n") - f.close() - env['ANSIBLE_CONFIG'] = tmp_config - cleanup = True - - elif os.path.isabs(ansible_config): - if os.path.exists(ansible_config): - env['ANSIBLE_CONFIG'] = ansible_config + if extra_env_variables: + if not isinstance(extra_env_variables, dict): + msg = "extra_env_variables must be a dict" + LOG.error(msg) + raise SystemError(msg) else: - raise RuntimeError('No such configuration file: %s' % - ansible_config) - elif os.path.exists(os.path.join(workdir, ansible_config)): - env['ANSIBLE_CONFIG'] = os.path.join(workdir, ansible_config) + env.update(extra_env_variables) - play = os.path.join(workdir, playbook) - - if os.path.exists(play): - cmd = ["ansible-playbook-{}".format(sys.version_info[0]), - '-u', ssh_user, - '-i', inventory - ] - - if 0 < verbosity < 6: - cmd.extend(['-' + ('v' * verbosity)]) - - if key: - cmd.extend(['--private-key=%s' % key]) - - if module_path: - cmd.extend(['--module-path=%s' % module_path]) - - if limit_hosts: - cmd.extend(['-l %s' % limit_hosts]) - - if tags: - cmd.extend(['-t %s' % tags]) + with TempDirs(chdir=False) as ansible_artifact_path: + r_opts = { + 'private_data_dir': workdir, + 'project_dir': playbook_dir, + 'inventory': inventory, + 'envvars': _encode_envvars(env=env), + 'playbook': playbook, + 'verbosity': verbosity, + 'quiet': quiet, + 'extravars': extravars, + 'fact_cache': ansible_fact_path, + 'fact_cache_type': 'jsonfile', + 'artifact_dir': ansible_artifact_path, + 'rotate_artifacts': 256 + } if skip_tags: - cmd.extend(['--skip_tags %s' % skip_tags]) + r_opts['skip_tags'] = skip_tags - if python_interpreter: - cmd.extend([ - '--extra-vars', - 'ansible_python_interpreter=%s' % python_interpreter - ]) + if tags: + r_opts['tags'] = tags - if extra_vars: - if isinstance(extra_vars, dict) and extra_vars: - cmd.extend(['--extra-vars', '%s' % convert(extra_vars)]) - elif os.path.exists(extra_vars) and os.path.isfile(extra_vars): - # We don't need to check if the content of the file is - # a valid YAML or JSON, the ansible-playbook command - # will do it better - cmd.extend(['--extra-vars', '@{}'.format(extra_vars)]) - else: - raise RuntimeError('No such extra vars file: %s' % extra_vars) + if limit_hosts: + r_opts['limit'] = limit_hosts - cmd.extend(['-c', connection, play]) + runner_config = ansible_runner.runner_config.RunnerConfig(**r_opts) + runner_config.prepare() + runner = ansible_runner.Runner(config=runner_config) + status, rc = runner.run() - proc = run_command_and_log(logger, cmd, env=env, retcode_only=False) - proc.wait() - cleanup and os.unlink(tmp_config) - if proc.returncode != 0: - raise RuntimeError(proc.stdout.read()) - return proc.returncode, proc.stdout.read() + if verbosity > 1: + status = runner.stdout + + if rc == 0: + LOG.info( + 'Ansible execution success. playbook: {}'.format( + playbook + ) + ) + return rc, status else: - cleanup and os.unlink(tmp_config) - raise RuntimeError('No such playbook: %s' % play) + err_msg = ( + 'Ansible execution failed. playbook: {},' + ' Run Status: {},' + ' Return Code: {}'.format( + playbook, + status, + rc + ) + ) + if not quiet: + LOG.error(err_msg) + raise RuntimeError(err_msg) def convert(data): @@ -398,13 +647,7 @@ def store_cli_param(command_name, parsed_args): command_name = command_name.replace(" ", "-") history_path = os.path.join(os.path.expanduser("~"), '.tripleo') - if not os.path.exists(history_path): - try: - os.mkdir(history_path) - except OSError as e: - messages = _("Unable to create TripleO history directory: " - "{0}, {1}").format(history_path, e) - raise OSError(messages) + makedirs(history_path) if os.path.isdir(history_path): try: with open(os.path.join(history_path, @@ -1190,14 +1433,14 @@ def run_update_ansible_action(log, clients, nodes, inventory, tags=tags, skip_tags=skip_tags, verbosity=verbosity, extra_vars=extra_vars) else: - run_ansible_playbook(logger=LOG, - workdir=workdir, - playbook=book, + run_ansible_playbook(playbook=book, inventory=inventory, + workdir=workdir, ssh_user=ssh_user, key=ssh_private_key(workdir, priv_key), module_path='/usr/share/ansible-modules', - limit_hosts=nodes, tags=tags, + limit_hosts=nodes, + tags=tags, skip_tags=skip_tags) @@ -1339,24 +1582,14 @@ def bulk_symlink(log, src, dst, tmpd='/tmp'): """ log.debug("Symlinking %s to %s, via temp dir %s" % (src, dst, tmpd)) - tmp = None - try: - if not os.path.exists(tmpd): - raise exceptions.NotFound("{} does not exist. Cannot create a " - "temp folder using this path".format( - tmpd)) - tmp = tempfile.mkdtemp(dir=tmpd) - subprocess.check_call(['mkdir', '-p', dst]) - os.chmod(tmp, 0o755) + + makedirs(dst) + with TempDirs(dir_path=tmpd) as tmp: for obj in os.listdir(src): - tmpf = os.path.join(tmp, obj) - os.symlink(os.path.join(src, obj), tmpf) - os.rename(tmpf, os.path.join(dst, obj)) - except Exception: - raise - finally: - if tmp: - shutil.rmtree(tmp, ignore_errors=True) + if not os.path.exists(os.path.join(dst, obj)): + tmpf = os.path.join(tmp, obj) + os.symlink(os.path.join(src, obj), tmpf) + os.rename(tmpf, os.path.join(dst, obj)) def run_command_and_log(log, cmd, cwd=None, env=None, retcode_only=True): @@ -1450,12 +1683,13 @@ def build_prepare_env(environment_files, environment_directories): env_url = heat_utils.normalise_file_path_to_url(path) return request.urlopen(env_url).read() - env_f, env = ( + return ( template_utils.process_multiple_environments_and_files( - env_files, env_path_is_object=lambda path: True, - object_request=get_env_file)) - - return env + env_files, + env_path_is_object=lambda path: True, + object_request=get_env_file + ) + )[1] def rel_or_abs_path(file_path, tht_root): @@ -1812,7 +2046,7 @@ def parse_all_validations_on_disk(path, groups=None): validations_abspath = glob.glob("{path}/*.yaml".format(path=path)) for pl in validations_abspath: - validation_id, ext = os.path.splitext(os.path.basename(pl)) + validation_id, _ext = os.path.splitext(os.path.basename(pl)) with open(pl, 'r') as val_playbook: contents = yaml.safe_load(val_playbook) @@ -1910,23 +2144,6 @@ def get_local_timezone(): return timezone -def ansible_symlink(): - # https://bugs.launchpad.net/tripleo/+bug/1812837 - python_version = sys.version_info[0] - ansible_playbook_cmd = "ansible-playbook-{}".format(python_version) - cmd = ['sudo', 'ln', '-s'] - if not os.path.exists('/usr/bin/ansible-playbook'): - if os.path.exists('/usr/bin/' + ansible_playbook_cmd): - cmd.extend(['/usr/bin/' + ansible_playbook_cmd, - '/usr/bin/ansible-playbook']) - run_command(cmd, name='ansible-playbook-symlink') - else: - if not os.path.exists('/usr/bin/' + ansible_playbook_cmd): - cmd.extend(['/usr/bin/ansible-playbook', - '/usr/bin/' + ansible_playbook_cmd]) - run_command(cmd, name='ansible-playbook-3-symlink') - - def check_file_for_enabled_service(env_file): """Checks environment file for the said service. @@ -1979,7 +2196,7 @@ def reset_cmdline(): output = '' try: output = run_command(['reset', '-I']) - except RuntimeError as e: + except RuntimeError: LOG.warning('Unable to reset command line. Try manually running ' '"reset" if the command line is broken.') sys.stdout.write(output) @@ -2003,7 +2220,7 @@ def safe_write(path, data): f.write(data) except OSError as error: if error.errno != errno.EEXIST: - msg = _('The output file %(file)s can not be ' - 'created. Error: %(msg)') % {'file': path, - 'msg': error.message} + msg = _( + 'The output file {file} can not be created. Error: {msg}' + ).format(file=path, msg=str(error)) raise oscexc.CommandError(msg) diff --git a/tripleoclient/v1/container_image.py b/tripleoclient/v1/container_image.py index b083c6bec..a69c0d771 100644 --- a/tripleoclient/v1/container_image.py +++ b/tripleoclient/v1/container_image.py @@ -194,45 +194,49 @@ class BuildImage(command.Command): tmp.write('list_dependencies=true') kolla_config_files = list(parsed_args.kolla_config_files) kolla_config_files.append(path) - kolla_tmp_dir = None - if parsed_args.use_buildah: - # A temporary directory is needed to let Kolla generates the - # Dockerfiles that will be used by Buildah to build the images. - kolla_tmp_dir = tempfile.mkdtemp(prefix='kolla-') + with utils.TempDirs(dir_prefix='kolla-') as kolla_tmp_dir: + try: + builder = kolla_builder.KollaImageBuilder( + parsed_args.config_files + ) + result = builder.build_images(kolla_config_files, + parsed_args.excludes, + parsed_args.use_buildah, + kolla_tmp_dir) - try: - builder = kolla_builder.KollaImageBuilder(parsed_args.config_files) - result = builder.build_images(kolla_config_files, - parsed_args.excludes, - parsed_args.use_buildah, - kolla_tmp_dir) - - if parsed_args.use_buildah: - deps = json.loads(result) - kolla_cfg = utils.get_read_config(kolla_config_files) - bb = buildah.BuildahBuilder( - kolla_tmp_dir, deps, - utils.get_from_cfg(kolla_cfg, "base"), - utils.get_from_cfg(kolla_cfg, "type"), - utils.get_from_cfg(kolla_cfg, "tag"), - utils.get_from_cfg(kolla_cfg, "namespace"), - utils.get_from_cfg(kolla_cfg, "registry"), - utils.getboolean_from_cfg(kolla_cfg, "push")) - bb.build_all() - elif parsed_args.list_dependencies: - deps = json.loads(result) - yaml.safe_dump(deps, self.app.stdout, indent=2, - default_flow_style=False) - elif parsed_args.list_images: - deps = json.loads(result) - images = [] - BuildImage.images_from_deps(images, deps) - yaml.safe_dump(images, self.app.stdout, - default_flow_style=False) - elif result: - self.app.stdout.write(result) - finally: - os.remove(path) + if parsed_args.use_buildah: + deps = json.loads(result) + kolla_cfg = utils.get_read_config(kolla_config_files) + bb = buildah.BuildahBuilder( + kolla_tmp_dir, deps, + utils.get_from_cfg(kolla_cfg, "base"), + utils.get_from_cfg(kolla_cfg, "type"), + utils.get_from_cfg(kolla_cfg, "tag"), + utils.get_from_cfg(kolla_cfg, "namespace"), + utils.get_from_cfg(kolla_cfg, "registry"), + utils.getboolean_from_cfg(kolla_cfg, "push")) + bb.build_all() + elif parsed_args.list_dependencies: + deps = json.loads(result) + yaml.safe_dump( + deps, + self.app.stdout, + indent=2, + default_flow_style=False + ) + elif parsed_args.list_images: + deps = json.loads(result) + images = [] + BuildImage.images_from_deps(images, deps) + yaml.safe_dump( + images, + self.app.stdout, + default_flow_style=False + ) + elif result: + self.app.stdout.write(result) + finally: + os.remove(path) class PrepareImageFiles(command.Command): diff --git a/tripleoclient/v1/minion_config.py b/tripleoclient/v1/minion_config.py index 9f26daf42..2148a2a9d 100644 --- a/tripleoclient/v1/minion_config.py +++ b/tripleoclient/v1/minion_config.py @@ -148,8 +148,7 @@ def prepare_minion_deploy(upgrade=False, no_validations=False, # we move any user provided environment files into this root later. tempdir = os.path.join(os.path.abspath(CONF['output_dir']), 'tripleo-config-generated-env-files') - if not os.path.isdir(tempdir): - os.mkdir(tempdir) + utils.makedirs(tempdir) env_data['PythonInterpreter'] = sys.executable @@ -275,8 +274,7 @@ def prepare_minion_deploy(upgrade=False, no_validations=False, deploy_args += ['--deployment-user', u] deploy_args += ['--output-dir=%s' % CONF['output_dir']] - if not os.path.isdir(CONF['output_dir']): - os.mkdir(CONF['output_dir']) + utils.makedirs(CONF['output_dir']) # TODO(aschultz): move this to a central class if CONF.get('net_config_override', None): @@ -345,7 +343,6 @@ def prepare_minion_deploy(upgrade=False, no_validations=False, utils.set_hostname(CONF.get('minion_hostname')) if CONF.get('minion_enable_validations') and not no_validations: - utils.ansible_symlink() undercloud_preflight.minion_check(verbose_level, upgrade) if CONF.get('custom_env_files'): diff --git a/tripleoclient/v1/overcloud_config.py b/tripleoclient/v1/overcloud_config.py index 398410970..72878c36f 100644 --- a/tripleoclient/v1/overcloud_config.py +++ b/tripleoclient/v1/overcloud_config.py @@ -19,6 +19,7 @@ from osc_lib.i18n import _ from oslo_concurrency import processutils from tripleoclient import command +from tripleoclient import utils from tripleoclient.workflows import deployment @@ -76,8 +77,7 @@ class DownloadConfig(command.Command): str(e)) raise OSError(message) - if not os.path.exists(config_dir): - os.makedirs(config_dir) + utils.makedirs(config_dir) def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index 88d562f79..63d1a1adf 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -24,7 +24,6 @@ import re import shutil import six import subprocess -import tempfile import time import yaml @@ -167,8 +166,7 @@ class DeployOvercloud(command.Command): user_env_path = os.path.join( user_env_dir, os.path.basename(abs_env_path)) self.log.debug("user_env_path=%s" % user_env_path) - if not os.path.exists(user_env_dir): - os.makedirs(user_env_dir) + utils.makedirs(user_env_dir) with open(user_env_path, 'w') as f: self.log.debug("Writing user environment %s" % user_env_path) f.write(contents) @@ -350,8 +348,7 @@ class DeployOvercloud(command.Command): if not os.path.isfile(file_path): self.log.debug("Missing in templates directory, downloading \ %s from swift into %s" % (pf, file_path)) - if not os.path.exists(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) + utils.makedirs(os.path.dirname(file_path)) # open in binary as the swiftclient get/put error under # python3 if opened as Text I/O with open(file_path, 'wb') as f: @@ -361,20 +358,16 @@ class DeployOvercloud(command.Command): # copy tht_root to temporary directory because we need to # download any missing (e.g j2 rendered) files from the plan tht_root = os.path.abspath(parsed_args.templates) - tht_tmp = tempfile.mkdtemp(prefix='tripleoclient-') - new_tht_root = "%s/tripleo-heat-templates" % tht_tmp - self.log.debug("Creating temporary templates tree in %s" - % new_tht_root) - try: + + with utils.TempDirs(dir_prefix='tripleoclient-', + cleanup=(not parsed_args.no_cleanup)) as tht_tmp: + new_tht_root = "%s/tripleo-heat-templates" % tht_tmp + self.log.debug( + "Creating temporary templates tree in %s" % new_tht_root + ) shutil.copytree(tht_root, new_tht_root, symlinks=True) self._deploy_tripleo_heat_templates(stack, parsed_args, new_tht_root, tht_root) - finally: - if parsed_args.no_cleanup: - self.log.warning("Not cleaning temporary directory %s" - % tht_tmp) - else: - shutil.rmtree(tht_tmp) def _deploy_tripleo_heat_templates(self, stack, parsed_args, tht_root, user_tht_root): diff --git a/tripleoclient/v1/tripleo_deploy.py b/tripleoclient/v1/tripleo_deploy.py index 92b5dbafd..eb11ff03d 100644 --- a/tripleoclient/v1/tripleo_deploy.py +++ b/tripleoclient/v1/tripleo_deploy.py @@ -103,7 +103,7 @@ class Deploy(command.Command): deployment_user = None ansible_dir = None python_version = sys.version_info[0] - ansible_playbook_cmd = "ansible-playbook-{}".format(python_version) + ansible_playbook_cmd = "ansible-playbook" python_cmd = "python{}".format(python_version) def _is_undercloud_deploy(self, parsed_args): @@ -256,13 +256,12 @@ class Deploy(command.Command): def _create_persistent_dirs(self): """Creates temporary working directories""" - if not os.path.exists(constants.STANDALONE_EPHEMERAL_STACK_VSTATE): - os.mkdir(constants.STANDALONE_EPHEMERAL_STACK_VSTATE) + utils.makedirs(constants.STANDALONE_EPHEMERAL_STACK_VSTATE) def _create_working_dirs(self, stack_name='undercloud'): """Creates temporary working directories""" - if self.output_dir and not os.path.exists(self.output_dir): - os.mkdir(self.output_dir) + if self.output_dir: + utils.makedirs(self.output_dir) if not self.tht_render: self.tht_render = os.path.join(self.output_dir, 'tripleo-heat-installer-templates') @@ -901,27 +900,6 @@ class Deploy(command.Command): yaml.safe_dump(output, f, default_flow_style=False) return output - # Never returns, calls exec() - def _launch_ansible(self, ansible_dir, extra_args=None, - operation="deploy"): - - if operation not in constants.DEPLOY_ANSIBLE_ACTIONS.keys(): - self.log.error(_('Operation %s is not allowed') % operation) - raise exceptions.DeploymentError('Invalid operation to run in ' - 'ansible.') - list_args = constants.DEPLOY_ANSIBLE_ACTIONS[operation].split() - - if extra_args: - list_args.extend(extra_args) - - self.log.warning(_('** Running ansible %s tasks **') % operation) - os.chdir(ansible_dir) - playbook_inventory = os.path.join(ansible_dir, 'inventory.yaml') - cmd = [self.ansible_playbook_cmd, '-i', playbook_inventory] + list_args - self.log.debug('Running Ansible %s tasks: %s' % (operation, ' ' - .join(cmd))) - return utils.run_command_and_log(self.log, cmd) - def get_parser(self, prog_name): parser = argparse.ArgumentParser( description=self.get_description(), @@ -1303,38 +1281,46 @@ class Deploy(command.Command): _('Using the existing %s for deployment') % ansible_config) shutil.copy(ansible_config, self.ansible_dir) - extra_args = [] + extra_args = dict() if not parsed_args.inflight: - extra_args = ['--skip-tags', 'opendev-validation'] + extra_args = {'skip_tags': 'opendev-validation'} # Kill heat, we're done with it now. if not parsed_args.keep_running: self._kill_heat(parsed_args) if not parsed_args.output_only: + operations = list() if parsed_args.upgrade: # Run Upgrade tasks before the deployment - rc = self._launch_ansible(self.ansible_dir, - operation='upgrade', - extra_args=extra_args) - if rc != 0: - raise exceptions.DeploymentError('Upgrade failed') - rc = self._launch_ansible(self.ansible_dir, - extra_args=extra_args) - if rc != 0: - raise exceptions.DeploymentError('Deployment failed') + operations.append( + constants.DEPLOY_ANSIBLE_ACTIONS['upgrade'] + ) + operations.append( + constants.DEPLOY_ANSIBLE_ACTIONS['deploy'] + ) if parsed_args.upgrade: # Run Post Upgrade tasks after the deployment - rc = self._launch_ansible(self.ansible_dir, - operation='post-upgrade', - extra_args=extra_args) - if rc != 0: - raise exceptions.DeploymentError('Post Upgrade failed') + operations.append( + constants.DEPLOY_ANSIBLE_ACTIONS['post-upgrade'] + ) # Run Online Upgrade tasks after the deployment - rc = self._launch_ansible(self.ansible_dir, - operation='online-upgrade', - extra_args=extra_args) - if rc != 0: - raise exceptions.DeploymentError( - 'Online Upgrade failed') + operations.append( + constants.DEPLOY_ANSIBLE_ACTIONS['online-upgrade'] + ) + with utils.Pushd(self.ansible_dir): + for operation in operations: + for k, v in extra_args.items(): + if k in operation: + operation[k] = ','.join([operation[k], v]) + else: + operation[k] = v + rc = utils.run_ansible_playbook( + inventory=os.path.join( + self.ansible_dir, + 'inventory.yaml' + ), + workdir=self.ansible_dir, + **operation + )[0] except Exception as e: self.log.error("Exception: %s" % six.text_type(e)) self.log.error(traceback.print_exc()) @@ -1415,7 +1401,6 @@ class Deploy(command.Command): def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) - utils.ansible_symlink() unconf_msg = _('User did not confirm upgrade, so exiting. ' 'Consider using the --yes parameter if you ' 'prefer to skip this warning in the future') diff --git a/tripleoclient/v1/tripleo_validator.py b/tripleoclient/v1/tripleo_validator.py index 9a947683d..4af6b3407 100644 --- a/tripleoclient/v1/tripleo_validator.py +++ b/tripleoclient/v1/tripleo_validator.py @@ -17,7 +17,6 @@ import argparse import json import logging import os -import pwd import six import sys import textwrap @@ -363,23 +362,6 @@ class TripleOValidatorRun(command.Command): return parser - def _run_ansible(self, logger, plan, workdir, log_path_dir, playbook, - inventory, retries, output_callback, extra_vars, - python_interpreter, gathering_policy): - rc, output = oooutils.run_ansible_playbook( - logger=logger, - plan=plan, - workdir=workdir, - log_path_dir=log_path_dir, - playbook=playbook, - inventory=inventory, - retries=retries, - output_callback=output_callback, - extra_vars=extra_vars, - python_interpreter=python_interpreter, - gathering_policy=gathering_policy) - return rc, output - def _run_validator_run(self, parsed_args): LOG = logging.getLogger(__name__ + ".ValidationsRunAnsible") playbooks = [] @@ -407,9 +389,6 @@ class TripleOValidatorRun(command.Command): for pb in parsed_args.validation_name: playbooks.append(pb + '.yaml') - python_interpreter = \ - "/usr/bin/python{}".format(sys.version_info[0]) - static_inventory = oooutils.get_tripleo_ansible_inventory( ssh_user='heat-admin', stack=parsed_args.plan, @@ -418,28 +397,27 @@ class TripleOValidatorRun(command.Command): failed_val = False - with ThreadPoolExecutor(max_workers=parsed_args.workers) as executor: - LOG.debug(_('Running the validations with Ansible')) - tasks_exec = { - executor.submit( - self._run_ansible, - logger=LOG, - plan=parsed_args.plan, - workdir=constants.ANSIBLE_VALIDATION_DIR, - log_path_dir=pwd.getpwuid(os.getuid()).pw_dir, - playbook=playbook, - inventory=static_inventory, - retries=False, - output_callback='validation_output', - extra_vars=extra_vars_input, - python_interpreter=python_interpreter, - gathering_policy='explicit'): playbook - for playbook in playbooks - } + with oooutils.TempDirs() as tmp: + with ThreadPoolExecutor(max_workers=parsed_args.workers) as exe: + LOG.debug(_('Running the validations with Ansible')) + tasks_exec = { + exe.submit( + oooutils.run_ansible_playbook, + plan=parsed_args.plan, + workdir=tmp, + playbook=playbook, + playbook_dir=constants.ANSIBLE_VALIDATION_DIR, + inventory=static_inventory, + output_callback='validation_output', + quiet=True, + extra_vars=extra_vars_input, + gathering_policy='explicit'): playbook + for playbook in playbooks + } for tk, pl in six.iteritems(tasks_exec): try: - rc, output = tk.result() + _rc, output = tk.result() print('[SUCCESS] - {}\n{}'.format(pl, oooutils.indent(output))) except Exception as e: failed_val = True diff --git a/tripleoclient/v1/undercloud_config.py b/tripleoclient/v1/undercloud_config.py index 60d9656f2..2350701ba 100644 --- a/tripleoclient/v1/undercloud_config.py +++ b/tripleoclient/v1/undercloud_config.py @@ -417,8 +417,7 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=True, # we move any user provided environment files into this root later. tempdir = os.path.join(os.path.abspath(CONF['output_dir']), 'tripleo-config-generated-env-files') - if not os.path.isdir(tempdir): - os.mkdir(tempdir) + utils.makedirs(tempdir) # Set the undercloud home dir parameter so that stackrc is produced in # the users home directory. @@ -705,8 +704,7 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=True, deploy_args += ['--deployment-user', u] deploy_args += ['--output-dir=%s' % CONF['output_dir']] - if not os.path.isdir(CONF['output_dir']): - os.mkdir(CONF['output_dir']) + utils.makedirs(CONF['output_dir']) if CONF.get('cleanup'): deploy_args.append('--cleanup') @@ -782,7 +780,6 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=True, deploy_args += ['--hieradata-override=%s' % data_file] if CONF.get('enable_validations') and not no_validations: - utils.ansible_symlink() undercloud_preflight.check(verbose_level, upgrade) deploy_args += ['-e', os.path.join( tht_templates, "environments/tripleo-validations.yaml")] diff --git a/tripleoclient/v1/undercloud_preflight.py b/tripleoclient/v1/undercloud_preflight.py index 3eff0ff27..59d02da6f 100644 --- a/tripleoclient/v1/undercloud_preflight.py +++ b/tripleoclient/v1/undercloud_preflight.py @@ -84,19 +84,19 @@ def _check_diskspace(upgrade=False): Second one checks minimal disk space for an upgrade. """ if upgrade: - playbook = 'undercloud-disk-space-pre-upgrade.yaml' + playbook_args = constants.DEPLOY_ANSIBLE_ACTIONS['preflight-upgrade'] else: - playbook = 'undercloud-disk-space.yaml' + playbook_args = constants.DEPLOY_ANSIBLE_ACTIONS['preflight-deploy'] - python_interpreter = "/usr/bin/python{}".format(sys.version_info[0]) - utils.run_ansible_playbook(logger=LOG, - workdir=constants.ANSIBLE_VALIDATION_DIR, - playbook=playbook, - inventory='undercloud,', - retries=False, - connection='local', - output_callback='validation_output', - python_interpreter=python_interpreter) + with utils.TempDirs() as tmp: + utils.run_ansible_playbook( + workdir=tmp, + inventory='undercloud,', + connection='local', + output_callback='validation_output', + playbook_dir=constants.ANSIBLE_VALIDATION_DIR, + **playbook_args + ) def _check_memory(): diff --git a/tripleoclient/workflows/deployment.py b/tripleoclient/workflows/deployment.py index 9ce243211..f004d7663 100644 --- a/tripleoclient/workflows/deployment.py +++ b/tripleoclient/workflows/deployment.py @@ -14,10 +14,8 @@ from __future__ import print_function import copy import os import pprint -import shutil import socket import subprocess -import tempfile import time from heatclient.common import event_utils @@ -219,7 +217,7 @@ def get_hosts_and_enable_ssh_admin(log, clients, stack, overcloud_ssh_network, "Check if the user/ip are corrects.\n".format(hosts)) else: log.error("Unknown error. " - "Original message is:\n{}".format(hosts, e.message)) + "Original message is:\n{} {}".format(hosts, e)) else: raise exceptions.DeploymentError("Cannot find any hosts on '{}'" @@ -239,12 +237,11 @@ def enable_ssh_admin(log, clients, plan_name, hosts, ssh_user, ssh_key): "-o StrictHostKeyChecking=no " "-o PasswordAuthentication=no " "-o UserKnownHostsFile=/dev/null") - tmp_key_dir = tempfile.mkdtemp() - tmp_key_private = os.path.join(tmp_key_dir, 'id_rsa') - tmp_key_public = os.path.join(tmp_key_dir, 'id_rsa.pub') - tmp_key_comment = "TripleO split stack short term key" - try: + with utils.TempDirs() as tmp_key_dir: + tmp_key_private = os.path.join(tmp_key_dir, 'id_rsa') + tmp_key_public = os.path.join(tmp_key_dir, 'id_rsa.pub') + tmp_key_comment = "TripleO split stack short term key" tmp_key_command = ["ssh-keygen", "-N", "", "-t", "rsa", "-b", "4096", "-f", tmp_key_private, "-C", tmp_key_comment] DEVNULL = open(os.devnull, 'w') @@ -324,9 +321,6 @@ def enable_ssh_admin(log, clients, plan_name, hosts, ssh_user, ssh_key): tmp_key_comment] print("Removing TripleO short term key from %s" % host) subprocess.check_call(rm_tmp_key_command, stderr=subprocess.STDOUT) - finally: - print("Removing short term keys locally") - shutil.rmtree(tmp_key_dir) print("Enabling ssh admin - COMPLETE.") diff --git a/tripleoclient/workflows/support.py b/tripleoclient/workflows/support.py index 2f9dab342..4080bd5b4 100644 --- a/tripleoclient/workflows/support.py +++ b/tripleoclient/workflows/support.py @@ -20,6 +20,7 @@ from osc_lib.i18n import _ from tripleoclient.exceptions import ContainerDeleteFailed from tripleoclient.exceptions import DownloadError from tripleoclient.exceptions import LogFetchError +from tripleoclient import utils from tripleoclient.workflows import base @@ -44,9 +45,10 @@ def download_files(clients, container_name, destination): if not os.path.dirname(destination): destination = os.path.join(os.sep, os.getcwd(), destination) - if not os.path.exists(destination): - print('Creating destination path: {}'.format(destination)) - os.makedirs(destination) + if utils.makedirs(destination): + print('Created destination path: {}'.format(destination)) + else: + print('Destination path exists: {}'.format(destination)) if not check_local_space(destination, object_list): raise DownloadError(_('Not enough local space to download files.'))