diff --git a/requirements.txt b/requirements.txt index 3675ff772..6e7279e54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ python-openstackclient>=5.2.0 # Apache-2.0 simplejson>=3.5.1 # MIT six>=1.10.0 # MIT osc-lib>=1.8.0 # Apache-2.0 -websocket-client>=0.44.0 # LGPLv2+ tripleo-common>=12.7.0 # Apache-2.0 cryptography>=2.1 # BSD/Apache-2.0 ansible-runner>=1.4.5 # Apache 2.0 diff --git a/tripleoclient/exceptions.py b/tripleoclient/exceptions.py index 9141c3e11..4ce941426 100644 --- a/tripleoclient/exceptions.py +++ b/tripleoclient/exceptions.py @@ -25,10 +25,6 @@ class WorkflowServiceError(Base): """The service type is unknown""" -class WebSocketTimeout(Base): - """Timed out waiting for messages on the websocket""" - - class WebSocketConnectionClosed(Base): """Websocket connection is closed before wait for messages""" diff --git a/tripleoclient/export.py b/tripleoclient/export.py index 3ba4b9d58..7cb03fab4 100644 --- a/tripleoclient/export.py +++ b/tripleoclient/export.py @@ -18,12 +18,12 @@ import json import logging import os import re -import sys import yaml from osc_lib.i18n import _ from tripleo_common import constants as tripleo_common_constants +from tripleo_common.utils import swift as swiftutils from tripleoclient import constants from tripleoclient import utils as oooutils @@ -35,13 +35,11 @@ def export_passwords(swift, stack, excludes=True): # Export the passwords from swift obj = 'plan-environment.yaml' container = stack - try: - resp_headers, content = swift.get_object(container, obj) - except Exception as e: - LOG.error("An error happened while exporting the password " - "file from swift: %s", str(e)) - sys.exit(1) - + content = swiftutils.get_object_string( + swift, + container=container, + object_name=obj + ) data = yaml.safe_load(content) # The "passwords" key in plan-environment.yaml are generated passwords, # they are not necessarily the actual password values used during the diff --git a/tripleoclient/plugin.py b/tripleoclient/plugin.py index 76c02449e..48cb81254 100644 --- a/tripleoclient/plugin.py +++ b/tripleoclient/plugin.py @@ -15,16 +15,10 @@ """OpenStackClient Plugin interface""" -import json import logging -import socket -import uuid from osc_lib import utils from swiftclient import client as swift_client -import websocket - -from tripleoclient import exceptions LOG = logging.getLogger(__name__) @@ -65,117 +59,6 @@ def build_option_parser(parser): return parser -class WebsocketClient(object): - - def __init__(self, instance, queue_name="tripleo", cacert=None): - self._project_id = None - self._ws = None - self._websocket_client_id = None - self._queue_name = queue_name - - endpoint = instance.get_endpoint_for_service_type( - 'messaging-websocket') - token = instance.auth.get_token(instance.session) - - self._project_id = instance.auth_ref.project_id - - self._websocket_client_id = str(uuid.uuid4()) - - LOG.debug('Instantiating messaging websocket client: %s', endpoint) - try: - if 'wss:' in endpoint and cacert: - OS_CACERT = {"ca_certs": cacert} - self._ws = websocket.create_connection(endpoint, - sslopt=OS_CACERT) - else: - self._ws = websocket.create_connection(endpoint) - except socket.error: - LOG.error("Could not establish a connection to the Zaqar " - "websocket. The command was sent but the answer " - "could not be read.") - raise - - self.send('authenticate', extra_headers={'X-Auth-Token': token}) - - # create and subscribe to a queue - # NOTE: if the queue exists it will 204 - self.send('queue_create', {'queue_name': queue_name}) - self.send('subscription_create', { - 'queue_name': queue_name, - 'ttl': 10000 - }) - - def cleanup(self): - self._ws.close() - - def send(self, action, body=None, extra_headers=None): - - headers = { - 'Client-ID': self._websocket_client_id, - 'X-Project-ID': self._project_id - } - - if extra_headers is not None: - headers.update(extra_headers) - - msg = {'action': action, 'headers': headers} - if body: - msg['body'] = body - self._ws.send(json.dumps(msg)) - data = self.recv() - if data['headers']['status'] not in (200, 201, 204): - raise RuntimeError(data) - return data - - def recv(self): - content = self._ws.recv() - try: - return json.loads(content) - except ValueError: - raise ValueError( - "No JSON object could be decoded; {0!r}".format(content)) - - def wait_for_messages(self, timeout=None): - """Wait for messages on a Zaqar queue - - A timeout can be provided in seconds, if no timeout is provided it - will block forever until a message is received. If no message is - received (for example, Zaqar is down) then it will block until manually - killed. - - If no timeout is provided this method will never stop waiting for new - messages. It is the responsibility of the consumer to stop consuming - messages. - """ - - if timeout is None: - LOG.warning("Waiting for messages on queue '{}' with no timeout." - .format(self._queue_name)) - - try: - self._ws.settimeout(timeout) - while True: - message = self.recv() - LOG.debug(message) - yield message['body']['payload'] - - except websocket.WebSocketTimeoutException: - raise exceptions.WebSocketTimeout() - except websocket.WebSocketConnectionClosedException: - raise exceptions.WebSocketConnectionClosed() - - def __enter__(self): - """Return self to allow usage as a context manager""" - return self - - def __exit__(self, *exc): - """Call cleanup when exiting the context manager""" - try: - self.cleanup() - except websocket.WebSocketConnectionClosedException: - pass - - class MistralContext(object): """MistralContext, a shim for calling Mistral actions directly @@ -283,11 +166,6 @@ class ClientWrapper(object): self._local_orchestration = client return self._local_orchestration - def messaging_websocket(self, queue_name='tripleo'): - """Returns a websocket for the messaging service""" - return WebsocketClient(self._instance, queue_name, - cacert=self._instance.cacert) - @property def object_store(self): """Returns an object_store service client diff --git a/tripleoclient/tests/fakes.py b/tripleoclient/tests/fakes.py index d19ccad54..a026e5240 100644 --- a/tripleoclient/tests/fakes.py +++ b/tripleoclient/tests/fakes.py @@ -88,21 +88,6 @@ class FakeClientWrapper(object): def __init__(self): self._instance = mock.Mock() self.object_store = FakeObjectClient() - self._mock_websocket = mock.Mock() - self._mock_websocket.__enter__ = mock.Mock( - return_value=self._mock_websocket) - # Return False to avoid silencing exceptions - self._mock_websocket.__exit__ = mock.Mock(return_value=False) - self._mock_websocket.wait_for_messages = mock.Mock( - return_value=iter([{ - "status": "SUCCESS", - "message": "Success", - "execution_id": "IDID" - }]) - ) - - def messaging_websocket(self): - return self._mock_websocket class FakeRunnerConfig(object): diff --git a/tripleoclient/tests/fixture_data/deployment.py b/tripleoclient/tests/fixture_data/deployment.py index e80ccf775..edfde553c 100644 --- a/tripleoclient/tests/fixture_data/deployment.py +++ b/tripleoclient/tests/fixture_data/deployment.py @@ -36,9 +36,6 @@ class PlanManagementFixture(fixtures.Fixture): def _setUp(self): super(PlanManagementFixture, self)._setUp() - self.mock_tarball = self.useFixture(fixtures.MockPatch( - 'tripleoclient.workflows.plan_management.tarball') - ).mock class UtilsOvercloudFixture(fixtures.Fixture): diff --git a/tripleoclient/tests/test_plugin.py b/tripleoclient/tests/test_plugin.py index 56200f6a4..c7bb907b4 100644 --- a/tripleoclient/tests/test_plugin.py +++ b/tripleoclient/tests/test_plugin.py @@ -12,9 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import json import mock -import socket from tripleoclient import plugin from tripleoclient.tests import base @@ -23,138 +21,16 @@ from tripleoclient.tests import fakes class TestPlugin(base.TestCase): - @mock.patch("websocket.create_connection") - def test_make_client(self, ws_create_connection): + def test_make_client(self): clientmgr = mock.MagicMock() clientmgr.get_endpoint_for_service_type.return_value = fakes.WS_URL clientmgr.auth.get_token.return_value = "TOKEN" clientmgr.auth_ref.project_id = "ID" clientmgr.cacert = None - ws_create_connection.return_value.recv.return_value = json.dumps({ - "headers": { - "status": 200 - } - }) - client = plugin.make_client(clientmgr) - - websocket = client.messaging_websocket() - # The second access should not return the same client: - self.assertIsNot(client.messaging_websocket(), websocket) plugin.make_client(clientmgr) # And the functions should only be called when the client is created: - self.assertEqual(clientmgr.auth.get_token.call_count, 2) - self.assertEqual(clientmgr.get_endpoint_for_service_type.call_count, 2) - ws_create_connection.assert_called_with("ws://0.0.0.0") - - @mock.patch("websocket.create_connection") - def test_invalid_json(self, ws_create_connection): - clientmgr = mock.MagicMock() - clientmgr.get_endpoint_for_service_type.return_value = fakes.WS_URL - - clientmgr.auth.get_token.return_value = "TOKEN" - clientmgr.auth_ref.project_id = "ID" - clientmgr.cacert = None - ws_create_connection.return_value.recv.return_value = "BAD JSON" - client = plugin.make_client(clientmgr) - - # NOTE(d0ugal): Manually catching errors rather than using assertRaises - # so we can assert the message. - try: - client.messaging_websocket() - self.assertFail() - except ValueError as e: - self.assertEqual( - str(e), "No JSON object could be decoded; 'BAD JSON'") - - @mock.patch.object(plugin.WebsocketClient, "recv") - @mock.patch("websocket.create_connection") - def test_handle_websocket_multiple(self, ws_create_connection, recv_mock): - - send_ack = { - "headers": { - "status": 200 - } - } - - # Creating the websocket sends three messages and closing sends one. - # The one being tested is wrapped between these - recv_mock.side_effect = [send_ack, send_ack, send_ack, { - "body": { - "payload": { - "status": 200, - "message": "Result for IDID", - "execution_id": "IDID", - } - } - }, send_ack] - - clientmgr = mock.MagicMock() - clientmgr.get_endpoint_for_service_type.return_value = fakes.WS_URL - clientmgr.auth.get_token.return_value = "TOKEN" - clientmgr.auth_ref.project_id = "ID" - clientmgr.cacert = None - - client = plugin.make_client(clientmgr) - - with client.messaging_websocket() as ws: - for payload in ws.wait_for_messages(): - self.assertEqual(payload, { - "status": 200, - "message": "Result for IDID", - "execution_id": "IDID", - }) - # We only want to test the first message, as there is only one. - # The last one, the send_ack will be used by the context - # manager when it is closed. - break - - @mock.patch("websocket.create_connection") - def test_websocket_creation_error(self, ws_create_connection): - - ws_create_connection.side_effect = socket.error - - clientmgr = mock.MagicMock() - clientmgr.get_endpoint_for_service_type.return_value = fakes.WS_URL - clientmgr.auth.get_token.return_value = "TOKEN" - clientmgr.auth_ref.project_id = "ID" - clientmgr.cacert = None - - client = plugin.make_client(clientmgr) - - msg = ("Could not establish a connection to the Zaqar websocket. The " - "command was sent but the answer could not be read.") - with mock.patch('tripleoclient.plugin.LOG') as mock_log: - self.assertRaises(socket.error, client.messaging_websocket) - mock_log.error.assert_called_once_with(msg) - - @mock.patch("websocket.create_connection") - def test_make_tls_client(self, ws_create_connection): - clientmgr = mock.MagicMock() - clientmgr.get_endpoint_for_service_type.return_value = fakes.WSS_URL - - clientmgr.auth.get_token.return_value = "TOKEN" - clientmgr.auth_ref.project_id = "ID" - clientmgr.cacert = '/etc/pki/ca-trust/source/anchors/cm-local-ca.pem' - ws_create_connection.return_value.recv.return_value = json.dumps({ - "headers": { - "status": 200 - } - }) - client = plugin.make_client(clientmgr) - - websocket = client.messaging_websocket() - # The second access should not return the same client: - self.assertIsNot(client.messaging_websocket(), websocket) - - plugin.make_client(clientmgr) - - # And the functions should only be called when the client is created: - self.assertEqual(clientmgr.auth.get_token.call_count, 2) - self.assertEqual(clientmgr.get_endpoint_for_service_type.call_count, 2) - ws_create_connection.assert_called_with( - "wss://0.0.0.0", - sslopt={'ca_certs': - '/etc/pki/ca-trust/source/anchors/cm-local-ca.pem'}) + self.assertEqual(clientmgr.auth.get_token.call_count, 0) + self.assertEqual(clientmgr.get_endpoint_for_service_type.call_count, 0) diff --git a/tripleoclient/tests/v1/baremetal/fakes.py b/tripleoclient/tests/v1/baremetal/fakes.py index e737f11ee..016f13dd2 100644 --- a/tripleoclient/tests/v1/baremetal/fakes.py +++ b/tripleoclient/tests/v1/baremetal/fakes.py @@ -48,14 +48,6 @@ class ClientWrapper(object): def __init__(self): self._instance = None - self._mock_websocket = mock.Mock() - self._mock_websocket.__enter__ = mock.Mock( - return_value=self._mock_websocket) - # Return False to avoid silencing exceptions - self._mock_websocket.__exit__ = mock.Mock(return_value=False) - - def messaging_websocket(self): - return self._mock_websocket class TestBaremetal(utils.TestCommand): diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py index 3b4cc406e..0aec5bfcc 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py +++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py @@ -25,7 +25,6 @@ import mock import openstack from osc_lib import exceptions as oscexc from osc_lib.tests import utils -from swiftclient.exceptions import ClientException as ObjectClientException from tripleoclient import constants from tripleoclient import exceptions @@ -102,8 +101,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): client = self.app.client_manager.tripleoclient = plugin.ClientWrapper( instance=ooofakes.FakeInstanceData ) - client.messaging_websocket = \ - ooofakes.FakeClientWrapper().messaging_websocket get_object = client.object_store.get_object = mock.Mock() get_object.return_value = ('f1', 'content') client.object_store.put_object = mock.Mock() @@ -165,11 +162,18 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): client.object_store.get_account = mock.MagicMock() + self.mock_tar = mock.patch( + 'tripleo_common.utils.tarball.create_tarball', + autospec=True + ) + self.mock_tar.start() + def tearDown(self): super(TestDeployOvercloud, self).tearDown() os.unlink(self.parameter_defaults_env_file) self.cmd._download_missing_files_from_plan = self.real_download_missing shutil.rmtree = self.real_shutil + self.mock_tar.stop() @mock.patch('tripleoclient.utils.check_nic_config_with_ansible') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' @@ -368,12 +372,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_validate_args.assert_called_once_with(parsed_args) - plane_management_fixture.mock_tarball.create_tarball.\ - assert_called_with( - self.tmp_dir.join('tripleo-heat-templates'), mock.ANY) - plane_management_fixture.mock_tarball.\ - tarball_extract_to_swift_container.assert_called_with( - clients.tripleoclient.object_store, mock.ANY, 'overcloud') self.assertFalse(mock_invoke_plan_env_wf.called) calls = [ @@ -393,146 +391,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): object_client = clients.tripleoclient.object_store object_client.put_object.assert_has_calls(calls) - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_get_ctlplane_attrs', autospec=True, return_value={}) - @mock.patch('os.chdir') - @mock.patch('tripleoclient.utils.copy_clouds_yaml') - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_get_undercloud_host_entry', autospec=True, - return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane') - @mock.patch('tripleoclient.utils.check_stack_network_matches_env_files') - @mock.patch('tripleoclient.utils.check_ceph_fsid_matches_env_files') - @mock.patch('shutil.rmtree', autospec=True) - @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True) - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_deploy_postconfig', autospec=True) - @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env', - autospec=True) - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_validate_args') - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env', autospec=True) - @mock.patch('tripleoclient.utils.create_tempest_deployer_input', - autospec=True) - @mock.patch('tripleoclient.workflows.deployment.create_overcloudrc', - autospec=True) - @mock.patch('heatclient.common.template_utils.get_template_contents', - autospec=True) - @mock.patch('tempfile.mkdtemp', autospec=True) - def test_tht_deploy_with_plan_environment_file( - self, mock_tmpdir, mock_get_template_contents, - mock_overcloudrc, - mock_create_tempest_deployer, mock_create_parameters_env, - mock_validate_args, - mock_breakpoints_cleanup, - mock_postconfig, mock_shutil_rmtree, - mock_invoke_plan_env_wf, - mock_stack_network_check, mock_ceph_fsid, - mock_get_undercloud_host_entry, mock_copy, mock_chdir, - mock_get_ctlplane_attrs): - fixture = deployment.DeploymentWorkflowFixture() - self.useFixture(fixture) - plane_management_fixture = deployment.PlanManagementFixture() - self.useFixture(plane_management_fixture) - utils_fixture = deployment.UtilsFixture() - self.useFixture(utils_fixture) - - arglist = ['--templates', '-p', 'the-plan-environment.yaml'] - verifylist = [ - ('templates', '/usr/share/openstack-tripleo-heat-templates/'), - ('plan_environment_file', 'the-plan-environment.yaml') - ] - - mock_tmpdir.return_value = "/tmp/tht" - - clients = self.app.client_manager - orchestration_client = clients.orchestration - mock_stack = fakes.create_tht_stack() - orchestration_client.stacks.get.side_effect = [None, mock.Mock()] - - def _orch_clt_create(**kwargs): - orchestration_client.stacks.get.return_value = mock_stack - - orchestration_client.stacks.create.side_effect = _orch_clt_create - - object_client = clients.tripleoclient.object_store - object_client.get_object = mock.Mock() - object_client.put_container = mock.Mock() - mock_env = yaml.safe_dump({'environments': []}) - object_client.get_object.return_value = ({}, mock_env) - - clients.network.api.find_attr.return_value = { - "id": "network id" - } - mock_get_template_contents.return_value = [{}, "template"] - - parsed_args = self.check_parser(self.cmd, arglist, verifylist) - - baremetal = clients.baremetal - baremetal.node.list.return_value = range(10) - - expected_parameters = { - 'CephClusterFSID': self.uuid1_value, - 'CephStorageCount': 3, - 'ExtraConfig': '{}', - 'HypervisorNeutronPhysicalBridge': 'br-ex', - 'HypervisorNeutronPublicInterface': 'nic1', - 'NeutronDnsmasqOptions': 'dhcp-option-force=26,1400', - 'NeutronFlatNetworks': 'datacentre', - 'NeutronNetworkType': 'gre', - 'NeutronPublicInterface': 'nic1', - 'NeutronTunnelTypes': 'gre', - 'NtpServer': '', - 'SnmpdReadonlyUserPassword': 'PASSWORD', - 'StackAction': 'CREATE', - 'DeployIdentifier': 12345678, - 'RootStackName': 'overcloud', - 'UndercloudHostsEntries': [ - '192.168.0.1 uc.ctlplane.localhost uc.ctlplane' - ], - 'CtlplaneNetworkAttributes': {}, - } - - testcase = self - - def _custom_create_params_env(_self, parameters, tht_root, - container_name): - for key, value in six.iteritems(parameters): - testcase.assertEqual(value, expected_parameters[key]) - parameter_defaults = {"parameter_defaults": parameters} - return parameter_defaults - - mock_create_parameters_env.side_effect = _custom_create_params_env - - mock_open_context = mock.mock_open() - - with mock.patch('six.moves.builtins.open', mock_open_context): - self.cmd.take_action(parsed_args) - - self.assertFalse(orchestration_client.stacks.create.called) - - mock_get_template_contents.assert_called_with( - object_request=mock.ANY, - template_object=constants.OVERCLOUD_YAML_NAME) - - mock_create_tempest_deployer.assert_called_with() - mock_validate_args.assert_called_once_with(parsed_args) - - plane_management_fixture.mock_tarball.create_tarball.\ - assert_called_with( - '/tmp/tht/tripleo-heat-templates', mock.ANY) - plane_management_fixture.mock_tarball.\ - tarball_extract_to_swift_container.assert_called_with( - clients.tripleoclient.object_store, mock.ANY, 'overcloud') - - mock_open_context.assert_has_calls( - [mock.call('the-plan-environment.yaml')]) - clients.tripleoclient.object_store.put_object.assert_called() - self.assertTrue(mock_invoke_plan_env_wf.called) - mock_copy.assert_called_once() - object_client.put_container.assert_called_once_with( - 'overcloud', headers={'x-container-meta-usage-tripleo': 'plan'}) - @mock.patch('tripleoclient.workflows.deployment.create_overcloudrc', autospec=True) @mock.patch('os.chdir') @@ -885,8 +743,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): object_client.put_container.assert_called_once_with( 'overcloud', headers={'x-container-meta-usage-tripleo': 'plan'}) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_deploy_postconfig', autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' @@ -895,8 +751,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): '_heat_deploy', autospec=True) def test_environment_dirs_env_dir_not_found(self, mock_deploy_heat, mock_update_parameters, - mock_post_config, - mock_tarball): + mock_post_config): utils_fixture = deployment.UtilsOvercloudFixture() self.useFixture(utils_fixture) @@ -966,7 +821,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): '_heat_deploy', autospec=True) def test_try_overcloud_deploy_with_no_templates_existing( self, mock_heat_deploy_func): - mock_heat_deploy_func.side_effect = ObjectClientException('error') + mock_heat_deploy_func.side_effect = Exception('error') self.assertRaises(ValueError, self.cmd._try_overcloud_deploy_with_compat_yaml, '/fake/path', mock.ANY, mock.ANY, mock.ANY, @@ -978,7 +833,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): def test_try_overcloud_deploy_show_missing_file( self, mock_heat_deploy_func): mock_heat_deploy_func.side_effect = \ - ObjectClientException('/fake/path not found') + Exception('/fake/path not found') try: self.cmd._try_overcloud_deploy_with_compat_yaml( '/fake/path', mock.ANY, mock.ANY, mock.ANY, diff --git a/tripleoclient/tests/v1/test_overcloud_bios.py b/tripleoclient/tests/v1/test_overcloud_bios.py index 44ade07cc..a4973224a 100644 --- a/tripleoclient/tests/v1/test_overcloud_bios.py +++ b/tripleoclient/tests/v1/test_overcloud_bios.py @@ -31,14 +31,7 @@ class Base(fakes.TestBaremetal): {"name": "hyperthreading", "value": "on"} ] } - tripleoclient = self.app.client_manager.tripleoclient self.app.client_manager.baremetal.node.list.return_value = [] - self.websocket = tripleoclient.messaging_websocket() - self.websocket.wait_for_messages.return_value = iter([{ - 'status': "SUCCESS", - 'execution_id': 'fake id', - 'root_execution_id': 'fake id', - }]) self.execution = self.workflow.executions.create.return_value self.execution.id = 'fake id' diff --git a/tripleoclient/tests/v1/test_overcloud_plan.py b/tripleoclient/tests/v1/test_overcloud_plan.py index f26ee8fd0..011f2e127 100644 --- a/tripleoclient/tests/v1/test_overcloud_plan.py +++ b/tripleoclient/tests/v1/test_overcloud_plan.py @@ -105,6 +105,15 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.swift = self.app.client_manager.tripleoclient.object_store self.swift.get_account = mock.MagicMock() + self.mock_tar = mock.patch( + 'tripleo_common.utils.tarball.create_tarball', + autospec=True + ) + self.mock_tar.start() + + def tearDown(self): + super(TestOvercloudCreatePlan, self).tearDown() + self.mock_tar.stop() @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) @mock.patch('os.chdir', autospec=True) @@ -123,7 +132,7 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.cmd.take_action(parsed_args) # Verify - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -138,10 +147,9 @@ class TestOvercloudCreatePlan(utils.TestCommand): ) @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch("tripleoclient.workflows.plan_management.tarball") @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) - def test_create_custom_plan(self, mock_tmp, mock_cd, mock_tarball, + def test_create_custom_plan(self, mock_tmp, mock_cd, mock_run_playbook): # Setup @@ -156,7 +164,7 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.cmd.take_action(parsed_args) # Verify - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -172,11 +180,10 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.swift.get_account.assert_called_once() @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch("tripleoclient.workflows.plan_management.tarball") @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_custom_plan_plan_environment_file( - self, mock_tmp, mock_cd, mock_tarball, mock_run_playbook): + self, mock_tmp, mock_cd, mock_run_playbook): # Setup arglist = ['overcast', '--templates', '/fake/path', '-p', 'the_plan_environment.yaml'] @@ -197,7 +204,7 @@ class TestOvercloudCreatePlan(utils.TestCommand): [mock.call('the_plan_environment.yaml', 'rb')]) # Verify - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -234,7 +241,7 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.app.options = fakes.FakeOptions() self.cmd.take_action(parsed_args) # Verify - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -267,7 +274,7 @@ class TestOvercloudCreatePlan(utils.TestCommand): self.cmd.take_action(parsed_args) # Verify - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -325,7 +332,7 @@ class TestOvercloudDeployPlan(utils.TestCommand): # Run self.cmd.take_action(parsed_args) - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-deploy-deployment-plan.yaml', 'undercloud,', mock.ANY, diff --git a/tripleoclient/tests/v1/test_overcloud_raid.py b/tripleoclient/tests/v1/test_overcloud_raid.py index d0956a874..e2fbe9ee5 100644 --- a/tripleoclient/tests/v1/test_overcloud_raid.py +++ b/tripleoclient/tests/v1/test_overcloud_raid.py @@ -37,14 +37,6 @@ class TestCreateRAID(fakes.TestBaremetal): {"foo2": "bar2"} ] } - tripleoclient = self.app.client_manager.tripleoclient - websocket = tripleoclient.messaging_websocket() - websocket.wait_for_messages.return_value = iter([ - {'status': "SUCCESS", - 'execution_id': 'IDID'} - ]) - self.websocket = websocket - execution = mock.MagicMock( output=json.dumps({ "result": None diff --git a/tripleoclient/tests/v2/overcloud_node/test_overcloud_node.py b/tripleoclient/tests/v2/overcloud_node/test_overcloud_node.py index 63fc49c34..91fc73c28 100644 --- a/tripleoclient/tests/v2/overcloud_node/test_overcloud_node.py +++ b/tripleoclient/tests/v2/overcloud_node/test_overcloud_node.py @@ -268,13 +268,6 @@ class TestIntrospectNode(fakes.TestOvercloudNode): self.cmd, argslist, verifylist) def _check_introspect_all_manageable(self, parsed_args, provide=False): - self.websocket.wait_for_messages.return_value = iter([{ - "status": "SUCCESS", - "message": "Success", - "introspected_nodes": {}, - "execution_id": "IDID" - }] * 2) - self.cmd.take_action(parsed_args) call_list = [mock.call( @@ -293,12 +286,6 @@ class TestIntrospectNode(fakes.TestOvercloudNode): 2 if provide else 1) def _check_introspect_nodes(self, parsed_args, nodes, provide=False): - self.websocket.wait_for_messages.return_value = [{ - "status": "SUCCESS", - "message": "Success", - "execution_id": "IDID", - }] - self.cmd.take_action(parsed_args) call_list = [mock.call( diff --git a/tripleoclient/tests/workflows/test_base.py b/tripleoclient/tests/workflows/test_base.py deleted file mode 100644 index 9e298046f..000000000 --- a/tripleoclient/tests/workflows/test_base.py +++ /dev/null @@ -1,183 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import mock - -from osc_lib.tests import utils - -from tripleoclient import exceptions as ex -from tripleoclient.workflows import base - - -class TestBaseWorkflows(utils.TestCommand): - - def test_wait_for_messages_success(self): - payload_a = { - 'status': 'ERROR', - 'execution_id': 2, - 'root_execution_id': 1 - } - payload_b = { - 'status': 'ERROR', - 'execution_id': 1, - 'root_execution_id': 1 - } - - mistral = mock.Mock() - websocket = mock.Mock() - websocket.wait_for_messages.return_value = iter([payload_a, payload_b]) - execution = mock.Mock() - execution.id = 1 - - messages = list(base.wait_for_messages(mistral, websocket, execution)) - - self.assertEqual([payload_a, payload_b], messages) - - self.assertFalse(mistral.executions.get.called) - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_wait_for_messages_timeout(self): - mistral = mock.Mock() - websocket = mock.Mock() - websocket.wait_for_messages.side_effect = ex.WebSocketTimeout - execution = mock.Mock() - execution.id = 1 - - messages = base.wait_for_messages(mistral, websocket, execution) - - self.assertRaises(ex.WebSocketTimeout, list, messages) - - self.assertTrue(mistral.executions.get.called) - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_wait_for_messages_connection_closed(self): - mistral = mock.Mock() - websocket = mock.Mock() - websocket.wait_for_messages.side_effect = ex.WebSocketConnectionClosed - - execution = mock.Mock() - execution.id = 1 - - messages = base.wait_for_messages(mistral, websocket, execution) - - self.assertRaises(ex.WebSocketConnectionClosed, list, messages) - - self.assertTrue(mistral.executions.get.called) - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_wait_for_messages_different_execution(self): - payload_a = { - 'status': 'RUNNING', - 'execution_id': 'aaaa', - 'root_execution_id': 'aaaa' - } - payload_b = { - 'status': 'RUNNING', - 'execution_id': 'bbbb', - 'root_execution_id': 'bbbb' - } - - mistral = mock.Mock() - mistral.executions.get.return_value.state = 'RUNNING' - websocket = mock.Mock() - websocket.wait_for_messages.return_value = iter([payload_a, payload_b]) - execution = mock.Mock() - execution.id = 'aaaa' - - messages = list(base.wait_for_messages(mistral, websocket, execution)) - - # Assert only payload_a was returned - self.assertEqual([payload_a], messages) - - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_backwards_compat_wait_for_messages_success(self): - payload_a = { - 'status': 'ERROR', - 'execution': {'id': 2, - 'root_execution_id': 1} - } - payload_b = { - 'status': 'ERROR', - 'execution': {'id': 1, - 'root_execution_id': 1} - } - - mistral = mock.Mock() - websocket = mock.Mock() - websocket.wait_for_messages.return_value = iter([payload_a, payload_b]) - execution = mock.Mock() - execution.id = 1 - - messages = list(base.wait_for_messages(mistral, websocket, execution)) - - self.assertEqual([payload_a, payload_b], messages) - - self.assertFalse(mistral.executions.get.called) - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_backwards_compatible_call_with_different_execution(self): - payload_a = { - 'status': 'RUNNING', - 'execution': {'id': 'aaaa', - 'root_execution_id': 'aaaa'} - } - payload_b = { - 'status': 'RUNNING', - 'execution': {'id': 'bbbb', - 'root_execution_id': 'bbbb'} - } - - mistral = mock.Mock() - mistral.executions.get.return_value.state = 'RUNNING' - websocket = mock.Mock() - websocket.wait_for_messages.return_value = iter([payload_a, payload_b]) - execution = mock.Mock() - execution.id = 'aaaa' - - messages = list(base.wait_for_messages(mistral, websocket, execution)) - - # Assert only payload_a was returned - self.assertEqual([payload_a], messages) - - websocket.wait_for_messages.assert_called_with(timeout=None) - - def test_wait_for_messages_execution_complete(self): - payload_a = { - 'status': 'RUNNING', - 'execution_id': 'aaaa', - 'root_execution_id': 'aaaa' - } - payload_b = { - 'status': 'SUCCESS', - 'execution_id': 'aaaa', - 'root_execution_id': 'aaaa' - } - - mistral = mock.Mock() - mistral.executions.get.return_value.state = 'SUCCESS' - mistral.executions.get.return_value.output = json.dumps(payload_b) - websocket = mock.Mock() - websocket.wait_for_messages.return_value = iter([payload_a]) - execution = mock.Mock() - execution.id = 'aaaa' - - messages = list(base.wait_for_messages(mistral, websocket, execution)) - - # Assert only payload_b was returned - self.assertEqual([payload_a, payload_b], messages) - mistral.executions.get.assert_called_with('aaaa') - - websocket.wait_for_messages.assert_called_with(timeout=None) diff --git a/tripleoclient/tests/workflows/test_parameters.py b/tripleoclient/tests/workflows/test_parameters.py index 6946998ad..449740368 100644 --- a/tripleoclient/tests/workflows/test_parameters.py +++ b/tripleoclient/tests/workflows/test_parameters.py @@ -16,7 +16,6 @@ import mock from osc_lib.tests import utils -from tripleoclient import exceptions from tripleoclient.workflows import parameters @@ -39,10 +38,6 @@ class TestParameterWorkflows(utils.TestCommand): self.orchestration = mock.Mock() self.tripleoclient = mock.Mock() self.tripleoclient.object_store = mock.MagicMock() - self.websocket = mock.Mock() - self.websocket.__enter__ = lambda s: self.websocket - self.websocket.__exit__ = lambda s, *exc: None - self.tripleoclient.messaging_websocket.return_value = self.websocket self.app.client_manager.tripleoclient = self.tripleoclient self.app.client_manager.orchestration = self.orchestration self.app.client_manager.baremetal = mock.Mock() @@ -78,39 +73,6 @@ class TestParameterWorkflows(utils.TestCommand): flatten.start() self.addCleanup(flatten.stop) - @mock.patch('yaml.safe_load') - @mock.patch("six.moves.builtins.open") - def test_invoke_plan_env_workflows(self, mock_open, - mock_safe_load): - plan_env_data = { - 'name': 'overcloud', - 'workflow_parameters': { - 'tripleo.derive_params.v1.derive_parameters': { - 'num_phy_cores_per_numa_node_for_pmd': 2 - } - } - } - mock_safe_load.return_value = plan_env_data - - self.websocket.wait_for_messages.return_value = iter([{ - "execution_id": "IDID", - "status": "SUCCESS", - "message": "", - "result": {} - }]) - - parameters.invoke_plan_env_workflows( - self.app.client_manager, - 'overcloud', - 'the-plan-environment.yaml') - - self.workflow.executions.create.assert_called_once_with( - 'tripleo.derive_params.v1.derive_parameters', - workflow_input={ - 'plan': 'overcloud', - 'user_inputs': { - 'num_phy_cores_per_numa_node_for_pmd': 2}}) - @mock.patch('yaml.safe_load') @mock.patch("six.moves.builtins.open") @mock.patch('tripleoclient.utils.run_ansible_playbook', autospec=True) @@ -194,67 +156,6 @@ class TestParameterWorkflows(utils.TestCommand): ] mock_playbook.assert_has_calls(calls, any_order=True) - @mock.patch('yaml.safe_load') - @mock.patch("six.moves.builtins.open") - def test_invoke_plan_env_workflow_failed(self, mock_open, - mock_safe_load): - plan_env_data = { - 'name': 'overcloud', - 'workflow_parameters': { - 'tripleo.derive_params.v1.derive_parameters': { - 'num_phy_cores_per_numa_node_for_pmd': 2 - } - } - } - mock_safe_load.return_value = plan_env_data - - self.websocket.wait_for_messages.return_value = iter([{ - "execution_id": "IDID", - "status": "FAILED", - "message": "workflow failure", - "result": "" - }]) - - self.assertRaises(exceptions.PlanEnvWorkflowError, - parameters.invoke_plan_env_workflows, - self.app.client_manager, 'overcloud', - 'the-plan-environment.yaml') - - self.workflow.executions.create.assert_called_once_with( - 'tripleo.derive_params.v1.derive_parameters', - workflow_input={ - 'plan': 'overcloud', - 'user_inputs': { - 'num_phy_cores_per_numa_node_for_pmd': 2}}) - - @mock.patch('yaml.safe_load') - @mock.patch("six.moves.builtins.open") - def test_invoke_plan_env_workflows_no_workflow_params( - self, mock_open, mock_safe_load): - plan_env_data = {'name': 'overcloud'} - mock_safe_load.return_value = plan_env_data - - parameters.invoke_plan_env_workflows( - self.app.client_manager, - 'overcloud', - 'the-plan-environment.yaml') - - self.workflow.executions.create.assert_not_called() - - @mock.patch('yaml.safe_load') - @mock.patch("six.moves.builtins.open") - def test_invoke_plan_env_workflows_no_plan_env_file( - self, mock_open, mock_safe_load): - - mock_open.side_effect = IOError('') - - self.assertRaises(exceptions.PlanEnvWorkflowError, - parameters.invoke_plan_env_workflows, - self.app.client_manager, 'overcloud', - 'the-plan-environment.yaml') - - self.workflow.executions.create.assert_not_called() - def test_check_deprecated_params_no_output(self): parameters.check_deprecated_parameters( self.app.client_manager, diff --git a/tripleoclient/tests/workflows/test_plan_management.py b/tripleoclient/tests/workflows/test_plan_management.py index 1a9f68d91..c50eb992f 100644 --- a/tripleoclient/tests/workflows/test_plan_management.py +++ b/tripleoclient/tests/workflows/test_plan_management.py @@ -14,7 +14,6 @@ import mock from osc_lib.tests import utils -from swiftclient import exceptions as swift_exc from tripleoclient import constants from tripleoclient.tests import base @@ -28,21 +27,27 @@ class TestPlanCreationWorkflows(utils.TestCommand): self.tripleoclient = mock.Mock() self.app.client_manager.tripleoclient = self.tripleoclient self.tripleoclient.object_store.get_account = mock.MagicMock() + self.mock_tar = mock.patch( + 'tripleo_common.utils.tarball.create_tarball', + autospec=True + ) + self.mock_tar.start() + + def tearDown(self): + super(TestPlanCreationWorkflows, self).tearDown() + self.mock_tar.stop() @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_plan_from_templates_success(self, mock_tmp, mock_cd, - mock_tarball, mock_run_playbook): plan_management.create_plan_from_templates( self.app.client_manager, 'test-overcloud', '/tht-root/') - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -58,12 +63,9 @@ class TestPlanCreationWorkflows(utils.TestCommand): @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) @mock.patch('tripleoclient.utils.rel_or_abs_path') - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_plan_from_templates_roles_data(self, mock_tmp, mock_cd, - mock_tarball, mock_norm_path, mock_run_playbook): mock_open_context = mock.mock_open() @@ -74,7 +76,7 @@ class TestPlanCreationWorkflows(utils.TestCommand): '/tht-root/', 'the_roles_file.yaml') - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -91,16 +93,13 @@ class TestPlanCreationWorkflows(utils.TestCommand): self.assertIn(mock.call('the_roles_file.yaml', '/tht-root/'), mock_norm_path.call_args_list) - self.tripleoclient.object_store.put_object.assert_called_once_with( + self.tripleoclient.object_store.put_object.assert_called_with( 'test-overcloud', 'roles_data.yaml', mock_open_context()) @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_plan_from_templates_plan_env_data(self, mock_tmp, mock_cd, - mock_tarball, mock_run_playbook): mock_open_context = mock.mock_open() with mock.patch('six.moves.builtins.open', mock_open_context): @@ -110,7 +109,7 @@ class TestPlanCreationWorkflows(utils.TestCommand): '/tht-root/', plan_env_file='the-plan-environment.yaml') - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -127,16 +126,13 @@ class TestPlanCreationWorkflows(utils.TestCommand): mock_open_context.assert_has_calls( [mock.call('the-plan-environment.yaml', 'rb')]) - self.tripleoclient.object_store.put_object.assert_called_once_with( + self.tripleoclient.object_store.put_object.assert_called_with( 'test-overcloud', 'plan-environment.yaml', mock_open_context()) @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_plan_from_templates_networks_data(self, mock_tmp, mock_cd, - mock_tarball, mock_run_playbook): mock_open_context = mock.mock_open() with mock.patch('six.moves.builtins.open', mock_open_context): @@ -146,7 +142,7 @@ class TestPlanCreationWorkflows(utils.TestCommand): '/tht-root/', networks_file='the-network-data.yaml') - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -162,16 +158,13 @@ class TestPlanCreationWorkflows(utils.TestCommand): mock_open_context.assert_has_calls( [mock.call('the-network-data.yaml', 'rb')]) - self.tripleoclient.object_store.put_object.assert_called_once_with( + self.tripleoclient.object_store.put_object.assert_called_with( 'test-overcloud', 'network_data.yaml', mock_open_context()) @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_create_plan_with_password_gen_disabled(self, mock_tmp, mock_cd, - mock_tarball, mock_run_playbook): plan_management.create_plan_from_templates( self.app.client_manager, @@ -180,7 +173,7 @@ class TestPlanCreationWorkflows(utils.TestCommand): generate_passwords=False, disable_image_params_prepare=True) - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-create-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -226,16 +219,23 @@ class TestPlanUpdateWorkflows(base.TestCommand): # e.g. 'plan-environment.yaml: mock content' return {}, '{0}: mock content\n'.format(args[1]) self.object_store.get_object.side_effect = get_object + self.mock_tar = mock.patch( + 'tripleo_common.utils.tarball.create_tarball', + autospec=True + ) + self.mock_tar.start() + + def tearDown(self): + super(TestPlanUpdateWorkflows, self).tearDown() + self.mock_tar.stop() @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('tripleo_common.utils.swift.empty_container', autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_update_plan_from_templates_keep_env( - self, mock_tmp, mock_cd, mock_empty_container, mock_tarball, + self, mock_tmp, mock_cd, mock_empty_container, mock_run_playbook): plan_management.update_plan_from_templates( @@ -266,7 +266,7 @@ class TestPlanUpdateWorkflows(base.TestCommand): ], any_order=True, ) - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-update-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -280,12 +280,10 @@ class TestPlanUpdateWorkflows(base.TestCommand): ) @mock.patch("tripleoclient.utils.run_ansible_playbook", autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('tripleo_common.utils.swift.empty_container', autospec=True) def test_update_plan_from_templates_recreate_env( - self, mock_empty_container, mock_tarball, mock_run_playbook): + self, mock_empty_container, mock_run_playbook): plan_management.update_plan_from_templates( self.app.client_manager, @@ -302,7 +300,7 @@ class TestPlanUpdateWorkflows(base.TestCommand): 'plan-environment.yaml: mock content\n' ) - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-update-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -320,14 +318,12 @@ class TestPlanUpdateWorkflows(base.TestCommand): autospec=True) @mock.patch('yaml.safe_load', autospec=True) - @mock.patch('tripleoclient.workflows.plan_management.tarball', - autospec=True) @mock.patch('tripleo_common.utils.swift.empty_container', autospec=True) @mock.patch('os.chdir', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_update_plan_from_templates_recreate_env_missing_passwords( - self, mock_tmp, mock_cd, mock_empty_container, mock_tarball, + self, mock_tmp, mock_cd, mock_empty_container, mock_yaml_safe_load, mock_update_passwords, mock_run_playbook): plan_management.update_plan_from_templates( self.app.client_manager, @@ -340,7 +336,7 @@ class TestPlanUpdateWorkflows(base.TestCommand): # Ensure that the passwords variable is passed with a value of None. mock_update_passwords.assert_called_with( mock.ANY, 'test-overcloud', None) - mock_run_playbook.assert_called_once_with( + mock_run_playbook.assert_called_with( 'cli-update-deployment-plan.yaml', 'undercloud,', mock.ANY, @@ -375,7 +371,6 @@ parameter_defaults: self.plan_name, {'AdminPassword': "1234"}) - self.swift_client.put_object.assert_called_once() result = self.swift_client.put_object.call_args_list[0][0][2] # Check new data is in @@ -393,7 +388,7 @@ parameter_defaults: def test_no_plan_environment(self): self.swift_client.get_object.side_effect = ( - swift_exc.ClientException("404")) + Exception("404")) plan_management._update_passwords(self.swift_client, self.plan_name, diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index 8fe8d7d8c..371941588 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -59,6 +59,7 @@ from heatclient import exc as hc_exc from six.moves.urllib import error as url_error from six.moves.urllib import request +from tripleo_common.utils import swift as swiftutils from tripleoclient import constants from tripleoclient import exceptions @@ -2361,12 +2362,12 @@ def update_deployment_status(clients, plan, status): # create {plan}-messages container if not there swift = clients.tripleoclient.object_store - swift.put_container(container) - - swift.put_object( - container, - 'deployment_status.yaml', - yaml.safe_dump( + swiftutils.create_container(swiftclient=swift, container=container) + swiftutils.put_object_string( + swift=swift, + container=container, + object_name='deployment_status.yaml', + contents=yaml.safe_dump( { 'deployment_status': status, 'workflow_status': { diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index c6c6bb66f..0f10a3eed 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -34,7 +34,6 @@ from keystoneauth1.exceptions.catalog import EndpointNotFound import openstack from osc_lib import exceptions as oscexc from osc_lib.i18n import _ -from swiftclient.exceptions import ClientException from tripleo_common import update from tripleoclient import command @@ -504,7 +503,7 @@ class DeployOvercloud(command.Command): user_env = yaml.safe_load(self.object_client.get_object( parsed_args.stack, constants.USER_ENVIRONMENT)[1]) template_utils.deep_update(env, user_env) - except ClientException: + except Exception: pass parameters.update(self._update_parameters(parsed_args, stack)) template_utils.deep_update(env, self._create_parameters_env( @@ -589,7 +588,7 @@ class DeployOvercloud(command.Command): run_validations, skip_deploy_identifier, plan_env_file, deployment_options=deployment_options) - except ClientException as e: + except Exception as e: messages = 'Failed to deploy: %s' % str(e) raise ValueError(messages) diff --git a/tripleoclient/workflows/base.py b/tripleoclient/workflows/base.py deleted file mode 100644 index 91cf9d9e6..000000000 --- a/tripleoclient/workflows/base.py +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import json -import keystoneauth1 -import logging - -from tripleoclient import exceptions - -LOG = logging.getLogger(__name__) - - -def start_workflow(workflow_client, identifier, workflow_input): - - execution = workflow_client.executions.create( - identifier, - workflow_input=workflow_input - ) - - LOG.debug("Started Mistral Workflow {}. Execution ID: {}".format( - identifier, execution.id)) - - return execution - - -def wait_for_messages(mistral, websocket, execution, timeout=None): - """Wait for messages on a websocket. - - Given an instance of mistral client, a websocket and a Mistral execution - wait for messages on that websocket queue that match the execution ID until - the timeout is reached. - - If no timeout is provided, this method will block forever. - - If a timeout is reached, called check_execution_status which will look up - the execution on Mistral and log information about it. - """ - try: - for payload in websocket.wait_for_messages(timeout=timeout): - # Ignore messages whose root_execution_id does not match the - # id of the execution for which we are waiting - - # New versions of tripleo-common don't sent the execution anymore - # but keeping the old way ot getting it is important to keep - # backwards compatibility. - - # TODO(apetrich) payload.execution is deprecated and will be - # removed from stein. We should keep this until payload.execution - # is removed from the LTS - payload_exec_id = payload.get('execution_id') or \ - payload.get('execution', {}).get('id') - - payload_root_exec_id = payload.get('root_execution_id', '') or \ - payload.get('execution', {}).get('root_execution_id', '') - - if payload_exec_id != execution.id and \ - payload_root_exec_id != execution.id: - - LOG.debug("Ignoring message from execution %s" - % payload_exec_id) - else: - yield payload - # If the message is from a sub-workflow, we just need to pass it - # on to be displayed. This should never be the last message - so - # continue and wait for the next. - if payload_exec_id != execution.id: - continue - # Check the status of the payload, if we are not given one - # default to running and assume it is just an "in progress" - # message from the workflow. - # Workflows should end with SUCCESS or ERROR statuses. - if payload.get('status', 'RUNNING') != "RUNNING": - return - try: - execution = mistral.executions.get(execution.id) - except keystoneauth1.exceptions.connection.ConnectFailure as e: - LOG.warning("Connection failure while fetching execution ID." - "Retrying: %s" % e) - continue - - if execution.state != "RUNNING": - # yield the output as the last payload which was missed - yield json.loads(execution.output) - return - except (exceptions.WebSocketTimeout, exceptions.WebSocketConnectionClosed): - check_execution_status(mistral, execution.id) - raise - - -def check_execution_status(workflow_client, execution_id): - """Check the status of a workflow that timeout when waiting for messages - - The status will be logged. - """ - - execution = workflow_client.executions.get(execution_id) - state = execution.state - - if state == 'RUNNING': - message = "The WebSocket timed out before the Workflow completed." - elif state == 'SUCCESS': - message = ("The Workflow finished successfully but no messages were " - "received before the WebSocket timed out.") - elif state == 'ERROR': - message = "The Workflow errored and no messages were received." - else: - message = "Unknown Execution state." - - LOG.error(("Timed out waiting for messages from Execution " - "(ID: {}, State: {}). {}").format(execution_id, state, message)) diff --git a/tripleoclient/workflows/deployment.py b/tripleoclient/workflows/deployment.py index 17f33d4b5..d559f4a5b 100644 --- a/tripleoclient/workflows/deployment.py +++ b/tripleoclient/workflows/deployment.py @@ -18,8 +18,7 @@ import yaml from heatclient.common import event_utils from heatclient import exc as heat_exc from openstackclient import shell -from swiftclient import exceptions as swiftexceptions -from tripleo_common.utils import swift as swift_utils +from tripleo_common.utils import swift as swiftutils from tripleoclient.constants import ANSIBLE_TRIPLEO_PLAYBOOKS from tripleoclient.constants import CLOUD_HOME_DIR @@ -515,13 +514,13 @@ def get_deployment_status(clients, plan): return None try: - body = swift_utils.get_object_string( + body = swiftutils.get_object_string( clients.tripleoclient.object_store, '%s-messages' % plan, 'deployment_status.yaml') return yaml.safe_load(body)['deployment_status'] - except swiftexceptions.ClientException: + except Exception: return None diff --git a/tripleoclient/workflows/parameters.py b/tripleoclient/workflows/parameters.py index 24575d332..1335dc97b 100644 --- a/tripleoclient/workflows/parameters.py +++ b/tripleoclient/workflows/parameters.py @@ -21,7 +21,6 @@ from tripleoclient.constants import ANSIBLE_TRIPLEO_PLAYBOOKS from tripleoclient.constants import UNUSED_PARAMETER_EXCLUDES_RE from tripleoclient import exceptions from tripleoclient import utils -from tripleoclient.workflows import base from tripleoclient.workflows import roles @@ -39,73 +38,37 @@ def invoke_plan_env_workflows(clients, stack_name, plan_env_file, raise exceptions.PlanEnvWorkflowError('File (%s) is not found: ' '%s' % (plan_env_file, exc)) - if plan_env_data and "playbook_parameters" in plan_env_data: - static_inventory = utils.get_tripleo_ansible_inventory( - ssh_user='heat-admin', - stack=stack_name, - undercloud_connection='local', - return_inventory_file_path=True - ) - with utils.TempDirs() as tmp: - for pb, pb_vars in plan_env_data["playbook_parameters"].items(): - print('Invoking playbook ({}) specified in plan-environment' - ' file'.format(pb)) - LOG.debug( - 'Running playbook "{}" with the' - ' following options {}.'.format( - pb, - pb_vars - ) + static_inventory = utils.get_tripleo_ansible_inventory( + ssh_user='heat-admin', + stack=stack_name, + undercloud_connection='local', + return_inventory_file_path=True + ) + with utils.TempDirs() as tmp: + for pb, pb_vars in plan_env_data["playbook_parameters"].items(): + print( + 'Invoking playbook ({}) specified in' + ' plan-environment file'.format(pb) + ) + LOG.debug( + 'Running playbook "{}" with the' + ' following options {}.'.format( + pb, + pb_vars ) - playbook_dir = os.path.dirname(pb) - if not playbook_dir: - playbook_dir = ANSIBLE_TRIPLEO_PLAYBOOKS + ) + playbook_dir = os.path.dirname(pb) + if not playbook_dir: + playbook_dir = ANSIBLE_TRIPLEO_PLAYBOOKS - utils.run_ansible_playbook( - playbook=os.path.basename(pb), - inventory=static_inventory, - workdir=tmp, - playbook_dir=playbook_dir, - verbosity=verbosity, - extra_vars=pb_vars - ) - - # NOTE(cloudnull): Remove this when mistral is gone. - elif plan_env_data and "workflow_parameters" in plan_env_data: - LOG.warning( - 'The workflow_parameters interface is deprecated, begin using' - ' playbook_parameters instead.' - ) - for wf_name, wf_inputs in plan_env_data["workflow_parameters"].items(): - print('Invoking workflow (%s) specified in plan-environment ' - 'file' % wf_name) - inputs = {'plan': stack_name, 'user_inputs': wf_inputs} - workflow_client = clients.workflow_engine - tripleoclients = clients.tripleoclient - with tripleoclients.messaging_websocket() as ws: - execution = base.start_workflow( - workflow_client, - wf_name, - workflow_input=inputs - ) - - # Getting the derive parameters timeout after 600 seconds. - for payload in base.wait_for_messages(workflow_client, - ws, execution, 600): - if ('message' in payload and - (payload.get('status', 'RUNNING') == "RUNNING")): - print(payload['message']) - - if payload.get('status', 'FAILED') == 'SUCCESS': - result = payload.get('result', '') - # Prints the workflow result - if result: - print('Workflow execution is completed. result:') - print(yaml.safe_dump(result, default_flow_style=False)) - else: - message = payload.get('message', '') - msg = ('Workflow execution is failed: %s' % (message)) - raise exceptions.PlanEnvWorkflowError(msg) + utils.run_ansible_playbook( + playbook=os.path.basename(pb), + inventory=static_inventory, + workdir=tmp, + playbook_dir=playbook_dir, + verbosity=verbosity, + extra_vars=pb_vars + ) def check_deprecated_parameters(clients, container): diff --git a/tripleoclient/workflows/plan_management.py b/tripleoclient/workflows/plan_management.py index ff1794226..ad4ff496b 100644 --- a/tripleoclient/workflows/plan_management.py +++ b/tripleoclient/workflows/plan_management.py @@ -11,14 +11,12 @@ # under the License. import logging import os -import tempfile import yaml -from swiftclient import exceptions as swift_exc from tripleo_common.actions import plan from tripleo_common.utils import plan as plan_utils from tripleo_common.utils import swift as swiftutils -from tripleo_common.utils import tarball +from tripleo_common.utils import template as temputils from tripleoclient import constants from tripleoclient import exceptions @@ -39,10 +37,11 @@ def _upload_templates(swift_client, container_name, tht_root, roles_file=None, plan_env_file=None, networks_file=None): """tarball up a given directory and upload it to Swift to be extracted""" - with tempfile.NamedTemporaryFile() as tmp_tarball: - tarball.create_tarball(tht_root, tmp_tarball.name) - tarball.tarball_extract_to_swift_container( - swift_client, tmp_tarball.name, container_name) + temputils.upload_templates_as_tarball( + swift=swift_client, + dir_to_upload=tht_root, + container=container_name + ) # Optional override of the roles_data.yaml file if roles_file: @@ -183,6 +182,19 @@ def create_plan_from_templates(clients, name, tht_root, roles_file=None, generate_passwords=True, plan_env_file=None, networks_file=None, verbosity_level=0, disable_image_params_prepare=False): + + with utils.TempDirs() as tmp: + utils.run_ansible_playbook( + "cli-undercloud-local-artifacts.yaml", + 'undercloud,', + workdir=tmp, + playbook_dir=constants.ANSIBLE_TRIPLEO_PLAYBOOKS, + extra_vars={ + "stack": name, + }, + verbosity=verbosity_level + ) + swift_client = clients.tripleoclient.object_store print("Creating Swift container to store the plan") @@ -210,6 +222,19 @@ def update_plan_from_templates(clients, name, tht_root, roles_file=None, networks_file=None, keep_env=False, verbosity_level=1, disable_image_params_prepare=False): + + with utils.TempDirs() as tmp: + utils.run_ansible_playbook( + "cli-undercloud-local-artifacts.yaml", + 'undercloud,', + workdir=tmp, + playbook_dir=constants.ANSIBLE_TRIPLEO_PLAYBOOKS, + extra_vars={ + "stack": name, + }, + verbosity=verbosity_level + ) + swift_client = clients.tripleoclient.object_store passwords = None keep_file_contents = {} @@ -308,13 +333,23 @@ def _list_plan_files(swift_client, container): def _upload_file(swift_client, container, filename, local_filename): with open(local_filename, 'rb') as file_content: - swift_client.put_object(container, filename, file_content) + swiftutils.put_object_string( + swift=swift_client, + container=container, + object_name=filename, + contents=file_content + ) # short function, just alias for interface parity with _upload_plan_file def _upload_file_content(swift_client, container, filename, content): LOG.debug("Uploading {0} to plan".format(filename)) - swift_client.put_object(container, filename, content) + swiftutils.put_object_string( + swift=swift_client, + container=container, + object_name=filename, + contents=content + ) def _load_passwords(swift_client, name): @@ -337,31 +372,41 @@ def _update_passwords(swift_client, name, passwords): env = yaml.safe_load(swift_client.get_object( name, constants.PLAN_ENVIRONMENT)[1]) env['passwords'] = passwords - swift_client.put_object(name, - constants.PLAN_ENVIRONMENT, - yaml.safe_dump(env, - default_flow_style=False)) - except swift_exc.ClientException: + swiftutils.put_object_string( + swift=swift_client, + container=name, + object_name=constants.PLAN_ENVIRONMENT, + contents=yaml.safe_dump(env, default_flow_style=False) + ) + except Exception as exp: # The plan likely has not been migrated to using Swift yet. - LOG.debug("Could not find plan environment %s in %s", - constants.PLAN_ENVIRONMENT, name) + LOG.debug("Could not find plan environment %s in %s - error: %s", + constants.PLAN_ENVIRONMENT, name, exp) def export_deployment_plan(clients, plan_name): - export_container = "plan-exports" - delete_after = 3600 + plan_path = os.path.join('/var/lib/tripleo/stacks', plan_name) + if os.path.exists(plan_path): + print("Plan information can be found here: {}".format(plan_path)) + return plan_path + else: + # NOTE(cloudnull): When swift is no longer in service remove this. + export_container = "plan-exports" + delete_after = 3600 - mistral_context = clients.tripleoclient.create_mistral_context() - action = plan.ExportPlanAction(plan_name, delete_after=delete_after, - exports_container=export_container) - result = action.run(mistral_context) - if result: - raise exceptions.WorkflowServiceError( - 'Exception exporting plan: {}'.format(result.error)) + mistral_context = clients.tripleoclient.create_mistral_context() + action = plan.ExportPlanAction(plan_name, delete_after=delete_after, + exports_container=export_container) + result = action.run(mistral_context) + if result: + raise exceptions.WorkflowServiceError( + 'Exception exporting plan: {}'.format(result.error)) - url = swiftutils.get_temp_url(clients.tripleoclient.object_store, - container=export_container, - object_name="{}.tar.gz".format(plan_name)) - print(url) - return url + url = swiftutils.get_temp_url( + clients.tripleoclient.object_store, + container=export_container, + object_name="{}.tar.gz".format(plan_name) + ) + print(url) + return url diff --git a/tripleoclient/workflows/roles.py b/tripleoclient/workflows/roles.py index 55ed3ee8c..9f55f0d65 100644 --- a/tripleoclient/workflows/roles.py +++ b/tripleoclient/workflows/roles.py @@ -15,6 +15,8 @@ import logging import yaml from tripleo_common.actions import plan +# TODO(cloudnull): Convert to a swiftutils in tripleo-common +# from tripleo_common.utils import swift as swiftutils LOG = logging.getLogger(__name__) @@ -36,6 +38,7 @@ def list_available_roles(clients, container='overcloud'): obj_client = clients.tripleoclient.object_store available_yaml_roles = list() LOG.info('Indexing roles from: {}'.format(container)) + # TODO(cloudnull): Convert to a swiftutils in tripleo-common for obj in obj_client.get_container(container)[-1]: name = obj['name'] if name.startswith('roles/') and name.endswith(('yml', 'yaml')):