Merge "Sync state if needed during retrieval"

2018-11-12 06:55:14 +00:00
parent e9c45aef8b de06f9373a
commit b8bfff895b
4 changed files with 312 additions and 6 deletions
@@ -88,6 +88,7 @@ mistral.actions =
    tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction
    tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction
    tripleo.deployment.get_deployment_failures = tripleo_common.actions.deployment:DeploymentFailuresAction
+    tripleo.deployment.get_deployment_status = tripleo_common.actions.deployment:DeploymentStatusAction
    tripleo.deployment.convert_status = tripleo_common.actions.deployment:ConvertStatusAction
    tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction
    tripleo.derive_params.convert_number_to_range_list = tripleo_common.actions.derive_params:ConvertNumberToRangeListAction
@@ -16,6 +16,7 @@ import json
 import logging
 import os
 import time
+import yaml

 from heatclient.common import deployment_utils
 from heatclient import exc as heat_exc
@@ -321,6 +322,89 @@ class DeploymentFailuresAction(base.TripleOAction):
                'Ansible errors file not found at %s' % failures_file)


+class DeploymentStatusAction(base.TripleOAction):
+    """Get the deployment status and update it if necessary
+
+    The status will be set based off of the stack status and any running
+    config_download workflow.
+    """
+
+    def __init__(self,
+                 plan=constants.DEFAULT_CONTAINER_NAME):
+        super(DeploymentStatusAction, self).__init__()
+        self.plan = plan
+
+    def run(self, context):
+        orchestration_client = self.get_orchestration_client(context)
+        workflow_client = self.get_workflow_client(context)
+        swift_client = self.get_object_client(context)
+
+        try:
+            stack = orchestration_client.stacks.get(self.plan)
+        except heat_exc.HTTPNotFound:
+            return dict(status_update=None,
+                        deployment_status=None)
+
+        try:
+            headers, body = swift_client.get_object(
+                '%s-messages' % self.plan,
+                'deployment_status.yaml')
+
+            deployment_status = yaml.safe_load(body)['deployment_status']
+        except swiftexceptions.ClientException:
+            deployment_status = None
+
+        stack_status = stack.stack_status
+        cd_status = None
+        ansible_status = None
+        # Will get set to new status if an update is required
+        status_update = None
+
+        cd_execs = workflow_client.executions.find(
+            workflow_name='tripleo.deployment.v1.config_download_deploy')
+        cd_execs.sort(key=lambda x: x.updated_at)
+        if cd_execs:
+            cd_exec = workflow_client.executions.get(cd_execs[-1].id)
+            cd_status = cd_exec.state
+            ansible_status = json.loads(cd_exec.output)['deployment_status']
+
+        def update_status(status):
+            # If we need to update the status return it
+            if deployment_status != status:
+                return status
+
+        # Update the status if needed. We do this since tripleoclient does not
+        # yet use a single API for overcloud deployment. Since there is no long
+        # running process to make sure the status is updated, we instead update
+        # the status if needed when we get it with this action.
+        #
+        # The logic here is:
+        #
+        # If stack or config_download is in progress, then the status is
+        # deploying.
+        #
+        # Else if stack is failed or config_download is failed or ansible is
+        # failed, then the status is failed.
+        #
+        # Else if config_download status is success and ansible is success
+        # then status is success.
+        #
+        # Else, we just return the read deployment_status from earlier.
+        if stack_status.endswith('IN_PROGRESS') or cd_status == 'RUNNING':
+            status_update = update_status('DEPLOYING')
+        elif stack_status.endswith('FAILED') or cd_status == 'FAILED' \
+                or ansible_status == 'DEPLOY_FAILED':
+            status_update = update_status('DEPLOY_FAILED')
+        elif cd_status == 'SUCCESS' and ansible_status == 'DEPLOY_SUCCESS':
+            status_update = update_status('DEPLOY_SUCCESS')
+
+        return dict(cd_status=cd_status,
+                    stack_status=stack_status,
+                    deployment_status=deployment_status,
+                    ansible_status=ansible_status,
+                    status_update=status_update)
+
+
 class ConvertStatusAction(base.TripleOAction):
    """Translate a Heat stack status into a config-download deployment status

@@ -503,6 +503,198 @@ class OvercloudRcActionTestCase(base.TestCase):
        self.assertEqual(result, {"overcloudrc": "fake overcloudrc"})


+class DeploymentStatusActionTest(base.TestCase):
+
+    def setUp(self):
+        super(DeploymentStatusActionTest, self).setUp()
+        self.plan = 'overcloud'
+        self.ctx = mock.MagicMock()
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'COMPLETE'
+        heat().stacks.get.return_value = mock_stack
+
+        body = 'deployment_status: DEPLOY_SUCCESS'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_SUCCESS"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['stack_status'], 'COMPLETE')
+        self.assertEqual(result['cd_status'], 'SUCCESS')
+        self.assertEqual(result['deployment_status'], 'DEPLOY_SUCCESS')
+        self.assertEqual(result['status_update'], None)
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status_update_failed(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'FAILED'
+        heat().stacks.get.return_value = mock_stack
+
+        body = 'deployment_status: DEPLOY_SUCCESS'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_SUCCESS"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['stack_status'], 'FAILED')
+        self.assertEqual(result['cd_status'], 'SUCCESS')
+        self.assertEqual(result['deployment_status'], 'DEPLOY_SUCCESS')
+        self.assertEqual(result['status_update'], 'DEPLOY_FAILED')
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status_update_deploying(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'IN_PROGRESS'
+        heat().stacks.get.return_value = mock_stack
+
+        body = 'deployment_status: DEPLOY_SUCCESS'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_SUCCESS"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['stack_status'], 'IN_PROGRESS')
+        self.assertEqual(result['cd_status'], 'SUCCESS')
+        self.assertEqual(result['deployment_status'], 'DEPLOY_SUCCESS')
+        self.assertEqual(result['status_update'], 'DEPLOYING')
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status_update_success(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'COMPLETE'
+        heat().stacks.get.return_value = mock_stack
+
+        body = 'deployment_status: DEPLOYING'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_SUCCESS"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['stack_status'], 'COMPLETE')
+        self.assertEqual(result['cd_status'], 'SUCCESS')
+        self.assertEqual(result['deployment_status'], 'DEPLOYING')
+        self.assertEqual(result['status_update'], 'DEPLOY_SUCCESS')
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status_ansible_failed(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'COMPLETE'
+        heat().stacks.get.return_value = mock_stack
+
+        body = 'deployment_status: DEPLOYING'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_FAILED"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['stack_status'], 'COMPLETE')
+        self.assertEqual(result['cd_status'], 'SUCCESS')
+        self.assertEqual(result['deployment_status'], 'DEPLOYING')
+        self.assertEqual(result['status_update'], 'DEPLOY_FAILED')
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_object_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_workflow_client')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.'
+                'get_orchestration_client')
+    def test_get_deployment_status_no_heat_stack(
+            self, heat, mistral, swift):
+
+        mock_stack = mock.Mock()
+        mock_stack.stack_status = 'COMPLETE'
+        heat().stacks.get.side_effect = heat_exc.HTTPNotFound()
+
+        body = 'deployment_status: DEPLOY_SUCCESS'
+        swift().get_object.return_value = [mock.Mock(), body]
+
+        execution = mock.Mock()
+        execution.updated_at = 1
+        execution.state = 'SUCCESS'
+        execution.output = '{"deployment_status":"DEPLOY_SUCCESS"}'
+        mistral().executions.get.return_value = execution
+        mistral().executions.find.return_value = [execution]
+
+        action = deployment.DeploymentStatusAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['status_update'], None)
+        self.assertEqual(result['deployment_status'], None)
+
+
 class DeploymentFailuresActionTest(base.TestCase):

    def setUp(self):
@@ -232,7 +232,7 @@ workflows:
          message: <% task().result %>
        on-success:
          - wait_for_stack_complete: <% $.config_download %>
-          - set_deployment_success: <% not $.config_download %>
+          - set_deployment_deploying: <% not $.config_download %>
        on-error: set_deployment_failed

      wait_for_stack_complete:
@@ -316,6 +316,12 @@ workflows:
          status: SUCCESS
          deployment_status: DEPLOY_SUCCESS

+      set_deployment_deploying:
+        on-success: send_message
+        publish:
+          status: SUCCESS
+          deployment_status: DEPLOYING
+
      send_message:
        workflow: tripleo.messaging.v1.send
        input:
@@ -702,7 +708,8 @@ workflows:

  get_deployment_status:
    description: >
-      Get deployment status
+      Get deployment status and update it if needed based on stack and
+      config_downlooad status.

    tags:
      - tripleo-common-managed
@@ -717,6 +724,32 @@ workflows:
    tasks:

      get_deployment_status:
+        action: tripleo.deployment.get_deployment_status
+        input:
+          plan: <% $.plan %>
+        publish:
+          status_update: <% yaml_parse(coalesce(task().result.status_update, '')) %>
+          deployment_status: <% task().result.deployment_status %>
+        on-complete:
+          - reload_deployment_status: <% $.status_update = null and $.deployment_status != null %>
+          - update_status: <% $.status_update != null and $.deployment_status != null %>
+          - send_message: <% $.deployment_status = null %>
+        publish-on-error:
+          message: No deployment status found for plan <% $.plan %>
+          deployment_status: ""
+
+      update_status:
+        workflow: tripleo.messaging.v1.send
+        input:
+          queue_name: <% $.queue_name %>
+          type: <% execution().name %>
+          status: RUNNING
+          execution: <% execution() %>
+          plan_name: <% $.plan %>
+          deployment_status: <% $.status_update %>
+        on-complete: reload_deployment_status
+
+      reload_deployment_status:
        action: swift.get_object
        input:
          container: <% $.plan %>-messages
@@ -724,9 +757,6 @@ workflows:
        publish:
          deployment_status: <% yaml_parse(task().result.last()) %>
        on-complete: send_message
-        publish-on-error:
-          message: No deployment status found for plan <% $.plan %>
-          deployment_status: ""

      send_message:
        workflow: tripleo.messaging.v1.send
@@ -735,7 +765,6 @@ workflows:
          type: <% execution().name %>
          execution: <% execution() %>
          status: <% $.get("status", "SUCCESS") %>
-          message: <% $.get("message", "") %>
          payload:
            deployment_status: <% $.get(deployment_status, "") %>