Workflow and action for deployment failures

Adds a new workflow and corresponding custom action for querying the
ansible failures that occurred during a plan deployment with
config-download.

Relies on the json-error.py ansible callback plugin to provide a
structured data format for the errors.

Change-Id: I16d2dd0b3022cd5964919d07dd0ec603490a3ed7
(cherry picked from commit e8c521acf1)

setup.cfg

@@ -85,6 +85,7 @@ mistral.actions =
     tripleo.container_images.prepare = tripleo_common.actions.container_images:PrepareContainerImageEnv
     tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction
     tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction
+    tripleo.deployment.get_deployment_failures = tripleo_common.actions.deployment:DeploymentFailuresAction
     tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction
     tripleo.derive_params.convert_number_to_range_list = tripleo_common.actions.derive_params:ConvertNumberToRangeListAction
     tripleo.derive_params.convert_range_to_number_list = tripleo_common.actions.derive_params:ConvertRangeToNumberListAction

tripleo_common/actions/deployment.py

@@ -14,7 +14,9 @@
 # under the License.
 import json
 import logging
+import os
 import time
+import yaml
 
 from heatclient.common import deployment_utils
 from heatclient import exc as heat_exc
@@ -264,3 +266,53 @@ class OvercloudRcAction(base.TripleOAction):
             return actions.Result(error=error)
 
         return overcloudrc.create_overcloudrc(stack, self.no_proxy, admin_pass)
+
+
+class DeploymentFailuresAction(base.TripleOAction):
+    """Return all of the failures (if any) from deploying the plan
+
+    :param plan: name of the Swift container / plan name
+    """
+
+    def __init__(self,
+                 plan=constants.DEFAULT_CONTAINER_NAME,
+                 deployment_status_file=constants.DEPLOYMENT_STATUS_FILE,
+                 work_dir=constants.MISTRAL_WORK_DIR,
+                 ansible_errors_file=constants.ANSIBLE_ERRORS_FILE):
+        super(DeploymentFailuresAction, self).__init__()
+        self.plan = plan
+        self.messages_container = "%s-messages" % self.plan
+        self.deployment_status_file = deployment_status_file
+        self.work_dir = work_dir
+        self.ansible_errors_file = ansible_errors_file
+
+    def _format_return(self, message, failures={}):
+        return dict(message=message,
+                    failures=failures)
+
+    def run(self, context):
+        swift = self.get_object_client(context)
+
+        try:
+            deployment_status_obj = swift.get_object(
+                self.messages_container, self.deployment_status_file)[1]
+        except swiftexceptions.ClientException:
+            return self._format_return(
+                'Swift container %s not found' % self.messages_container)
+
+        try:
+            deployment_status = yaml.safe_load(deployment_status_obj)
+            execution_id = \
+                deployment_status['workflow_status']['payload']['execution_id']
+        except KeyError:
+            return self._format_return(
+                'Execution not found in %s' % self.deployment_status_file)
+
+        try:
+            failures_file = os.path.join(self.work_dir, execution_id,
+                                         self.ansible_errors_file)
+            failures = json.loads(open(failures_file).read())
+            return self._format_return('', failures)
+        except IOError:
+            return self._format_return(
+                'Ansible errors file not found at %s' % failures_file)

tripleo_common/constants.py

@@ -169,3 +169,7 @@ HOST_NETWORK = 'ctlplane'
 EXTERNAL_TASKS = ['external_deploy_tasks']
 
 ANSIBLE_ERRORS_FILE = 'ansible-errors.json'
+
+DEPLOYMENT_STATUS_FILE = 'deployment_status.yaml'
+
+MISTRAL_WORK_DIR = '/var/lib/mistral'

tripleo_common/tests/actions/test_deployment.py

@@ -12,6 +12,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import json
 import mock
 import yaml
 
@@ -537,3 +538,77 @@ class OvercloudRcActionTestCase(base.TestCase):
         result = action.run(mock_ctx)
 
         self.assertEqual(result, {"overcloudrc": "fake overcloudrc"})
+
+
+class DeploymentFailuresActionTest(base.TestCase):
+
+    def setUp(self):
+        super(DeploymentFailuresActionTest, self).setUp()
+        self.plan = 'overcloud'
+        self.ctx = mock.MagicMock()
+
+    @mock.patch('tripleo_common.actions.deployment.yaml.safe_load')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('tripleo_common.actions.deployment.open')
+    def test_get_deployment_failures(
+            self, mock_open, mock_obj_client, mock_yaml_load):
+
+        test_result = dict(host0=["a", "b", "c"])
+        mock_read = mock.MagicMock()
+        mock_read.read.return_value = json.dumps(test_result)
+        mock_open.return_value = mock_read
+        action = deployment.DeploymentFailuresAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result['failures'], test_result)
+
+    @mock.patch('tripleo_common.actions.deployment.yaml.safe_load')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('tripleo_common.actions.deployment.open')
+    def test_get_deployment_failures_no_container(
+            self, mock_open, mock_obj_client, mock_yaml_load):
+
+        test_result = dict(
+            failures={},
+            message='Swift container overcloud-messages not found')
+
+        swift = mock.MagicMock()
+        swift.get_object.side_effect = swiftexceptions.ClientException("404")
+        mock_obj_client.return_value = swift
+
+        action = deployment.DeploymentFailuresAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result, test_result)
+
+    @mock.patch('tripleo_common.actions.deployment.yaml.safe_load')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('tripleo_common.actions.deployment.open')
+    def test_get_deployment_failures_no_execution(
+            self, mock_open, mock_obj_client, mock_yaml_load):
+
+        test_result = dict(
+            failures={},
+            message='Execution not found in deployment_status.yaml')
+
+        mock_yaml_load.side_effect = KeyError()
+
+        action = deployment.DeploymentFailuresAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertEqual(result, test_result)
+
+    @mock.patch('tripleo_common.actions.deployment.yaml.safe_load')
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('tripleo_common.actions.deployment.open')
+    def test_get_deployment_failures_no_file(
+            self, mock_open, mock_obj_client, mock_yaml_load):
+
+        mock_open.side_effect = IOError()
+
+        action = deployment.DeploymentFailuresAction(self.plan)
+        result = action.run(self.ctx)
+
+        self.assertTrue(result['message'].startswith(
+                        "Ansible errors file not found at"))
+        self.assertEqual({}, result['failures'])

workbooks/deployment.yaml

@@ -438,3 +438,41 @@ workflows:
                 execution: <% execution() %>
         on-success:
           - fail: <% $.get('status') = "FAILED" %>
+
+  get_deployment_failures:
+    description: >
+      Get deployment failures
+
+    tags:
+      - tripleo-common-managed
+
+    input:
+      - plan: overcloud
+      - queue_name: tripleo
+
+    output:
+      deployment_failures: <% $.deployment_failures %>
+
+    tasks:
+
+      get_deployment_failures:
+        action: tripleo.deployment.get_deployment_failures
+        input:
+          plan: <% $.plan %>
+        publish:
+          deployment_failures: <% task().result %>
+        on-complete: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      send_message:
+        workflow: tripleo.messaging.v1.send
+        input:
+          queue_name: <% $.queue_name %>
+          type: <% execution().name %>
+          execution: <% execution() %>
+          status: <% $.get("status", "SUCCESS") %>
+          message: <% $.get("message", "") %>
+          payload:
+            deployment_failures: <% $.get(deployment_failures, "") %>