Workflows to run validations

This commit adds a workflow to run ansible validations.

Validations can be run individually or by group, and the result is sent
over the specified zaqar queue.

Change-Id: Iba8b210c83e94eaa340d34e17ad2b0278ecc4e53
Depends-On: I807fed12e9a42a71c130f393370ff4152831c27b

README.rst

@@ -21,6 +21,7 @@ code to accomplish these tasks. ::
 
     sudo rm -Rf /usr/lib/python2.7/site-packages/tripleo_common*
     sudo python setup.py install
+    sudo cp /usr/share/tripleo-common/sudoers /etc/sudoers.d/tripleo-common
     sudo systemctl restart openstack-mistral-executor
     sudo systemctl restart openstack-mistral-engine
     # this loads the actions via entrypoints
@@ -45,3 +46,31 @@ Finally you need to generate an SSH keypair for the validation user and copy
 it to the overcloud's authorized_keys files::
 
     $ mistral execution-create tripleo.validations.v1.copy_ssh_key
+
+Running validations using the mistral workflow
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create a context.json file containing the arguments passed to the workflow::
+
+    {
+      "validation_names": ["512e", "rabbitmq-limits"]
+    }
+
+Run the ``tripleo.validations.v1.run_validations`` workflow with mistral
+client::
+
+    mistral execution-create tripleo.validations.v1.run_validations context.json
+
+
+Running groups of validations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create a context.json file containing the arguments passed to the workflow::
+
+    {
+      "group_names": ["network", "post-deployment"]
+    }
+
+Run the ``tripleo.validations.v1.run_groups`` workflow with mistral client::
+
+    mistral execution-create tripleo.validations.v1.run_groups context.json

scripts/run-validation

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+VALIDATION_FILE=$1
+IDENTITY_FILE=$2
+
+if [[ -z "$VALIDATION_FILE" ]]; then
+    echo "Missing required validation file"
+    exit 1
+fi
+
+if [[ ! -r "$VALIDATION_FILE" ]]; then
+    echo "Can not find validation at $VALIDATION_FILE"
+    exit 1
+fi
+
+if [[ -z "$IDENTITY_FILE" ]]; then
+    echo "Missing required identity file"
+    exit 1
+fi
+
+# Make sure ssh is not asking interactively for hosts it can't check the key
+# authenticity
+export ANSIBLE_HOST_KEY_CHECKING=False
+
+export ANSIBLE_PRIVATE_KEY_FILE=$IDENTITY_FILE
+
+export ANSIBLE_INVENTORY=$(which tripleo-ansible-inventory)
+
+ansible-playbook $VALIDATION_FILE

setup.cfg

@@ -27,9 +27,11 @@ scripts =
     scripts/tripleo-build-images
     scripts/upload-puppet-modules
     scripts/upload-swift-artifacts
+    scripts/run-validation
 
 data_files =
     lib/heat/undercloud_heat_plugins = undercloud_heat_plugins/*
+    share/tripleo-common = sudoers
     share/tripleo-common/image-yaml = image-yaml/*
     share/tripleo-common/workbooks = workbooks/*
 
@@ -74,3 +76,4 @@ mistral.actions =
     tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction
     tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction
     tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction
+    tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction

sudoers

@@ -0,0 +1,7 @@
+Defaults!/usr/bin/run-validation !requiretty
+Defaults:validations !requiretty
+Defaults:mistral !requiretty
+mistral ALL = (validations) NOPASSWD:SETENV: /usr/bin/run-validation
+mistral ALL = NOPASSWD: /usr/bin/chown validations\: /tmp/validations_identity_*
+mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_*
+validations ALL = NOPASSWD: ALL

tripleo_common/actions/validations.py

@@ -16,6 +16,9 @@ import os
 import shutil
 import tempfile
 
+from mistral.workflow import utils as mistral_workflow_utils
+from oslo_concurrency.processutils import ProcessExecutionError
+
 from tripleo_common.actions import base
 from tripleo_common.utils import validations as utils
 
@@ -76,3 +79,29 @@ class ListGroupsAction(base.TripleOAction):
             group for validation in validations
             for group in validation['groups']
         }
+
+
+class RunValidationAction(base.TripleOAction):
+    """Run the given validation"""
+    def __init__(self, validation):
+        super(RunValidationAction, self).__init__()
+        self.validation = validation
+
+    def run(self):
+        mc = self._get_workflow_client()
+        try:
+            env = mc.environments.get('ssh_keys')
+            private_key = env.variables['private_key']
+            identity_file = utils.write_identity_file(private_key)
+
+            stdout, stderr = utils.run_validation(self.validation,
+                                                  identity_file)
+            return_value = {'stdout': stdout, 'stderr': stderr}
+            mistral_result = (return_value, None)
+        except ProcessExecutionError as e:
+            return_value = {'stdout': e.stdout, 'stderr': e.stderr}
+            # Indicates to Mistral there was a failure
+            mistral_result = (None, return_value)
+        finally:
+            utils.cleanup_identity_file(identity_file)
+        return mistral_workflow_utils.Result(*mistral_result)

tripleo_common/tests/actions/test_validations.py

@@ -15,6 +15,9 @@
 import collections
 import mock
 
+from mistral.workflow import utils as mistral_workflow_utils
+from oslo_concurrency.processutils import ProcessExecutionError
+
 from tripleo_common.actions import validations
 from tripleo_common.tests import base
 from tripleo_common.tests.utils import test_validations
@@ -88,3 +91,65 @@ class ListGroupsActionTest(base.TestCase):
         action = validations.ListGroupsAction()
         self.assertEqual(set(['group1', 'group2']), action.run())
         mock_load_validations.assert_called_once_with()
+
+
+class RunValidationActionTest(base.TestCase):
+
+    @mock.patch(
+        'tripleo_common.actions.base.TripleOAction._get_workflow_client')
+    @mock.patch('tripleo_common.utils.validations.write_identity_file')
+    @mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
+    @mock.patch('tripleo_common.utils.validations.run_validation')
+    def test_run(self, mock_run_validation, mock_cleanup_identity_file,
+                 mock_write_identity_file, get_workflow_client_mock):
+        mistral = mock.MagicMock()
+        get_workflow_client_mock.return_value = mistral
+        environment = collections.namedtuple('environment', ['variables'])
+        mistral.environments.get.return_value = environment(variables={
+            'private_key': 'shhhh'
+        })
+        mock_write_identity_file.return_value = 'identity_file_path'
+        mock_run_validation.return_value = 'output', 'error'
+        action = validations.RunValidationAction('validation')
+        expected = mistral_workflow_utils.Result(
+            data={
+                'stdout': 'output',
+                'stderr': 'error'
+            },
+            error=None)
+        self.assertEqual(expected, action.run())
+        mock_write_identity_file.assert_called_once_with('shhhh')
+        mock_run_validation.assert_called_once_with('validation',
+                                                    'identity_file_path')
+        mock_cleanup_identity_file.assert_called_once_with(
+            'identity_file_path')
+
+    @mock.patch(
+        'tripleo_common.actions.base.TripleOAction._get_workflow_client')
+    @mock.patch('tripleo_common.utils.validations.write_identity_file')
+    @mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
+    @mock.patch('tripleo_common.utils.validations.run_validation')
+    def test_run_failing(self, mock_run_validation, mock_cleanup_identity_file,
+                         mock_write_identity_file, get_workflow_client_mock):
+        mistral = mock.MagicMock()
+        get_workflow_client_mock.return_value = mistral
+        environment = collections.namedtuple('environment', ['variables'])
+        mistral.environments.get.return_value = environment(variables={
+            'private_key': 'shhhh'
+        })
+        mock_write_identity_file.return_value = 'identity_file_path'
+        mock_run_validation.side_effect = ProcessExecutionError(
+            stdout='output', stderr='error')
+        action = validations.RunValidationAction('validation')
+        expected = mistral_workflow_utils.Result(
+            data=None,
+            error={
+                'stdout': 'output',
+                'stderr': 'error'
+            })
+        self.assertEqual(expected, action.run())
+        mock_write_identity_file.assert_called_once_with('shhhh')
+        mock_run_validation.assert_called_once_with('validation',
+                                                    'identity_file_path')
+        mock_cleanup_identity_file.assert_called_once_with(
+            'identity_file_path')

tripleo_common/tests/utils/test_validations.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import namedtuple
 import mock
 import yaml
 
@@ -94,6 +95,26 @@ class ValidationsKeyTest(base.TestCase):
             '/usr/bin/ssh-keygen', '-t', 'rsa', '-N', '',
             '-f', '/path/to/key', '-C', 'tripleo-validations')
 
+    @mock.patch("oslo_concurrency.processutils.execute")
+    @mock.patch('tempfile.mkstemp')
+    def test_write_identity_file(self, mock_mkstemp, mock_execute):
+        mock_open_context = mock.mock_open()
+        mock_mkstemp.return_value = 'fd', 'tmp_path'
+        with mock.patch('os.fdopen',
+                        mock_open_context):
+            validations.write_identity_file('private_key')
+
+        mock_open_context.assert_called_once_with('fd', 'w')
+        mock_open_context().write.assert_called_once_with('private_key')
+        mock_execute.assert_called_once_with(
+            '/usr/bin/sudo', '/usr/bin/chown', 'validations:', 'tmp_path')
+
+    @mock.patch("oslo_concurrency.processutils.execute")
+    def test_cleanup_identity_file(self, mock_execute):
+        validations.cleanup_identity_file('/path/to/key')
+        mock_execute.assert_called_once_with(
+            '/usr/bin/sudo', '/usr/bin/rm', '-f', '/path/to/key')
+
 
 class LoadValidationsTest(base.TestCase):
 
@@ -153,3 +174,35 @@ class LoadValidationsTest(base.TestCase):
 
         expected = [VALIDATION_GROUPS_1_2_PARSED, VALIDATION_GROUP_1_PARSED]
         self.assertEqual(expected, my_validations)
+
+
+class RunValidationTest(base.TestCase):
+
+    @mock.patch('tripleo_common.utils.validations.find_validation')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('mistral.context.ctx')
+    def test_run_validation(self, mock_ctx, mock_execute,
+                            mock_find_validation):
+        Ctx = namedtuple('Ctx', 'auth_uri user_name auth_token project_name')
+        mock_ctx.return_value = Ctx(
+            auth_uri='auth_uri',
+            user_name='user_name',
+            auth_token='auth_token',
+            project_name='project_name'
+        )
+        mock_execute.return_value = 'output'
+        mock_find_validation.return_value = 'validation_path'
+
+        result = validations.run_validation('validation', 'identity_file')
+        self.assertEqual('output', result)
+        mock_execute.assert_called_once_with(
+            '/usr/bin/sudo', '-u', 'validations',
+            'OS_AUTH_URL=auth_uri',
+            'OS_USERNAME=user_name',
+            'OS_AUTH_TOKEN=auth_token',
+            'OS_TENANT_NAME=project_name',
+            '/usr/bin/run-validation',
+            'validation_path',
+            'identity_file'
+        )
+        mock_find_validation.assert_called_once_with('validation')

tripleo_common/utils/validations.py

@@ -15,8 +15,10 @@
 import glob
 import logging
 import os
+import tempfile
 import yaml
 
+from mistral import context
 from oslo_concurrency import processutils
 
 from tripleo_common import constants
@@ -71,8 +73,43 @@ def get_remaining_metadata(validation):
         return dict()
 
 
+def find_validation(validation):
+    return '{}/{}.yaml'.format(constants.DEFAULT_VALIDATIONS_PATH, validation)
+
+
+def run_validation(validation, identity_file):
+    ctx = context.ctx()
+    return processutils.execute(
+        '/usr/bin/sudo', '-u', 'validations',
+        'OS_AUTH_URL={}'.format(ctx.auth_uri),
+        'OS_USERNAME={}'.format(ctx.user_name),
+        'OS_AUTH_TOKEN={}'.format(ctx.auth_token),
+        'OS_TENANT_NAME={}'.format(ctx.project_name),
+        '/usr/bin/run-validation',
+        find_validation(validation),
+        identity_file
+    )
+
+
 def create_ssh_keypair(key_path):
     """Create SSH keypair"""
     LOG.debug('Creating SSH keypair at %s', key_path)
     processutils.execute('/usr/bin/ssh-keygen', '-t', 'rsa', '-N', '',
                          '-f', key_path, '-C', 'tripleo-validations')
+
+
+def write_identity_file(key):
+    """Write the SSH private key to disk"""
+    fd, path = tempfile.mkstemp(prefix='validations_identity_')
+    LOG.debug('Writing SSH key to disk at %s', path)
+    with os.fdopen(fd, 'w') as tmp:
+        tmp.write(key)
+    processutils.execute('/usr/bin/sudo', '/usr/bin/chown', 'validations:',
+                         path)
+    return path
+
+
+def cleanup_identity_file(path):
+    """Write the SSH private key to disk"""
+    LOG.debug('Cleaning up identity file at %s', path)
+    processutils.execute('/usr/bin/sudo', '/usr/bin/rm', '-f', path)

workbooks/validations.yaml

@@ -5,6 +5,157 @@ description: TripleO Validations Workflows v1
 
 workflows:
 
+  run_validation:
+    type: direct
+    input:
+      - validation_name
+      - queue_name: tripleo
+
+    tasks:
+
+      notify_running:
+        on-complete: run_validation
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_validation
+              payload:
+                validation_name: <% $.validation_name %>
+                status: RUNNING
+                execution: <% execution() %>
+
+      run_validation:
+        on-success: send_message
+        on-error: set_status_failed
+        action: tripleo.validations.run_validation validation=<% $.validation_name %>
+        publish:
+          status: SUCCESS
+          stdout: <% task(run_validation).result.stdout %>
+          stderr: <% task(run_validation).result.stderr %>
+
+      set_status_failed:
+        on-complete: send_message
+        publish:
+          status: FAILED
+          stdout: <% task(run_validation).result.stdout %>
+          stderr: <% task(run_validation).result.stderr %>
+
+      send_message:
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_validation
+              payload:
+                validation_name: <% $.validation_name %>
+                status: <% $.get('status', 'SUCCESS') %>
+                stdout: <% $.stdout %>
+                stderr: <% $.stderr %>
+                execution: <% execution() %>
+
+  run_validations:
+    type: direct
+    input:
+      - validation_names: []
+      - queue_name: tripleo
+
+    tasks:
+
+      notify_running:
+        on-complete: run_validations
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_validations
+              payload:
+                validation_names: <% $.validation_names %>
+                status: RUNNING
+                execution: <% execution() %>
+
+      run_validations:
+        on-success: send_message
+        on-error: set_status_failed
+        workflow: tripleo.validations.v1.run_validation validation_name=<% $.validation %> queue_name=<% $.queue_name %>
+        with-items: validation in <% $.validation_names %>
+        publish:
+          status: SUCCESS
+
+      set_status_failed:
+        on-complete: send_message
+        publish:
+          status: FAILED
+
+      send_message:
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_validations
+              payload:
+                validation_names: <% $.validation_names %>
+                status: <% $.get('status', 'SUCCESS') %>
+                execution: <% execution() %>
+
+  run_groups:
+    type: direct
+    input:
+      - group_names: []
+      - queue_name: tripleo
+
+    tasks:
+
+      find_validations:
+        on-success: notify_running
+        action: tripleo.list_validations groups=<% $.group_names %>
+        publish:
+          validations: <% task(find_validations).result %>
+
+      notify_running:
+        on-complete: run_validation_group
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_validations
+              payload:
+                group_names: <% $.group_names %>
+                validation_names: <% $.validations.id %>
+                status: RUNNING
+                execution: <% execution() %>
+
+      run_validation_group:
+        on-success: send_message
+        on-error: set_status_failed
+        workflow: tripleo.validations.v1.run_validation validation_name=<% $.validation %> queue_name=<% $.queue_name %>
+        with-items: validation in <% $.validations.id %>
+        publish:
+          status: SUCCESS
+
+      set_status_failed:
+        on-complete: send_message
+        publish:
+          status: FAILED
+
+      send_message:
+        action: zaqar.queue_post
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.run_groups
+              payload:
+                group_names: <% $.group_names %>
+                validation_names: <% $.validations.id %>
+                status: <% $.get('status', 'SUCCESS') %>
+                execution: <% execution() %>
+
   list:
     type: direct
     input: