Make validation inputs configurable via Mistral

The validations have certain values (e.g. the amount of RAM necessary
for the undercloud node) configurable, but these were not exposed
through Mistral.

This adds a new `--inputs` parameter to the `run-validation` script
which takes a path to a YAML or JSON file with the additional
inputs (i.e. Ansible extra-args) defined.

And the `run_validation` action now optionally takes an `inputs`
dictionary which gets passed to `run-validation`.

Closes-Bug: #1625547
Implements: blueprint validation-framework
Signed-off-by: Gael Chamoulaud <gchamoul@redhat.com>

Change-Id: I8944cf7133d47869d26974fd123cd93c98425f17
Co-authored: Tomas Sedovic <tsedovic@redhat.com>

scripts/run-validation

@@ -3,6 +3,27 @@
 set -e
 set -o pipefail
 
+SCRIPT_NAME=$(basename $0)
+OPTS=`getopt -o i: -l inputs: -n $SCRIPT_NAME -- "$@"`
+
+if [ $? != 0 ]; then
+    echo "Terminating..." >&2
+    exit 1
+fi
+
+EXTRA_VARS_FILE=""
+
+# Note the quotes around `$OPTS': they are essential!
+eval set -- "$OPTS"
+
+while true ; do
+    case "$1" in
+        -i | --inputs ) EXTRA_VARS_FILE="$2" ; shift 2 ;;
+        -- ) shift ; break ;;
+        * ) echo "Error: unsupported option $1." ; exit 1 ;;
+    esac
+done
+
 VALIDATION_FILE=$1
 IDENTITY_FILE=$2
 PLAN_NAME=$3
@@ -22,6 +43,15 @@ if [[ -z "$IDENTITY_FILE" ]]; then
     exit 1
 fi
 
+if [[ -z "$EXTRA_VARS_FILE" ]]; then
+    EXTRA_VARS_ARGS=""
+elif [[ -r "$EXTRA_VARS_FILE" ]]; then
+    EXTRA_VARS_ARGS="--extra-vars @$EXTRA_VARS_FILE"
+else
+    echo "Can not find the inputs file at $EXTRA_VARS_FILE"
+    exit 1
+fi
+
 # Make sure ssh is not asking interactively for hosts it can't check the key
 # authenticity
 export ANSIBLE_HOST_KEY_CHECKING=False
@@ -55,4 +85,4 @@ if [ -z "${OS_PASSWORD:-}" ]; then
     export OS_AUTH_URL=${OS_AUTH_URL/%v3/v2.0}
 fi
 
-ansible-playbook $VALIDATION_FILE
+ansible-playbook $EXTRA_VARS_ARGS $VALIDATION_FILE

sudoers

@@ -4,8 +4,13 @@ Defaults:mistral !requiretty
 mistral ALL = (validations) NOPASSWD:SETENV: /usr/bin/run-validation
 mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
         !/usr/bin/chown /tmp/validations_identity_* *, !/usr/bin/chown /tmp/validations_identity_*..*
+mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
+        /usr/bin/chown validations\: /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
+        !/usr/bin/chown /tmp/validations_inputs_* *, !/usr/bin/chown /tmp/validations_inputs_*..*
 mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
         !/usr/bin/rm /tmp/validations_identity_* *, !/usr/bin/rm /tmp/validations_identity_*..*
+mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
+        !/usr/bin/rm /tmp/validations_inputs_* *, !/usr/bin/rm /tmp/validations_inputs_*..*
 mistral ALL = NOPASSWD: /bin/nova-manage cell_v2 discover_hosts *
 mistral ALL = NOPASSWD: /usr/bin/tar --xattrs --ignore-failed-read -C / -cf /var/tmp/undercloud-backup-*.tar *
 mistral ALL = NOPASSWD: /usr/bin/chown mistral. /var/tmp/undercloud-backup-*/filesystem-*.tar

tripleo_common/actions/validations.py

@@ -121,16 +121,19 @@ class ListGroupsAction(base.TripleOAction):
 
 class RunValidationAction(base.TripleOAction):
     """Run the given validation"""
-    def __init__(self, validation, plan=constants.DEFAULT_CONTAINER_NAME):
+    def __init__(self, validation, plan=constants.DEFAULT_CONTAINER_NAME,
+                 inputs=None):
         super(RunValidationAction, self).__init__()
         self.validation = validation
         self.plan = plan
+        self.inputs = inputs if inputs else {}
 
     def run(self, context):
         mc = self.get_workflow_client(context)
         swift = self.get_object_client(context)
 
         identity_file = None
+        inputs_file = None
 
         # Make sure the ssh_keys environment exists
         try:
@@ -146,11 +149,13 @@ class RunValidationAction(base.TripleOAction):
         try:
             private_key = env.variables['private_key']
             identity_file = utils.write_identity_file(private_key)
+            inputs_file = utils.write_inputs_file(self.inputs)
 
             stdout, stderr = utils.run_validation(swift,
                                                   self.validation,
                                                   identity_file,
                                                   self.plan,
+                                                  inputs_file,
                                                   context)
             return_value = {'stdout': stdout, 'stderr': stderr}
             mistral_result = {"data": return_value}
@@ -164,6 +169,8 @@ class RunValidationAction(base.TripleOAction):
         finally:
             if identity_file:
                 utils.cleanup_identity_file(identity_file)
+            if inputs_file:
+                utils.cleanup_inputs_file(inputs_file)
         return actions.Result(**mistral_result)
 
 

tripleo_common/tests/actions/test_validations.py

@@ -165,11 +165,17 @@ class RunValidationActionTest(base.TestCase):
     @mock.patch(
         'tripleo_common.actions.base.TripleOAction.get_workflow_client')
     @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('tripleo_common.utils.validations.write_inputs_file')
+    @mock.patch('tripleo_common.utils.validations.cleanup_inputs_file')
     @mock.patch('tripleo_common.utils.validations.write_identity_file')
     @mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
     @mock.patch('tripleo_common.utils.validations.run_validation')
-    def test_run(self, mock_run_validation, mock_cleanup_identity_file,
+    def test_run(self, mock_run_validation,
-                 mock_write_identity_file, mock_get_object_client,
+                 mock_cleanup_identity_file,
+                 mock_write_identity_file,
+                 mock_cleanup_inputs_file,
+                 mock_write_inputs_file,
+                 mock_get_object_client,
                  get_workflow_client_mock):
         mock_ctx = mock.MagicMock()
         mistral = mock.MagicMock()
@@ -181,6 +187,7 @@ class RunValidationActionTest(base.TestCase):
         swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
         mock_get_object_client.return_value = swiftclient
         mock_write_identity_file.return_value = 'identity_file_path'
+        mock_write_inputs_file.return_value = 'inputs_file_path'
         mock_run_validation.return_value = 'output', 'error'
         action = validations.RunValidationAction('validation')
         expected = actions.Result(
@@ -196,18 +203,26 @@ class RunValidationActionTest(base.TestCase):
             'validation',
             'identity_file_path',
             constants.DEFAULT_CONTAINER_NAME,
+            'inputs_file_path',
             mock_ctx)
         mock_cleanup_identity_file.assert_called_once_with(
             'identity_file_path')
+        mock_cleanup_inputs_file.assert_called_once_with('inputs_file_path')
 
     @mock.patch(
         'tripleo_common.actions.base.TripleOAction.get_workflow_client')
+    @mock.patch('tripleo_common.utils.validations.write_inputs_file')
+    @mock.patch('tripleo_common.utils.validations.cleanup_inputs_file')
     @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
     @mock.patch('tripleo_common.utils.validations.write_identity_file')
     @mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
     @mock.patch('tripleo_common.utils.validations.run_validation')
-    def test_run_failing(self, mock_run_validation, mock_cleanup_identity_file,
+    def test_run_failing(self, mock_run_validation,
-                         mock_write_identity_file, mock_get_object_client,
+                         mock_cleanup_identity_file,
+                         mock_write_identity_file,
+                         mock_get_object_client,
+                         mock_cleanup_inputs_file,
+                         mock_write_inputs_file,
                          get_workflow_client_mock):
         mock_ctx = mock.MagicMock()
         mistral = mock.MagicMock()
@@ -219,6 +234,7 @@ class RunValidationActionTest(base.TestCase):
         swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
         mock_get_object_client.return_value = swiftclient
         mock_write_identity_file.return_value = 'identity_file_path'
+        mock_write_inputs_file.return_value = 'inputs_file_path'
         mock_run_validation.side_effect = ProcessExecutionError(
             stdout='output', stderr='error')
         action = validations.RunValidationAction('validation')
@@ -235,6 +251,8 @@ class RunValidationActionTest(base.TestCase):
             'validation',
             'identity_file_path',
             constants.DEFAULT_CONTAINER_NAME,
+            'inputs_file_path',
             mock_ctx)
         mock_cleanup_identity_file.assert_called_once_with(
             'identity_file_path')
+        mock_cleanup_inputs_file.assert_called_once_with('inputs_file_path')

tripleo_common/tests/utils/test_validations.py

@@ -249,7 +249,7 @@ class RunValidationTest(base.TestCase):
 
         result = validations.run_validation(mock_get_object_client(),
                                             'validation', 'identity_file',
-                                            'plan', mock_ctx)
+                                            'plan', 'inputs_file', mock_ctx)
         self.assertEqual('output', result)
         mock_execute.assert_called_once_with(
             '/usr/bin/sudo', '-u', 'validations',
@@ -258,6 +258,7 @@ class RunValidationTest(base.TestCase):
             'OS_AUTH_TOKEN=auth_token',
             'OS_TENANT_NAME=project_name',
             '/usr/bin/run-validation',
+            '--inputs', 'inputs_file',
             'validation_path',
             'identity_file',
             'plan'

tripleo_common/utils/validations.py

@@ -140,7 +140,8 @@ def download_validation(swift, plan, validation):
     return dst_path
 
 
-def run_validation(swift, validation, identity_file, plan, context):
+def run_validation(swift, validation, identity_file,
+                   plan, inputs_file, context):
     return processutils.execute(
         '/usr/bin/sudo', '-u', 'validations',
         'OS_AUTH_URL={}'.format(context.auth_uri),
@@ -148,6 +149,7 @@ def run_validation(swift, validation, identity_file, plan, context):
         'OS_AUTH_TOKEN={}'.format(context.auth_token),
         'OS_TENANT_NAME={}'.format(context.project_name),
         '/usr/bin/run-validation',
+        '--inputs', inputs_file,
         download_validation(swift, plan, validation),
         identity_file,
         plan
@@ -176,3 +178,20 @@ def pattern_validator(pattern, value):
     if not re.match(pattern, value):
         return False
     return True
+
+
+def write_inputs_file(inputs):
+    """Serialise the validation inputs to a file on disk."""
+    fd, path = tempfile.mkstemp(prefix='validation_inputs_')
+    LOG.debug("Writing the validation inputs to %s", path)
+    with os.fdopen(fd, 'w') as tmp:
+        tmp.write(yaml.dump(inputs))
+    processutils.execute('/usr/bin/sudo', '/usr/bin/chown', 'validations:',
+                         path)
+    return path
+
+
+def cleanup_inputs_file(path):
+    """Remove the temporary validation inputs file."""
+    LOG.debug("Cleaning up the validation inputs at %s", path)
+    processutils.execute('/usr/bin/sudo', '/usr/bin/rm', '-f', path)

workbooks/validations.yaml

@@ -9,6 +9,7 @@ workflows:
     input:
       - validation_name
       - plan: overcloud
+      - validation_inputs: {}
       - queue_name: tripleo
 
     tags:
@@ -32,7 +33,7 @@ workflows:
       run_validation:
         on-success: send_message
         on-error: set_status_failed
-        action: tripleo.validations.run_validation validation=<% $.validation_name %> plan=<% $.plan %>
+        action: tripleo.validations.run_validation validation=<% $.validation_name %> plan=<% $.plan %> inputs=<% $.validation_inputs %>
         publish:
           status: SUCCESS
           validation: <% $.validation_name %>
@@ -65,6 +66,7 @@ workflows:
     input:
       - validation_names: []
       - plan: overcloud
+      - validation_inputs: {}
       - queue_name: tripleo
 
     tags:
@@ -91,6 +93,7 @@ workflows:
         workflow: tripleo.validations.v1.run_validation
         input:
           validation_name: <% $.validation %>
+          validation_inputs: <% $.validation_inputs %>
           plan: <% $.plan %>
           queue_name: <% $.queue_name %>
         with-items: validation in <% $.validation_names %>
@@ -121,6 +124,7 @@ workflows:
     input:
       - group_names: []
       - plan: overcloud
+      - validation_inputs: {}
       - queue_name: tripleo
 
     tags:
@@ -154,6 +158,7 @@ workflows:
         workflow: tripleo.validations.v1.run_validation
         input:
           validation_name: <% $.validation %>
+          validation_inputs: <% $.validation_inputs %>
           plan: <% $.plan %>
           queue_name: <% $.queue_name %>
         with-items: validation in <% $.validations.id %>