Merge "Remove Validations Mistral Workflows"

This commit is contained in:
Zuul 2020-06-23 11:44:13 +00:00 committed by Gerrit Code Review
commit a4f244284f
9 changed files with 10 additions and 1094 deletions

View File

@ -1,111 +0,0 @@
#!/bin/bash
set -e
set -o pipefail
SCRIPT_NAME=$(basename $0)
OPTS=`getopt -o i: -l inputs: -n $SCRIPT_NAME -- "$@"`
if [ $? != 0 ]; then
echo "Terminating..." >&2
exit 1
fi
EXTRA_VARS_FILE=""
# Note the quotes around `$OPTS': they are essential!
eval set -- "$OPTS"
while true ; do
case "$1" in
-i | --inputs ) EXTRA_VARS_FILE="$2" ; shift 2 ;;
-- ) shift ; break ;;
* ) echo "Error: unsupported option $1." ; exit 1 ;;
esac
done
VALIDATION_FILE=$1
IDENTITY_FILE=$2
PLAN_NAME=$3
VALIDATIONS_BASEDIR=$4
if [[ -z "$VALIDATION_FILE" ]]; then
echo "Missing required validation file"
exit 1
fi
if [[ ! -r "$VALIDATION_FILE" ]]; then
echo "Can not find validation at $VALIDATION_FILE"
exit 1
fi
if [[ -z "$IDENTITY_FILE" ]]; then
echo "Missing required identity file"
exit 1
fi
if [[ -z "$EXTRA_VARS_FILE" ]]; then
EXTRA_VARS_ARGS=""
elif [[ -r "$EXTRA_VARS_FILE" ]]; then
EXTRA_VARS_ARGS="--extra-vars @$EXTRA_VARS_FILE"
else
echo "Can not find the inputs file at $EXTRA_VARS_FILE"
exit 1
fi
if [[ -z "$VALIDATIONS_BASEDIR" ]]; then
echo "Missing required tripleo-validations basedir"
exit 1
fi
if [[ ! -d "$VALIDATIONS_BASEDIR" ]]; then
echo "Can not find tripleo-validations basedir at $VALIDATION_BASEDIR"
exit 1
fi
# Make sure ssh is not asking interactively for hosts it can't check the key
# authenticity
export ANSIBLE_HOST_KEY_CHECKING=False
# Disable retry files until we find a good use and location for them
export ANSIBLE_RETRY_FILES_ENABLED=False
# Use a fact cache to improve performance
# The defaults from tripleo_common/actions/ansible.py cannot be used
# beause they will clash with the 'mistral' user owning the fact cache
# path.
export ANSIBLE_GATHERING=smart
export ANSIBLE_CACHE_PLUGIN=jsonfile
export ANSIBLE_CACHE_PLUGIN_CONNECTION=~/ansible_fact_cache
export ANSIBLE_GATHER_TIMEOUT=7200 # 7200s = 2h
# ANSIBLE_SUDO_FLAGS is deprecated in favor of become and will be removed in
# version 2.8. We now have to use ANSIBLE_BECOME_FLAGS to pass Flags to the
# privilege escalation sudo executable.
export ANSIBLE_BECOME_FLAGS="-H -S -n -E"
export ANSIBLE_PRIVATE_KEY_FILE=$IDENTITY_FILE
export ANSIBLE_INVENTORY=$(which tripleo-ansible-inventory)
# Use the custom validation-specific formatter
export ANSIBLE_STDOUT_CALLBACK=validation_output
export ANSIBLE_CALLBACK_PLUGINS="${VALIDATIONS_BASEDIR}/callback_plugins"
export ANSIBLE_ROLES_PATH="${VALIDATIONS_BASEDIR}/roles"
export ANSIBLE_LOOKUP_PLUGINS="${VALIDATIONS_BASEDIR}/lookup_plugins"
export ANSIBLE_LIBRARY="${VALIDATIONS_BASEDIR}/library"
# Environment variable is the easiest way to pass variables to an Ansible
# dynamic inventory script
export TRIPLEO_PLAN_NAME=${PLAN_NAME}
# NOTE(shadower): set up token-based environment for the openstack
# client when the password doesn't exist. This happens when called
# from mistral:
if [ -z "${OS_PASSWORD:-}" ]; then
# The auth type must be explicitly set to token
export OS_AUTH_TYPE=token
# The openstack client expects the token as OS_TOKEN
export OS_TOKEN=$OS_AUTH_TOKEN
# TODO(shadower): I could not get the token auth working with v3:
export OS_AUTH_URL=${OS_AUTH_URL/%v3/v2.0}
fi
ansible-playbook $EXTRA_VARS_ARGS $VALIDATION_FILE

View File

@ -30,7 +30,6 @@ scripts =
scripts/bootstrap_host_only_eval
scripts/bootstrap_host_only_exec
scripts/pull-puppet-modules
scripts/run-validation
scripts/tripleo-build-images
scripts/tripleo-config-download
scripts/tripleo-container-image-prepare
@ -92,9 +91,6 @@ mistral.actions =
tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction
tripleo.validations.get_privkey = tripleo_common.actions.validations:GetPrivkeyAction
tripleo.validations.enabled = tripleo_common.actions.validations:Enabled
tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction
tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction
tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction
# deprecated for pike release, will be removed in queens
tripleo.ansible-playbook = tripleo_common.actions.ansible:AnsiblePlaybookAction
# deprecated for rocky release, will be removed in the "S" cycle

16
sudoers
View File

@ -1,21 +1,5 @@
Defaults!/usr/bin/run-validation !requiretty
Defaults:validations !requiretty
Defaults:mistral !requiretty
mistral ALL = (validations) NOPASSWD:SETENV: /usr/bin/run-validation
mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
/usr/bin/chown -h validations\: /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
!/usr/bin/chown /tmp/validations_identity_* *, !/usr/bin/chown /tmp/validations_identity_*..*
mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
/usr/bin/chown -h validations\: /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
!/usr/bin/chown /tmp/validations_inputs_* *, !/usr/bin/chown /tmp/validations_inputs_*..*
mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
/usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
!/usr/bin/rm /tmp/validations_identity_* *, !/usr/bin/rm /tmp/validations_identity_*..*
mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
/usr/bin/rm -f /tmp/validations_inputs_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
!/usr/bin/rm /tmp/validations_inputs_* *, !/usr/bin/rm /tmp/validations_inputs_*..*
mistral ALL = NOPASSWD: /bin/nova-manage cell_v2 discover_hosts *
mistral ALL = NOPASSWD: /usr/bin/tar --xattrs --ignore-failed-read -C / -cf /var/tmp/undercloud-backup-*.tar *
mistral ALL = NOPASSWD: /usr/bin/chown mistral. /var/tmp/undercloud-backup-*/filesystem-*.tar
mistral ALL = NOPASSWD: /usr/bin/tripleo-container-image-prepare *
validations ALL = NOPASSWD: ALL

View File

@ -12,17 +12,11 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from mistral_lib import actions
from mistralclient.api import base as mistralclient_api
from oslo_concurrency.processutils import ProcessExecutionError
from swiftclient import exceptions as swiftexceptions
from tripleo_common.actions import base
from tripleo_common import constants
from tripleo_common.utils import passwords as password_utils
from tripleo_common.utils import validations as utils
class GetSshKeyAction(base.TripleOAction):
@ -81,94 +75,3 @@ class Enabled(base.TripleOAction):
return_value['stdout'] = 'Validations are disabled'
mistral_result = {"error": return_value}
return actions.Result(**mistral_result)
class ListValidationsAction(base.TripleOAction):
"""Return a set of TripleO validations"""
def __init__(self, plan=constants.DEFAULT_CONTAINER_NAME, groups=None):
super(ListValidationsAction, self).__init__()
self.groups = groups
self.plan = plan
def run(self, context):
swift = self.get_object_client(context)
try:
return utils.load_validations(
swift, plan=self.plan, groups=self.groups)
except swiftexceptions.ClientException as err:
msg = "Error loading validations from Swift: %s" % err
return actions.Result(error={"msg": six.text_type(msg)})
class ListGroupsAction(base.TripleOAction):
"""Return a set of TripleO validation groups"""
def __init__(self, plan=constants.DEFAULT_CONTAINER_NAME):
super(ListGroupsAction, self).__init__()
self.plan = plan
def run(self, context):
swift = self.get_object_client(context)
try:
validations = utils.load_validations(swift, plan=self.plan)
except swiftexceptions.ClientException as err:
msg = "Error loading validations from Swift: %s" % err
return actions.Result(error={"msg": six.text_type(msg)})
return {
group for validation in validations
for group in validation['groups']
}
class RunValidationAction(base.TripleOAction):
"""Run the given validation"""
def __init__(self, validation, plan=constants.DEFAULT_CONTAINER_NAME,
inputs=None):
super(RunValidationAction, self).__init__()
self.validation = validation
self.plan = plan
self.inputs = inputs if inputs else {}
def run(self, context):
mc = self.get_workflow_client(context)
swift = self.get_object_client(context)
identity_file = None
inputs_file = None
# Make sure the ssh_keys environment exists
try:
env = mc.environments.get('ssh_keys')
except Exception:
workflow_env = {
'name': 'ssh_keys',
'description': 'SSH keys for TripleO validations',
'variables': password_utils.create_ssh_keypair()
}
env = mc.environments.create(**workflow_env)
try:
private_key = env.variables['private_key']
identity_file = utils.write_identity_file(private_key)
inputs_file = utils.write_inputs_file(self.inputs)
stdout, stderr = utils.run_validation(swift,
self.validation,
identity_file,
self.plan,
inputs_file,
context)
return_value = {'stdout': stdout, 'stderr': stderr}
mistral_result = {"data": return_value}
except mistralclient_api.APIException as e:
return_value = {'stdout': '', 'stderr': e.error_message}
mistral_result = {"error": return_value}
except ProcessExecutionError as e:
return_value = {'stdout': e.stdout, 'stderr': e.stderr}
# Indicates to Mistral there was a failure
mistral_result = {"error": return_value}
finally:
if identity_file:
utils.cleanup_identity_file(identity_file)
if inputs_file:
utils.cleanup_inputs_file(inputs_file)
return actions.Result(**mistral_result)

View File

@ -15,13 +15,8 @@
import collections
from unittest import mock
from mistral_lib import actions
from oslo_concurrency.processutils import ProcessExecutionError
from tripleo_common.actions import validations
from tripleo_common import constants
from tripleo_common.tests import base
from tripleo_common.tests.utils import test_validations
class GetPubkeyActionTest(base.TestCase):
@ -108,151 +103,3 @@ class Enabled(base.TestCase):
self.assertIsNone(action_result.data)
self.assertEqual('Validations are disabled',
action_result.error['stdout'])
class ListValidationsActionTest(base.TestCase):
@mock.patch('tripleo_common.utils.validations.load_validations')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_run_default(self, mock_get_object_client, mock_load_validations):
mock_ctx = mock.MagicMock()
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
mock_load_validations.return_value = 'list of validations'
action = validations.ListValidationsAction(plan='overcloud')
self.assertEqual('list of validations', action.run(mock_ctx))
mock_load_validations.assert_called_once_with(
mock_get_object_client(), plan='overcloud', groups=None)
@mock.patch('tripleo_common.utils.validations.load_validations')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_run_groups(self, mock_get_object_client, mock_load_validations):
mock_ctx = mock.MagicMock()
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
mock_load_validations.return_value = 'list of validations'
action = validations.ListValidationsAction(
plan='overcloud', groups=['group1', 'group2'])
self.assertEqual('list of validations', action.run(mock_ctx))
mock_load_validations.assert_called_once_with(
mock_get_object_client(), plan='overcloud',
groups=['group1', 'group2'])
class ListGroupsActionTest(base.TestCase):
@mock.patch('tripleo_common.utils.validations.load_validations')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_run(self, mock_get_object_client, mock_load_validations):
mock_ctx = mock.MagicMock()
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
mock_load_validations.return_value = [
test_validations.VALIDATION_GROUPS_1_2_PARSED,
test_validations.VALIDATION_GROUP_1_PARSED,
test_validations.VALIDATION_WITH_METADATA_PARSED]
action = validations.ListGroupsAction(plan='overcloud')
self.assertEqual({'group1', 'group2'}, action.run(mock_ctx))
mock_load_validations.assert_called_once_with(
mock_get_object_client(), plan='overcloud')
class RunValidationActionTest(base.TestCase):
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_workflow_client')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('tripleo_common.utils.validations.write_inputs_file')
@mock.patch('tripleo_common.utils.validations.cleanup_inputs_file')
@mock.patch('tripleo_common.utils.validations.write_identity_file')
@mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
@mock.patch('tripleo_common.utils.validations.run_validation')
def test_run(self, mock_run_validation,
mock_cleanup_identity_file,
mock_write_identity_file,
mock_cleanup_inputs_file,
mock_write_inputs_file,
mock_get_object_client,
get_workflow_client_mock):
mock_ctx = mock.MagicMock()
mistral = mock.MagicMock()
get_workflow_client_mock.return_value = mistral
environment = collections.namedtuple('environment', ['variables'])
mistral.environments.get.return_value = environment(variables={
'private_key': 'shhhh'
})
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
mock_write_identity_file.return_value = 'identity_file_path'
mock_write_inputs_file.return_value = 'inputs_file_path'
mock_run_validation.return_value = 'output', 'error'
action = validations.RunValidationAction('validation')
expected = actions.Result(
data={
'stdout': 'output',
'stderr': 'error'
},
error=None)
self.assertEqual(expected, action.run(mock_ctx))
mock_write_identity_file.assert_called_once_with('shhhh')
mock_run_validation.assert_called_once_with(
mock_get_object_client(),
'validation',
'identity_file_path',
constants.DEFAULT_CONTAINER_NAME,
'inputs_file_path',
mock_ctx)
mock_cleanup_identity_file.assert_called_once_with(
'identity_file_path')
mock_cleanup_inputs_file.assert_called_once_with('inputs_file_path')
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_workflow_client')
@mock.patch('tripleo_common.utils.validations.write_inputs_file')
@mock.patch('tripleo_common.utils.validations.cleanup_inputs_file')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('tripleo_common.utils.validations.write_identity_file')
@mock.patch('tripleo_common.utils.validations.cleanup_identity_file')
@mock.patch('tripleo_common.utils.validations.run_validation')
def test_run_failing(self, mock_run_validation,
mock_cleanup_identity_file,
mock_write_identity_file,
mock_get_object_client,
mock_cleanup_inputs_file,
mock_write_inputs_file,
get_workflow_client_mock):
mock_ctx = mock.MagicMock()
mistral = mock.MagicMock()
get_workflow_client_mock.return_value = mistral
environment = collections.namedtuple('environment', ['variables'])
mistral.environments.get.return_value = environment(variables={
'private_key': 'shhhh'
})
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
mock_write_identity_file.return_value = 'identity_file_path'
mock_write_inputs_file.return_value = 'inputs_file_path'
mock_run_validation.side_effect = ProcessExecutionError(
stdout='output', stderr='error')
action = validations.RunValidationAction('validation')
expected = actions.Result(
data=None,
error={
'stdout': 'output',
'stderr': 'error'
})
self.assertEqual(expected, action.run(mock_ctx))
mock_write_identity_file.assert_called_once_with('shhhh')
mock_run_validation.assert_called_once_with(
mock_get_object_client(),
'validation',
'identity_file_path',
constants.DEFAULT_CONTAINER_NAME,
'inputs_file_path',
mock_ctx)
mock_cleanup_identity_file.assert_called_once_with(
'identity_file_path')
mock_cleanup_inputs_file.assert_called_once_with('inputs_file_path')

View File

@ -1,284 +0,0 @@
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import mock
from collections import namedtuple
import yaml
from tripleo_common.constants import PLAN_NAME_PATTERN
from tripleo_common.tests import base
from tripleo_common.utils import validations
VALIDATION_DEFAULT = """---
- hosts: overcloud
vars:
metadata:
name: First validation
description: Default validation
tasks:
- name: Ping the nodes
ping:
"""
VALIDATION_CUSTOM = """---
- hosts: overcloud
vars:
metadata:
name: First validation
description: Custom validation
tasks:
- name: Ping the nodes
ping:
"""
VALIDATION_GROUP_1 = """---
- hosts: overcloud
vars:
metadata:
name: First validation
description: A validation belonging to group1
groups:
- group1
tasks:
- name: Ping the nodes
ping:
"""
VALIDATION_WITH_METADATA = """---
- hosts: undercloud
vars:
metadata:
name: Validation with metadata
description: A validation with extra metadata
foo: a foo metadata
bar: 42
tasks:
- name: Do something useful
watch_tv:
"""
VALIDATION_GROUPS_1_2 = """---
- hosts: undercloud
vars:
metadata:
name: Validation from many groups
description: A validation belonging to groups 1 and 2
groups:
- group1
- group2
tasks:
- name: Do something useful
watch_tv:
"""
VALIDATION_GROUP_1_PARSED = {
'description': 'A validation belonging to group1',
'groups': ['group1'],
'id': 'VALIDATION_GROUP_1',
'parameters': {},
'name': 'First validation',
}
VALIDATION_WITH_METADATA_PARSED = {
'description': 'A validation with extra metadata',
'groups': [],
'id': 'VALIDATION_WITH_METADATA',
'parameters': {'foo': 'a foo metadata', 'bar': 42},
'name': 'Validation with metadata',
}
VALIDATION_GROUPS_1_2_PARSED = {
'description': 'A validation belonging to groups 1 and 2',
'groups': ['group1', 'group2'],
'id': 'VALIDATION_GROUPS_1_2',
'parameters': {},
'name': 'Validation from many groups',
}
class ValidationsKeyTest(base.TestCase):
@mock.patch("oslo_concurrency.processutils.execute")
@mock.patch('tempfile.mkstemp')
def test_write_identity_file(self, mock_mkstemp, mock_execute):
mock_open_context = mock.mock_open()
mock_mkstemp.return_value = 'fd', 'tmp_path'
with mock.patch('os.fdopen',
mock_open_context):
validations.write_identity_file('private_key')
mock_open_context.assert_called_once_with('fd', 'w')
mock_open_context().write.assert_called_once_with('private_key')
mock_execute.assert_called_once_with(
'/usr/bin/sudo', '/usr/bin/chown', '-h', 'validations:',
'tmp_path')
@mock.patch("oslo_concurrency.processutils.execute")
def test_cleanup_identity_file(self, mock_execute):
validations.cleanup_identity_file('/path/to/key')
mock_execute.assert_called_once_with(
'/usr/bin/sudo', '/usr/bin/rm', '-f', '/path/to/key')
class LoadValidationsTest(base.TestCase):
def test_get_validation_metadata(self):
validation = yaml.safe_load(VALIDATION_GROUP_1)
value = validations.get_validation_metadata(validation, 'name')
self.assertEqual('First validation', value)
@mock.patch('tripleo_common.utils.validations.DEFAULT_METADATA')
def test_get_validation_metadata_default_value(self, mock_metadata):
mock_metadata.get.return_value = 'default_value'
value = validations.get_validation_metadata({}, 'missing')
self.assertEqual('default_value', value)
def test_get_validation_parameters(self):
validation = yaml.safe_load(VALIDATION_WITH_METADATA)
value = validations.get_validation_parameters(validation)
expected = {
'foo': 'a foo metadata',
'bar': 42
}
self.assertEqual(expected, value)
def test_get_validation_parameters_no_extra(self):
validation = yaml.safe_load(VALIDATION_GROUP_1)
value = validations.get_validation_parameters(validation)
self.assertEqual({}, value)
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_load_validations_no_group(self, mock_get_object_client):
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
swiftclient.get_container.side_effect = (
({}, []), # no custom validations
({},
[{'name': 'VALIDATION_GROUP_1.yaml', 'groups': ['group1']},
{'name': 'VALIDATION_WITH_METADATA.yaml'}]))
swiftclient.get_object.side_effect = (
({}, VALIDATION_GROUP_1),
({}, VALIDATION_WITH_METADATA),
)
mock_get_object_client.return_value = swiftclient
my_validations = validations.load_validations(
mock_get_object_client(), plan='overcloud')
expected = [VALIDATION_GROUP_1_PARSED, VALIDATION_WITH_METADATA_PARSED]
self.assertEqual(expected, my_validations)
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_load_validations_group(self, mock_get_object_client):
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
swiftclient.get_container.side_effect = (
({}, []), # no custom validations
({},
[
{'name': 'VALIDATION_GROUPS_1_2.yaml',
'groups': ['group1', 'group2']},
{'name': 'VALIDATION_GROUP_1.yaml', 'groups': ['group1']},
{'name': 'VALIDATION_WITH_METADATA.yaml'}
]
)
)
swiftclient.get_object.side_effect = (
({}, VALIDATION_GROUPS_1_2),
({}, VALIDATION_GROUP_1),
({}, VALIDATION_WITH_METADATA),
)
mock_get_object_client.return_value = swiftclient
my_validations = validations.load_validations(
mock_get_object_client(), plan='overcloud', groups=['group1'])
expected = [VALIDATION_GROUPS_1_2_PARSED, VALIDATION_GROUP_1_PARSED]
self.assertEqual(expected, my_validations)
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_load_validations_custom_gets_picked_over_default(
self, mock_get_object_client):
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
swiftclient.get_container.side_effect = (
({}, [{'name': 'FIRST_VALIDATION.yaml'}]),
({}, [{'name': 'FIRST_VALIDATION.yaml'}])
)
swiftclient.get_object.side_effect = (
({}, VALIDATION_CUSTOM),
({}, VALIDATION_DEFAULT)
)
mock_get_object_client.return_value = swiftclient
my_validations = validations.load_validations(
mock_get_object_client(), plan='overcloud')
self.assertEqual(len(my_validations), 1)
self.assertEqual('Custom validation', my_validations[0]['description'])
class RunValidationTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('tripleo_common.utils.validations.download_validation')
@mock.patch('oslo_concurrency.processutils.execute')
def test_run_validation(self, mock_execute,
mock_download_validation, mock_get_object_client):
swiftclient = mock.MagicMock(url='http://swift:8080/v1/AUTH_test')
mock_get_object_client.return_value = swiftclient
Ctx = namedtuple('Ctx', 'auth_uri user_name auth_token project_name')
mock_ctx = Ctx(
auth_uri='auth_uri',
user_name='user_name',
auth_token='auth_token',
project_name='project_name'
)
mock_execute.return_value = 'output'
mock_download_validation.return_value = 'validation_path'
result = validations.run_validation(mock_get_object_client(),
'validation', 'identity_file',
'plan', 'inputs_file', mock_ctx)
self.assertEqual('output', result)
mock_execute.assert_called_once_with(
'/usr/bin/sudo', '-u', 'validations',
'OS_AUTH_URL=auth_uri',
'OS_USERNAME=user_name',
'OS_AUTH_TOKEN=auth_token',
'OS_TENANT_NAME=project_name',
'/usr/bin/run-validation',
'--inputs', 'inputs_file',
'validation_path',
'identity_file',
'plan',
'/usr/share/openstack-tripleo-validations'
)
mock_download_validation.assert_called_once_with(
mock_get_object_client(), 'plan', 'validation')
class RunPatternValidatorTest(base.TestCase):
def test_valid_patterns(self):
self.assertTrue(validations.pattern_validator("^$", ""))
self.assertTrue(
validations.pattern_validator(PLAN_NAME_PATTERN, "foo"))
self.assertTrue(
validations.pattern_validator(PLAN_NAME_PATTERN, "Foo-1"))
def test_invalid_patterns(self):
self.assertFalse(
validations.pattern_validator("^$", "foo"))
self.assertFalse(
validations.pattern_validator(PLAN_NAME_PATTERN, "foo_1"))

View File

@ -17,6 +17,7 @@
import json
import logging
import os
import re
import requests
import sys
import tempfile
@ -32,7 +33,8 @@ from tripleo_common import constants
from tripleo_common.image import kolla_builder
from tripleo_common.utils import passwords as password_utils
from tripleo_common.utils import swift as swiftutils
from tripleo_common.utils.validations import pattern_validator
LOG = logging.getLogger(__name__)
LOG = logging.getLogger(__name__)
@ -560,3 +562,10 @@ def purge_excess_keys(max_keys, keys_map):
for key_path in key_paths[1:keys_to_be_purged + 1]:
del keys_map[key_path]
return keys_map
def pattern_validator(pattern, value):
LOG.debug('Validating %s with pattern %s', value, pattern)
if not re.match(pattern, value):
return False
return True

View File

@ -1,204 +0,0 @@
# Copyright 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import re
import tempfile
import yaml
from oslo_concurrency import processutils
from swiftclient import exceptions as swiftexceptions
from tripleo_common import constants
import tripleo_common.utils.swift as swift_utils
LOG = logging.getLogger(__name__)
DEFAULT_METADATA = {
'name': 'Unnamed',
'description': 'No description',
'stage': 'No stage',
'groups': [],
}
def get_validation_metadata(validation, key):
try:
return validation[0]['vars']['metadata'][key]
except KeyError:
return DEFAULT_METADATA.get(key)
except TypeError:
LOG.exception("Failed to get validation metadata.")
def _get_validations_from_swift(swift, container, objects, groups, results,
skip_existing=False):
existing_ids = [validation['id'] for validation in results]
for obj in objects:
validation_id, ext = os.path.splitext(obj['name'])
if ext != '.yaml':
continue
if skip_existing and validation_id in existing_ids:
continue
contents = swift_utils.get_object_string(swift, container, obj['name'])
validation = yaml.safe_load(contents)
validation_groups = get_validation_metadata(validation, 'groups') or []
if not groups or set.intersection(set(groups), set(validation_groups)):
results.append({
'id': validation_id,
'name': get_validation_metadata(validation, 'name'),
'groups': get_validation_metadata(validation, 'groups'),
'description': get_validation_metadata(validation,
'description'),
'parameters': get_validation_parameters(validation)
})
return results
def load_validations(swift, plan, groups=None):
"""Loads all validations.
Retrieves all of default and custom validations for a given plan and
returns a list of dicts, with each dict representing a single validation.
If both a default and a custom validation with the same name are found,
the custom validation is picked.
"""
results = []
# Get custom validations first
container = plan
try:
objects = swift.get_container(
container, prefix=constants.CUSTOM_VALIDATIONS_FOLDER)[1]
except swiftexceptions.ClientException:
pass
else:
results = _get_validations_from_swift(
swift, container, objects, groups, results)
# Get default validations
container = constants.VALIDATIONS_CONTAINER_NAME
objects = swift.get_container(container)[1]
results = _get_validations_from_swift(swift, container, objects, groups,
results, skip_existing=True)
return results
def get_validation_parameters(validation):
try:
return {
k: v
for k, v in validation[0]['vars'].items()
if k != 'metadata'
}
except KeyError:
return dict()
def download_validation(swift, plan, validation):
"""Downloads validations from Swift to a temporary location"""
dst_dir = '/tmp/{}-validations'.format(plan)
# Download the whole default validations container
swift_utils.download_container(
swift,
constants.VALIDATIONS_CONTAINER_NAME,
dst_dir,
overwrite_only_newer=True
)
filename = '{}.yaml'.format(validation)
swift_path = os.path.join(constants.CUSTOM_VALIDATIONS_FOLDER, filename)
dst_path = os.path.join(dst_dir, filename)
# If a custom validation with that name exists, get it from the plan
# container and override. Otherwise, the default one will be used.
try:
contents = swift_utils.get_object_string(swift, plan, swift_path)
except swiftexceptions.ClientException:
pass
else:
with open(dst_path, 'w') as f:
f.write(contents)
return dst_path
def run_validation(swift, validation, identity_file,
plan, inputs_file, context):
return processutils.execute(
'/usr/bin/sudo', '-u', 'validations',
'OS_AUTH_URL={}'.format(context.auth_uri),
'OS_USERNAME={}'.format(context.user_name),
'OS_AUTH_TOKEN={}'.format(context.auth_token),
'OS_TENANT_NAME={}'.format(context.project_name),
'/usr/bin/run-validation',
'--inputs', inputs_file,
download_validation(swift, plan, validation),
identity_file,
plan,
constants.DEFAULT_VALIDATIONS_BASEDIR
)
def write_identity_file(key):
"""Write the SSH private key to disk"""
fd, path = tempfile.mkstemp(prefix='validations_identity_')
LOG.debug('Writing SSH key to disk at %s', path)
with os.fdopen(fd, 'w') as tmp:
tmp.write(key)
processutils.execute('/usr/bin/sudo', '/usr/bin/chown', '-h',
'validations:', path)
return path
def cleanup_identity_file(path):
"""Remove the SSH private key from disk"""
LOG.debug('Cleaning up identity file at %s', path)
processutils.execute('/usr/bin/sudo', '/usr/bin/rm', '-f', path)
def pattern_validator(pattern, value):
LOG.debug('Validating %s with pattern %s', value, pattern)
if not re.match(pattern, value):
return False
return True
def write_inputs_file(inputs):
"""Serialise the validation inputs to a file on disk."""
fd, path = tempfile.mkstemp(prefix='validations_inputs_')
LOG.debug("Writing the validation inputs to %s", path)
with os.fdopen(fd, 'w') as tmp:
tmp.write(yaml.dump(inputs))
processutils.execute('/usr/bin/sudo',
'/usr/bin/chown',
'-h',
'validations:',
path)
return path
def cleanup_inputs_file(path):
"""Remove the temporary validation inputs file."""
LOG.debug("Cleaning up the validation inputs at %s", path)
processutils.execute('/usr/bin/sudo', '/usr/bin/rm', '-f', path)

View File

@ -5,230 +5,6 @@ description: TripleO Validations Workflows v1
workflows:
run_validation:
input:
- validation_name
- plan: overcloud
- validation_inputs: {}
- queue_name: tripleo
tags:
- tripleo-common-managed
tasks:
notify_running:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: RUNNING
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
validation_name: <% $.validation_name %>
plan: <% $.plan %>
on-complete: run_validation
run_validation:
on-success: send_message
on-error: set_status_failed
action: tripleo.validations.run_validation validation=<% $.validation_name %> plan=<% $.plan %> inputs=<% $.validation_inputs %>
publish:
status: SUCCESS
validation: <% $.validation_name %>
stdout: <% task().result.stdout %>
stderr: <% task().result.stderr %>
set_status_failed:
on-complete: send_message
publish:
status: FAILED
validation: <% $.validation_name %>
stdout: <% task(run_validation).result.stdout %>
stderr: <% task(run_validation).result.stderr %>
send_message:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: <% $.get('status', 'SUCCESS') %>
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
validation_name: <% $.validation_name %>
plan: <% $.plan %>
stdout: <% $.stdout %>
stderr: <% $.stderr %>
run_validations:
input:
- validation_names: []
- plan: overcloud
- validation_inputs: {}
- queue_name: tripleo
tags:
- tripleo-common-managed
tasks:
notify_running:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: RUNNING
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
validation_names: <% $.validation_names %>
plan: <% $.plan %>
on-complete: run_validations
run_validations:
on-success: send_message
on-error: set_status_failed
workflow: tripleo.validations.v1.run_validation
input:
validation_name: <% $.validation %>
validation_inputs: <% $.validation_inputs %>
plan: <% $.plan %>
queue_name: <% $.queue_name %>
with-items: validation in <% $.validation_names %>
publish:
status: SUCCESS
message: <% task().result %>
set_status_failed:
on-complete: send_message
publish:
status: FAILED
message: <% task(run_validations).result.where($.status = 'FAILED') %>
send_message:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: <% $.get('status', 'SUCCESS') %>
message: <% $.get('message', '') %>
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
validation_names: <% $.validation_names %>
plan: <% $.plan %>
run_groups:
input:
- group_names: []
- plan: overcloud
- validation_inputs: {}
- queue_name: tripleo
tags:
- tripleo-common-managed
tasks:
find_validations:
on-success: notify_running
action: tripleo.validations.list_validations plan=<% $.plan %> groups=<% $.group_names %>
publish:
validations: <% task().result %>
notify_running:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: RUNNING
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
group_names: <% $.group_names %>
validation_names: <% $.validations %>
plan: <% $.plan %>
on-complete: run_validation_group
run_validation_group:
on-success: send_message
on-error: set_status_failed
workflow: tripleo.validations.v1.run_validation
input:
validation_name: <% $.validation %>
validation_inputs: <% $.validation_inputs %>
plan: <% $.plan %>
queue_name: <% $.queue_name %>
with-items: validation in <% $.validations.id %>
publish:
status: SUCCESS
message: <% task().result %>
set_status_failed:
on-complete: send_message
publish:
status: FAILED
message: <% task(run_validation_group).result.where($.status = 'FAILED') %>
send_message:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
status: <% $.get('status', 'SUCCESS') %>
message: <% $.get('message', '') %>
execution: <% execution() %>
plan_name: <% $.plan %>
payload:
group_names: <% $.group_names %>
validation_names: <% $.validations %>
plan: <% $.plan %>
list:
input:
- group_names: []
- plan: overcloud
- queue_name: tripleo
output:
validations: <% $.validations %>
tags:
- tripleo-common-managed
tasks:
find_validations:
on-success: send_message
action: tripleo.validations.list_validations plan=<% $.plan %> groups=<% $.group_names %>
publish:
status: SUCCESS
message: <% task().result %>
validations: <% task().result %>
publish-on-error:
status: FAILED
message: <% task().result %>
send_message:
workflow: tripleo.messaging.v1.send
input:
queue_name: <% $.queue_name %>
type: <% execution().name %>
execution: <% execution() %>
status: <% $.status %>
message: <% $.get('message', '') %>
payload:
validations: <% $.get('validations', []) %>
list_groups:
input:
- plan: overcloud
output:
groups: <% task(find_groups).result %>
tags:
- tripleo-common-managed
tasks:
find_groups:
action: tripleo.validations.list_groups plan=<% $.plan %>
add_validation_ssh_key_parameter:
input:
- container