openstack
/
validations-libs
Code Issues Proposed changes

Run validations with parameters from a file 

Resolves: rhbz#2122209

Signed-off-by: Veronika Fisarova <vfisarov@redhat.com>
Change-Id: Ifc6c28003c4c2c5f3dd6198e650f9713a02dc82d
This commit is contained in:
Veronika Fisarova 2022-11-11 15:22:09 +01:00
parent 9a2bcee59f
commit 278d0e9d80
8 changed files with 482 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
preliminary-file-structure.yaml Normal file
View File

@ -0,0 +1,59 @@
---
# This file is generated by the `validation ... ` CLI.
#
# As shown in this template, you can specify validation(s) of you choice by the
# following options:
#
# validation(s), group(s), product(s) and category(ies) included in the run
# + parameters to each specific validation,
# validation, group(s), product(s), category(ies) excluded in the run,
#
# optional arguments for the run,
# e.g.:
# --config
# --limit
# --validation-dir
# and other.
#
# Delete the comment sign for the use of the required action. Add the '-' sign for
# including, respectively excluding, more items on the list.
#
#
#Example:
#
#Note: Skip list isn't included in the run_arguments list because it has the same
#functionality as the exclude_validation parameter.
#
#
include_validation:
- check-rhsm-version
include_group:
- prep
- pre-deployment
# include_category:
# -
# include_product:
# -
exclude_validation:
- fips-enabled
# exclude_group:
# -
# exclude_category:
# -
# exclude_product:
# -
config: CONFIG_PATH
limit:
- undercloud-0
- undercloud-1
ssh-user: SSH_USER
validation-dir: VALIDATION_DIR
ansible-base-dir: ANSIBLE_BASE_DIR
validation-log-dir: VALIDATION_LOG_DIR
inventory: INVENTORY_DIR
output-log: foo
python-interpreter: PYTHON_INTERPRETER_PATH
extra-env-vars:
key1: val1
key2: val2
extra-vars-file: JSON/YAML_PATH

1
setup.cfg
View File

@ -40,6 +40,7 @@ validation.cli:
show_group = validations_libs.cli.show:ShowGroup
show_parameter = validations_libs.cli.show:ShowParameter
run = validations_libs.cli.run:Run
file = validations_libs.cli.file:File
history_list = validations_libs.cli.history:ListHistory
history_get = validations_libs.cli.history:GetHistory
init = validations_libs.cli.community:CommunityValidationInit

84
validations_libs/cli/file.py Normal file
View File

@ -0,0 +1,84 @@
import os
from validations_libs import utils
from validations_libs.cli import common
from validations_libs.cli.base import BaseCommand
from validations_libs.validation_actions import ValidationActions
from validations_libs.exceptions import ValidationRunException
class File(BaseCommand):
"""Include validations by name(s), group(s), category(ies) or by product(s),
or exclude validations by name(s), group(s), category(ies) or by product(s),
and run them from File"""
def get_parser(self, parser):
"""Argument parser ..."""
parser = super(File, self).get_parser(parser)
parser.add_argument(
'--path-to-file',
dest='path_to_file',
required=True,
default=None,
help=("The path where the YAML file is stored.\n"))
parser.add_argument(
'--junitxml',
dest='junitxml',
default=None,
help=("Path where the run result in JUnitXML format will be stored.\n"))
return parser
def take_action(self, parsed_args):
"""Take action"""
self.base.set_argument_parser(self, parsed_args)
if parsed_args.path_to_file:
try:
yaml_file = common.read_cli_data_file(parsed_args.path_to_file)
if not isinstance(yaml_file, dict):
raise ValidationRunException("Wrong format of the File.")
except (FileNotFoundError) as e:
raise FileNotFoundError(e)
self.base.config = utils.load_config(os.path.abspath(yaml_file['config']))
v_actions = ValidationActions(yaml_file.get('validation-dir'),
log_path=yaml_file.get('validation-log-dir',
'/home/stack/validations'))
hosts = yaml_file.get('limit')
hosts_converted = ",".join(hosts)
try:
results = v_actions.run_validations(
validation_name=yaml_file.get('include_validation'),
group=yaml_file.get('include_group'),
category=yaml_file.get('include_category'),
product=yaml_file.get('include_product'),
exclude_validation=yaml_file.get('exclude_validation'),
exclude_group=yaml_file.get('exclude_group'),
exclude_category=yaml_file.get('exclude_category'),
exclude_product=yaml_file.get('exclude_product'),
validation_config=self.base.config,
limit_hosts=hosts_converted,
ssh_user=yaml_file.get('ssh-user', 'stack'),
inventory=yaml_file.get('inventory', 'localhost'),
base_dir=yaml_file.get('ansible-base-dir', '/usr/share/ansible'),
python_interpreter=yaml_file.get('python-interpreter', '/usr/bin/python3'),
extra_vars=yaml_file.get('extra-vars-file'),
extra_env_vars=yaml_file.get('extra-env-vars'))
except (RuntimeError, ValidationRunException) as e:
raise ValidationRunException(e)
if results:
failed_rc = any([r for r in results if r['Status'] == 'FAILED'])
if yaml_file.get('output-log'):
common.write_output(yaml_file.get('output-log'), results)
if parsed_args.junitxml:
common.write_junitxml(parsed_args.junitxml, results)
common.print_dict(results)
if failed_rc:
raise ValidationRunException("One or more validations have failed.")
else:
msg = ("No validation has been run, please check "
"log in the Ansible working directory.")
raise ValidationRunException(msg)

4
validations_libs/cli/run.py
View File

@ -197,8 +197,8 @@ class Run(BaseCommand):
extra_vars = common.read_cli_data_file(
parsed_args.extra_vars_file)
skip_list = None
# skip_list is {} so it could be properly processed in the ValidationAction class
skip_list = {}
if parsed_args.skip_list:
skip_list = common.read_cli_data_file(parsed_args.skip_list)
if not isinstance(skip_list, dict):

145
validations_libs/tests/cli/test_file.py Normal file
View File

@ -0,0 +1,145 @@
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import sys
import copy
try:
from unittest import mock
except ImportError:
import mock
from validations_libs.cli import file
from validations_libs.exceptions import ValidationRunException
from validations_libs.tests import fakes
from validations_libs.tests.cli.fakes import BaseCommand
class TestRun(BaseCommand):
maxDiff = None
def setUp(self):
super(TestRun, self).setUp()
self.cmd = file.File(self.app, None)
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN),
autospec=True)
def test_run_validation_success(self, mock_run):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.cmd.take_action(parsed_args)
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN),
autospec=True)
@mock.patch('validations_libs.utils.create_log_dir', return_value='/foo/bar/logdir/')
@mock.patch('validations_libs.utils.parse_all_validations_on_disk', return_value=[])
def test_run_validation_success_full(self, mock_parse_all_validations_on_disk, mock_create_log_dir, mock_run):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.cmd.take_action(parsed_args)
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN),
autospec=True)
@mock.patch('validations_libs.utils.parse_all_validations_on_disk')
def test_run_validation_success_validations_on_disk_exists(self, mock_validation_dir, mock_run):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml')]
mock_validation_dir.return_value = [{'id': 'foo',
'description': 'foo',
'groups': ['prep', 'pre-deployment'],
'categories': ['os', 'storage'],
'products': ['product1'],
'name': 'Advanced Format 512e Support',
'path': '/tmp'}]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.cmd.take_action(parsed_args)
@mock.patch('os.path.exists', return_value=True)
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN),
autospec=True)
def test_run_validation_success_with_junitxml(self, mock_run, mock_exists):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml',
'--junitxml', 'foo'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml'),
('junitxml', 'foo')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.cmd.take_action(parsed_args)
def test_run_validation_cmd_parser_error(self):
args = self._set_args(['foo', 'preliminary-file-structure.yaml'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml')]
self.assertRaises(Exception, self.check_parser, self.cmd, args, verifylist)
@mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN),
autospec=True)
def test_run_validation_failed_validation(self, mock_run, mock_exists):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml'])
verifylist = [
('path_to_file', 'preliminary-file-structure.yaml')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.assertRaises(ValidationRunException,
self.cmd.take_action, parsed_args)
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN),
autospec=True)
@mock.patch('os.path.exists', return_values=True)
def test_run_validation_failed_validation_junitxml_module_disabled(self, mock_exists,
mock_run):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml',
'--junitxml', 'foo'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml'),
('junitxml', 'foo')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.assertRaises(ValidationRunException, self.cmd.take_action, parsed_args)
@mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN),
autospec=True)
@mock.patch('validations_libs.cli.common.write_junitxml', return_value={})
@mock.patch('os.path.exists', return_values=True)
def test_run_validation_failed_validation_junitxml_success(self, mock_junitxml,
mock_junitxml_module,
mock_run, mock_log_dir):
args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml',
'--junitxml', 'foo'])
verifylist = [('path_to_file', 'preliminary-file-structure.yaml'),
('junitxml', 'foo')]
parsed_args = self.check_parser(self.cmd, args, verifylist)
self.assertRaises(ValidationRunException,
self.cmd.take_action, parsed_args)

36
validations_libs/tests/fakes.py
View File

@ -470,6 +470,42 @@ FAKE_PLAYBOOK_TEMPLATE = \
- my_val
"""
PARSED_YAML_FILE = {
'include_validation': ['check-rhsm-version'],
'include_group': ['prep', 'pre-deployment'],
'exclude_validation': ['fips-enabled'],
'config': 'CONFIG_PATH',
'limit': ['undercloud-0', 'undercloud-1'],
'ssh-user': 'stack',
'validation-dir': 'VALIDATION_DIR',
'ansible-base-dir': '/usr/share/ansible',
'validation-log-dir': 'VALIDATION_LOG_DIR',
'inventory': 'tmp/inventory.yaml',
'output-log': 'foo',
'python-interpreter': '/usr/bin/python',
'extra-env-vars': {'key1': 'val1', 'key2': 'val2'},
'extra-vars-file': '/tmp/extra-vars-file.yaml'}
PARSED_YAML_FILE_WRONG_VALIDATION_NAME = {
'include_validation': ['this-validation-doesnt-exist'],
'include_group': ['prep', 'pre-deployment'],
'exclude_validation': ['fips-enabled'],
'config': 'CONFIG_PATH',
'limit': ['undercloud-0', 'undercloud-1'],
'ssh-user': 'stack',
'validation-dir': 'VALIDATION_DIR',
'ansible-base-dir': '/usr/share/ansible',
'validation-log-dir': 'VALIDATION_LOG_DIR',
'inventory': 'tmp/inventory.yaml',
'output-log': 'foo',
'python-interpreter': '/usr/bin/python',
'extra-env-vars': ['key1=val1', 'key2=val2'],
'extra-vars-file': '/tmp/extra-vars-file.yaml'}
WRONG_INVENTORY_FORMAT = {
'inventory': ['is', 'not', 'dictionary']
}
def fake_ansible_runner_run_return(status='successful', rc=0):
return status, rc

100
validations_libs/tests/test_validation_actions.py
View File

@ -25,6 +25,7 @@ from unittest import TestCase
from validations_libs.tests import fakes
from validations_libs.validation_actions import ValidationActions
from validations_libs.exceptions import ValidationRunException, ValidationShowException
import copy
class TestValidationActions(TestCase):
@ -54,7 +55,7 @@ class TestValidationActions(TestCase):
@mock.patch('validations_libs.utils.os.path.exists', return_value=True)
@mock.patch('validations_libs.utils.get_validations_playbook',
return_value=['/tmp/foo/fake.yaml'])
def test_validation_skip_validation(self, mock_validation_play, mock_exists, mock_access):
def test_validation_skip_validation_invalid_operation(self, mock_validation_play, mock_exists, mock_access):
playbook = ['fake.yaml']
inventory = 'tmp/inventory.yaml'
@ -64,11 +65,31 @@ class TestValidationActions(TestCase):
}}
run = ValidationActions()
run_return = run.run_validations(playbook, inventory,
validations_dir='/tmp/foo',
skip_list=skip_list,
self.assertRaises(ValidationRunException, run.run_validations, playbook, inventory,
validations_dir='/tmp/foo', skip_list=skip_list, limit_hosts=None)
@mock.patch('validations_libs.utils.os.access', return_value=True)
@mock.patch('validations_libs.utils.os.path.exists', return_value=True)
@mock.patch('validations_libs.utils.get_validations_playbook',
return_value=['/tmp/foo/fake.yaml', '/tmp/foo/fake1.yaml'])
@mock.patch('validations_libs.utils.os.makedirs')
@mock.patch('validations_libs.ansible.Ansible.run', return_value=('fake1.yaml', 0, 'successful'))
def test_validation_skip_validation_success(self, mock_ansible_run,
mock_makedirs, mock_validation_play,
mock_exists, mock_access):
playbook = ['fake.yaml', 'fake1.yaml']
inventory = 'tmp/inventory.yaml'
skip_list = {'fake': {'hosts': 'ALL',
'reason': None,
'lp': None
}}
run = ValidationActions()
return_run = run.run_validations(playbook, inventory,
validations_dir='/tmp/foo', skip_list=skip_list,
limit_hosts=None)
self.assertEqual(run_return, [])
self.assertEqual(return_run, [])
@mock.patch('validations_libs.utils.current_time',
return_value='time')
@ -246,6 +267,75 @@ class TestValidationActions(TestCase):
validation_cfg_file=None
)
@mock.patch('validations_libs.utils.os.makedirs')
@mock.patch('validations_libs.utils.os.access', return_value=True)
@mock.patch('validations_libs.utils.os.path.exists', return_value=True)
@mock.patch('validations_libs.validation_actions.ValidationLogs.get_results',
side_effect=fakes.FAKE_SUCCESS_RUN)
@mock.patch('validations_libs.utils.parse_all_validations_on_disk')
@mock.patch('validations_libs.ansible.Ansible.run')
def test_validation_run_from_file_success(self, mock_ansible_run,
mock_validation_dir,
mock_results, mock_exists, mock_access,
mock_makedirs):
mock_validation_dir.return_value = [{
'description': 'My Validation One Description',
'groups': ['prep', 'pre-deployment'],
'id': 'foo',
'name': 'My Validition One Name',
'parameters': {},
'path': '/tmp/foobar/validation-playbooks'}]
mock_ansible_run.return_value = ('foo.yaml', 0, 'successful')
expected_run_return = fakes.FAKE_SUCCESS_RUN[0]
yaml_file = fakes.PARSED_YAML_FILE
run = ValidationActions()
run_return = run.run_validations(
validation_name=yaml_file.get('include_validation'),
group=yaml_file.get('include_group'),
category=yaml_file.get('include_category'),
product=yaml_file.get('include_product'),
exclude_validation=yaml_file.get('exclude_validation'),
exclude_group=yaml_file.get('exclude_group'),
exclude_category=yaml_file.get('exclude_category'),
exclude_product=yaml_file.get('exclude_product'),
validation_config=fakes.DEFAULT_CONFIG,
limit_hosts=yaml_file.get('limit'),
ssh_user=yaml_file.get('ssh-user'),
validations_dir=yaml_file.get('validation-dir'),
inventory=yaml_file.get('inventory'),
base_dir=yaml_file.get('ansible-base-dir'),
python_interpreter=yaml_file.get('python-interpreter'),
extra_vars=yaml_file.get('extra-vars-file'),
extra_env_vars=yaml_file.get('extra-env-vars'))
self.assertEqual(run_return, expected_run_return)
mock_ansible_run.assert_called_with(
workdir=ANY,
playbook='/tmp/foobar/validation-playbooks/foo.yaml',
base_dir='/usr/share/ansible',
playbook_dir='/tmp/foobar/validation-playbooks',
parallel_run=True,
inventory='tmp/inventory.yaml',
output_callback='vf_validation_stdout',
callback_whitelist=None,
quiet=True,
extra_vars='/tmp/extra-vars-file.yaml',
limit_hosts=['undercloud-0', 'undercloud-1'],
extra_env_variables={'key1': 'val1', 'key2': 'val2'},
ansible_cfg_file=None,
gathering_policy='explicit',
ansible_artifact_path=ANY,
log_path=ANY,
run_async=False,
python_interpreter='/usr/bin/python',
ssh_user='stack',
validation_cfg_file=fakes.DEFAULT_CONFIG)
@mock.patch('validations_libs.utils.get_validations_playbook')
def test_validation_run_wrong_validation_name(self, mock_validation_play):
mock_validation_play.return_value = []

61
validations_libs/validation_actions.py
View File

@ -21,6 +21,7 @@ import yaml
from validations_libs.ansible import Ansible as v_ansible
from validations_libs.group import Group
from validations_libs.cli.common import Spinner
from validations_libs.validation import Validation
from validations_libs.validation_logs import ValidationLogs, ValidationLog
from validations_libs import constants
from validations_libs import utils as v_utils
@ -314,6 +315,50 @@ class ValidationActions:
return [path[1] for path in logs[-history_limit:]]
def _retrieve_validation_to_exclude(self, skip_list, validations, validations_dir, validation_config,
exclude_validation=None, exclude_group=None,
exclude_category=None, exclude_product=None, limit_hosts=None):
if exclude_validation is None:
exclude_validation = []
if limit_hosts is None:
limit_hosts = []
validations = [
os.path.basename(os.path.splitext(play)[0]) for play in validations]
if exclude_validation:
for validation in exclude_validation:
skip_list[validation] = {'hosts': 'ALL', 'reason': 'CLI override',
'lp': None}
if exclude_group or exclude_category or exclude_product:
exclude_validation.extend(v_utils.parse_all_validations_on_disk(
path=validations_dir, groups=exclude_group,
categories=exclude_category, products=exclude_product,
validation_config=validation_config))
for validation in exclude_validation:
skip_list[validation] = {'hosts': 'ALL', 'reason': 'CLI override',
'lp': None}
if skip_list is None:
return skip_list
# Returns False if validation is skipped on all hosts ('hosts' = ALL)
# Return False if validation validation should be run on hosts that are also defined in skip_list (invalid operation)
# Return True if there is any hosts where validation will be run
def _retrieve_validation_hosts(validation):
if validation['hosts'] == 'ALL':
return False
if not set(limit_hosts).difference(set(validation['hosts'])):
return False
return True
# There can be validations we want to run on only on some hosts (limit_hosts)
# validation_difference is all validations that will be run
validation_difference = set(validations).difference(set(skip_list.keys()))
if any([_retrieve_validation_hosts(skip_list[val]) for val in skip_list]) or validation_difference:
return skip_list
else:
raise ValidationRunException("Invalid operation, there is no validation to run.")
def run_validations(self, validation_name=None, inventory='localhost',
group=None, category=None, product=None,
extra_vars=None, validations_dir=None,
@ -323,7 +368,9 @@ class ValidationActions:
python_interpreter=None, skip_list=None,
callback_whitelist=None,
output_callback='vf_validation_stdout', ssh_user=None,
validation_config=None):
validation_config=None, exclude_validation=None,
exclude_group=None, exclude_category=None,
exclude_product=None):
"""Run one or multiple validations by name(s), by group(s) or by
product(s)
@ -467,6 +514,18 @@ class ValidationActions:
'Gathered playbooks:\n -{}').format(
'\n -'.join(playbooks)))
if skip_list is None:
skip_list = {}
skip_list = self._retrieve_validation_to_exclude(validations_dir=validations_dir,
exclude_validation=exclude_validation,
exclude_group=exclude_group,
exclude_category=exclude_category,
exclude_product=exclude_product,
validation_config=validation_config,
skip_list=skip_list, validations=playbooks,
limit_hosts=limit_hosts)
results = []
for playbook in playbooks:
# Check if playbook should be skipped and on which hosts