From 278d0e9d803ca085798bc99eefa028d23e29bc78 Mon Sep 17 00:00:00 2001 From: Veronika Fisarova Date: Fri, 11 Nov 2022 15:22:09 +0100 Subject: [PATCH] Run validations with parameters from a file Resolves: rhbz#2122209 Signed-off-by: Veronika Fisarova Change-Id: Ifc6c28003c4c2c5f3dd6198e650f9713a02dc82d --- preliminary-file-structure.yaml | 59 +++++++ setup.cfg | 1 + validations_libs/cli/file.py | 84 ++++++++++ validations_libs/cli/run.py | 4 +- validations_libs/tests/cli/test_file.py | 145 ++++++++++++++++++ validations_libs/tests/fakes.py | 36 +++++ .../tests/test_validation_actions.py | 100 +++++++++++- validations_libs/validation_actions.py | 61 +++++++- 8 files changed, 482 insertions(+), 8 deletions(-) create mode 100644 preliminary-file-structure.yaml create mode 100644 validations_libs/cli/file.py create mode 100644 validations_libs/tests/cli/test_file.py diff --git a/preliminary-file-structure.yaml b/preliminary-file-structure.yaml new file mode 100644 index 00000000..5a769e15 --- /dev/null +++ b/preliminary-file-structure.yaml @@ -0,0 +1,59 @@ +--- +# This file is generated by the `validation ... ` CLI. +# +# As shown in this template, you can specify validation(s) of you choice by the +# following options: +# +# validation(s), group(s), product(s) and category(ies) included in the run +# + parameters to each specific validation, +# validation, group(s), product(s), category(ies) excluded in the run, +# +# optional arguments for the run, +# e.g.: +# --config +# --limit +# --validation-dir +# and other. +# +# Delete the comment sign for the use of the required action. Add the '-' sign for +# including, respectively excluding, more items on the list. +# +# +#Example: +# +#Note: Skip list isn't included in the run_arguments list because it has the same +#functionality as the exclude_validation parameter. +# +# +include_validation: + - check-rhsm-version +include_group: + - prep + - pre-deployment +# include_category: +# - +# include_product: +# - +exclude_validation: + - fips-enabled +# exclude_group: +# - +# exclude_category: +# - +# exclude_product: +# - +config: CONFIG_PATH +limit: + - undercloud-0 + - undercloud-1 +ssh-user: SSH_USER +validation-dir: VALIDATION_DIR +ansible-base-dir: ANSIBLE_BASE_DIR +validation-log-dir: VALIDATION_LOG_DIR +inventory: INVENTORY_DIR +output-log: foo +python-interpreter: PYTHON_INTERPRETER_PATH +extra-env-vars: + key1: val1 + key2: val2 +extra-vars-file: JSON/YAML_PATH diff --git a/setup.cfg b/setup.cfg index 0597c5e4..ffe7b243 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,6 +40,7 @@ validation.cli: show_group = validations_libs.cli.show:ShowGroup show_parameter = validations_libs.cli.show:ShowParameter run = validations_libs.cli.run:Run + file = validations_libs.cli.file:File history_list = validations_libs.cli.history:ListHistory history_get = validations_libs.cli.history:GetHistory init = validations_libs.cli.community:CommunityValidationInit diff --git a/validations_libs/cli/file.py b/validations_libs/cli/file.py new file mode 100644 index 00000000..a08c2b8b --- /dev/null +++ b/validations_libs/cli/file.py @@ -0,0 +1,84 @@ +import os +from validations_libs import utils +from validations_libs.cli import common +from validations_libs.cli.base import BaseCommand +from validations_libs.validation_actions import ValidationActions +from validations_libs.exceptions import ValidationRunException + + +class File(BaseCommand): + """Include validations by name(s), group(s), category(ies) or by product(s), + or exclude validations by name(s), group(s), category(ies) or by product(s), + and run them from File""" + + def get_parser(self, parser): + """Argument parser ...""" + parser = super(File, self).get_parser(parser) + + parser.add_argument( + '--path-to-file', + dest='path_to_file', + required=True, + default=None, + help=("The path where the YAML file is stored.\n")) + + parser.add_argument( + '--junitxml', + dest='junitxml', + default=None, + help=("Path where the run result in JUnitXML format will be stored.\n")) + return parser + + def take_action(self, parsed_args): + """Take action""" + self.base.set_argument_parser(self, parsed_args) + + if parsed_args.path_to_file: + try: + yaml_file = common.read_cli_data_file(parsed_args.path_to_file) + if not isinstance(yaml_file, dict): + raise ValidationRunException("Wrong format of the File.") + except (FileNotFoundError) as e: + raise FileNotFoundError(e) + self.base.config = utils.load_config(os.path.abspath(yaml_file['config'])) + v_actions = ValidationActions(yaml_file.get('validation-dir'), + log_path=yaml_file.get('validation-log-dir', + '/home/stack/validations')) + hosts = yaml_file.get('limit') + hosts_converted = ",".join(hosts) + + try: + results = v_actions.run_validations( + validation_name=yaml_file.get('include_validation'), + group=yaml_file.get('include_group'), + category=yaml_file.get('include_category'), + product=yaml_file.get('include_product'), + exclude_validation=yaml_file.get('exclude_validation'), + exclude_group=yaml_file.get('exclude_group'), + exclude_category=yaml_file.get('exclude_category'), + exclude_product=yaml_file.get('exclude_product'), + validation_config=self.base.config, + limit_hosts=hosts_converted, + ssh_user=yaml_file.get('ssh-user', 'stack'), + inventory=yaml_file.get('inventory', 'localhost'), + base_dir=yaml_file.get('ansible-base-dir', '/usr/share/ansible'), + python_interpreter=yaml_file.get('python-interpreter', '/usr/bin/python3'), + extra_vars=yaml_file.get('extra-vars-file'), + extra_env_vars=yaml_file.get('extra-env-vars')) + + except (RuntimeError, ValidationRunException) as e: + raise ValidationRunException(e) + + if results: + failed_rc = any([r for r in results if r['Status'] == 'FAILED']) + if yaml_file.get('output-log'): + common.write_output(yaml_file.get('output-log'), results) + if parsed_args.junitxml: + common.write_junitxml(parsed_args.junitxml, results) + common.print_dict(results) + if failed_rc: + raise ValidationRunException("One or more validations have failed.") + else: + msg = ("No validation has been run, please check " + "log in the Ansible working directory.") + raise ValidationRunException(msg) diff --git a/validations_libs/cli/run.py b/validations_libs/cli/run.py index 485ebadd..3795b12c 100644 --- a/validations_libs/cli/run.py +++ b/validations_libs/cli/run.py @@ -197,8 +197,8 @@ class Run(BaseCommand): extra_vars = common.read_cli_data_file( parsed_args.extra_vars_file) - - skip_list = None + # skip_list is {} so it could be properly processed in the ValidationAction class + skip_list = {} if parsed_args.skip_list: skip_list = common.read_cli_data_file(parsed_args.skip_list) if not isinstance(skip_list, dict): diff --git a/validations_libs/tests/cli/test_file.py b/validations_libs/tests/cli/test_file.py new file mode 100644 index 00000000..dc56dd56 --- /dev/null +++ b/validations_libs/tests/cli/test_file.py @@ -0,0 +1,145 @@ +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import sys +import copy +try: + from unittest import mock +except ImportError: + import mock + +from validations_libs.cli import file +from validations_libs.exceptions import ValidationRunException +from validations_libs.tests import fakes +from validations_libs.tests.cli.fakes import BaseCommand + + +class TestRun(BaseCommand): + + maxDiff = None + + def setUp(self): + super(TestRun, self).setUp() + self.cmd = file.File(self.app, None) + + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN), + autospec=True) + def test_run_validation_success(self, mock_run): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml']) + + verifylist = [('path_to_file', 'preliminary-file-structure.yaml')] + + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.cmd.take_action(parsed_args) + + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN), + autospec=True) + @mock.patch('validations_libs.utils.create_log_dir', return_value='/foo/bar/logdir/') + @mock.patch('validations_libs.utils.parse_all_validations_on_disk', return_value=[]) + def test_run_validation_success_full(self, mock_parse_all_validations_on_disk, mock_create_log_dir, mock_run): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml']) + + verifylist = [('path_to_file', 'preliminary-file-structure.yaml')] + + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.cmd.take_action(parsed_args) + + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN), + autospec=True) + @mock.patch('validations_libs.utils.parse_all_validations_on_disk') + def test_run_validation_success_validations_on_disk_exists(self, mock_validation_dir, mock_run): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml']) + verifylist = [('path_to_file', 'preliminary-file-structure.yaml')] + + mock_validation_dir.return_value = [{'id': 'foo', + 'description': 'foo', + 'groups': ['prep', 'pre-deployment'], + 'categories': ['os', 'storage'], + 'products': ['product1'], + 'name': 'Advanced Format 512e Support', + 'path': '/tmp'}] + + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.cmd.take_action(parsed_args) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN), + autospec=True) + def test_run_validation_success_with_junitxml(self, mock_run, mock_exists): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml', + '--junitxml', 'foo']) + verifylist = [('path_to_file', 'preliminary-file-structure.yaml'), + ('junitxml', 'foo')] + + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.cmd.take_action(parsed_args) + + def test_run_validation_cmd_parser_error(self): + args = self._set_args(['foo', 'preliminary-file-structure.yaml']) + verifylist = [('path_to_file', 'preliminary-file-structure.yaml')] + + self.assertRaises(Exception, self.check_parser, self.cmd, args, verifylist) + + @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR') + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN), + autospec=True) + def test_run_validation_failed_validation(self, mock_run, mock_exists): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml']) + verifylist = [ + ('path_to_file', 'preliminary-file-structure.yaml')] + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.assertRaises(ValidationRunException, + self.cmd.take_action, parsed_args) + + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN), + autospec=True) + @mock.patch('os.path.exists', return_values=True) + def test_run_validation_failed_validation_junitxml_module_disabled(self, mock_exists, + mock_run): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml', + '--junitxml', 'foo']) + verifylist = [('path_to_file', 'preliminary-file-structure.yaml'), + ('junitxml', 'foo')] + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.assertRaises(ValidationRunException, self.cmd.take_action, parsed_args) + + @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR') + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_FAILED_RUN), + autospec=True) + @mock.patch('validations_libs.cli.common.write_junitxml', return_value={}) + @mock.patch('os.path.exists', return_values=True) + def test_run_validation_failed_validation_junitxml_success(self, mock_junitxml, + mock_junitxml_module, + mock_run, mock_log_dir): + args = self._set_args(['--path-to-file', 'preliminary-file-structure.yaml', + '--junitxml', 'foo']) + verifylist = [('path_to_file', 'preliminary-file-structure.yaml'), + ('junitxml', 'foo')] + parsed_args = self.check_parser(self.cmd, args, verifylist) + self.assertRaises(ValidationRunException, + self.cmd.take_action, parsed_args) diff --git a/validations_libs/tests/fakes.py b/validations_libs/tests/fakes.py index fe5d9d34..2140848a 100644 --- a/validations_libs/tests/fakes.py +++ b/validations_libs/tests/fakes.py @@ -470,6 +470,42 @@ FAKE_PLAYBOOK_TEMPLATE = \ - my_val """ +PARSED_YAML_FILE = { + 'include_validation': ['check-rhsm-version'], + 'include_group': ['prep', 'pre-deployment'], + 'exclude_validation': ['fips-enabled'], + 'config': 'CONFIG_PATH', + 'limit': ['undercloud-0', 'undercloud-1'], + 'ssh-user': 'stack', + 'validation-dir': 'VALIDATION_DIR', + 'ansible-base-dir': '/usr/share/ansible', + 'validation-log-dir': 'VALIDATION_LOG_DIR', + 'inventory': 'tmp/inventory.yaml', + 'output-log': 'foo', + 'python-interpreter': '/usr/bin/python', + 'extra-env-vars': {'key1': 'val1', 'key2': 'val2'}, + 'extra-vars-file': '/tmp/extra-vars-file.yaml'} + +PARSED_YAML_FILE_WRONG_VALIDATION_NAME = { + 'include_validation': ['this-validation-doesnt-exist'], + 'include_group': ['prep', 'pre-deployment'], + 'exclude_validation': ['fips-enabled'], + 'config': 'CONFIG_PATH', + 'limit': ['undercloud-0', 'undercloud-1'], + 'ssh-user': 'stack', + 'validation-dir': 'VALIDATION_DIR', + 'ansible-base-dir': '/usr/share/ansible', + 'validation-log-dir': 'VALIDATION_LOG_DIR', + 'inventory': 'tmp/inventory.yaml', + 'output-log': 'foo', + 'python-interpreter': '/usr/bin/python', + 'extra-env-vars': ['key1=val1', 'key2=val2'], + 'extra-vars-file': '/tmp/extra-vars-file.yaml'} + +WRONG_INVENTORY_FORMAT = { + 'inventory': ['is', 'not', 'dictionary'] +} + def fake_ansible_runner_run_return(status='successful', rc=0): return status, rc diff --git a/validations_libs/tests/test_validation_actions.py b/validations_libs/tests/test_validation_actions.py index 951a0b3d..9d83089c 100644 --- a/validations_libs/tests/test_validation_actions.py +++ b/validations_libs/tests/test_validation_actions.py @@ -25,6 +25,7 @@ from unittest import TestCase from validations_libs.tests import fakes from validations_libs.validation_actions import ValidationActions from validations_libs.exceptions import ValidationRunException, ValidationShowException +import copy class TestValidationActions(TestCase): @@ -54,7 +55,7 @@ class TestValidationActions(TestCase): @mock.patch('validations_libs.utils.os.path.exists', return_value=True) @mock.patch('validations_libs.utils.get_validations_playbook', return_value=['/tmp/foo/fake.yaml']) - def test_validation_skip_validation(self, mock_validation_play, mock_exists, mock_access): + def test_validation_skip_validation_invalid_operation(self, mock_validation_play, mock_exists, mock_access): playbook = ['fake.yaml'] inventory = 'tmp/inventory.yaml' @@ -64,11 +65,31 @@ class TestValidationActions(TestCase): }} run = ValidationActions() - run_return = run.run_validations(playbook, inventory, - validations_dir='/tmp/foo', - skip_list=skip_list, + self.assertRaises(ValidationRunException, run.run_validations, playbook, inventory, + validations_dir='/tmp/foo', skip_list=skip_list, limit_hosts=None) + + @mock.patch('validations_libs.utils.os.access', return_value=True) + @mock.patch('validations_libs.utils.os.path.exists', return_value=True) + @mock.patch('validations_libs.utils.get_validations_playbook', + return_value=['/tmp/foo/fake.yaml', '/tmp/foo/fake1.yaml']) + @mock.patch('validations_libs.utils.os.makedirs') + @mock.patch('validations_libs.ansible.Ansible.run', return_value=('fake1.yaml', 0, 'successful')) + def test_validation_skip_validation_success(self, mock_ansible_run, + mock_makedirs, mock_validation_play, + mock_exists, mock_access): + + playbook = ['fake.yaml', 'fake1.yaml'] + inventory = 'tmp/inventory.yaml' + skip_list = {'fake': {'hosts': 'ALL', + 'reason': None, + 'lp': None + }} + + run = ValidationActions() + return_run = run.run_validations(playbook, inventory, + validations_dir='/tmp/foo', skip_list=skip_list, limit_hosts=None) - self.assertEqual(run_return, []) + self.assertEqual(return_run, []) @mock.patch('validations_libs.utils.current_time', return_value='time') @@ -246,6 +267,75 @@ class TestValidationActions(TestCase): validation_cfg_file=None ) + @mock.patch('validations_libs.utils.os.makedirs') + @mock.patch('validations_libs.utils.os.access', return_value=True) + @mock.patch('validations_libs.utils.os.path.exists', return_value=True) + @mock.patch('validations_libs.validation_actions.ValidationLogs.get_results', + side_effect=fakes.FAKE_SUCCESS_RUN) + @mock.patch('validations_libs.utils.parse_all_validations_on_disk') + @mock.patch('validations_libs.ansible.Ansible.run') + def test_validation_run_from_file_success(self, mock_ansible_run, + mock_validation_dir, + mock_results, mock_exists, mock_access, + mock_makedirs): + + mock_validation_dir.return_value = [{ + 'description': 'My Validation One Description', + 'groups': ['prep', 'pre-deployment'], + 'id': 'foo', + 'name': 'My Validition One Name', + 'parameters': {}, + 'path': '/tmp/foobar/validation-playbooks'}] + + mock_ansible_run.return_value = ('foo.yaml', 0, 'successful') + + expected_run_return = fakes.FAKE_SUCCESS_RUN[0] + + yaml_file = fakes.PARSED_YAML_FILE + + run = ValidationActions() + run_return = run.run_validations( + validation_name=yaml_file.get('include_validation'), + group=yaml_file.get('include_group'), + category=yaml_file.get('include_category'), + product=yaml_file.get('include_product'), + exclude_validation=yaml_file.get('exclude_validation'), + exclude_group=yaml_file.get('exclude_group'), + exclude_category=yaml_file.get('exclude_category'), + exclude_product=yaml_file.get('exclude_product'), + validation_config=fakes.DEFAULT_CONFIG, + limit_hosts=yaml_file.get('limit'), + ssh_user=yaml_file.get('ssh-user'), + validations_dir=yaml_file.get('validation-dir'), + inventory=yaml_file.get('inventory'), + base_dir=yaml_file.get('ansible-base-dir'), + python_interpreter=yaml_file.get('python-interpreter'), + extra_vars=yaml_file.get('extra-vars-file'), + extra_env_vars=yaml_file.get('extra-env-vars')) + self.assertEqual(run_return, expected_run_return) + + mock_ansible_run.assert_called_with( + workdir=ANY, + playbook='/tmp/foobar/validation-playbooks/foo.yaml', + base_dir='/usr/share/ansible', + playbook_dir='/tmp/foobar/validation-playbooks', + parallel_run=True, + inventory='tmp/inventory.yaml', + output_callback='vf_validation_stdout', + callback_whitelist=None, + quiet=True, + extra_vars='/tmp/extra-vars-file.yaml', + limit_hosts=['undercloud-0', 'undercloud-1'], + extra_env_variables={'key1': 'val1', 'key2': 'val2'}, + ansible_cfg_file=None, + gathering_policy='explicit', + ansible_artifact_path=ANY, + log_path=ANY, + run_async=False, + python_interpreter='/usr/bin/python', + ssh_user='stack', + validation_cfg_file=fakes.DEFAULT_CONFIG) + @mock.patch('validations_libs.utils.get_validations_playbook') def test_validation_run_wrong_validation_name(self, mock_validation_play): mock_validation_play.return_value = [] diff --git a/validations_libs/validation_actions.py b/validations_libs/validation_actions.py index 2ac923dc..4ce3d78b 100644 --- a/validations_libs/validation_actions.py +++ b/validations_libs/validation_actions.py @@ -21,6 +21,7 @@ import yaml from validations_libs.ansible import Ansible as v_ansible from validations_libs.group import Group from validations_libs.cli.common import Spinner +from validations_libs.validation import Validation from validations_libs.validation_logs import ValidationLogs, ValidationLog from validations_libs import constants from validations_libs import utils as v_utils @@ -314,6 +315,50 @@ class ValidationActions: return [path[1] for path in logs[-history_limit:]] + def _retrieve_validation_to_exclude(self, skip_list, validations, validations_dir, validation_config, + exclude_validation=None, exclude_group=None, + exclude_category=None, exclude_product=None, limit_hosts=None): + if exclude_validation is None: + exclude_validation = [] + if limit_hosts is None: + limit_hosts = [] + + validations = [ + os.path.basename(os.path.splitext(play)[0]) for play in validations] + + if exclude_validation: + for validation in exclude_validation: + skip_list[validation] = {'hosts': 'ALL', 'reason': 'CLI override', + 'lp': None} + + if exclude_group or exclude_category or exclude_product: + exclude_validation.extend(v_utils.parse_all_validations_on_disk( + path=validations_dir, groups=exclude_group, + categories=exclude_category, products=exclude_product, + validation_config=validation_config)) + for validation in exclude_validation: + skip_list[validation] = {'hosts': 'ALL', 'reason': 'CLI override', + 'lp': None} + if skip_list is None: + return skip_list + + # Returns False if validation is skipped on all hosts ('hosts' = ALL) + # Return False if validation validation should be run on hosts that are also defined in skip_list (invalid operation) + # Return True if there is any hosts where validation will be run + def _retrieve_validation_hosts(validation): + if validation['hosts'] == 'ALL': + return False + if not set(limit_hosts).difference(set(validation['hosts'])): + return False + return True + # There can be validations we want to run on only on some hosts (limit_hosts) + # validation_difference is all validations that will be run + validation_difference = set(validations).difference(set(skip_list.keys())) + if any([_retrieve_validation_hosts(skip_list[val]) for val in skip_list]) or validation_difference: + return skip_list + else: + raise ValidationRunException("Invalid operation, there is no validation to run.") + def run_validations(self, validation_name=None, inventory='localhost', group=None, category=None, product=None, extra_vars=None, validations_dir=None, @@ -323,7 +368,9 @@ class ValidationActions: python_interpreter=None, skip_list=None, callback_whitelist=None, output_callback='vf_validation_stdout', ssh_user=None, - validation_config=None): + validation_config=None, exclude_validation=None, + exclude_group=None, exclude_category=None, + exclude_product=None): """Run one or multiple validations by name(s), by group(s) or by product(s) @@ -467,6 +514,18 @@ class ValidationActions: 'Gathered playbooks:\n -{}').format( '\n -'.join(playbooks))) + if skip_list is None: + skip_list = {} + + skip_list = self._retrieve_validation_to_exclude(validations_dir=validations_dir, + exclude_validation=exclude_validation, + exclude_group=exclude_group, + exclude_category=exclude_category, + exclude_product=exclude_product, + validation_config=validation_config, + skip_list=skip_list, validations=playbooks, + limit_hosts=limit_hosts) + results = [] for playbook in playbooks: # Check if playbook should be skipped and on which hosts