From b37015ee1f8dcb6dccbf73abf5037b1c6829a1bf Mon Sep 17 00:00:00 2001 From: matbu Date: Wed, 15 Sep 2021 23:01:13 +0200 Subject: [PATCH] Expose skip list mechanism via the CLI The skip list feature was available only via the API, this patch allow the user to pass a file with the list of the validations which he wants to skip during the run, example: check-ram: hosts: all reason: Wrong ram value lp: https://lp.fake.net check-cpu: hosts: undercloud reason: Unstable validation lp: https://lp.fake.net Change-Id: I04059f9339085e0dcef1f018cad1be511ee7d3c7 --- README.rst | 27 +++++++ skiplist-example.yaml | 11 +++ validations_libs/cli/common.py | 14 ++-- validations_libs/cli/run.py | 18 ++++- validations_libs/tests/cli/test_common.py | 47 +++++++++++ validations_libs/tests/cli/test_run.py | 95 ++++++++++++++++++++--- validations_libs/validation_actions.py | 15 +++- 7 files changed, 204 insertions(+), 23 deletions(-) create mode 100644 skiplist-example.yaml create mode 100644 validations_libs/tests/cli/test_common.py diff --git a/README.rst b/README.rst index 9c83e1ef..b492b398 100644 --- a/README.rst +++ b/README.rst @@ -59,4 +59,31 @@ Then run validations:: validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts + +Skip list +========= + +You can provide a file with a list of Validations to skip via the run command:: + + validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts --skiplist my-skip-list.yaml + +This file should be formed as:: + + validation-name: + hosts: targeted_hostname + reason: reason to ignore the file + lp: bug number + +The framework will skip the validation against the ``hosts`` key. +In order to skip the validation on every hosts, you can set ``all`` value such +as:: + + hosts: all + +If no hosts key is provided for a given validation, it will be considered as ``hosts: all``. + +.. note:: + The ``reason`` and ``lp`` key are for tracking and documentation purposes, + the framework won't use those keys. + .. _Apache_license: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/skiplist-example.yaml b/skiplist-example.yaml new file mode 100644 index 00000000..8798d8f1 --- /dev/null +++ b/skiplist-example.yaml @@ -0,0 +1,11 @@ +check-ram: + hosts: all + # reason and lp key is not mandatory for the VF. Those values are in the list + # in order to track the reason and eventually the related bug number of the + # skipped validation. + reason: Wrong ram value + lp: https://lp.fake.net +check-cpu: + hosts: undercloud + reason: Unstable validation + lp: https://lp.fake.net diff --git a/validations_libs/cli/common.py b/validations_libs/cli/common.py index 889c6ba9..b7322336 100644 --- a/validations_libs/cli/common.py +++ b/validations_libs/cli/common.py @@ -102,16 +102,16 @@ def write_junitxml(output_junitxml, results): output.write(to_xml_report_string([ts])) -def read_extra_vars_file(extra_vars_file): - """Read file containing extra variables. +def read_cli_data_file(data_file): + """Read CLI data (YAML/JSON) file. """ try: - with open(extra_vars_file, 'r') as env_file: - return yaml.safe_load(env_file.read()) - except yaml.YAMLError as error: + with open(data_file, 'r') as _file: + return yaml.safe_load(_file.read()) + except (yaml.YAMLError, IOError) as error: error_msg = ( - "The extra_vars file must be properly formatted YAML/JSON." - "Details: {}.").format(error) + "The file {} must be properly formatted YAML/JSON." + "Details: {}.").format(data_file, error) raise RuntimeError(error_msg) diff --git a/validations_libs/cli/run.py b/validations_libs/cli/run.py index ebc5161b..444c4aa5 100644 --- a/validations_libs/cli/run.py +++ b/validations_libs/cli/run.py @@ -97,6 +97,13 @@ class Run(BaseCommand): "KEY multiple times, the last given VALUE for that same KEY " "will override the other(s)")) + parser.add_argument('--skiplist', dest='skip_list', + default=None, + help=("Path where the skip list is stored. " + "An example of the skiplist format could " + "be found at the root of the " + "validations-libs repository.")) + extra_vars_group = parser.add_mutually_exclusive_group(required=False) extra_vars_group.add_argument( '--extra-vars', @@ -184,9 +191,15 @@ class Run(BaseCommand): "Loading extra vars file {}".format( parsed_args.extra_vars_file)) - extra_vars = common.read_extra_vars_file( + extra_vars = common.read_cli_data_file( parsed_args.extra_vars_file) + skip_list = None + if parsed_args.skip_list: + skip_list = common.read_cli_data_file(parsed_args.skip_list) + if not isinstance(skip_list, dict): + raise RuntimeError("Wrong format for the skiplist.") + try: results = v_actions.run_validations( inventory=parsed_args.inventory, @@ -203,7 +216,8 @@ class Run(BaseCommand): quiet=quiet_mode, ssh_user=parsed_args.ssh_user, log_path=parsed_args.validation_log_dir, - validation_config=config) + validation_config=config, + skip_list=skip_list) except RuntimeError as e: raise RuntimeError(e) diff --git a/validations_libs/tests/cli/test_common.py b/validations_libs/tests/cli/test_common.py new file mode 100644 index 00000000..f36debf7 --- /dev/null +++ b/validations_libs/tests/cli/test_common.py @@ -0,0 +1,47 @@ +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from unittest import TestCase +import yaml + +try: + from unittest import mock +except ImportError: + import mock + +from validations_libs.cli import common + + +class TestCommon(TestCase): + + def setUp(self): + super(TestCommon, self).setUp() + + def test_read_cli_data_file_with_example_file(self): + example_data = {'check-cpu': {'hosts': 'undercloud', + 'lp': 'https://lp.fake.net', + 'reason': 'Unstable validation'}, + 'check-ram': {'hosts': 'all', + 'lp': 'https://lp.fake.net', + 'reason': 'Wrong ram value'}} + data = common.read_cli_data_file('skiplist-example.yaml') + self.assertEqual(data, example_data) + + @mock.patch('six.moves.builtins.open', side_effect=IOError) + def test_read_cli_data_file_ioerror(self, mock_open): + self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo') + + @mock.patch('yaml.safe_load', side_effect=yaml.YAMLError) + def test_read_cli_data_file_yaml_error(self, mock_yaml): + self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo') diff --git a/validations_libs/tests/cli/test_run.py b/validations_libs/tests/cli/test_run.py index cb1159e7..2ef08121 100644 --- a/validations_libs/tests/cli/test_run.py +++ b/validations_libs/tests/cli/test_run.py @@ -83,7 +83,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -121,7 +122,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -172,7 +174,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -208,7 +211,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -248,7 +252,8 @@ class TestRun(BaseCommand): 'python_interpreter': sys.executable, 'quiet': False, 'ssh_user': 'doe', - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -285,7 +290,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -325,7 +331,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo', @@ -372,7 +379,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo'] @@ -405,7 +413,8 @@ class TestRun(BaseCommand): 'python_interpreter': sys.executable, 'quiet': True, 'ssh_user': 'doe', - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } arglist = ['--validation', 'foo'] @@ -442,7 +451,8 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } self._set_args(arglist) @@ -479,10 +489,73 @@ class TestRun(BaseCommand): 'quiet': True, 'ssh_user': 'doe', 'log_path': mock_log_dir, - 'validation_config': {} + 'validation_config': {}, + 'skip_list': None } self._set_args(arglist) parsed_args = self.check_parser(self.cmd, arglist, verifylist) self.cmd.take_action(parsed_args) mock_run.assert_called_with(**run_called_args) + + @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR') + @mock.patch('yaml.safe_load', return_value={'key': 'value'}) + @mock.patch('six.moves.builtins.open') + @mock.patch('getpass.getuser', + return_value='doe') + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN)) + @mock.patch('validations_libs.utils.load_config', return_value={}) + def test_run_command_with_skip_list(self, mock_config, mock_run, + mock_user, mock_open, + mock_yaml, mock_log_dir): + + run_called_args = { + 'inventory': 'localhost', + 'limit_hosts': None, + 'group': [], + 'category': [], + 'product': [], + 'extra_vars': None, + 'validations_dir': '/usr/share/ansible/validation-playbooks', + 'base_dir': '/usr/share/ansible', + 'validation_name': ['foo'], + 'extra_env_vars': None, + 'python_interpreter': sys.executable, + 'quiet': True, + 'ssh_user': 'doe', + 'log_path': mock_log_dir, + 'validation_config': {}, + 'skip_list': {'key': 'value'} + } + + arglist = ['--validation', 'foo', + '--skiplist', '/foo/skip.yaml'] + verifylist = [('validation_name', ['foo']), + ('skip_list', '/foo/skip.yaml')] + self._set_args(arglist) + parsed_args = self.check_parser(self.cmd, arglist, verifylist) + self.cmd.take_action(parsed_args) + mock_run.assert_called_with(**run_called_args) + + @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR') + @mock.patch('yaml.safe_load', return_value=[{'key': 'value'}]) + @mock.patch('six.moves.builtins.open') + @mock.patch('getpass.getuser', + return_value='doe') + @mock.patch('validations_libs.validation_actions.ValidationActions.' + 'run_validations', + return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN)) + @mock.patch('validations_libs.utils.load_config', return_value={}) + def test_run_command_with_skip_list_bad_format(self, mock_config, mock_run, + mock_user, mock_open, + mock_yaml, mock_log_dir): + + arglist = ['--validation', 'foo', + '--skiplist', '/foo/skip.yaml'] + verifylist = [('validation_name', ['foo']), + ('skip_list', '/foo/skip.yaml')] + self._set_args(arglist) + parsed_args = self.check_parser(self.cmd, arglist, verifylist) + self.assertRaises(RuntimeError, self.cmd.take_action, parsed_args) diff --git a/validations_libs/validation_actions.py b/validations_libs/validation_actions.py index 1eddb43e..02e2ff9a 100644 --- a/validations_libs/validation_actions.py +++ b/validations_libs/validation_actions.py @@ -188,8 +188,8 @@ class ValidationActions(object): 'cloud2,!cloud1' """ - hosts = skip_list[playbook].get('hosts') - if hosts == 'ALL' or hosts is None: + hosts = skip_list[playbook].get('hosts', 'all') + if hosts.lower() == 'all': return None else: _hosts = ['!{}'.format(hosts)] @@ -464,8 +464,17 @@ class ValidationActions(object): 'validations': _playbook.split('.')[0], 'UUID': validation_uuid, }) + # Print hosts which has been skipped: + if _hosts: + skipped_hosts = [h.replace('!', '') + for h in _hosts.split(',') if '!' in h] + if skipped_hosts: + msg = ("Validation {} has been skipped " + "on hosts: {}").format(_play, + ','.join(skipped_hosts)) + self.log.info(msg) else: - self.log.debug('Skipping Validations: {}'.format(playbook)) + self.log.info('Skipping Validations: {}'.format(playbook)) if run_async: return results