Expose skip list mechanism via the CLI

The skip list feature was available only via the API, this patch
allow the user to pass a file with the list of the validations
which he wants to skip during the run, example:

check-ram: hosts: all
  reason: Wrong ram value
  lp: https://lp.fake.net
check-cpu: hosts: undercloud
  reason: Unstable validation
  lp: https://lp.fake.net
Change-Id: I04059f9339085e0dcef1f018cad1be511ee7d3c7
This commit is contained in:
matbu 2021-09-15 23:01:13 +02:00 committed by mbu
parent 11488cd88d
commit b37015ee1f
7 changed files with 204 additions and 23 deletions

View File

@ -59,4 +59,31 @@ Then run validations::
validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts
Skip list
=========
You can provide a file with a list of Validations to skip via the run command::
validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts --skiplist my-skip-list.yaml
This file should be formed as::
validation-name:
hosts: targeted_hostname
reason: reason to ignore the file
lp: bug number
The framework will skip the validation against the ``hosts`` key.
In order to skip the validation on every hosts, you can set ``all`` value such
as::
hosts: all
If no hosts key is provided for a given validation, it will be considered as ``hosts: all``.
.. note::
The ``reason`` and ``lp`` key are for tracking and documentation purposes,
the framework won't use those keys.
.. _Apache_license: http://www.apache.org/licenses/LICENSE-2.0

11
skiplist-example.yaml Normal file
View File

@ -0,0 +1,11 @@
check-ram:
hosts: all
# reason and lp key is not mandatory for the VF. Those values are in the list
# in order to track the reason and eventually the related bug number of the
# skipped validation.
reason: Wrong ram value
lp: https://lp.fake.net
check-cpu:
hosts: undercloud
reason: Unstable validation
lp: https://lp.fake.net

View File

@ -102,16 +102,16 @@ def write_junitxml(output_junitxml, results):
output.write(to_xml_report_string([ts]))
def read_extra_vars_file(extra_vars_file):
"""Read file containing extra variables.
def read_cli_data_file(data_file):
"""Read CLI data (YAML/JSON) file.
"""
try:
with open(extra_vars_file, 'r') as env_file:
return yaml.safe_load(env_file.read())
except yaml.YAMLError as error:
with open(data_file, 'r') as _file:
return yaml.safe_load(_file.read())
except (yaml.YAMLError, IOError) as error:
error_msg = (
"The extra_vars file must be properly formatted YAML/JSON."
"Details: {}.").format(error)
"The file {} must be properly formatted YAML/JSON."
"Details: {}.").format(data_file, error)
raise RuntimeError(error_msg)

View File

@ -97,6 +97,13 @@ class Run(BaseCommand):
"KEY multiple times, the last given VALUE for that same KEY "
"will override the other(s)"))
parser.add_argument('--skiplist', dest='skip_list',
default=None,
help=("Path where the skip list is stored. "
"An example of the skiplist format could "
"be found at the root of the "
"validations-libs repository."))
extra_vars_group = parser.add_mutually_exclusive_group(required=False)
extra_vars_group.add_argument(
'--extra-vars',
@ -184,9 +191,15 @@ class Run(BaseCommand):
"Loading extra vars file {}".format(
parsed_args.extra_vars_file))
extra_vars = common.read_extra_vars_file(
extra_vars = common.read_cli_data_file(
parsed_args.extra_vars_file)
skip_list = None
if parsed_args.skip_list:
skip_list = common.read_cli_data_file(parsed_args.skip_list)
if not isinstance(skip_list, dict):
raise RuntimeError("Wrong format for the skiplist.")
try:
results = v_actions.run_validations(
inventory=parsed_args.inventory,
@ -203,7 +216,8 @@ class Run(BaseCommand):
quiet=quiet_mode,
ssh_user=parsed_args.ssh_user,
log_path=parsed_args.validation_log_dir,
validation_config=config)
validation_config=config,
skip_list=skip_list)
except RuntimeError as e:
raise RuntimeError(e)

View File

@ -0,0 +1,47 @@
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from unittest import TestCase
import yaml
try:
from unittest import mock
except ImportError:
import mock
from validations_libs.cli import common
class TestCommon(TestCase):
def setUp(self):
super(TestCommon, self).setUp()
def test_read_cli_data_file_with_example_file(self):
example_data = {'check-cpu': {'hosts': 'undercloud',
'lp': 'https://lp.fake.net',
'reason': 'Unstable validation'},
'check-ram': {'hosts': 'all',
'lp': 'https://lp.fake.net',
'reason': 'Wrong ram value'}}
data = common.read_cli_data_file('skiplist-example.yaml')
self.assertEqual(data, example_data)
@mock.patch('six.moves.builtins.open', side_effect=IOError)
def test_read_cli_data_file_ioerror(self, mock_open):
self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo')
@mock.patch('yaml.safe_load', side_effect=yaml.YAMLError)
def test_read_cli_data_file_yaml_error(self, mock_yaml):
self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo')

View File

@ -83,7 +83,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -121,7 +122,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -172,7 +174,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -208,7 +211,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -248,7 +252,8 @@ class TestRun(BaseCommand):
'python_interpreter': sys.executable,
'quiet': False,
'ssh_user': 'doe',
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -285,7 +290,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -325,7 +331,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo',
@ -372,7 +379,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo']
@ -405,7 +413,8 @@ class TestRun(BaseCommand):
'python_interpreter': sys.executable,
'quiet': True,
'ssh_user': 'doe',
'validation_config': {}
'validation_config': {},
'skip_list': None
}
arglist = ['--validation', 'foo']
@ -442,7 +451,8 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
self._set_args(arglist)
@ -479,10 +489,73 @@ class TestRun(BaseCommand):
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {}
'validation_config': {},
'skip_list': None
}
self._set_args(arglist)
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_run.assert_called_with(**run_called_args)
@mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
@mock.patch('yaml.safe_load', return_value={'key': 'value'})
@mock.patch('six.moves.builtins.open')
@mock.patch('getpass.getuser',
return_value='doe')
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN))
@mock.patch('validations_libs.utils.load_config', return_value={})
def test_run_command_with_skip_list(self, mock_config, mock_run,
mock_user, mock_open,
mock_yaml, mock_log_dir):
run_called_args = {
'inventory': 'localhost',
'limit_hosts': None,
'group': [],
'category': [],
'product': [],
'extra_vars': None,
'validations_dir': '/usr/share/ansible/validation-playbooks',
'base_dir': '/usr/share/ansible',
'validation_name': ['foo'],
'extra_env_vars': None,
'python_interpreter': sys.executable,
'quiet': True,
'ssh_user': 'doe',
'log_path': mock_log_dir,
'validation_config': {},
'skip_list': {'key': 'value'}
}
arglist = ['--validation', 'foo',
'--skiplist', '/foo/skip.yaml']
verifylist = [('validation_name', ['foo']),
('skip_list', '/foo/skip.yaml')]
self._set_args(arglist)
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_run.assert_called_with(**run_called_args)
@mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
@mock.patch('yaml.safe_load', return_value=[{'key': 'value'}])
@mock.patch('six.moves.builtins.open')
@mock.patch('getpass.getuser',
return_value='doe')
@mock.patch('validations_libs.validation_actions.ValidationActions.'
'run_validations',
return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN))
@mock.patch('validations_libs.utils.load_config', return_value={})
def test_run_command_with_skip_list_bad_format(self, mock_config, mock_run,
mock_user, mock_open,
mock_yaml, mock_log_dir):
arglist = ['--validation', 'foo',
'--skiplist', '/foo/skip.yaml']
verifylist = [('validation_name', ['foo']),
('skip_list', '/foo/skip.yaml')]
self._set_args(arglist)
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.assertRaises(RuntimeError, self.cmd.take_action, parsed_args)

View File

@ -188,8 +188,8 @@ class ValidationActions(object):
'cloud2,!cloud1'
"""
hosts = skip_list[playbook].get('hosts')
if hosts == 'ALL' or hosts is None:
hosts = skip_list[playbook].get('hosts', 'all')
if hosts.lower() == 'all':
return None
else:
_hosts = ['!{}'.format(hosts)]
@ -464,8 +464,17 @@ class ValidationActions(object):
'validations': _playbook.split('.')[0],
'UUID': validation_uuid,
})
# Print hosts which has been skipped:
if _hosts:
skipped_hosts = [h.replace('!', '')
for h in _hosts.split(',') if '!' in h]
if skipped_hosts:
msg = ("Validation {} has been skipped "
"on hosts: {}").format(_play,
','.join(skipped_hosts))
self.log.info(msg)
else:
self.log.debug('Skipping Validations: {}'.format(playbook))
self.log.info('Skipping Validations: {}'.format(playbook))
if run_async:
return results