diff --git a/README.rst b/README.rst index 07cab59..f4212b1 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Validation After edit your file, you can check if it is valid with the following command:: - $ tempest-skiplist validate --file file.yaml + $ tempest-skip validate --file file.yaml Examples -------- @@ -68,4 +68,4 @@ It will also be skipped in the releases master, train and ussuri, specifically in jobs job1 and job2. It will not be skipped in any other job, no matter what the release is. -Removing the list of jobs, means it will be skipped everywhere. \ No newline at end of file +Removing the list of jobs, means it will be skipped everywhere. diff --git a/doc/requirements.txt b/doc/requirements.txt index 236d5dc..cace379 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,3 +3,4 @@ reno>=3.1.0 # Apache-2.0 sphinx>=2.0.0,!=2.1.0 # BSD sphinx-argparse>=0.2.2 # MIT sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD +validators==0.18.2 diff --git a/doc/source/addtest/addtest.rst b/doc/source/addtest/addtest.rst new file mode 100644 index 0000000..744f928 --- /dev/null +++ b/doc/source/addtest/addtest.rst @@ -0,0 +1,86 @@ +======================== +Adding tests to skiplist +======================== + +Adding tests +------------ + +Of course it is possible to directly edit the yaml file and add the test +itself, but that may lead to failures, as for example duplicated entries, +identation issues, having a namespace test instead of full path test name. + +In order to avoid that, you can use the `addtest` command that will identify if +the test is already on the skiplist, avoiding duplication. It will also +generate the yaml file properly as well as avoid the use of test namespace. For +example, if a user tries to add the tempest.scenario, it will skip all the +tests under tempest.scenario, which is not the desirable behavior. However, it +is a lot of work to add each entry under tempest.scenario, since we must repeat +all the reasons, bugzilla, releases, etc. + +The addtest command solves all these problems for you. First of all, when you +try to add a test passing a test namespace, addtest will give you a list of +tests under that particular namespace, where you can choose from the list which +ones you would like to add. Select the ones you want and you are done! + +Examples +-------- + +The command below add the test tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks:: + + $ tempest-skip addtest \ + --file roles/validate-tempest/vars/tempest_skip.yml \ + --release master \ + --test tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks \ + --reason 'Failing on network' \ + --lp https://launchpad.net/bug/12345 + +In this example, we are adding the full path test, and so, the command will not +prompt you a list of tests that you want to choose. If for example, only the +namespace be parsed, you will be prompted with a list of tests under that +namespace:: + + $ tempest-skip addtest \ + --file roles/validate-tempest/vars/tempest_skip.yml \ + --release master \ + --test tempest.scenario.test_network_basic_ops \ + --reason 'Failing on network' \ + --lp https://launchpad.net/bug/12345 + +And this is the output: + +.. code-block:: + + [?] These are the tests available on the namespace, choose which ones you want to add. Press space to select: + > o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_preserve_preexisting_port + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_instance_port_admin_state + o tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_router_admin_state + +You can use the arrow keys on your keyboard to navigate through the list, and +space to select. Once you are done, just press enter, the command will go +through each test, check if the test exists or not. If it exists, it will also +check the release exists, and properly add the test. + +Once you are done, you can validate if the yaml file was generated properly +with the command:: + + $ tempest-skip validate --file roles/validate-tempest/vars/tempest_skip.yml + +There are some arguments you can pass to the addtest, the required ones are: + +* --file +* --lp or --bz +* --reason + +All the other arguments have default values, for example, `--release` default +value is master and `--deployment` default value is overcloud + +For more information, use:: + + $ tempest-skip addtest --help diff --git a/doc/source/addtest/index.rst b/doc/source/addtest/index.rst new file mode 100644 index 0000000..b8e05f9 --- /dev/null +++ b/doc/source/addtest/index.rst @@ -0,0 +1,5 @@ +.. toctree:: + :maxdepth: 2 + :includehidden: + + addtest diff --git a/doc/source/index.rst b/doc/source/index.rst index c31ebdf..6980791 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,5 +15,6 @@ Content: yaml/index validate/index listyaml/index + addtest/index * :ref:`search` diff --git a/requirements.txt b/requirements.txt index 7c86436..ea0a3f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,6 @@ cliff -voluptuous \ No newline at end of file +inquirer +ruamel.yaml==0.16.12 +tempest +validators +voluptuous diff --git a/setup.cfg b/setup.cfg index a997473..01d2105 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,7 @@ console_scripts = tempest_skip.cm = validate = tempest_skip.validate:Validate list = tempest_skip.list_yaml:ListYaml + addtest = tempest_skip.add_test:AddTest [build_sphinx] source-dir = doc/source diff --git a/tempest_skip/add_test.py b/tempest_skip/add_test.py new file mode 100644 index 0000000..35d7676 --- /dev/null +++ b/tempest_skip/add_test.py @@ -0,0 +1,189 @@ + +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import tempfile +from argparse import ArgumentTypeError + +import inquirer +import ruamel.yaml as yaml +import validators +from cliff.command import Command +from stestr import config_file +from tempest.cmd.init import TempestInit +from tempest.cmd.run import TempestRun + + +class AddTest(Command): + """Command to add a test into the skiplist""" + + log = logging.getLogger(__name__) + + def take_action(self, parsed_args): + self.log.debug('Running add_test command') + self.abs_path = os.path.abspath(parsed_args.file) + self.yaml_file = yaml.round_trip_load(open(self.abs_path), + preserve_quotes=True) + test_list = self._get_tests_list() + filter_tests = list(filter(lambda x: parsed_args.test in x, test_list)) + if len(filter_tests) > 1: + question = [inquirer.Checkbox('list_tests', + 'These are the tests available on ' + 'the namespace, choose which ones ' + 'you want to add. Press space to ' + 'select', + choices=filter_tests)] + answers = inquirer.prompt(question) + for test in answers.get('list_tests', []): + self._add_test_in_yaml(test, parsed_args.deployment, + parsed_args.release, + parsed_args.reason, lp=parsed_args.lp, + bz=parsed_args.bz, job=parsed_args.job) + else: + self._add_test_in_yaml(parsed_args.test, parsed_args.deployment, + parsed_args.release, + parsed_args.reason, lp=parsed_args.lp, + bz=parsed_args.bz, job=parsed_args.job) + + with open(self.abs_path, 'w') as f: + yaml.round_trip_dump(self.yaml_file, f, + Dumper=yaml.RoundTripDumper, + indent=4, block_seq_indent=2) + + def _add_test_in_yaml(self, test_name, deployment, release, reason, + lp=None, bz=None, job=None): + was_inserted = False + release_exist = False + deployment_exist = False + job_exist = False + jobs = [] + for test in self.yaml_file.get('known_failures'): + if test.get('test') == test_name: + # Test already exist + for d in test.get('deployment'): + if d == deployment: + deployment_exist = True + + for r in test.get('releases'): + if r.get('name') == release: + was_inserted = True + release_exist = True + + for j in test.get('jobs'): + if j == job: + job_exist = True + + if not release_exist: + entry = {'name': release, + 'reason': reason} + if lp: + entry['lp'] = lp + if bz: + entry['bz'] = bz + test.get('releases').append(entry) + was_inserted = True + if not deployment_exist: + test.get('deployment').append(deployment) + if not job_exist: + test.get('jobs').append(job) + + if not was_inserted: + release = {'name': release, 'reason': reason} + if lp: + release['lp'] = lp + if bz: + release['bz'] = bz + if job: + jobs = [job] + entry = {'test': test_name, 'deployment': [deployment], + 'releases': [release], + 'jobs': jobs} + self.yaml_file.get('known_failures').append(entry) + self.log.debug('Test {} added with release {}'.format( + test_name, release)) + else: + self.log.warning( + 'Test {} already exist for release {}, doing nothing'.format( + test_name, release)) + + def _get_tests_list(self): + tempest_run = TempestRun(__name__, []) + tempest_init = TempestInit(__name__, []) + + path = tempfile.mkdtemp(dir='/tmp') + + parser = tempest_init.get_parser(__name__) + parser.workspace_path = None + parser.name = path.split(os.path.sep)[-1] + parser.config_dir = None + parser.dir = path + parser.show_global_dir = False + + tempest_init.take_action(parser) + + parser = tempest_run.get_parser('tempest') + parser.list_tests = True + parser.config_file = None + parser.workspace = path.split(os.path.sep)[-1] + parser.state = None + parser.smoke = False + parser.regex = '' + parser.whitelist_file = None + parser.blacklist_file = None + parser.black_regex = None + parser.workspace_path = None + tempest_run._create_stestr_conf() + + # config = os.path.join(path, '.stestr.conf') + os.chdir(path) + conf = config_file.TestrConf('.stestr.conf') + cmd = conf.get_run_command() + try: + cmd.setUp() + ids = cmd.list_tests() + finally: + cmd.cleanUp() + + return [i.split('[')[0] for i in ids] + + def _validate_url(self, url): + if not validators.url(url): + raise ArgumentTypeError('{} is not a valid url'.format(url)) + return url + + def get_parser(self, prog_name): + parser = super(AddTest, self).get_parser(prog_name) + parser.add_argument('--file', dest='file', required=True, + help='Skiplist config file') + parser.add_argument('--release', dest='release', default='master', + help='Release where the test will be added') + parser.add_argument('--job', dest='job', + help='Specify in which job this test will be ' + 'skipped') + parser.add_argument('--test', dest='test', required=True, + help='Test to be skipped') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--lp', dest='lp', + help='Launchpad bug', type=self._validate_url) + group.add_argument('--bz', dest='bz', + help='Bugzilla bug', type=self._validate_url) + parser.add_argument('--reason', dest='reason', required=True, + help='Reason to test be skipped') + parser.add_argument('--deployment', dest='deployment', required=False, + default='overcloud', + choices=['overcloud', 'undercloud']) + return parser diff --git a/tempest_skip/tests/base.py b/tempest_skip/tests/base.py index f6ff92c..097f6c4 100644 --- a/tempest_skip/tests/base.py +++ b/tempest_skip/tests/base.py @@ -13,8 +13,20 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import tempfile + from oslotest import base class TestCase(base.BaseTestCase): - pass + + def write_yaml_file(self, file_content): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + + yaml_file = os.fdopen(fd, 'w') + yaml_file.write(file_content) + yaml_file.close() + + return path diff --git a/tempest_skip/tests/test_add_test.py b/tempest_skip/tests/test_add_test.py new file mode 100644 index 0000000..daf60a3 --- /dev/null +++ b/tempest_skip/tests/test_add_test.py @@ -0,0 +1,107 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from tempest_skip.add_test import AddTest +from tempest_skip.tests import base +import yaml + + +class TestAddFile(base.TestCase): + def setUp(self): + super(TestAddFile, self).setUp() + self.list_file = """ + known_failures: + - test: 'tempest_skip.tests.test_list_yaml' + deployment: + - 'overcloud' + releases: + - name: 'master' + reason: 'It can be removed when bug for OVN is repaired' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + jobs: [] + - test: 'tempest_skip.tests.test_list_yaml_2' + deployment: + - 'overcloud' + releases: + - name: 'master' + reason: 'This test was enabled recently on ovn' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + jobs: [] + - test: 'tempest_skip.tests.test_list_yaml_3' + deployment: + - 'overcloud' + - 'undercloud' + releases: + - name: 'train' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + jobs: + - 'job1' + """ + + self.path = self.write_yaml_file(self.list_file) + self.cmd = AddTest(__name__, []) + self.parser = self.cmd.get_parser(__name__) + self.parser.file = self.path + + self.parser.job = None + self.parser.release = 'master' + self.parser.lp = 'https://launchpad.net/bug/12345' + self.parser.bz = None + self.parser.reason = 'A good reason' + self.parser.deployment = 'overcloud' + self.tests_list = ['tempest_skip.tests.test1', + 'tempest_skip.tests.test2', + 'tempest_skip.tests.test3'] + + @mock.patch('inquirer.Checkbox') + @mock.patch('inquirer.prompt') + @mock.patch('tempest_skip.add_test.AddTest._get_tests_list') + def test_add_test_new_test(self, tests_list, prompt_mock, checkbox_mock): + self.parser.test = 'tempest_skip.tests' + tests_list.return_value = self.tests_list + prompt_mock.return_value = {'list_tests': ['tempest_skip.tests.test3']} + self.cmd.take_action(self.parser) + checkbox_mock.assert_called_once() + prompt_mock.assert_called_once() + yaml_file = yaml.safe_load(open(self.path)) + # There are 3 tests in the skip file, adding one more test, should be 4 + self.assertEqual(4, len(yaml_file['known_failures'])) + + @mock.patch('tempest_skip.add_test.AddTest._get_tests_list') + def test_add_test_already_exist(self, tests_list): + self.parser.test = 'tempest_skip.tests.test_list_yaml' + + tests_list.return_value = self.tests_list + + self.cmd.take_action(self.parser) + yaml_file = yaml.safe_load(open(self.path)) + self.assertEqual(3, len(yaml_file['known_failures'])) + + @mock.patch('inquirer.Checkbox') + @mock.patch('inquirer.prompt') + @mock.patch('tempest_skip.add_test.AddTest._get_tests_list') + def test_add_test_no_prompt(self, tests_list, prompt_mock, checkbox_mock): + self.parser.test = 'tempest_skip.tests.test1' + tests_list.return_value = self.tests_list + prompt_mock.return_value = {'list_tests': ['tempest_skip.tests.test3']} + self.cmd.take_action(self.parser) + checkbox_mock.assert_not_called() + prompt_mock.assert_not_called() + yaml_file = yaml.safe_load(open(self.path)) + # There are 3 tests in the skip file, adding one more test, should be 4 + self.assertEqual(4, len(yaml_file['known_failures'])) diff --git a/tempest_skip/tests/test_list_yaml.py b/tempest_skip/tests/test_list_yaml.py index 252e26c..da07193 100644 --- a/tempest_skip/tests/test_list_yaml.py +++ b/tempest_skip/tests/test_list_yaml.py @@ -13,9 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import os -import tempfile - from cliff.lister import Lister from tempest_skip.tests import base from tempest_skip.list_yaml import ListYaml @@ -74,16 +71,6 @@ class TestListYaml(base.TestCase): self.parser.deployment = None self.parser.job = None - def write_yaml_file(self, file_content): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - - yaml_file = os.fdopen(fd, 'w') - yaml_file.write(file_content) - yaml_file.close() - - return path - def test_list_yaml(self): cmd_result = self.cmd.take_action(self.parser) diff --git a/test-requirements.txt b/test-requirements.txt index e059622..37d7a21 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,5 +3,6 @@ # process, which may cause wedges in the gate later. flake8==3.8.3 # MIT +mock oslotest>=3.2.0 # Apache-2.0 stestr>=1.1.0 # Apache-2.0