From 79eae8ee763053074606e88bba352d591b850247 Mon Sep 17 00:00:00 2001 From: Arx Cruz Date: Tue, 7 Apr 2020 13:23:18 +0200 Subject: [PATCH] Create validate command This patch adds the validate command in the tempest-skip project and tox-functional jobs since we now have python code. This will be necessary in order to validate the yaml file, and can also be used in a job. Also, documentation update and added new packages in the requirements. Change-Id: I38b9b98d66c50b2da98e316f856f210ea0d99fd1 --- .zuul.yaml | 4 ++ doc/source/index.rst | 1 + doc/source/validate/index.rst | 5 ++ doc/source/validate/validate.rst | 17 +++++ doc/source/yaml/formatting.rst | 104 +++++++++++++++++++++++++--- requirements.txt | 2 + setup.cfg | 2 + tempest_skip/main.py | 34 ++++++++- tempest_skip/tests/base.py | 20 ++++++ tempest_skip/tests/test_validate.py | 96 +++++++++++++++++++++++++ tempest_skip/validate.py | 58 ++++++++++++++++ test-requirements.txt | 2 + 12 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 doc/source/validate/index.rst create mode 100644 doc/source/validate/validate.rst create mode 100644 tempest_skip/tests/base.py create mode 100644 tempest_skip/tests/test_validate.py create mode 100644 tempest_skip/validate.py diff --git a/.zuul.yaml b/.zuul.yaml index 6864773..fddac39 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,7 +3,11 @@ jobs: - openstack-tox-pep8 - openstack-tox-docs + - openstack-tox-py36 + - openstack-tox-py37 gate: jobs: - openstack-tox-pep8 - openstack-tox-docs + - openstack-tox-py36 + - openstack-tox-py37 diff --git a/doc/source/index.rst b/doc/source/index.rst index a237829..415cea8 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -13,5 +13,6 @@ Content: overview install/index yaml/index + validate/index * :ref:`search` \ No newline at end of file diff --git a/doc/source/validate/index.rst b/doc/source/validate/index.rst new file mode 100644 index 0000000..d49e484 --- /dev/null +++ b/doc/source/validate/index.rst @@ -0,0 +1,5 @@ +.. toctree:: + :maxdepth: 2 + :includehidden: + + validate \ No newline at end of file diff --git a/doc/source/validate/validate.rst b/doc/source/validate/validate.rst new file mode 100644 index 0000000..2e8a215 --- /dev/null +++ b/doc/source/validate/validate.rst @@ -0,0 +1,17 @@ +====================== +Validate the yaml file +====================== + +Validation +---------- + +You can use :command:`tempest-skip` validate command to validate if the yaml +file is in the expected format:: + + $ tempest-skip validate --file good_file.yaml + + +This will return nothing if the file is valid, or an error otherwise:: + + $ tempest-skip validate --file bad_file.yaml + required key not provided @ data['known_failures'][0]['releases'][2]['reason'] \ No newline at end of file diff --git a/doc/source/yaml/formatting.rst b/doc/source/yaml/formatting.rst index 96f2b41..1fb8b5a 100644 --- a/doc/source/yaml/formatting.rst +++ b/doc/source/yaml/formatting.rst @@ -9,21 +9,23 @@ The YAML file used by `openstack-tempest-skiplist` use the following pattern: .. code-block:: yaml + known_failures: - test: 'full.tempest.test' - bz: 'bugzilla bug' - lp: 'launchpad bug' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' deployment: 'undercloud or overcloud' jobs: - job1 - job2 reason: 'default reason' releases: - - master - lp: 'master launchpad' + - name: master + lp: 'https://launchpad.net/bugs/2' reason: 'Some reason' - - train - bz: 'train bugzilla' - - ussuri + - name: train + bz: 'https://bugzilla.redhat.com/train1' + - name: ussuri + bz: 'https://bugzilla.redhat.com/ussuri1' YAML values @@ -40,6 +42,7 @@ and the second is based on the release. This is required because it's possible to have the test failing in two different releases but with different reasons. If the lp or bz is set on test level, you don't need to add it per release, it will use the test level. +The value of both lp and bz are their respective URL Deployment @@ -70,4 +73,89 @@ Releases Releases contain a list of releases that the test will be skipped. It's very common that in a release the test is passing, but in another don't, so we can -manage it here. \ No newline at end of file +manage it here. +Here, it's also required a reason, and a lp or a bz + +Examples +-------- + +Below are some examples of valid yaml files that can be used: + +No releases ++++++++++++ + +Since there's no releases, this will be valid for all releases: + +.. code-block:: yaml + + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + reason: 'This test will be skipped in any release' + + +With releases ++++++++++++++ + +As release is set, the test will be skipped only on the matching releases + +.. code-block:: yaml + + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + reason: 'This test will be skipped in any release' + releases: + - name: rocky + reason: 'Test failing in rock because of network' + lp: 'https://launchpad.net/bugs/1' + - name: ussuri + reason: 'Test is failing in ussuri because of storage bug' + bz: 'https://bugzilla.redhat.com/1' + + +With jobs ++++++++++ + +If a list of jobs is set, the test will be skipped only in the matching jobs + +.. code-block:: yaml + + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + reason: 'This test will be skipped in any release' + jobs: + - tempest-test-job-skip1 + - tempest-test-job-skip2 + + +With jobs and releases +++++++++++++++++++++++ + +This test will be skipped only when it matches both, job and release + +.. code-block:: yaml + + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + reason: 'This test will be skipped in all releases' + releases: + - name: rocky + reason: 'Test failing in rock because of network' + lp: 'https://launchpad.net/bugs/1' + - name: ussuri + reason: 'Test is failing in ussuri because of storage bug' + bz: 'https://bugzilla.redhat.com/1' + jobs: + - tempest-test-job-skip1 + - tempest-test-job-skip2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..7c86436 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,2 @@ +cliff +voluptuous \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index cd02926..afe3641 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,8 @@ packages = [entry_points] console_scripts = tempest-skip = tempest_skip.main:main +tempest_skip.cm = + validate = tempest_skip.validate:Validate [build_sphinx] source-dir = doc/source diff --git a/tempest_skip/main.py b/tempest_skip/main.py index e82d1fc..998c263 100644 --- a/tempest_skip/main.py +++ b/tempest_skip/main.py @@ -13,10 +13,38 @@ # License for the specific language governing permissions and limitations # under the License. +import sys -def main(): - pass +from cliff.app import App +from cliff.commandmanager import CommandManager + + +class TempestSkip(App): + + def __init__(self): + super(TempestSkip, self).__init__( + description='Tempest skiplist tool', + version='0.1', + command_manager=CommandManager('tempest_skip.cm'), + deferred_help=True + ) + + def initialize_app(self, argv): + self.LOG.debug('Initializing tempest skiplist tool') + + def prepare_to_run_command(self, cmd): + self.LOG.debug('prepare_to_run_command %s', cmd.__class__.__name__) + + def clean_up(self, cmd, result, err): + self.LOG.debug('clean_up %s', cmd.__class__.__name__) + if err: + self.LOG.debug('Error: %s', err) + + +def main(argv=sys.argv[1:]): + tempest_skip = TempestSkip() + return tempest_skip.run(argv) if __name__ == "__main__": - main() + sys.exit(main(sys.argv[1:])) diff --git a/tempest_skip/tests/base.py b/tempest_skip/tests/base.py new file mode 100644 index 0000000..f6ff92c --- /dev/null +++ b/tempest_skip/tests/base.py @@ -0,0 +1,20 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslotest import base + + +class TestCase(base.BaseTestCase): + pass diff --git a/tempest_skip/tests/test_validate.py b/tempest_skip/tests/test_validate.py new file mode 100644 index 0000000..bbfccbe --- /dev/null +++ b/tempest_skip/tests/test_validate.py @@ -0,0 +1,96 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import subprocess +import tempfile + +from tempest_skip.tests import base + + +class TestValidate(base.TestCase): + def setUp(self): + super(TestValidate, self).setUp() + + def assertRunExit(self, cmd, expected): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + msg = ("Running %s got an unexpected returncode\n" + "Stdout: %s\nStderr: %s" % (' '.join(cmd), out, err)) + self.assertEqual(p.returncode, expected, msg) + return out, err + + def test_validate_passes(self): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + + valid_yaml = """ + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + jobs: + - openstack-tempest-skip-job1 + - openstack-tempest-skip-job2 + reason: 'This test is failing' + releases: + - name: 'master' + lp: 'https://launchpad.net/bugs/1' + reason: 'Test with launchpad' + - name: 'train' + bz: 'https://bugzilla.redhat.com/1' + reason: 'Test with bugzilla' + - name: 'ussuri' + reason: 'Test without launchpad or bugzilla' + lp: 'https://launchpad.net/bugs/1' + """ + yaml_file = os.fdopen(fd, 'w') + yaml_file.write(valid_yaml) + yaml_file.close() + + self.assertRunExit(['tempest-skip', 'validate', '--file', path], 0) + + def test_validate_fails(self): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + + valid_yaml = """ + known_failures: + - test: 'tempest_skip.tests.test_validate' + bz: 'https://bugzilla.redhat.com/1' + lp: 'https://launchpad.net/bugs/1' + deployment: 'undercloud' + jobs: + - openstack-tempest-skip-job1: + option: '1' + - openstack-tempest-skip-job2 + reason: 'This test is failing' + releases: + - name: 'master' + lp: 'https://launchpad.net/bugs/1' + reason: 'Test with launchpad' + - name: 'train' + bz: 'https://bugzilla.redhat.com/1' + reason: 'Test with bugzilla' + - name: 'ussuri' + reason: 'Test without launchpad or bugzilla' + """ + yaml_file = os.fdopen(fd, 'w') + yaml_file.write(valid_yaml) + yaml_file.close() + + self.assertRunExit(['tempest-skip', 'validate', '--file', path], 1) diff --git a/tempest_skip/validate.py b/tempest_skip/validate.py new file mode 100644 index 0000000..19d66e1 --- /dev/null +++ b/tempest_skip/validate.py @@ -0,0 +1,58 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from cliff.command import Command +import voluptuous as v +import yaml + + +class Validate(Command): + "Command to validate the yaml file parsed to tempest skiplist" + + log = logging.getLogger(__name__) + + validate = v.Schema({ + 'known_failures': [{ + v.Required('test'): str, + v.Optional('bz'): v.Url(), + v.Optional('lp'): v.Url(), + v.Required('deployment'): v.Any('undercloud', 'overcloud'), + v.Optional('reason'): str, + v.Optional('releases'): [ + v.Schema({ + v.Required('name'): str, + v.Required(v.SomeOf( + validators=[v.Any('lp', 'bz')], min_valid=1)): v.Url(), + v.Required('reason'): str + }) + ], + v.Optional('jobs'): [str] + }] + }) + + def take_action(self, parsed_args): + self.log.debug('Running validate command') + yaml_file = yaml.safe_load(open(parsed_args.filename)) + self.validate(yaml_file) + + def get_parser(self, prog_name): + parser = super(Validate, self).get_parser(prog_name) + parser.add_argument('--file', dest='filename', + help='Path to the YAML file to be validate', + required=True) + + return parser diff --git a/test-requirements.txt b/test-requirements.txt index 93fe66f..b66ea3e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,5 @@ # process, which may cause wedges in the gate later. flake8<3.0.0 # MIT +oslotest>=3.2.0 # Apache-2.0 +stestr>=1.1.0 # Apache-2.0 \ No newline at end of file