diff --git a/tools/checktests.py b/tools/checktests.py index eeac86a3..e321cde3 100644 --- a/tools/checktests.py +++ b/tools/checktests.py @@ -1,4 +1,5 @@ # Copyright 2018, OpenStack Foundation +# Copyright 2021, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,12 +14,12 @@ # under the License. import argparse -import ast import importlib import json import os import re import sys +import unittest def get_v1_version(guideline): @@ -90,7 +91,7 @@ def get_submodules(parent_module_name): if not os.path.exists(os.path.join(root, '__init__.py')): continue for f in files: - if f.endswith('.py'): + if f.endswith('.py') and not f == "__init__.py": module_path = root + '/' + f module_name = root_name + '.' + os.path.splitext(f)[0] if module_name not in submodules: @@ -98,26 +99,42 @@ def get_submodules(parent_module_name): return submodules -def get_tests(module_name): - submodules = get_submodules(module_name) - tests = {} - for module_name in submodules: - filename = submodules[module_name] - with open(filename, 'r') as f: - source = f.read() - parsed = ast.parse(source) - for node in parsed.body: - if node.__class__ is ast.ClassDef: - for classnode in node.body: - if (classnode.__class__ is ast.FunctionDef and - classnode.name.startswith('test_')): - for decorator in classnode.decorator_list: - if hasattr(decorator, 'func'): - if decorator.func.attr == 'idempotent_id': - tests['id-' + decorator.args[0].s] = \ - module_name + "." + node.name + "." + \ - classnode.name - return tests +def get_module_tests(tests, parsed_test): + if isinstance(tests, unittest.TestCase): + test_description = tests.id() + test_uuid_regex = r'id-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}' + test_id = re.search(test_uuid_regex, test_description) + + if not test_id: + return parsed_test + + test_id = test_id.group(0) + test_name = test_description.split("[")[0] + test_list = parsed_test.get(test_id, []) + test_list.append(test_name) + parsed_test[test_id] = test_list + + return parsed_test + elif not isinstance(tests, unittest.suite.TestSuite): + return + + for test in tests: + parsed_test = get_module_tests(test, parsed_test) + + return parsed_test + + +def get_tests(submodules): + loader = unittest.TestLoader() + parsed_tests = {} + for submodule in submodules: + try: + tests = loader.loadTestsFromName(submodule) + parsed_tests = get_module_tests(tests, parsed_tests) + except Exception as e: + print("Unable to load: {}. Exception: {}".format(submodule, e)) + + return parsed_tests def run(): @@ -133,10 +150,10 @@ def run(): 'against') args = parser.parse_args() - guideline = load_guideline(args.guideline_file) required = get_required_tests(guideline) - tests = get_tests(args.testlib) + submodules = get_submodules(args.testlib) + lib_tests = get_tests(submodules) missing_uuids = [] missing_tests = {} @@ -144,10 +161,11 @@ def run(): for test in required: uuid = test[0] testnames = test[1] - if uuid not in tests: + if uuid not in lib_tests: missing_uuids.append(test) else: - if tests[uuid] not in testnames: + in_testnames = [test in lib_tests[uuid] for test in testnames] + if not any(in_testnames): missing_tests[uuid] = test exit_code = 0 @@ -182,7 +200,7 @@ def run(): " idempotent_id:\n" " %s\n" " names: " % (args.testlib, - uuid, tests[uuid], + uuid, lib_tests[uuid], args.guideline_file, missing_tests[uuid][0])) for testname in missing_tests[uuid][1]: diff --git a/tools/consistency.sh b/tools/consistency.sh index f69713f0..3a42e1bb 100755 --- a/tools/consistency.sh +++ b/tools/consistency.sh @@ -77,17 +77,21 @@ if [[ -z $SFSDIR ]]; then git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR fi +pip install $TEMPESTDIR +pip install $DNSDIR +pip install $ORCHESTRATIONDIR +pip install $SFSDIR export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR python3 ./tools/checktests.py --guideline guidelines/next.json exit_1=$? -python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin -exit_2=$? -# TODO(kopecmartin) In order to unblock gates, skip check of manila tempest plugin until the following bug is resolved: -# https://storyboard.openstack.org/#!/story/2009146 -# python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests -# exit_3=$? +# TODO(lpiwowar) The consistency check of designate_tempest_plugin is omitted until +# https://bugs.launchpad.net/designate/+bug/1943115 is fixed. +# python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin +# exit_2=$? +python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests +exit_3=$? # TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the # checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id # call to track the id @@ -96,10 +100,10 @@ exit_2=$? python3 ./tools/checktests.py --guideline current_guideline exit_5=$? -python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin -exit_6=$? -# python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests -# exit_7=$? +# python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin +# exit_6=$? +python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests +exit_7=$? # python3 ./tools/checktests.py --guideline add-ons/orchestration_current_guideline --testlib heat_tempest_plugin # exit_8=$? @@ -111,6 +115,5 @@ if [[ "${CLEANTEMPEST}" ]]; then rm -rf $SFSDIR fi - #! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 )) -! (( $exit_1 || $exit_2 || $exit_5 || $exit_6 )) +! (( $exit_1 || $exit_3 || $exit_5 || $exit_7 ))