# Copyright 2018, OpenStack Foundation # Copyright 2021, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import argparse import importlib import json import os import re import sys import unittest def get_v1_version(guideline): schema = guideline.get('schema') if schema: v1regex = re.compile("^1.*") if v1regex.match(schema): return "1" return None def get_v2_version(guideline): metadata = guideline.get('metadata') if metadata and isinstance(metadata, dict): schema = metadata.get('schema') if schema: v2regex = re.compile("^2.*") if v2regex.match(schema): return "2" return None def get_guideline_version(guideline): return get_v1_version(guideline) or get_v2_version(guideline) def get_required_tests_v1(guideline): capabilities = guideline.get('capabilities') tests = [] for capability_name in capabilities: capability = capabilities.get(capability_name) for test_name in capability.get('tests'): test = capability.get('tests').get(test_name) if not test.get('flagged'): idempotent_id = test.get('idempotent_id') test_set = {test_name} if test.get('aliases'): test_set = test_set.union(set(test.get('aliases'))) tests.append((idempotent_id, test_set)) return tests def get_required_tests_v2(guideline): return get_required_tests_v1(guideline) def get_required_tests(guideline): if get_v1_version(guideline): return get_required_tests_v1(guideline) elif get_v2_version(guideline): return get_required_tests_v2(guideline) return None def load_guideline(guideline_file): with open(guideline_file) as f: guideline = json.load(f) return guideline def get_submodules(parent_module_name): module = importlib.import_module(parent_module_name) base_path = os.path.abspath(os.path.dirname(module.__file__)) base_index = len(base_path.split('/')) - 1 submodules = {} for root, dirnames, files in os.walk(base_path): root_name = '.'.join(root.split('/')[base_index:]) if not os.path.exists(os.path.join(root, '__init__.py')): continue for f in files: if f.endswith('.py') and not f == "__init__.py": module_path = root + '/' + f module_name = root_name + '.' + os.path.splitext(f)[0] if module_name not in submodules: submodules[module_name] = module_path return submodules def get_module_tests(tests, parsed_test): if isinstance(tests, unittest.TestCase): test_description = tests.id() test_uuid_regex = r'id-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}' test_id = re.search(test_uuid_regex, test_description) if not test_id: return parsed_test test_id = test_id.group(0) test_name = test_description.split("[")[0] test_list = parsed_test.get(test_id, []) test_list.append(test_name) parsed_test[test_id] = test_list return parsed_test elif not isinstance(tests, unittest.suite.TestSuite): return for test in tests: parsed_test = get_module_tests(test, parsed_test) return parsed_test def get_tests(submodules): loader = unittest.TestLoader() parsed_tests = {} for submodule in submodules: try: tests = loader.loadTestsFromName(submodule) parsed_tests = get_module_tests(tests, parsed_tests) except Exception as e: print("Unable to load: {}. Exception: {}".format(submodule, e)) return parsed_tests def run(): parser = argparse.ArgumentParser() parser.add_argument('--guideline', action='store', dest='guideline_file', default='next.json', type=str, help='The name of the guideline file to to check') parser.add_argument('--testlib', action='store', dest='testlib', default='tempest', type=str, help='The test library, in the PYTHONPATH, to ' 'check the consistency of the guideline ' 'against') args = parser.parse_args() guideline = load_guideline(args.guideline_file) required = get_required_tests(guideline) submodules = get_submodules(args.testlib) lib_tests = get_tests(submodules) missing_uuids = [] missing_tests = {} for test in required: uuid = test[0] testnames = test[1] if uuid not in lib_tests: missing_uuids.append(test) else: in_testnames = [test in lib_tests[uuid] for test in testnames] if not any(in_testnames): missing_tests[uuid] = test exit_code = 0 if len(missing_uuids) > 0: exit_code = 1 print("### Idempotent ID Errors Detected. To resolve these errors, " "fix the uuid name (format id-) in the guideline to " "match the id in the test suite:") for test in missing_uuids: print("Idempotent ID in guideline '%s' does not appear in test " "library '%s'\n" " idempotent_id:\n" " %s\n" " names: " % (args.guideline_file, args.testlib, test[0])) for testname in test[1]: print(" %s" % (testname)) print("") if len(missing_tests) > 0: exit_code = 1 print("### Test Name Errors Detected. " "To resolve these errors, update " "the Interop guideline with the missing " "test names:") for uuid in missing_tests: print("Test found in test library '%s'\n" " idempotent_id:\n" " %s\n" " name:\n" " %s\n" "Entry in guideline '%s'\n" " idempotent_id:\n" " %s\n" " names: " % (args.testlib, uuid, lib_tests[uuid], args.guideline_file, missing_tests[uuid][0])) for testname in missing_tests[uuid][1]: print(" %s" % (testname)) print("") sys.exit(exit_code) if __name__ == '__main__': run()