cf024be535
Chectest.py wasn't able to catch bug [1] as it didn't take into consideration inheritance during the test discovery. The cause of the issue is the fact that chectest.py parsed python files by itself to discover the tests. This patch fixes this by using unittest library for test discovery instead. [1] https://review.opendev.org/c/osf/interop/+/806178 Story: 2009146 Task: 43097 Change-Id: I6e1b11eeb3ca1915ca41b6af88f9f568e6d674eb
215 lines
6.9 KiB
Python
215 lines
6.9 KiB
Python
# Copyright 2018, OpenStack Foundation
|
|
# Copyright 2021, Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import importlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import unittest
|
|
|
|
|
|
def get_v1_version(guideline):
|
|
schema = guideline.get('schema')
|
|
if schema:
|
|
v1regex = re.compile("^1.*")
|
|
if v1regex.match(schema):
|
|
return "1"
|
|
return None
|
|
|
|
|
|
def get_v2_version(guideline):
|
|
metadata = guideline.get('metadata')
|
|
if metadata and isinstance(metadata, dict):
|
|
schema = metadata.get('schema')
|
|
if schema:
|
|
v2regex = re.compile("^2.*")
|
|
if v2regex.match(schema):
|
|
return "2"
|
|
return None
|
|
|
|
|
|
def get_guideline_version(guideline):
|
|
return get_v1_version(guideline) or get_v2_version(guideline)
|
|
|
|
|
|
def get_required_tests_v1(guideline):
|
|
capabilities = guideline.get('capabilities')
|
|
tests = []
|
|
for capability_name in capabilities:
|
|
capability = capabilities.get(capability_name)
|
|
for test_name in capability.get('tests'):
|
|
test = capability.get('tests').get(test_name)
|
|
if not test.get('flagged'):
|
|
idempotent_id = test.get('idempotent_id')
|
|
test_set = {test_name}
|
|
if test.get('aliases'):
|
|
test_set = test_set.union(set(test.get('aliases')))
|
|
tests.append((idempotent_id, test_set))
|
|
return tests
|
|
|
|
|
|
def get_required_tests_v2(guideline):
|
|
return get_required_tests_v1(guideline)
|
|
|
|
|
|
def get_required_tests(guideline):
|
|
if get_v1_version(guideline):
|
|
return get_required_tests_v1(guideline)
|
|
elif get_v2_version(guideline):
|
|
return get_required_tests_v2(guideline)
|
|
return None
|
|
|
|
|
|
def load_guideline(guideline_file):
|
|
with open(guideline_file) as f:
|
|
guideline = json.load(f)
|
|
return guideline
|
|
|
|
|
|
def get_submodules(parent_module_name):
|
|
module = importlib.import_module(parent_module_name)
|
|
base_path = os.path.abspath(os.path.dirname(module.__file__))
|
|
base_index = len(base_path.split('/')) - 1
|
|
|
|
submodules = {}
|
|
for root, dirnames, files in os.walk(base_path):
|
|
root_name = '.'.join(root.split('/')[base_index:])
|
|
if not os.path.exists(os.path.join(root, '__init__.py')):
|
|
continue
|
|
for f in files:
|
|
if f.endswith('.py') and not f == "__init__.py":
|
|
module_path = root + '/' + f
|
|
module_name = root_name + '.' + os.path.splitext(f)[0]
|
|
if module_name not in submodules:
|
|
submodules[module_name] = module_path
|
|
return submodules
|
|
|
|
|
|
def get_module_tests(tests, parsed_test):
|
|
if isinstance(tests, unittest.TestCase):
|
|
test_description = tests.id()
|
|
test_uuid_regex = r'id-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}'
|
|
test_id = re.search(test_uuid_regex, test_description)
|
|
|
|
if not test_id:
|
|
return parsed_test
|
|
|
|
test_id = test_id.group(0)
|
|
test_name = test_description.split("[")[0]
|
|
test_list = parsed_test.get(test_id, [])
|
|
test_list.append(test_name)
|
|
parsed_test[test_id] = test_list
|
|
|
|
return parsed_test
|
|
elif not isinstance(tests, unittest.suite.TestSuite):
|
|
return
|
|
|
|
for test in tests:
|
|
parsed_test = get_module_tests(test, parsed_test)
|
|
|
|
return parsed_test
|
|
|
|
|
|
def get_tests(submodules):
|
|
loader = unittest.TestLoader()
|
|
parsed_tests = {}
|
|
for submodule in submodules:
|
|
try:
|
|
tests = loader.loadTestsFromName(submodule)
|
|
parsed_tests = get_module_tests(tests, parsed_tests)
|
|
except Exception as e:
|
|
print("Unable to load: {}. Exception: {}".format(submodule, e))
|
|
|
|
return parsed_tests
|
|
|
|
|
|
def run():
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--guideline', action='store', dest='guideline_file',
|
|
default='next.json', type=str,
|
|
help='The name of the guideline file to to check')
|
|
parser.add_argument('--testlib', action='store', dest='testlib',
|
|
default='tempest', type=str,
|
|
help='The test library, in the PYTHONPATH, to '
|
|
'check the consistency of the guideline '
|
|
'against')
|
|
|
|
args = parser.parse_args()
|
|
guideline = load_guideline(args.guideline_file)
|
|
required = get_required_tests(guideline)
|
|
submodules = get_submodules(args.testlib)
|
|
lib_tests = get_tests(submodules)
|
|
|
|
missing_uuids = []
|
|
missing_tests = {}
|
|
|
|
for test in required:
|
|
uuid = test[0]
|
|
testnames = test[1]
|
|
if uuid not in lib_tests:
|
|
missing_uuids.append(test)
|
|
else:
|
|
in_testnames = [test in lib_tests[uuid] for test in testnames]
|
|
if not any(in_testnames):
|
|
missing_tests[uuid] = test
|
|
|
|
exit_code = 0
|
|
if len(missing_uuids) > 0:
|
|
exit_code = 1
|
|
print("### Idempotent ID Errors Detected. To resolve these errors, "
|
|
"fix the uuid name (format id-<uuid>) in the guideline to "
|
|
"match the id in the test suite:")
|
|
for test in missing_uuids:
|
|
print("Idempotent ID in guideline '%s' does not appear in test "
|
|
"library '%s'\n"
|
|
" idempotent_id:\n"
|
|
" %s\n"
|
|
" names: " % (args.guideline_file, args.testlib, test[0]))
|
|
for testname in test[1]:
|
|
print(" %s" % (testname))
|
|
print("")
|
|
|
|
if len(missing_tests) > 0:
|
|
exit_code = 1
|
|
print("### Test Name Errors Detected. "
|
|
"To resolve these errors, update "
|
|
"the Interop guideline with the missing "
|
|
"test names:")
|
|
for uuid in missing_tests:
|
|
print("Test found in test library '%s'\n"
|
|
" idempotent_id:\n"
|
|
" %s\n"
|
|
" name:\n"
|
|
" %s\n"
|
|
"Entry in guideline '%s'\n"
|
|
" idempotent_id:\n"
|
|
" %s\n"
|
|
" names: " % (args.testlib,
|
|
uuid, lib_tests[uuid],
|
|
args.guideline_file,
|
|
missing_tests[uuid][0]))
|
|
for testname in missing_tests[uuid][1]:
|
|
print(" %s" % (testname))
|
|
print("")
|
|
|
|
sys.exit(exit_code)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run()
|