934a9a30bf
In the script for results reporting we have an option which allows to use latest started jenkins job (tests runner) build while getting tests data. Added appropriate option to the cases uploader script to prevent 'not found' errors when new test threads are just added. Change-Id: I35d60155772f7db76a5c1a2515e69effbbbaf0f5
434 lines
16 KiB
Python
434 lines
16 KiB
Python
# Copyright 2015 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
import re
|
|
import string
|
|
|
|
from logging import DEBUG
|
|
from optparse import OptionParser
|
|
from proboscis import TestPlan
|
|
from proboscis.decorators import DEFAULT_REGISTRY
|
|
|
|
from fuelweb_test.testrail.builds import Build
|
|
from fuelweb_test.testrail.settings import GROUPS_TO_EXPAND
|
|
from fuelweb_test.testrail.settings import logger
|
|
from fuelweb_test.testrail.settings import TestRailSettings
|
|
from fuelweb_test.testrail.testrail_client import TestRailProject
|
|
from fuelweb_test.testrail import datetime_util
|
|
from system_test import define_custom_groups
|
|
from system_test import discover_import_tests
|
|
from system_test import register_system_test_cases
|
|
from system_test import tests_directory
|
|
from system_test import get_basepath
|
|
from system_test.tests.base import ActionTest
|
|
|
|
|
|
GROUP_FIELD = 'custom_test_group'
|
|
|
|
STEP_NUM_PATTERN = re.compile(r'^(\d{1,3})[.].+')
|
|
DURATION_PATTERN = re.compile(r'Duration:?\s+(\d+(?:[sm]|\s?m))(?:in)?\b')
|
|
|
|
|
|
def get_tests_descriptions(milestone_id, tests_include, tests_exclude, groups,
|
|
default_test_priority):
|
|
plan = _create_test_plan_from_registry(groups=groups)
|
|
all_plan_tests = plan.tests[:]
|
|
|
|
tests = []
|
|
|
|
for jenkins_suffix in groups:
|
|
group = groups[jenkins_suffix]
|
|
plan.filter(group_names=[group])
|
|
for case in plan.tests:
|
|
if not _is_case_processable(case=case, tests=tests):
|
|
continue
|
|
|
|
case_name = test_group = _get_test_case_name(case)
|
|
|
|
if _is_not_included(case_name, tests_include) or \
|
|
_is_excluded(case_name, tests_exclude):
|
|
continue
|
|
|
|
docstring = _get_docstring(parent_home=case.entry.parent.home,
|
|
case_state=case.state,
|
|
home=case.entry.home)
|
|
|
|
title, steps, duration = _parse_docstring(docstring, case)
|
|
|
|
if case.entry.home.func_name in GROUPS_TO_EXPAND:
|
|
"""Expand specified test names with the group names that are
|
|
used in jenkins jobs where this test is started.
|
|
"""
|
|
title = ' - '.join([title, jenkins_suffix])
|
|
test_group = '_'.join([case.entry.home.func_name,
|
|
jenkins_suffix])
|
|
|
|
test_case = {
|
|
"title": title,
|
|
"type_id": 1,
|
|
"milestone_id": milestone_id,
|
|
"priority_id": default_test_priority,
|
|
"estimate": duration,
|
|
"refs": "",
|
|
"custom_test_group": test_group,
|
|
"custom_test_case_description": docstring or " ",
|
|
"custom_test_case_steps": steps
|
|
}
|
|
|
|
if not any([x[GROUP_FIELD] == test_group for x in tests]):
|
|
tests.append(test_case)
|
|
else:
|
|
logger.warning("Testcase '{0}' run in multiple Jenkins jobs!"
|
|
.format(test_group))
|
|
|
|
plan.tests = all_plan_tests[:]
|
|
|
|
return tests
|
|
|
|
|
|
def upload_tests_descriptions(testrail_project, section_id,
|
|
tests, check_all_sections):
|
|
tests_suite = testrail_project.get_suite_by_name(
|
|
TestRailSettings.tests_suite)
|
|
check_section = None if check_all_sections else section_id
|
|
cases = testrail_project.get_cases(suite_id=tests_suite['id'],
|
|
section_id=check_section)
|
|
existing_cases = [case[GROUP_FIELD] for case in cases]
|
|
custom_cases_fields = _get_custom_cases_fields(
|
|
case_fields=testrail_project.get_case_fields(),
|
|
project_id=testrail_project.project['id'])
|
|
|
|
for test_case in tests:
|
|
if test_case[GROUP_FIELD] in existing_cases:
|
|
testrail_case = _get_testrail_case(testrail_cases=cases,
|
|
test_case=test_case,
|
|
group_field=GROUP_FIELD)
|
|
fields_to_update = _get_fields_to_update(test_case, testrail_case)
|
|
|
|
if fields_to_update:
|
|
logger.debug('Updating test "{0}" in TestRail project "{1}", '
|
|
'suite "{2}", section "{3}". Updated fields: {4}'
|
|
.format(
|
|
test_case[GROUP_FIELD],
|
|
TestRailSettings.project,
|
|
TestRailSettings.tests_suite,
|
|
TestRailSettings.tests_section,
|
|
', '.join(fields_to_update.keys())))
|
|
testrail_project.update_case(case_id=testrail_case['id'],
|
|
fields=fields_to_update)
|
|
else:
|
|
logger.debug('Skipping "{0}" test case uploading because '
|
|
'it is up-to-date in "{1}" suite'
|
|
.format(test_case[GROUP_FIELD],
|
|
TestRailSettings.tests_suite))
|
|
|
|
else:
|
|
for case_field, default_value in custom_cases_fields.items():
|
|
if case_field not in test_case:
|
|
test_case[case_field] = default_value
|
|
|
|
logger.debug('Uploading test "{0}" to TestRail project "{1}", '
|
|
'suite "{2}", section "{3}"'.format(
|
|
test_case[GROUP_FIELD],
|
|
TestRailSettings.project,
|
|
TestRailSettings.tests_suite,
|
|
TestRailSettings.tests_section))
|
|
testrail_project.add_case(section_id=section_id, case=test_case)
|
|
|
|
|
|
def get_tests_groups_from_jenkins(runner_name, build_number, distros):
|
|
runner_build = Build(runner_name, build_number)
|
|
res = {}
|
|
for b in runner_build.build_data['subBuilds']:
|
|
|
|
if b['result'] is None:
|
|
logger.debug("Skipping '{0}' job (build #{1}) because it's still "
|
|
"running...".format(b['jobName'], b['buildNumber'],))
|
|
continue
|
|
|
|
# Get the test group from the console of the job
|
|
z = Build(b['jobName'], b['buildNumber'])
|
|
console = z.get_job_console()
|
|
groups = [keyword.split('=')[1]
|
|
for line in console
|
|
for keyword in line.split()
|
|
if 'run_system_test.py' in line and '--group=' in keyword]
|
|
if not groups:
|
|
logger.error("No test group found in console of the job {0}/{1}"
|
|
.format(b['jobName'], b['buildNumber']))
|
|
continue
|
|
# Use the last group (there can be several groups in upgrade jobs)
|
|
test_group = groups[-1]
|
|
|
|
# Get the job suffix
|
|
job_name = b['jobName']
|
|
for distro in distros:
|
|
if distro in job_name:
|
|
sep = '.' + distro + '.'
|
|
job_suffix = job_name.split(sep)[-1]
|
|
break
|
|
else:
|
|
job_suffix = job_name.split('.')[-1]
|
|
res[job_suffix] = test_group
|
|
return res
|
|
|
|
|
|
def _create_test_plan_from_registry(groups):
|
|
discover_import_tests(get_basepath(), tests_directory)
|
|
define_custom_groups()
|
|
for one in groups:
|
|
register_system_test_cases(one)
|
|
return TestPlan.create_from_registry(DEFAULT_REGISTRY)
|
|
|
|
|
|
def _is_case_processable(case, tests):
|
|
if not case.entry.info.enabled or not hasattr(case.entry, 'parent'):
|
|
return False
|
|
|
|
parent_home = case.entry.parent.home
|
|
if issubclass(parent_home, ActionTest) and \
|
|
any([test[GROUP_FIELD] == parent_home.__name__ for test in tests]):
|
|
return False
|
|
|
|
# Skip @before_class methods without doc strings:
|
|
# they are just pre-checks, not separate tests cases
|
|
if case.entry.info.before_class:
|
|
if case.entry.home.func_doc is None:
|
|
logger.debug('Skipping method "{0}", because it is not a '
|
|
'test case'.format(case.entry.home.func_name))
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def _get_test_case_name(case):
|
|
"""Returns test case name
|
|
"""
|
|
parent_home = case.entry.parent.home
|
|
return parent_home.__name__ if issubclass(parent_home, ActionTest) \
|
|
else case.entry.home.func_name
|
|
|
|
|
|
def _is_not_included(case_name, include):
|
|
if include and include not in case_name:
|
|
logger.debug("Skipping '{0}' test because it doesn't "
|
|
"contain '{1}' in method name".format(case_name, include))
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def _is_excluded(case_name, exclude):
|
|
if exclude and exclude in case_name:
|
|
logger.debug("Skipping '{0}' test because it contains"
|
|
" '{1}' in method name".format(case_name, exclude))
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def _get_docstring(parent_home, case_state, home):
|
|
if issubclass(parent_home, ActionTest):
|
|
docstring = parent_home.__doc__.split('\n')
|
|
case_state.instance._load_config()
|
|
configuration = case_state.instance.config_name
|
|
docstring[0] = '{0} on {1}'.format(docstring[0], configuration)
|
|
docstring = '\n'.join(docstring)
|
|
else:
|
|
docstring = home.func_doc or ''
|
|
return docstring
|
|
|
|
|
|
def _parse_docstring(s, case):
|
|
split_s = s.strip().split('\n\n')
|
|
title_r, steps_r, duration_r = _unpack_docstring(split_s)
|
|
title = _parse_title(title_r, case) if title_r else ''
|
|
steps = _parse_steps(steps_r) if steps_r else ''
|
|
duration = _parse_duration(duration_r)
|
|
return title, steps, duration
|
|
|
|
|
|
def _unpack_docstring(items):
|
|
count = len(items)
|
|
title = steps = duration = ''
|
|
if count > 3:
|
|
title, steps, duration, _ = _unpack_list(*items)
|
|
elif count == 3:
|
|
title, steps, duration = items
|
|
elif count == 2:
|
|
title, steps = items
|
|
elif count == 1:
|
|
title = items
|
|
return title, steps, duration
|
|
|
|
|
|
def _unpack_list(title, steps, duration, *other):
|
|
return title, steps, duration, other
|
|
|
|
|
|
def _parse_title(s, case):
|
|
title = ' '.join(map(string.strip, s.split('\n')))
|
|
return title if title else case.entry.home.func_name
|
|
|
|
|
|
def _parse_steps(strings):
|
|
steps = []
|
|
index = -1
|
|
for s_raw in strings.strip().split('\n'):
|
|
s = s_raw.strip()
|
|
_match = STEP_NUM_PATTERN.search(s)
|
|
if _match:
|
|
steps.append({'content': _match.group(), 'expected': 'pass'})
|
|
index += 1
|
|
else:
|
|
if index > -1:
|
|
steps[index]['content'] = ' '.join([steps[index]['content'],
|
|
s])
|
|
return steps
|
|
|
|
|
|
def _parse_duration(s):
|
|
match = DURATION_PATTERN.search(s)
|
|
return match.group(1).replace(' ', '') if match else '3m'
|
|
|
|
|
|
def _get_custom_cases_fields(case_fields, project_id):
|
|
custom_cases_fields = {}
|
|
for field in case_fields:
|
|
for config in field['configs']:
|
|
if ((project_id in
|
|
config['context']['project_ids'] or
|
|
not config['context']['project_ids']) and
|
|
config['options']['is_required']):
|
|
try:
|
|
custom_cases_fields[field['system_name']] = \
|
|
int(config['options']['items'].split(',')[0])
|
|
except:
|
|
logger.error("Couldn't find default value for required "
|
|
"field '{0}', setting '1' (index)!".format(
|
|
field['system_name']))
|
|
custom_cases_fields[field['system_name']] = 1
|
|
return custom_cases_fields
|
|
|
|
|
|
def _get_fields_to_update(test_case, testrail_case):
|
|
"""Produces dictionary with fields to be updated
|
|
"""
|
|
fields_to_update = {}
|
|
for field in ('title', 'estimate', 'custom_test_case_description',
|
|
'custom_test_case_steps'):
|
|
if test_case[field] and \
|
|
test_case[field] != testrail_case[field]:
|
|
if field == 'estimate':
|
|
testcase_estimate_raw = int(test_case[field][:-1])
|
|
testcase_estimate = \
|
|
datetime_util.duration_to_testrail_estimate(
|
|
testcase_estimate_raw)
|
|
if testrail_case[field] == testcase_estimate:
|
|
continue
|
|
elif field == 'custom_test_case_description' and \
|
|
test_case[field] == testrail_case[field].replace('\r', ''):
|
|
continue
|
|
fields_to_update[field] = test_case[field]
|
|
return fields_to_update
|
|
|
|
|
|
def _get_testrail_case(testrail_cases, test_case, group_field):
|
|
"""Returns testrail case that corresponds to test case from repo
|
|
"""
|
|
return next((case for case in testrail_cases
|
|
if case[group_field] == test_case[group_field]))
|
|
|
|
|
|
def main():
|
|
parser = OptionParser(
|
|
description="Upload tests cases to TestRail. "
|
|
"See settings.py for configuration."
|
|
)
|
|
parser.add_option("-v", "--verbose",
|
|
action="store_true", dest="verbose", default=False,
|
|
help="Enable debug output")
|
|
parser.add_option('-j', '--job-name', dest='job_name', default=None,
|
|
help='Jenkins swarm runner job name')
|
|
parser.add_option('-N', '--build-number', dest='build_number',
|
|
default='latest',
|
|
help='Jenkins swarm runner build number')
|
|
parser.add_option('-o', '--check_one_section', action="store_true",
|
|
dest='check_one_section', default=False,
|
|
help='Look for existing test case only in specified '
|
|
'section of test suite.')
|
|
parser.add_option("-l", "--live", dest="live_upload", action="store_true",
|
|
help="Get tests results from running swarm")
|
|
|
|
(options, _) = parser.parse_args()
|
|
|
|
if options.verbose:
|
|
logger.setLevel(DEBUG)
|
|
|
|
if options.live_upload and options.build_number == 'latest':
|
|
options.build_number = 'latest_started'
|
|
|
|
project = TestRailProject(
|
|
url=TestRailSettings.url,
|
|
user=TestRailSettings.user,
|
|
password=TestRailSettings.password,
|
|
project=TestRailSettings.project
|
|
)
|
|
|
|
testrail_section = project.get_section_by_name(
|
|
suite_id=project.get_suite_by_name(TestRailSettings.tests_suite)['id'],
|
|
section_name=TestRailSettings.tests_section
|
|
)
|
|
|
|
testrail_milestone = project.get_milestone_by_name(
|
|
name=TestRailSettings.milestone)
|
|
|
|
testrail_default_test_priority = [priority['id'] for priority in
|
|
project.get_priorities() if
|
|
priority['is_default'] is True][0]
|
|
|
|
distros = [config['name'].split()[0].lower()
|
|
for config in project.get_config_by_name(
|
|
'Operation System')['configs']
|
|
if config['name'] in TestRailSettings.operation_systems]
|
|
|
|
tests_groups = get_tests_groups_from_jenkins(
|
|
options.job_name,
|
|
options.build_number,
|
|
distros) if options.job_name else []
|
|
|
|
# If Jenkins job build is specified, but it doesn't have downstream builds
|
|
# with tests groups in jobs names, then skip tests cases uploading because
|
|
# ALL existing tests cases will be uploaded
|
|
if options.job_name and not tests_groups:
|
|
return
|
|
|
|
tests_descriptions = get_tests_descriptions(
|
|
milestone_id=testrail_milestone['id'],
|
|
tests_include=TestRailSettings.tests_include,
|
|
tests_exclude=TestRailSettings.tests_exclude,
|
|
groups=tests_groups,
|
|
default_test_priority=testrail_default_test_priority
|
|
)
|
|
|
|
upload_tests_descriptions(testrail_project=project,
|
|
section_id=testrail_section['id'],
|
|
tests=tests_descriptions,
|
|
check_all_sections=not options.check_one_section)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|