761 lines
26 KiB
Python
Executable File
761 lines
26 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""
|
|
CI Status Tool
|
|
|
|
Used for a quick stats collection on Third Party CIs for various OpenStack
|
|
projects.
|
|
|
|
Example usage:
|
|
$ ./ci-status.py -v -u datera-ci \\
|
|
-k /home/user/.ssh/id_rsa \\
|
|
-c "Datera CI" -a datera-dsvm-full -t 2 \\
|
|
-j openstack/cinder \\
|
|
--failures --number-of-reports --is-reporting \\
|
|
--jenkins_disagreement
|
|
|
|
Output:
|
|
Gerrit Query: ssh -i /home/user/.ssh/id_rsa -p 29418 dater
|
|
a-ci@review.openstack.org "gerrit query --format=JSON --comments --current-
|
|
patch-set project:openstack/cinder NOT age:2d reviewer:Datera CI "
|
|
|
|
##### DATERA-DSVM-FULL #####
|
|
|
|
####### --number-of-reports arg result #######
|
|
|
|
40 results in 2 days
|
|
|
|
###### --is-reporting arg result #######
|
|
|
|
Review: 263026 --> 2016-07-07T17:02:15+00:00
|
|
|
|
###### --failures arg result #######
|
|
|
|
20% failures
|
|
|
|
###### --jenkins-disagreement arg result #######
|
|
|
|
0% -1 Jenkins && +1 CI
|
|
20% +1 Jenkins && -1 CI
|
|
|
|
Minimal usage:
|
|
$ ./ci-status.py -u datera-ci -k /home/user/.ssh/id_rsa \\
|
|
-j openstack/cinder -c "Datera CI" -a datera-dsvm-full \\
|
|
--is-reporting
|
|
|
|
Output:
|
|
##### DATERA-DSVM-FULL #####
|
|
Review: 263026 --> 2016-07-07T17:02:15+00:00
|
|
|
|
Passthrough query usage:
|
|
$ ./ci-status.py -u datera-ci -k /home/user/.ssh/id_rsa \\
|
|
-q "reviewer:{Some Body} -j openstack/cinder"
|
|
|
|
Output:
|
|
Will be a large dictionary
|
|
|
|
Config example:
|
|
|
|
# In .gerritqueryrc file in your $HOME directory
|
|
# (or passed in via config option)
|
|
|
|
[DEFAULT]
|
|
verbose=True
|
|
host=review.openstack.org
|
|
username=datera-ci
|
|
port=29418
|
|
query_project=openstack/cinder
|
|
keyfile=/home/user/.ssh/id_rsa
|
|
|
|
# I would not recommend putting any other flags into this config
|
|
# file otherwise you could introduce silent errors
|
|
# For example:
|
|
|
|
# Adding these fields
|
|
ci_account=datera-ci
|
|
ci_runner_name=datera-dsvm-full
|
|
|
|
# Then running this command
|
|
$ ./ci-status.py -c mellanox-ci --is-reporting
|
|
|
|
# Would report a false negative for Datera. A CI
|
|
# will show as non-reporting if you provide the
|
|
# ci_account name of one CI and the ci_runner_name of
|
|
# a different CI. The tool has no way to tell that
|
|
# these values do not belong together and will just
|
|
# report that the CI has not posted within the specified
|
|
# timeframe.
|
|
|
|
The "--all" flag:
|
|
# In order to use this flag, you must first run this command:
|
|
$ ./ci-status.py --scrape-wiki --force -j openstack/your_project
|
|
|
|
# It will fill your .gerritquerycache file with information about
|
|
# the various CIs for your desired OpenStack project
|
|
|
|
# Now you're free to run commands with the --all flag
|
|
$ ./ci-status -j openstack/you_project --all --is-reporting
|
|
"""
|
|
|
|
from __future__ import print_function, unicode_literals
|
|
|
|
import re
|
|
import sys
|
|
import functools
|
|
import subprocess
|
|
import shlex
|
|
import os.path
|
|
import threading
|
|
import Queue
|
|
from urlparse import urljoin
|
|
from StringIO import StringIO
|
|
|
|
import arrow
|
|
import simplejson as json
|
|
import requests
|
|
from lxml import etree
|
|
|
|
from oslo_config import cfg
|
|
|
|
opts = [
|
|
cfg.BoolOpt('verbose',
|
|
short='v',
|
|
default=False),
|
|
cfg.StrOpt('host',
|
|
short='s',
|
|
default=None,
|
|
help=('Eg. review.openstack.org')),
|
|
cfg.StrOpt('username',
|
|
short='u',
|
|
default=None,
|
|
help=('Gerrit CI username')),
|
|
cfg.StrOpt('keyfile',
|
|
short='k',
|
|
default=None,
|
|
help=('Gerrit CI ssh private keyfile')),
|
|
cfg.StrOpt('port',
|
|
short='p',
|
|
default=None,
|
|
help=('Gerrit CI ssh port')),
|
|
cfg.StrOpt('query-project',
|
|
short='j',
|
|
default=None,
|
|
help=('Gerrit CI project to query')),
|
|
cfg.IntOpt('time',
|
|
short='t',
|
|
default=2,
|
|
help=('Time in days for query, default=2')),
|
|
cfg.StrOpt('query',
|
|
short='q',
|
|
default='',
|
|
help=('Passthrough Query')),
|
|
cfg.StrOpt('ci-account',
|
|
short='c',
|
|
default='',
|
|
help=('Filter result by this CI account (name or username)'
|
|
'eg. "Datera CI" or "datera-ci"')),
|
|
cfg.StrOpt('ci-runner-name',
|
|
short='a',
|
|
default='',
|
|
help=("Specific CI runner (eg. datera-dsvm-full) "
|
|
"to use for filtering results. Useful if a "
|
|
"single CI account posts results for multiple "
|
|
"drivers. Can only be used if '--all' is NOT used")),
|
|
cfg.BoolOpt('is-reporting',
|
|
default=False,
|
|
help=('Report if the CI specified by "-c/--ci-account"'
|
|
'reported within the time specified by "-t/--time"')),
|
|
cfg.BoolOpt('number-of-reports',
|
|
default=False,
|
|
help=("Show number of reports within the last "
|
|
"'-t/--time' days")),
|
|
cfg.BoolOpt('failures',
|
|
default=False,
|
|
help=("Show number and percentage of failures for last "
|
|
"'-t/--time' days")),
|
|
cfg.BoolOpt('jenkins-disagreement',
|
|
default=False,
|
|
help=("Print percentage of disagreements between "
|
|
"CI and Jenkins")),
|
|
cfg.BoolOpt('number-of-rechecks',
|
|
default=False,
|
|
help=("Print number of rechecks for a ci within the last"
|
|
"'-t/--time' days. Heavily relies on information from "
|
|
"the ThirdPartyWiki. Should only be used for "
|
|
"heruistic purposes")),
|
|
cfg.BoolOpt('show-contacts',
|
|
default=False,
|
|
help=("Print contacts for each CI displayed")),
|
|
cfg.BoolOpt('contact-list',
|
|
default=False,
|
|
help=("Print all contacts in project cache")),
|
|
cfg.BoolOpt('contact-list-compact',
|
|
default=False,
|
|
help=("Print all contacts in project cache compactly")),
|
|
cfg.BoolOpt('not-reporting-list',
|
|
default=False,
|
|
help=("Print all CIs for the project that have NOT reported "
|
|
"in the last '-t/--time' days")),
|
|
cfg.BoolOpt('scrape-wiki',
|
|
default=False,
|
|
help=("Scrapes the wiki for CIs matching the project "
|
|
"specified by '-j/--query-project'. Use '-f/--force' "
|
|
"to update from wiki instead of displaying cache")),
|
|
cfg.BoolOpt('force',
|
|
short='f',
|
|
default=False,
|
|
help=("Forces cache update")),
|
|
cfg.BoolOpt('all',
|
|
default=False,
|
|
help=("Forces switches to be run on all detected CIs")),
|
|
]
|
|
|
|
|
|
CONF = cfg.ConfigOpts()
|
|
CONF.register_opts(opts)
|
|
CONF.register_cli_opts(opts)
|
|
|
|
CONFIG_FILE_NAME = os.path.join(os.path.expanduser("~"), '.gerritqueryrc')
|
|
CACHE_FILE_NAME = os.path.join(os.path.expanduser("~"), '.gerritquerycache')
|
|
WIKI_BASE_URL = "http://wiki.openstack.org"
|
|
THIRD_PARTY_WIKI_URL = urljoin(WIKI_BASE_URL, "/wiki/ThirdPartySystems")
|
|
WORKERS = 5
|
|
|
|
# From emailregex.com. Should be fine for heuristically extracting emails
|
|
# we're not doing any validation :)
|
|
EMAIL_REGEX = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
|
|
|
|
|
|
def main():
|
|
CONF(default_config_files=[CONFIG_FILE_NAME])
|
|
|
|
nargs = CONF
|
|
pname = nargs.query_project.split("/")[-1]
|
|
gquery = functools.partial(_gquery_base,
|
|
nargs.verbose,
|
|
nargs.keyfile,
|
|
nargs.port,
|
|
nargs.username,
|
|
nargs.host,
|
|
nargs.query_project,
|
|
nargs.time)
|
|
|
|
# ###############################################
|
|
# ############ Argument Handling ################
|
|
# ###############################################
|
|
|
|
if nargs.scrape_wiki:
|
|
if nargs.verbose:
|
|
print("\n####### --scrape-wiki result ######\n")
|
|
from pprint import pprint
|
|
pprint(get_reporting_dict(
|
|
pname,
|
|
force_update=nargs.force))
|
|
|
|
if nargs.contact_list or nargs.contact_list_compact:
|
|
contacts = []
|
|
cis = get_reporting_dict(
|
|
pname,
|
|
force_update=nargs.force).keys()
|
|
for ci in cis:
|
|
contacts.extend(get_email_contacts(ci, pname))
|
|
curr = contacts[0].split('@')[1]
|
|
prev = curr
|
|
for c in sorted(contacts, key=lambda x: x.split('@')[1]):
|
|
prev = curr
|
|
curr = c.split('@')[1]
|
|
if prev != curr and not nargs.contact_list_compact:
|
|
print()
|
|
print(c)
|
|
|
|
if not any((nargs.query,
|
|
nargs.is_reporting,
|
|
nargs.failures,
|
|
nargs.jenkins_disagreement,
|
|
nargs.number_of_reports,
|
|
nargs.number_of_rechecks,
|
|
nargs.show_contacts,
|
|
nargs.not_reporting_list)):
|
|
exit(0)
|
|
|
|
# Handle defaults depending on if --all is passed
|
|
if not nargs.all:
|
|
cis = [(nargs.ci_account, nargs.ci_runner_name)]
|
|
runner = nargs.ci_runner_name
|
|
results = gquery(
|
|
" ".join((nargs.query,
|
|
"reviewer:{}".format(nargs.ci_account))))
|
|
else:
|
|
# We don't want to hit the wiki twice this one should always
|
|
# be from the cache
|
|
cis = [(k, v['name']) for (k, v) in get_reporting_dict(
|
|
pname, force_update=False).items()]
|
|
runner = ''
|
|
results = gquery(
|
|
" ".join((nargs.query)))
|
|
|
|
for ci, name in sorted(cis, key=lambda x: x[1]):
|
|
if not nargs.not_reporting_list:
|
|
print("\n##### {} #####".format(name.upper()))
|
|
if nargs.number_of_reports:
|
|
if nargs.verbose:
|
|
print("\n####### --number-of-reports arg result #######\n")
|
|
print_number_of_reports(results,
|
|
ci,
|
|
runner,
|
|
nargs.time)
|
|
|
|
if nargs.is_reporting:
|
|
if nargs.verbose:
|
|
print("\n###### --is-reporting arg result #######\n")
|
|
print_is_reporting(results,
|
|
ci,
|
|
runner,
|
|
nargs.time)
|
|
|
|
if nargs.failures:
|
|
if nargs.verbose:
|
|
print("\n###### --failures arg result #######\n")
|
|
print_failure_results(results,
|
|
ci,
|
|
runner,
|
|
nargs.time)
|
|
|
|
if nargs.jenkins_disagreement:
|
|
if nargs.verbose:
|
|
print("\n###### --jenkins-disagreement arg result #######\n")
|
|
print_jenkins_disagreement(results,
|
|
ci,
|
|
runner,
|
|
nargs.time)
|
|
|
|
if nargs.number_of_rechecks:
|
|
if nargs.verbose:
|
|
print("\n###### --number-of-rechecks arg result #######\n")
|
|
print_number_of_rechecks(results,
|
|
ci,
|
|
runner,
|
|
nargs.time,
|
|
pname)
|
|
|
|
if nargs.show_contacts:
|
|
if nargs.verbose:
|
|
print("\n####### --show-contacts result ######\n")
|
|
print_email_contacts(ci, pname)
|
|
|
|
if nargs.query:
|
|
if nargs.verbose:
|
|
print("\n####### --query result ######\n")
|
|
print(gquery(nargs.query))
|
|
|
|
if nargs.not_reporting_list and nargs.all:
|
|
if nargs.verbose:
|
|
print("\n####### --not-reporting-list result ######\n")
|
|
not_reporting = []
|
|
for ci, name in cis:
|
|
result = get_is_reporting(results, ci, runner, nargs.time)
|
|
if all(result):
|
|
not_reporting.append(name)
|
|
print("These CIs have not reported in {} days".format(nargs.time))
|
|
for nr in not_reporting:
|
|
print(nr.upper())
|
|
# #######################################################
|
|
|
|
exit(0)
|
|
|
|
|
|
def _base(keyfile, port, username, host):
|
|
return 'ssh -i {keyfile} -p {port} {username}@{host}'.format(
|
|
keyfile=keyfile, port=port, username=username, host=host)
|
|
|
|
|
|
def _gquery_base(verbose, keyfile, port, username, host, project, time, query):
|
|
"""
|
|
Makes a large bulk-query to gerrit that can be further filtered
|
|
after-the-fact so we play nice and don't hammer gerrit
|
|
|
|
The size of the query largly depends on the "time" argument. It's used
|
|
to determine how many days old results can be.
|
|
|
|
Eg. time == 1 --> Only results >= 1 day old
|
|
|
|
The time argument is also reliant on the "last modified" date, not
|
|
"creation date". So a review created six months ago, but modified two
|
|
days ago will show up in a "time == 2" query
|
|
"""
|
|
cmd = " ".join((_base(keyfile, port, username, host),
|
|
("\"gerrit query --format=JSON --comments "
|
|
"--current-patch-set project:{} NOT age:{}d".format(
|
|
project, time)),
|
|
query,
|
|
"\""))
|
|
if verbose:
|
|
print("Gerrit Query: {}".format(cmd))
|
|
# Strip the last result line which is just query metadata
|
|
result = subprocess.check_output(shlex.split(cmd)).splitlines()[:-1]
|
|
return [json.loads(elem) for elem in result]
|
|
|
|
|
|
def get_ci_comments(result, ci):
|
|
"""
|
|
Returns a list of comments that match the specified CI as the reviewer in
|
|
either name or username
|
|
"""
|
|
return filter(
|
|
lambda x: ci.upper() in (x['reviewer'].get('username', "").upper(),
|
|
x['reviewer'].get('name', "").upper()),
|
|
result['comments'])
|
|
|
|
|
|
def get_recheck_comments(result, ci_check):
|
|
"""
|
|
ci_check is the recheck/rerun string for a specific ci as listed on
|
|
its wiki page
|
|
|
|
Makes the assumption that CI reviewer names will end with CI and
|
|
filters out those comments to ensure we're not accidentally counting
|
|
a comment made by a CI
|
|
|
|
For example: Mellanox CI n the following example post a comment
|
|
containing their recheck syntax, so this would be a false positive
|
|
recheck request if we did not filter out all CI comments:
|
|
|
|
"Build succeeded. Cinder-ISER-ISCSI SUCCESS in 35m 11s
|
|
Cinder-ISER-LIO SUCCESS in 35m 10s To re-run the job post
|
|
'recheck cinder-mlnx' comment. For more information visit
|
|
https://wiki.openstack.org/wiki/ThirdPartySystems/Mellanox_CI"
|
|
"""
|
|
return filter(
|
|
lambda x: (not x['reviewer']['name'].upper().endswith("CI") and
|
|
ci_check in x['message']),
|
|
result['comments'])
|
|
|
|
|
|
def most_recent_ci_comment_timestamp(result, ci):
|
|
""" Returns (review_id, timestamp, message) if comments exist for the CI
|
|
Otherwise returns (review_id, timestamp(0), '')"""
|
|
filtered = get_ci_comments(result, ci)
|
|
try:
|
|
body = sorted(filtered,
|
|
key=lambda x: x['timestamp'],
|
|
reverse=True)[0]
|
|
|
|
return (result['number'],
|
|
arrow.get(body['timestamp']),
|
|
body['message'])
|
|
|
|
except IndexError:
|
|
return (result['number'],
|
|
arrow.get(0),
|
|
'')
|
|
|
|
# ###############################################
|
|
# ############# Report Functions ################
|
|
# ###############################################
|
|
|
|
"""
|
|
The 'results' argument accepted by these functions is an array of dicts
|
|
with these keys:
|
|
|
|
['status',
|
|
'topic',
|
|
'currentPatchSet',
|
|
'url',
|
|
'commitMessage',
|
|
'createdOn',
|
|
'number',
|
|
'lastUpdated',
|
|
'project',
|
|
'comments',
|
|
'branch',
|
|
'owner',
|
|
'open',
|
|
'id',
|
|
'subject']
|
|
|
|
This array is pulled directly from the Gerrit server via the constructed
|
|
query above. Most of the logic is done on the 'comments' section of the
|
|
data structure since Third-Party CI results are left as comments on gerrit
|
|
reviews.
|
|
|
|
Only the most recent comments of any CI including Jenkins are taken into
|
|
account for printed statistics. This might be beefed up in the future but for
|
|
not it's the easiest implementation
|
|
"""
|
|
|
|
|
|
def get_number_of_reports(results, ci, runner, argtime):
|
|
"""
|
|
Report generator for --number-of-reports cli option
|
|
"""
|
|
count = 0
|
|
for result in results:
|
|
review, tstamp, message = most_recent_ci_comment_timestamp(
|
|
result, ci)
|
|
if (arrow.now() - tstamp <=
|
|
arrow.now() - arrow.now().replace(days=-argtime) and
|
|
runner in message):
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def get_is_reporting(results, ci, runner, argtime):
|
|
"""
|
|
Report generator for --is-reporting cli option
|
|
"""
|
|
if not results:
|
|
return None
|
|
rtstamp = arrow.get(0)
|
|
rreview = 0
|
|
for result in results:
|
|
review, tstamp, message = most_recent_ci_comment_timestamp(
|
|
result, ci)
|
|
if tstamp > rtstamp:
|
|
rtstamp = tstamp
|
|
rreview = review
|
|
return (rreview, rtstamp, message)
|
|
|
|
|
|
def get_failure_results(results, ci, runner, argtime):
|
|
"""
|
|
Report generator for --failure cli option
|
|
"""
|
|
count = 0
|
|
fail_count = 0
|
|
for result in results:
|
|
review, tstamp, message = most_recent_ci_comment_timestamp(
|
|
result, ci)
|
|
if (arrow.now() - tstamp <=
|
|
arrow.now() - arrow.now().replace(days=-argtime) and
|
|
runner in message):
|
|
count += 1
|
|
if "FAILURE" in message:
|
|
fail_count += 1
|
|
return (count, fail_count)
|
|
|
|
|
|
def get_jenkins_disagreement(results, ci, runner, argtime):
|
|
count = 0
|
|
negative_disagree_count = 0
|
|
positive_disagree_count = 0
|
|
for result in results:
|
|
jenkins_failure = False
|
|
_, jtstamp, jmessage = most_recent_ci_comment_timestamp(
|
|
result, 'jenkins')
|
|
review, tstamp, message = most_recent_ci_comment_timestamp(
|
|
result, ci)
|
|
|
|
# Rough way of determining that both Jenkins and CI are referring
|
|
# to the same patchset
|
|
if tstamp >= jtstamp:
|
|
|
|
# We only care if the result is within '-t' time
|
|
if (arrow.now() - tstamp <=
|
|
arrow.now() - arrow.now().replace(days=-argtime) and
|
|
runner in message):
|
|
count += 1
|
|
|
|
# Filter out non-voting/stylistic/unit checks
|
|
# 3rd party CIs don't run these checks, so we don't
|
|
# care if they disagree
|
|
jmessages = [elem for elem in jmessage.splitlines()
|
|
if not any([f in elem for f in ("non-voting",
|
|
"python",
|
|
"pylint",
|
|
"pep8",
|
|
"docs",)])]
|
|
if any("FAILURE" in jm for jm in jmessages):
|
|
jenkins_failure = True
|
|
|
|
if "FAILURE" not in message and jenkins_failure:
|
|
positive_disagree_count += 1
|
|
|
|
if "FAILURE" in message and not jenkins_failure:
|
|
negative_disagree_count += 1
|
|
return (count, negative_disagree_count, positive_disagree_count)
|
|
|
|
|
|
def get_rechecks(results, ci, runner, argtime, project):
|
|
check = get_reporting_dict(project)[ci].get('retry')
|
|
if not check:
|
|
return None, None
|
|
rechecks = 0
|
|
for result in results:
|
|
comments = get_recheck_comments(result, check)
|
|
for comment in comments:
|
|
tstamp = arrow.get(comment['timestamp'])
|
|
if (arrow.now() - tstamp <=
|
|
arrow.now() - arrow.now().replace(days=-argtime)):
|
|
rechecks += 1
|
|
return (rechecks, check)
|
|
|
|
|
|
def get_email_contacts(ci, project):
|
|
return get_reporting_dict(project)[ci].get('contact')
|
|
|
|
|
|
def get_reporting_dict(project, force_update=False):
|
|
"""
|
|
Report generator for --scrape-wiki cli option
|
|
"""
|
|
# Handle cache creation and loading
|
|
cache = {}
|
|
if os.path.exists(CACHE_FILE_NAME):
|
|
with open(CACHE_FILE_NAME) as f:
|
|
cache = json.load(f)
|
|
else:
|
|
with open(CACHE_FILE_NAME, 'w') as f:
|
|
json.dump(cache, f)
|
|
|
|
# By default we're just going to return the cache
|
|
if not force_update:
|
|
return cache.get(project)
|
|
|
|
# Heavy update requests depending on wiki page links
|
|
with requests.session() as c:
|
|
resp = c.get(THIRD_PARTY_WIKI_URL)
|
|
parser = etree.HTMLParser()
|
|
root = etree.parse(StringIO(resp.text), parser).getroot()
|
|
links = root.xpath('//a')
|
|
|
|
# Now that we have all the links, we'll spawn a bunch of
|
|
# worker threads and request them in parallel
|
|
q = Queue.Queue()
|
|
[q.put(urljoin(WIKI_BASE_URL, elem.attrib['href']))
|
|
for elem in links
|
|
if elem.text and elem.text.upper().endswith("CI")]
|
|
workers = []
|
|
results = {}
|
|
for _ in xrange(WORKERS):
|
|
workers.append(
|
|
threading.Thread(target=_link_checker,
|
|
args=(project, c, parser, q, results)))
|
|
for thread in workers:
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
q.join()
|
|
cache[project] = results
|
|
|
|
# Truncate file since we're writing the whole cache
|
|
with open(CACHE_FILE_NAME, 'w') as f:
|
|
json.dump(cache, f)
|
|
return results
|
|
|
|
|
|
# ###############################################
|
|
# ####### CommandLine Print Functions ###########
|
|
# ###############################################
|
|
|
|
|
|
def print_failure_results(results, ci, runner, argtime):
|
|
count, fail_count = get_failure_results(results, ci, runner, argtime)
|
|
if count > 0:
|
|
print("{}% failures".format(
|
|
int(float(fail_count)/count * 100)))
|
|
|
|
|
|
def print_is_reporting(results, ci, runner, argtime):
|
|
try:
|
|
rreview, rtstamp, message = \
|
|
get_is_reporting(results, ci, runner, argtime)
|
|
except ValueError:
|
|
print("No results found, ci: {}".format(ci))
|
|
return
|
|
|
|
if (arrow.now() - rtstamp <=
|
|
arrow.now() - arrow.now().replace(days=-argtime) and
|
|
runner in message):
|
|
print("Review: {} --> {}".format(
|
|
rreview, arrow.get(rtstamp)))
|
|
else:
|
|
print("{}+ days".format(argtime))
|
|
|
|
|
|
def print_number_of_reports(results, ci, runner, argtime):
|
|
count = get_number_of_reports(results, ci, runner, argtime)
|
|
print("{} results in {} days".format(
|
|
count, argtime))
|
|
|
|
|
|
def print_jenkins_disagreement(results, ci, runner, argtime):
|
|
count, negative_disagree_count, positive_disagree_count = \
|
|
get_jenkins_disagreement(results, ci, runner, argtime)
|
|
if count > 0:
|
|
print("{}% -1 Jenkins && +1 CI".format(
|
|
int(float(positive_disagree_count)/count * 100)))
|
|
print("{}% +1 Jenkins && -1 CI".format(
|
|
int(float(negative_disagree_count)/count * 100)))
|
|
|
|
|
|
def print_number_of_rechecks(results, ci, runner, argtime, project):
|
|
recheck_count, check = get_rechecks(results, ci, runner, argtime, project)
|
|
if recheck_count is None:
|
|
print("No recheck string found on Wiki page")
|
|
else:
|
|
print("{} rechecks in {} days, recheck string: {}".format(
|
|
recheck_count, argtime, check))
|
|
|
|
|
|
def print_email_contacts(ci, project):
|
|
print("Contact: {}".format(get_email_contacts(ci, project)))
|
|
|
|
# ###############################################
|
|
|
|
|
|
# Used for Thread workers
|
|
def _link_checker(project, session, parser, q, results):
|
|
while True:
|
|
try:
|
|
link = q.get()
|
|
resp = session.get(link)
|
|
root = etree.parse(StringIO(resp.text), parser).getroot()
|
|
# Grab all the table data entries
|
|
tds = root.xpath("//td/b")
|
|
try:
|
|
# Filter for supported openstack services
|
|
text = [elem.tail for elem in tds
|
|
if elem.text.upper() == "OPENSTACK PROGRAMS"][0]
|
|
# Filter for gerrit account name
|
|
if project.upper() in text.upper():
|
|
name = [elem.tail for elem in tds
|
|
if elem.text.upper() == "GERRIT ACCOUNT"][0]
|
|
# Handle oddly formatted names
|
|
name = name.lstrip(": ").split()[0].split("@")[0]
|
|
results[name] = {}
|
|
results[name]['name'] = link.split("/")[-1].upper()
|
|
# Try and get a retry syntax from the page
|
|
try:
|
|
# Ugly stripping to handle the weird inconsistent
|
|
# formats used on the wiki
|
|
retry_elem = root.xpath(
|
|
"//p/b")[0].tail.strip().strip(
|
|
"\"'").split("(")[0].strip().strip("\"")
|
|
for char in (":", "="):
|
|
if char in retry_elem:
|
|
retry_elem = retry_elem.split(
|
|
char)[1].strip().strip("\"")
|
|
# Any CI with this sting hasn't updated
|
|
# their wiki page with a retry syntax
|
|
if "please update" not in retry_elem.lower():
|
|
results[name]['retry'] = retry_elem
|
|
except IndexError:
|
|
pass
|
|
# Try and get maintaner info from the page
|
|
try:
|
|
contact_elem = [
|
|
elem.tail for elem in tds
|
|
if elem.text.upper() == "CONTACT INFORMATION"][0]
|
|
emails = re.findall(EMAIL_REGEX, contact_elem)
|
|
results[name]['contact'] = emails
|
|
except IndexError:
|
|
pass
|
|
except IndexError:
|
|
print("No 'OpenStack Programs' or 'Gerrit Account' field",
|
|
link)
|
|
q.task_done()
|
|
except Queue.Empty:
|
|
return
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|