openstack
/
ci-log-processing
Code Issues Proposed changes

Add job with 'timed_out' status to fetch; add hosts_region info 

Before that commit, losgcraper was pulling logs from job status
that was "SUCCESS" or "FAILURE". It was because job with other
statuses sometimes it does not contain important information to parse,
so the logscraper was creating "custom-job-result" to avoid situation
that logsender might fail to parse the build content.
This commit extends job to be pulled from logserver with field
"timed_out".
Also added new field into the document: "hosts_region" that provides
combination of host "cloud" and host "region" information.
It might be useful for infra team to check the infrastructure issues.

Change-Id: I25808709008f7e3a457d6cb4b73920619b292132
This commit is contained in:
Daniel Pawlik 2023-03-03 10:16:52 +01:00
parent b70fd37657
commit 3db7ed17dd
3 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
logscraper/logscraper.py
View File

@ -561,7 +561,7 @@ def download_file(url, directory, insecure, timeout):
def is_job_with_result(job_result):
results_with_status = ['failure', 'success']
results_with_status = ['failure', 'success', 'timed_out']
if (job_result["result"].lower() in results_with_status and
job_result["log_url"]):
return True

7
logscraper/logsender.py
View File

@ -172,13 +172,16 @@ def get_ready_directories(directory):
def get_hosts_id(build_inventory):
hosts_id = []
hosts_region = []
if 'all' not in build_inventory:
return hosts_id
for _, host_info in build_inventory['all']['hosts'].items():
if 'host_id' in host_info.get('nodepool', {}):
hosts_id.append(host_info['nodepool']['host_id'])
return hosts_id
hosts_region.append("%s-%s" % (host_info['nodepool']['cloud'],
host_info['nodepool']['region']))
return hosts_id, list(set(hosts_region))
def remove_directory(dir_path):
@ -212,7 +215,7 @@ def makeFields(build_inventory, buildinfo):
fields["node_provider"] = "local"
fields["log_url"] = buildinfo.get("log_url")
fields["tenant"] = buildinfo.get("tenant")
fields["hosts_id"] = get_hosts_id(build_inventory)
fields["hosts_id"], fields["hosts_region"] = get_hosts_id(build_inventory)
if "executor" in build_details and "hostname" in build_details["executor"]:
fields["zuul_executor"] = build_details["executor"]["hostname"]
return fields

4
logscraper/tests/test_logsender.py
View File

@ -244,6 +244,7 @@ parsed_fields = {
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_region': ['rax-DFW'],
'hosts_id': ['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
@ -357,6 +358,7 @@ class TestSender(base.TestCase):
'build_patchset': '3', 'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_region': ['rax-DFW'],
'hosts_id': [
'ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'
],
@ -453,6 +455,7 @@ class TestSender(base.TestCase):
'build_patchset': '3', 'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_region': ['rax-DFW'],
'hosts_id': [
'ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'
],
@ -711,6 +714,7 @@ class TestSender(base.TestCase):
'db_nova_cell1_update': 12,
'db_placement_select': 4,
'hostname': 'ubuntu-focal-ovh-gra1-0029678038',
'hosts_region': ['rax-DFW'],
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':