openstack/ci-log-processing
Code Issues Proposed changes

Merge "Improve fields in logsender"

Browse Source
This commit is contained in:
Zuul
2022-03-23 13:29:30 +00:00
committed by Gerrit Code Review
parent 3d7653fd24 ab6b050af6
commit bf72326260
2 changed files with 43 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
logscraper/logsender.py
View File

@@ -114,8 +114,7 @@ def read_yaml_file(file_path):
def get_inventory_info(directory):
try:
build_inventory = read_yaml_file("%s/inventory.yaml" % directory)
return build_inventory['all']['vars']['zuul']
return read_yaml_file("%s/inventory.yaml" % directory)
except FileNotFoundError:
logging.warning("Can not find inventory.yaml in build "
"dir %s" % directory)
@@ -147,31 +146,40 @@ def get_ready_directories(directory):
return log_files
def get_hosts_id(build_inventory):
hosts_id = []
for _, host_info in build_inventory['all']['hosts'].items():
if 'host_id' in host_info.get('nodepool', {}):
hosts_id.append(host_info['nodepool']['host_id'])
return hosts_id
def remove_directory(dir_path):
logging.debug("Removing directory %s" % dir_path)
shutil.rmtree(dir_path)
def makeFields(build_details, buildinfo):
def makeFields(build_inventory, buildinfo):
fields = {}
build_details = build_inventory['all']['vars']['zuul']
fields["build_node"] = "zuul-executor"
# NOTE: that field is added later
# fields["filename"] = build_file
fields["build_name"] = buildinfo.get("job_name")
fields["build_status"] = buildinfo["result"]
fields["project"] = buildinfo.get('project')
fields["voting"] = int(build_details["voting"])
fields["build_set"] = build_details["buildset"]
fields["build_set"] = str(build_details["buildset"])
fields["build_queue"] = build_details["pipeline"]
fields["build_ref"] = buildinfo.get("ref")
fields["build_branch"] = buildinfo.get("branch")
fields["build_change"] = buildinfo.get("change")
fields["build_patchset"] = buildinfo.get("patchset")
fields["build_newrev"] = build_details.get("newrev", "UNKNOWN")
fields["build_uuid"] = buildinfo.get("uuid")
fields["build_uuid"] = str(buildinfo.get("uuid"))
fields["node_provider"] = "local"
fields["log_url"] = buildinfo.get("log_url")
fields["tenant"] = buildinfo.get("tenant")
fields["hosts_id"] = get_hosts_id(build_inventory)
if "executor" in build_details and "hostname" in build_details["executor"]:
fields["zuul_executor"] = build_details["executor"]["hostname"]
return fields
@@ -273,6 +281,7 @@ def send(ready_directory, args, directory, index, workers):
for build_file in build_files:
es_fields["filename"] = build_file
es_fields["log_url"] = es_fields["log_url"] + build_file
send_status = send_to_es("%s/%s" % (build_dir, build_file),
es_fields, es_client, index, workers,
args.chunk_size, args.doc_type)

27
logscraper/tests/test_logsender.py
View File

@@ -242,6 +242,7 @@ parsed_fields = {
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id': ['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
@@ -303,11 +304,31 @@ class TestSender(base.TestCase):
directory = '/tmp/testdir'
index = 'logstash-index'
workers = 1
mock_build_info.return_value = parsed_fields
mock_es_client.return_value = 'fake_client_object'
expected_fields = {
'build_node': 'zuul-executor', 'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS', 'project': 'openstack/neutron',
'voting': 1, 'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check', 'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master', 'build_change': 829161,
'build_patchset': '3', 'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local', 'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url': 'https://somehost/829161/3/check/openstack-tox-py39/'
'38bf2cd/job-result.txt',
'tenant': 'openstack', 'zuul_executor': 'ze07.opendev.org',
'filename': 'job-result.txt'
}
args = logsender.get_arguments()
mock_send_to_es.return_value = True
logsender.send((build_uuid, build_files), args, directory, index,
workers)
self.assertTrue(mock_remove_dir.called)
mock_send_to_es.assert_called_with(
"%s/%s/job-result.txt" % (directory, build_uuid), expected_fields,
'fake_client_object', index, workers, None, '_doc')
@mock.patch('logscraper.logsender.remove_directory')
@mock.patch('logscraper.logsender.send_to_es')
@@ -384,6 +405,8 @@ class TestSender(base.TestCase):
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
@@ -409,6 +432,8 @@ class TestSender(base.TestCase):
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
@@ -434,6 +459,8 @@ class TestSender(base.TestCase):
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',