openstack/ci-log-processing
Code Issues Proposed changes
eee765f8c6
ci-log-processing/logscraper/tests/test_logsender.py
Lukáš Piwowarski c44de5158b Add index with individual test results 
This patch introduces a new option --subunit_index_prefix. When this
option is specified new index is created. This index stores individual
test results that are obtained by parsing the testrepository.subunit
file. Each document in this index contains among other the following
fields:
- test name
- test status (success, fail, skip)
- test duration

Change-Id: I7ed36ac6b22f7f5b1de958605164785d20a837aa
2022-09-22 09:35:20 +02:00

1202 lines
52 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#
# Copyright (C) 2022 Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import datetime
import io
import json
import os
from logscraper import logsender
from logscraper.tests import base
from opensearchpy.exceptions import TransportError
from ruamel.yaml import YAML
from unittest import mock
buildinfo = """
_id: 17428524
branch: master
build_args:
checkpoint_file: /tmp/results-checkpoint
debug: false
directory: /tmp/logscraper
download: true
follow: false
gearman_port: 4730
gearman_server: null
insecure: true
job_name: null
logstash_url: null
max_skipped: 500
workers: 32
zuul_api_url:
- https://zuul.opendev.org/api/tenant/openstack
buildset:
uuid: 52b29e0e716a4436bd20eed47fa396ce
change: 829161
duration: 1707.0
end_time: '2022-02-28T10:07:36'
error_detail: null
event_id: dda0cbf9caaa496b9127a7646b8a28a8
event_timestamp: '2022-02-28T09:32:08'
final: true
held: false
job_name: openstack-tox-py39
log_url: https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/
newrev: null
nodeset: fedora-35
patchset: '3'
pipeline: check
project: openstack/neutron
provides: []
ref: refs/changes/61/829161/3
ref_url: https://review.opendev.org/829161
result: SUCCESS
start_time: '2022-02-28T09:39:09'
tenant: openstack
uuid: 38bf2cdc947643c9bb04f11f40a0f211
voting: true
"""
inventory_info = """
all:
hosts:
fedora-35:
ansible_connection: ssh
ansible_host: 127.0.0.1
ansible_port: 22
ansible_python_interpreter: auto
ansible_user: zuul
ara_compress_html: false
ara_report_path: ara-report
ara_report_type: html
bindep_profile: test py39
enable_fips: false
nodepool:
az: null
cloud: rax
external_id: 3b2da968-7ec3-4356-b12c-b55b574902f8
host_id: ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21
interface_ip: 127.0.0.2
label: fedora-35
private_ipv4: 127.0.0.3
private_ipv6: null
provider: rax-dfw
public_ipv4: 127.0.0.2
public_ipv6: ''
region: DFW
python_version: 3.9
tox_constraints_file: 'requirements/upper-constraints.txt'
tox_environment:
NOSE_HTML_OUT_FILE: nose_results.html
NOSE_WITH_HTML_OUTPUT: 1
NOSE_WITH_XUNIT: 1
tox_envlist: py39
vars:
ara_compress_html: false
ara_report_path: ara-report
ara_report_type: html
bindep_profile: test py39
enable_fips: false
python_version: 3.9
tox_constraints_file: 'requirements/upper-constraints.txt'
tox_environment:
NOSE_HTML_OUT_FILE: nose_results.html
NOSE_WITH_HTML_OUTPUT: 1
NOSE_WITH_XUNIT: 1
tox_envlist: py39
zuul:
_inheritance_path:
- 'some_path'
- 'some_path_2'
attempts: 1
branch: master
build: 38bf2cdc947643c9bb04f11f40a0f211
buildset: 52b29e0e716a4436bd20eed47fa396ce
change: '829161'
change_url: https://review.opendev.org/829161
child_jobs: []
event_id: dda0cbf9caaa496b9127a7646b8a28a8
executor:
hostname: ze07.opendev.org
inventory_file: /var/lib/zuul/builds/build/ansible/inventory.yaml
log_root: /var/lib/zuul/builds/build/work/logs
result_data_file: /var/lib/zuul/builds/build/work/results.json
src_root: /var/lib/zuul/builds/build/work/src
work_root: /var/lib/zuul/builds/build/work
items:
- branch: master
change: '828673'
change_url: https://review.opendev.org/828673
patchset: '4'
project:
canonical_hostname: opendev.org
canonical_name: opendev.org/openstack/neutron
name: openstack/neutron
short_name: neutron
src_dir: src/opendev.org/openstack/neutron
- branch: master
change: '829161'
change_url: https://review.opendev.org/829161
patchset: '3'
project:
canonical_hostname: opendev.org
canonical_name: opendev.org/openstack/neutron
name: openstack/neutron
short_name: neutron
src_dir: src/opendev.org/openstack/neutron
job: openstack-tox-py39
jobtags: []
message: Q3YmM0Y2QzNzhkMWZhOWE5ODYK
patchset: '3'
pipeline: check
playbook_context:
playbook_projects:
trusted/project_0/opendev.org/opendev/base-jobs:
canonical_name: opendev.org/opendev/base-jobs
checkout: master
commit: 19dc53290a26b20d5c2c5b1bb25f029c4b04a716
trusted/project_1/opendev.org/zuul/zuul-jobs:
canonical_name: opendev.org/zuul/zuul-jobs
checkout: master
commit: e160f59e0e76c7e8625ec2d174b044a7c92cd32e
untrusted/project_0/opendev.org/zuul/zuul-jobs:
canonical_name: opendev.org/zuul/zuul-jobs
checkout: master
commit: e160f59e0e76c7e8625ec2d174b044a7c92cd32e
untrusted/project_1/opendev.org/opendev/base-jobs:
canonical_name: opendev.org/opendev/base-jobs
checkout: master
commit: 19dc53290a26b20d5c2c5b1bb25f029c4b04a716
playbooks:
- path: untrusted/project/opendev/zuul/zuul-jobs/playbooks/tox/run.yaml
roles:
- checkout: master
checkout_description: zuul branch
link_name: ansible/playbook_0/role_0/base-jobs
link_target: untrusted/project_1/opendev.org/opendev/base-jobs
role_path: ansible/playbook_0/role_0/base-jobs/roles
- checkout: master
checkout_description: playbook branch
link_name: ansible/playbook_0/role_1/zuul-jobs
link_target: untrusted/project_0/opendev.org/zuul/zuul-jobs
role_path: ansible/playbook_0/role_1/zuul-jobs/roles
post_review: false
project:
canonical_hostname: opendev.org
canonical_name: opendev.org/openstack/neutron
name: openstack/neutron
short_name: neutron
src_dir: src/opendev.org/openstack/neutron
projects:
opendev.org/openstack/neutron:
canonical_hostname: opendev.org
canonical_name: opendev.org/openstack/neutron
checkout: master
checkout_description: zuul branch
commit: 7be5a0aff1123b381674191f3baa1ec9c128e0f3
name: openstack/neutron
required: false
short_name: neutron
src_dir: src/opendev.org/openstack/neutron
opendev.org/openstack/requirements:
canonical_hostname: opendev.org
canonical_name: opendev.org/openstack/requirements
checkout: master
checkout_description: zuul branch
commit: 48fb5c24764d91833d8ca7084ee9f183785becd6
name: openstack/requirements
required: true
short_name: requirements
src_dir: src/opendev.org/openstack/requirements
ref: refs/changes/61/829161/3
resources: {}
tenant: openstack
timeout: 3600
voting: true
"""
parsed_fields = {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS',
'project': 'openstack/neutron',
'voting': 1,
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master',
'build_change': 829161,
'build_patchset': '3',
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id': ['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
'zuul_executor': 'ze07.opendev.org'
}
performance_json = open(os.path.join(os.path.dirname(__file__),
'performance-example.json')).read()
def _parse_get_yaml(text):
yaml = YAML()
return yaml.load(text)
class _MockedPoolMapAsyncResult:
def __init__(self, func, iterable):
self.func = func
self.iterable = iterable
self.wait = mock.Mock()
# mocked results
self._value = [self.func(i) for i in iterable]
def get(self, timeout=0):
return self._value
class FakeArgs(object):
def __init__(self, config=None, directory=None, host=None, port=None,
username=None, password=None, index_prefix=None, index=None,
insecure=None, follow=None, workers=None,
chunk_size=None, skip_debug=None, keep=None, debug=None,
wait_time=None, file_list=None,
performance_index_prefix=None, subunit_index_prefix=None):
self.config = config
self.directory = directory
self.host = host
self.port = port
self.username = username
self.password = password
self.index_prefix = index_prefix
self.index = index
self.insecure = insecure
self.follow = follow
self.workers = workers
self.chunk_size = chunk_size
self.skip_debug = skip_debug
self.keep = keep
self.debug = debug
self.wait_time = wait_time
self.file_list = file_list
self.performance_index_prefix = performance_index_prefix
self.subunit_index_prefix = subunit_index_prefix
class TestSender(base.TestCase):
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.remove_directory')
@mock.patch('logscraper.logsender.send_to_es')
@mock.patch('logscraper.logsender.get_build_information')
@mock.patch('logscraper.logsender.get_es_client')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir",
config='config.yaml'))
def test_send(self, mock_args, mock_es_client, mock_build_info,
mock_send_to_es, mock_remove_dir, mock_info):
build_uuid = '38bf2cdc947643c9bb04f11f40a0f211'
build_files = ['job-result.txt']
directory = '/tmp/testdir'
index = 'logstash-index'
perf_index = 'performance-index'
subunit_index = 'subunit-index'
mock_build_info.return_value = parsed_fields
mock_es_client.return_value = 'fake_client_object'
tags = ['test', 'info']
mock_info.return_value = ('job-result.txt', tags)
expected_fields = {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS', 'project': 'openstack/neutron',
'voting': 1, 'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master', 'build_change': 829161,
'build_patchset': '3', 'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id': [
'ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'
],
'log_url': 'https://somehost/829161/3/check/'
'openstack-tox-py39/38bf2cd/job-result.txt',
'tenant': 'openstack', 'zuul_executor': 'ze07.opendev.org',
'filename': 'job-result.txt', 'tags': tags}
args = logsender.get_arguments()
mock_send_to_es.return_value = True
logsender.send((build_uuid, build_files), args, directory, index,
perf_index, subunit_index)
self.assertTrue(mock_remove_dir.called)
mock_send_to_es.assert_called_with(
"%s/%s/job-result.txt" % (directory, build_uuid), expected_fields,
'fake_client_object', index, None, None, perf_index, subunit_index)
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.remove_directory')
@mock.patch('logscraper.logsender.send_to_es')
@mock.patch('logscraper.logsender.get_build_information')
@mock.patch('logscraper.logsender.get_es_client')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", keep=True))
def test_send_keep_dir(self, mock_args, mock_es_client, mock_build_info,
mock_send_to_es, mock_remove_dir, mock_info):
build_uuid = '38bf2cdc947643c9bb04f11f40a0f211'
build_files = ['job-result.txt']
directory = '/tmp/testdir'
index = 'logstash-index'
perf_index = 'performance-index'
subunit_index = 'subunit-index'
args = logsender.get_arguments()
mock_info.return_value = ('somefile.txt', ['somefile.txt'])
# No metter what is ES status, it should keep dir
mock_send_to_es.return_value = None
logsender.send((build_uuid, build_files), args, directory, index,
perf_index, subunit_index)
self.assertFalse(mock_remove_dir.called)
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.remove_directory')
@mock.patch('logscraper.logsender.send_to_es')
@mock.patch('logscraper.logsender.get_build_information')
@mock.patch('logscraper.logsender.get_es_client')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", keep=False))
def test_send_error_keep_dir(self, mock_args, mock_es_client,
mock_build_info, mock_send_to_es,
mock_remove_dir, mock_info):
build_uuid = '38bf2cdc947643c9bb04f11f40a0f211'
build_files = ['job-result.txt']
directory = '/tmp/testdir'
index = 'logstash-index'
perf_index = 'performance-index'
subunit_index = 'subunit-index'
args = logsender.get_arguments()
mock_info.return_value = ('somefile.txt', ['somefile.txt'])
mock_send_to_es.return_value = None
logsender.send((build_uuid, build_files), args, directory, index,
perf_index, subunit_index)
self.assertFalse(mock_remove_dir.called)
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.doc_iter')
@mock.patch('logscraper.logsender.logline_iter')
@mock.patch('opensearchpy.helpers.bulk')
@mock.patch('logscraper.logsender.open_file')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", index="myindex", workers=1,
chunk_size=1000,
config='config.yaml', skip_debug=False,
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_send_to_es(self, mock_args, mock_text, mock_bulk, mock_doc_iter,
mock_logline_chunk, mock_file_info):
build_file = 'job-result.txt'
es_fields = parsed_fields
es_client = mock.Mock()
args = logsender.get_arguments()
text = ["2022-02-28 09:39:09.596010 | Job console starting...",
"2022-02-28 09:39:09.610160 | Updating repositories",
"2022-02-28 09:39:09.996235 | Preparing job workspace"]
mock_text.return_value = io.StringIO("\n".join(text))
es_doc = [{
'_index': 'myindex',
'_source': {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS',
'project': 'openstack/neutron',
'voting': 1,
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master',
'build_change': 829161,
'build_patchset': '3',
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
'zuul_executor': 'ze07.opendev.org',
'@timestamp': '2022-02-28T09:39:09.596000',
'message': ' Job console starting...'
}
}, {
'_index': 'myindex',
'_source': {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS',
'project': 'openstack/neutron',
'voting': 1,
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master',
'build_change': 829161,
'build_patchset': '3',
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
'zuul_executor': 'ze07.opendev.org',
'@timestamp': '2022-02-28T09:39:09.610000',
'message': ' Updating repositories'
}
}, {
'_index': 'myindex',
'_source': {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS',
'project': 'openstack/neutron',
'voting': 1,
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master',
'build_change': 829161,
'build_patchset': '3',
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
'zuul_executor': 'ze07.opendev.org',
'@timestamp': '2022-02-28T09:39:09.996000',
'message': ' Preparing job workspace'
}
}]
mock_doc_iter.return_value = es_doc
logsender.send_to_es(build_file, es_fields, es_client, args.index,
args.chunk_size, args.skip_debug,
args.performance_index_prefix,
args.subunit_index_prefix)
self.assertEqual(1, mock_bulk.call_count)
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.doc_iter')
@mock.patch('logscraper.logsender.logline_iter')
@mock.patch('opensearchpy.helpers.bulk')
@mock.patch('logscraper.logsender.open_file')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", index="myindex", workers=1,
chunk_size=1000, config='test.yaml', skip_debug=False,
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_send_to_es_error(self, mock_args, mock_text, mock_bulk,
mock_logline, mock_doc_iter, mock_file_info):
build_file = 'job-result.txt'
es_fields = parsed_fields
es_client = mock.Mock()
args = logsender.get_arguments()
text = ["2022-02-28 09:39:09.596010 | Job console starting...",
"2022-02-28 09:39:09.610160 | Updating repositories",
"2022-02-28 09:39:09.996235 | Preparing job workspace"]
mock_text.return_value = io.StringIO("\n".join(text))
es_doc = [{
'_index': 'myindex',
'_source': {
'build_node': 'zuul-executor',
'build_name': 'openstack-tox-py39',
'build_status': 'SUCCESS',
'project': 'openstack/neutron',
'voting': 1,
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_branch': 'master',
'build_change': 829161,
'build_patchset': '3',
'build_newrev': 'UNKNOWN',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'node_provider': 'local',
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'tenant': 'openstack',
'zuul_executor': 'ze07.opendev.org',
'@timestamp': '2022-02-28T09:39:09.596000',
'message': ' Job console starting...'
}
}]
mock_doc_iter.return_value = es_doc
mock_bulk.side_effect = TransportError(500, "InternalServerError", {
"error": {
"root_cause": [{
"type": "error",
"reason": "error reason"
}]
}
})
send_status = logsender.send_to_es(build_file, es_fields, es_client,
args.index, args.chunk_size,
args.skip_debug,
args.performance_index_prefix,
args.subunit_index_prefix)
self.assertIsNone(send_status)
@mock.patch('json.load')
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('opensearchpy.helpers.bulk')
@mock.patch('logscraper.logsender.open_file')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", index="myindex", workers=1,
chunk_size=1000, config='test.yaml', skip_debug=False,
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_send_to_es_performance(self, mock_args, mock_text, mock_bulk,
mock_file_info, mock_json_load):
build_file = 'performance.json'
es_fields = parsed_fields
es_client = mock.Mock()
args = logsender.get_arguments()
mock_json_load.return_value = json.loads(performance_json)
mock_text.new_callable = mock.mock_open(
read_data=str(performance_json))
es_doc = {
'_index': 'perf',
'_source': {
'@timestamp': '2022-05-17T22:49:50',
'api_compute_get': 17,
'api_compute_largest': 2568,
'api_compute_post': 20,
'api_identity_get': 1174,
'api_identity_largest': 3856,
'api_identity_post': 283,
'api_identity_put': 34,
'api_image_get': 2,
'api_image_largest': 1410,
'api_image_post': 1,
'api_image_put': 1,
'api_info_get': 4,
'api_info_largest': 1659,
'api_placement_get': 9,
'api_placement_largest': 1609,
'api_placement_post': 1,
'api_placement_put': 2,
'api_v2.0_get': 19,
'api_v2.0_largest': 10862,
'api_v2.0_post': 9,
'api_v2.0_put': 3,
'api_volume_get': 2,
'api_volume_largest': 758,
'api_volume_post': 2,
'build_branch': 'master',
'build_change': 829161,
'build_name': 'openstack-tox-py39',
'build_newrev': 'UNKNOWN',
'build_node': 'zuul-executor',
'build_patchset': '3',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_status': 'SUCCESS',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'db_cinder_delete': 1,
'db_cinder_insert': 1,
'db_cinder_select': 52,
'db_cinder_update': 7,
'db_keystone_select': 59,
'db_neutron_delete': 1,
'db_neutron_select': 10,
'db_neutron_update': 1,
'db_nova_cell0_select': 33,
'db_nova_cell0_update': 12,
'db_nova_cell1_select': 32,
'db_nova_cell1_update': 12,
'db_placement_select': 4,
'hostname': 'ubuntu-focal-ovh-gra1-0029678038',
'hosts_id':
['ed82a4a59ac22bf396288f0b93bf1c658af932130f9d336aad528f21'],
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'message':
'{"services": [{"service": '
'"devstack@s-object.service", "MemoryCurrent": '
'64589824}, {"service": "devstack@keystone.service", '
'"MemoryCurrent": 260157440}, {"service": '
'"devstack@q-ovn-metadata-agent.service", '
'"MemoryCurrent": 222007296}, {"service": '
'"devstack@n-super-cond.service", "MemoryCurrent": '
'162463744}, {"service": "devstack@c-vol.service", '
'"MemoryCurrent": 150151168}, {"service": '
'"devstack@n-api-meta.service", "MemoryCurrent": '
'223076352}, {"service": "devstack@c-bak.service", '
'"MemoryCurrent": 135827456}, {"service": '
'"devstack@n-novnc-cell1.service", "MemoryCurrent": '
'110931968}, {"service": "devstack@n-api.service", '
'"MemoryCurrent": 261332992}, {"service": '
'"devstack@memory_tracker.service", "MemoryCurrent": '
'5562368}, {"service": "devstack@etcd.service", '
'"MemoryCurrent": 80420864}, {"service": '
'"devstack@q-svc.service", "MemoryCurrent": '
'422805504}, {"service": '
'"devstack@s-container-sync.service", "MemoryCurrent": '
'45375488}, {"service": "devstack@g-api.service", '
'"MemoryCurrent": 241049600}, {"service": '
'"devstack@s-proxy.service", "MemoryCurrent": '
'81174528}, {"service": "devstack@s-account.service", '
'"MemoryCurrent": 62246912}, {"service": '
'"devstack@c-sch.service", "MemoryCurrent": '
'112668672}, {"service": "devstack@n-sch.service", '
'"MemoryCurrent": 162050048}, {"service": '
'"devstack@s-container.service", "MemoryCurrent": '
'64376832}, {"service": "devstack@c-api.service", '
'"MemoryCurrent": 256569344}, {"service": '
'"devstack@n-cpu.service", "MemoryCurrent": '
'138313728}, {"service": '
'"devstack@placement-api.service", "MemoryCurrent": '
'166539264}, {"service": '
'"devstack@n-cond-cell1.service", "MemoryCurrent": '
'171405312}], "db": [{"db": "placement", "op": '
'"SELECT", "count": 4}, {"db": "nova_cell0", "op": '
'"UPDATE", "count": 12}, {"db": "nova_cell0", "op": '
'"SELECT", "count": 33}, {"db": "neutron", "op": '
'"UPDATE", "count": 1}, {"db": "neutron", "op": '
'"DELETE", "count": 1}, {"db": "neutron", "op": '
'"SELECT", "count": 10}, {"db": "nova_cell1", "op": '
'"SELECT", "count": 32}, {"db": "cinder", "op": '
'"DELETE", "count": 1}, {"db": "keystone", "op": '
'"SELECT", "count": 59}, {"db": "nova_cell1", "op": '
'"UPDATE", "count": 12}, {"db": "cinder", "op": '
'"INSERT", "count": 1}, {"db": "cinder", "op": '
'"UPDATE", "count": 7}, {"db": "cinder", "op": '
'"SELECT", "count": 52}], "processes": [{"cmd": '
'"/usr/lib/erlang/erts-10.6.4/bin/beam.smp", "pid": '
'34343, "args": "-W w -A 128 -MBas ageffcbf -MHas '
'ageffcbf", "rss": 86900736}, {"cmd": '
'"/usr/sbin/mysqld", "pid": 62703, "args": "", "rss": '
'739282944}, {"cmd": "/opt/stack/bin/etcd", "pid": '
'63704, "args": "--name '
'ubuntu-focal-ovh-gra1-0029678038", "rss": 19349504}, '
'{"cmd": "/usr/local/bin/privsep-helper", "pid": '
'91395, "args": "--config-file '
'/etc/neutron/neutron_ovn_metadata_agent.ini", "rss": '
'82690048}], "api": [{"service": "identity", "log": '
'"tls-proxy_access.log", "largest": 3402, "GET": 1173, '
'"POST": 283}, {"service": "info", "log": '
'"tls-proxy_access.log", "largest": 1659, "GET": 4}, '
'{"service": "placement", "log": '
'"tls-proxy_access.log", "largest": 1315, "GET": 9, '
'"POST": 1, "PUT": 2}, {"service": "v2.0", "log": '
'"tls-proxy_access.log", "largest": 10862, "GET": 19, '
'"POST": 9, "PUT": 3}, {"service": "compute", "log": '
'"tls-proxy_access.log", "largest": 2146, "GET": 16, '
'"POST": 20}, {"service": "volume", "log": '
'"tls-proxy_access.log", "largest": 390, "GET": 2, '
'"POST": 2}, {"service": "image", "log": '
'"tls-proxy_access.log", "largest": 1235, "GET": 2, '
'"POST": 1}, {"service": "identity", "log": '
'"access.log", "largest": 3856, "GET": 1174, "POST": '
'283, "PUT": 34}, {"service": "compute", "log": '
'"access.log", "largest": 2568, "GET": 17, "POST": '
'20}, {"service": "placement", "log": "access.log", '
'"largest": 1609, "GET": 9, "POST": 1, "PUT": 2}, '
'{"service": "volume", "log": "access.log", "largest": '
'758, "GET": 2, "POST": 2}, {"service": "image", '
'"log": "access.log", "largest": 1410, "GET": 2, '
'"POST": 1, "PUT": 1}], "report": {"timestamp": '
'"2022-05-17T22:49:50.871392", "hostname": '
'"ubuntu-focal-ovh-gra1-0029678038"}}',
'node_provider': 'local',
'project': 'openstack/neutron',
'service_devstack@c-api.service_memorycurrent': 256569344,
'service_devstack@c-bak.service_memorycurrent': 135827456,
'service_devstack@c-sch.service_memorycurrent': 112668672,
'service_devstack@c-vol.service_memorycurrent': 150151168,
'service_devstack@etcd.service_memorycurrent': 80420864,
'service_devstack@g-api.service_memorycurrent': 241049600,
'service_devstack@keystone.service_memorycurrent': 260157440,
'service_devstack@memory_tracker.service_memorycurrent':
5562368,
'service_devstack@n-api-meta.service_memorycurrent': 223076352,
'service_devstack@n-api.service_memorycurrent': 261332992,
'service_devstack@n-cond-cell1.service_memorycurrent':
171405312,
'service_devstack@n-cpu.service_memorycurrent': 138313728,
'service_devstack@n-novnc-cell1.service_memorycurrent':
110931968,
'service_devstack@n-sch.service_memorycurrent': 162050048,
'service_devstack@n-super-cond.service_memorycurrent':
162463744,
'service_devstack@placement-api.service_memorycurrent':
166539264,
'service_devstack@q-ovn-metadata-agent.service_memorycurrent':
222007296,
'service_devstack@q-svc.service_memorycurrent': 422805504,
'service_devstack@s-account.service_memorycurrent': 62246912,
'service_devstack@s-container-sync.service_memorycurrent':
45375488,
'service_devstack@s-container.service_memorycurrent': 64376832,
'service_devstack@s-object.service_memorycurrent': 64589824,
'service_devstack@s-proxy.service_memorycurrent': 81174528,
'tenant': 'openstack',
'voting': 1,
'zuul_executor': 'ze07.opendev.org'},
}
logsender.send_to_es(build_file, es_fields, es_client, args.index,
args.chunk_size, args.skip_debug,
args.performance_index_prefix,
args.subunit_index_prefix)
self.assertEqual(es_doc, list(mock_bulk.call_args.args[1])[0])
self.assertEqual(1, mock_bulk.call_count)
@mock.patch('logscraper.logsender.get_file_info')
@mock.patch('logscraper.logsender.doc_iter')
@mock.patch('logscraper.logsender.logline_iter')
@mock.patch('opensearchpy.helpers.bulk')
@mock.patch('logscraper.logsender.open_file')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", index="myindex", workers=1,
chunk_size=1000, config='test.yaml', skip_debug=True,
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_send_to_es_skip_debug(self, mock_args, mock_text, mock_bulk,
mock_logline, mock_doc_iter,
mock_file_info):
build_file = 'job-result.txt'
es_fields = parsed_fields
es_client = mock.Mock()
args = logsender.get_arguments()
text = ["2022-02-28 09:39:09.596010 | Job console starting...",
"2022-02-28 09:39:09.610160 | DEBUG Updating repositories",
"2022-02-28 09:39:09.996235 | DEBUG Preparing job workspace"]
mock_text.return_value = io.StringIO("\n".join(text))
es_doc = [{
'_index': 'myindex',
'_source': {
'@timestamp': '2022-02-28T09:39:09.596000',
'build_branch': 'master',
'build_change': 829161,
'build_name': 'openstack-tox-py39',
'build_newrev': 'UNKNOWN',
'build_node': 'zuul-executor',
'build_patchset': '3',
'build_queue': 'check',
'build_ref': 'refs/changes/61/829161/3',
'build_set': '52b29e0e716a4436bd20eed47fa396ce',
'build_status': 'SUCCESS',
'build_uuid': '38bf2cdc947643c9bb04f11f40a0f211',
'log_url':
'https://somehost/829161/3/check/openstack-tox-py39/38bf2cd/',
'message': ' Job console starting...',
'node_provider': 'local',
'project': 'openstack/neutron',
'tenant': 'openstack',
'voting': 1,
'zuul_executor': 'ze07.opendev.org'}
}]
mock_doc_iter.return_value = es_doc
logsender.send_to_es(build_file, es_fields, es_client, args.index,
args.chunk_size, args.skip_debug,
args.performance_index_prefix,
args.subunit_index_prefix)
self.assertEqual(es_doc, list(mock_bulk.call_args.args[1]))
self.assertEqual(1, mock_bulk.call_count)
@mock.patch('logscraper.logsender.logline_iter')
def test_doc_iter(self, mock_logline):
text = [(datetime.datetime(2022, 2, 28, 9, 39, 9, 596000),
'2022-02-28 09:39:09.596010 | Job console starting...\n'),
(datetime.datetime(2022, 2, 28, 9, 39, 9, 610000),
'2022-02-28 09:39:09.610160 | Updating repositories\n')]
expected_chunk = [{
'_index': 'someindex',
'_source': {
'@timestamp': '2022-02-28T09:39:09.596000',
'field': 'test',
'message': 'Job console starting...'
}
}, {
'_index': 'someindex',
'_source': {
'@timestamp': '2022-02-28T09:39:09.610000',
'field': 'test',
'message': 'Updating repositories'
}
}]
chunk_text = list(logsender.doc_iter(
text, 'someindex', {'field': 'test'}))
self.assertEqual(expected_chunk, chunk_text)
def test_logline_iter(self):
text = """2022-02-28 09:39:09.596 | Job console starting...
2022-02-28 09:39:09.610 | Updating repositories
2022-02-28 09:39:09.996 | Preparing job workspace"""
expected_data = [
(datetime.datetime(2022, 2, 28, 9, 39, 9, 596000),
'2022-02-28 09:39:09.596 | Job console starting...\n'),
(datetime.datetime(2022, 2, 28, 9, 39, 9, 610000),
'2022-02-28 09:39:09.610 | Updating repositories\n'),
(datetime.datetime(2022, 2, 28, 9, 39, 9, 996000),
'2022-02-28 09:39:09.996 | Preparing job workspace')
]
skip_debug = False
readed_data = mock.mock_open(read_data=text)
with mock.patch('builtins.open', readed_data) as mocked_open_file:
generated_text = list(logsender.logline_iter('nofile', skip_debug))
self.assertEqual(expected_data, generated_text)
self.assertTrue(mocked_open_file.called)
@mock.patch('json.load')
@mock.patch('logscraper.logsender.open_file')
def test_json_iter(self, mock_open_file, mock_json_load):
text = {
"transient": {
"cluster.index_state_management.coordinator.sweep_period": "1m"
},
"report": {
"timestamp": "2022-04-18T19:51:55.394370",
"hostname": "ubuntu-focal-rax-dfw-0029359041"
}
}
mock_json_load.return_value = text
result = logsender.json_iter('somefile')
self.assertEqual(datetime.datetime(2022, 4, 18, 19, 51, 55),
list(result)[0][0])
result = logsender.json_iter('somefile')
self.assertEqual(str(text).replace("\'", "\""), list(result)[0][1])
@mock.patch('logscraper.logsender.read_yaml_file',
side_effect=[_parse_get_yaml(buildinfo),
_parse_get_yaml(inventory_info)])
def test_makeFields(self, mock_read_yaml_file):
buildinfo_yaml = logsender.get_build_info('fake_dir')
inventory_info_yaml = logsender.get_inventory_info('other_fake_dir')
generated_info = logsender.makeFields(inventory_info_yaml,
buildinfo_yaml)
self.assertEqual(parsed_fields, generated_info)
def test_makeJsonFields(self):
expected_fields = {
'api_compute_get': 17,
'api_compute_largest': 2568,
'api_compute_post': 20,
'api_identity_get': 1174,
'api_identity_largest': 3856,
'api_identity_post': 283,
'api_identity_put': 34,
'api_image_get': 2,
'api_image_largest': 1410,
'api_image_post': 1,
'api_image_put': 1,
'api_info_get': 4,
'api_info_largest': 1659,
'api_placement_get': 9,
'api_placement_largest': 1609,
'api_placement_post': 1,
'api_placement_put': 2,
'api_v2.0_get': 19,
'api_v2.0_largest': 10862,
'api_v2.0_post': 9,
'api_v2.0_put': 3,
'api_volume_get': 2,
'api_volume_largest': 758,
'api_volume_post': 2,
'db_cinder_delete': 1,
'db_cinder_insert': 1,
'db_cinder_select': 52,
'db_cinder_update': 7,
'db_keystone_select': 59,
'db_neutron_delete': 1,
'db_neutron_select': 10,
'db_neutron_update': 1,
'db_nova_cell0_select': 33,
'db_nova_cell0_update': 12,
'db_nova_cell1_select': 32,
'db_nova_cell1_update': 12,
'db_placement_select': 4,
'hostname': 'ubuntu-focal-ovh-gra1-0029678038',
'service_devstack@c-api.service_memorycurrent': 256569344,
'service_devstack@c-bak.service_memorycurrent': 135827456,
'service_devstack@c-sch.service_memorycurrent': 112668672,
'service_devstack@c-vol.service_memorycurrent': 150151168,
'service_devstack@etcd.service_memorycurrent': 80420864,
'service_devstack@g-api.service_memorycurrent': 241049600,
'service_devstack@keystone.service_memorycurrent': 260157440,
'service_devstack@memory_tracker.service_memorycurrent': 5562368,
'service_devstack@n-api-meta.service_memorycurrent': 223076352,
'service_devstack@n-api.service_memorycurrent': 261332992,
'service_devstack@n-cond-cell1.service_memorycurrent': 171405312,
'service_devstack@n-cpu.service_memorycurrent': 138313728,
'service_devstack@n-novnc-cell1.service_memorycurrent': 110931968,
'service_devstack@n-sch.service_memorycurrent': 162050048,
'service_devstack@n-super-cond.service_memorycurrent': 162463744,
'service_devstack@placement-api.service_memorycurrent': 166539264,
'service_devstack@q-ovn-metadata-agent.service_memorycurrent':
222007296,
'service_devstack@q-svc.service_memorycurrent': 422805504,
'service_devstack@s-account.service_memorycurrent': 62246912,
'service_devstack@s-container-sync.service_memorycurrent':
45375488,
'service_devstack@s-container.service_memorycurrent': 64376832,
'service_devstack@s-object.service_memorycurrent': 64589824,
'service_devstack@s-proxy.service_memorycurrent': 81174528}
fields = logsender.makeJsonFields(performance_json)
self.assertEqual(expected_fields, fields)
def test_makeJsonFields_incorrect_values(self):
expected_fields = {
'api_placement_largest': 2151,
'hostname': 'ubuntu-focal-rax-iad-0030685864'
}
json_content = {
"services": [
{"service": "apache2.service", "MemoryCurrent": "[not set]"}],
"db": [{"db": "glance", "op": "DELETE", "count": "[not set]"}],
"api": [{
"service": "placement",
"largest": 2151,
"nova-scheduler-GET": "[not set]"
}],
"report": {
"timestamp": "2022-08-10T13:51:50.928521",
"hostname": "ubuntu-focal-rax-iad-0030685864",
"version": 2
}
}
fields = logsender.makeJsonFields(json.dumps(json_content))
self.assertEqual(expected_fields, fields)
def test_get_message(self):
line_1 = "28-02-2022 09:44:58.839036 | Some message"
line_2 = "2022-02-28 09:44:58.839036 | Other message | other log info"
self.assertEqual("Some message", logsender.get_message(line_1))
self.assertEqual("Other message | other log info",
logsender.get_message(line_2))
def test_get_timestamp(self):
for (line, expected) in [
("2022-02-28 09:44:58.839036 | Other message",
datetime.datetime(2022, 2, 28, 9, 44, 58, 839036)),
("2022-03-21T08:39:18.220547Z | Last metadata expiration",
datetime.datetime(2022, 3, 21, 8, 39, 18, 220547)),
("Mar 31 04:50:23.795709 nested-virt some log",
datetime.datetime(2022, 3, 31, 4, 50, 23, 795700)),
("Mar 21 09:33:23 fedora-rax-dfw-0028920567 sudo[2786]: zuul ",
datetime.datetime(datetime.date.today().year, 3, 21, 9, 33, 23)),
("2022-03-23T13:09:08.644Z|00040|connmgr|INFO|br-int: added",
datetime.datetime(2022, 3, 23, 13, 9, 8)),
("Friday 25 February 2022 09:27:51 +0000 (0:00:00.056)",
datetime.datetime(2022, 2, 25, 9, 27, 51)),
]:
got = logsender.get_timestamp(line)
self.assertEqual(expected, got)
@mock.patch('ruamel.yaml.YAML.load')
@mock.patch('logscraper.logsender.open_file')
def test_get_file_info(self, mock_open_file, mock_yaml):
config = {'files': [{
'name': 'job-output.txt',
'tags': ['console', 'console.html']
}, {'name': 'logs/undercloud/var/log/extra/logstash.txt',
'tags': ['console', 'postpci']}]}
expected_output_1 = ('logs/undercloud/var/log/extra/logstash.txt',
['console', 'postpci', 'logstash.txt'])
expected_output_2 = ('job-output.txt',
['console', 'console.html', 'job-output.txt'])
expected_output_3 = ('somejob.txt', ['somejob.txt'])
mock_yaml.return_value = config
self.assertEqual(expected_output_1, logsender.get_file_info(
config, './9e7bbfb1a4614bc4be06776658fa888f/logstash.txt'))
self.assertEqual(expected_output_2, logsender.get_file_info(
config, './9e7bbfb1a4614bc4be06776658fa888f/job-output.txt'))
self.assertEqual(expected_output_3, logsender.get_file_info(
config, './9e7bbfb1a4614bc4be06776658fa888f/somejob.txt'))
@mock.patch('logscraper.logsender.get_es_client')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
index_prefix="my-index-", workers=2))
def test_get_index(self, mock_args, mock_es_client):
args = logsender.get_arguments()
expected_index = ("my-index-%s" %
datetime.datetime.today().strftime('%Y.%m.%d'))
index = logsender.get_index(args)
self.assertEqual((expected_index, None, None), index)
@mock.patch('logscraper.logsender.send')
@mock.patch('logscraper.logsender.get_index')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", workers=2, index='myindex',
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_prepare_and_send(self, mock_args, mock_index, mock_send):
args = logsender.get_arguments()
ready_directories = {'builduuid': ['job-result.txt']}
mock_index.return_value = (args.index, args.performance_index_prefix,
args.subunit_index_prefix)
with mock.patch(
'multiprocessing.pool.Pool.starmap_async',
lambda self, func, iterable, chunksize=None,
callback=None,
error_callback=None: _MockedPoolMapAsyncResult(func, iterable),
):
logsender.prepare_and_send(ready_directories, args)
self.assertTrue(mock_send.called)
mock_send.assert_called_with((('builduuid', ['job-result.txt']),
args, args.directory, args.index,
args.performance_index_prefix,
args.subunit_index_prefix))
class TestSubunit(base.TestCase):
def setUp(self):
super().setUp()
subunit_parsed_fields_1 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_1.update({
'test_name':
'setUpClass (neutron_tempest_plugin.scenario.'
'test_dns_integration.'
'DNSIntegrationDomainPerProjectTests)',
'test_duration': '0.0',
'test_status': 'skip'
})
subunit_parsed_fields_2 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_2.update({
'test_name':
'neutron_tempest_plugin.scenario.test_dns_integration.'
'DNSIntegrationAdminTests.'
'test_fip_admin_delete',
'test_duration': '7.103220',
'test_status': 'success'
})
subunit_parsed_fields_3 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_3.update({
'test_name':
'neutron_tempest_plugin.scenario.test_dns_integration.'
'DNSIntegrationExtraTests.'
'test_port_with_publishing_subnet',
'test_duration': '9.188214',
'test_status': 'success'
})
subunit_parsed_fields_4 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_4.update({
'test_name':
'neutron_tempest_plugin.scenario.test_dns_integration.'
'DNSIntegrationTests.'
'test_fip',
'test_duration': '6.738004',
'test_status': 'success'
})
subunit_parsed_fields_5 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_5.update({
'test_name':
'neutron_tempest_plugin.scenario.test_dns_integration.'
'DNSIntegrationAdminTests.'
'test_port_on_special_network',
'test_duration': '6.611149',
'test_status': 'success'
})
subunit_parsed_fields_6 = copy.deepcopy(parsed_fields)
subunit_parsed_fields_6.update({
'test_name':
'neutron_tempest_plugin.scenario.test_dns_integration.'
'DNSIntegrationTests.'
'test_server_with_fip',
'test_duration': '30.278503',
'test_status': 'success'
})
self.subunit_docs = [
{
'_index': 'subunit',
'_source': subunit_parsed_fields_1
},
{
'_index': 'subunit',
'_source': subunit_parsed_fields_2
},
{
'_index': 'subunit',
'_source': subunit_parsed_fields_3
},
{
'_index': 'subunit',
'_source': subunit_parsed_fields_4
},
{
'_index': 'subunit',
'_source': subunit_parsed_fields_5
},
{
'_index': 'subunit',
'_source': subunit_parsed_fields_6
}
]
def test_subunit_iter(self):
subunit_file_name = os.path.join(os.path.dirname(__file__),
"testrepository.subunit")
subunit_index_name = "subunit"
result = list(logsender.subunit_iter(file_name=subunit_file_name,
index=subunit_index_name,
es_fields=parsed_fields))
self.assertEqual(result, self.subunit_docs)
@mock.patch('opensearchpy.helpers.bulk')
@mock.patch('argparse.ArgumentParser.parse_args', return_value=FakeArgs(
directory="/tmp/testdir", index="myindex", workers=1,
chunk_size=1000, config='test.yaml', skip_debug=False,
performance_index_prefix="perf",
subunit_index_prefix="subunit"))
def test_send_to_es_subunit(self, mock_args, mock_bulk):
build_file = os.path.join(os.path.dirname(__file__),
"testrepository.subunit")
es_fields = parsed_fields
es_client = mock.Mock()
args = logsender.get_arguments()
logsender.send_to_es(build_file, es_fields, es_client, args.index,
args.chunk_size, args.skip_debug,
args.performance_index_prefix,
args.subunit_index_prefix)
bulk_arg_docs = list(mock_bulk.call_args.args[1])
self.assertEqual(bulk_arg_docs, self.subunit_docs)
View Git Blame Copy Permalink