x
/
tobiko
Code Issues Proposed changes

Merge "Check if log files exist in containers and can be accessible from node"

This commit is contained in:
Zuul 2020-09-14 20:58:30 +00:00 committed by Gerrit Code Review
parent 4e56fcf9a9 c060690e14
commit 85829c96cc
3 changed files with 279 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
tobiko/tests/faults/containers/container_ops.py Normal file
View File

@ -0,0 +1,229 @@
# Copyright (c) 2020 Red Hat
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import random
import re
from oslo_log import log
from tobiko import fail
from tobiko.openstack import topology
from tobiko.shell import sh
LOG = log.getLogger(__name__)
def get_filtered_node_containers(node, containers_regex):
"""Search all containers are matched with containers_regex list
:param node: Node to search containers on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:param containers_regex: List of regex that can be matched to containers
:type containers_regex: list of strings
:return: List of contatiner names
:rtype: list of strings
"""
filtered_containers = []
all_node_containers = sh.execute(
'sudo podman ps --format "{{.Names}}"',
ssh_client=node.ssh_client,
expect_exit_status=None
).stdout.strip().split('\n')
for container in all_node_containers:
container = container.strip('"')
if any(re.fullmatch(reg, container) for reg in containers_regex):
filtered_containers.append(container)
return filtered_containers
def get_nodes_for_groups(groups):
"""Search for all nodes that are matched with the specified groups
:param groups: List of groups nodes can belong to
:type groups: list
:return: List of nodes that belong to the specified groups
:rtype: list of tobiko.openstack.topology.OpenStackTopologyNode
"""
nodes = []
for node in topology.list_openstack_nodes():
for group in node.groups:
if group in groups:
nodes.append(node)
return(nodes)
def get_config_files(node, kolla_jsons, conf_ignorelist, scripts_to_check):
"""Return the list of configuration files according to kolla JSONs
Kolla has the container execution commands in the matched JSON files.
Need to get all the '--config-file <file_name>' parameters from those
JSON files.
It is possible that the container is executed with the set of commands
are written in a separate script instead of a single command so need
to follow those scripts to get configuration files from there too.
:param node: Node to search configuration files on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:param kolla_jsons: Path to kolla json files to get commands from
:type kolla_jsons: list
:param conf_ignorelist: Configuration files to ignore
:type conf_ignorelist: list
:param scripts_to_check: Sripts to look for configuration files in if those
scripts will be found in kolla json files. Dictionary contains script
path on the container as the key and the path on the overcloud node
as the value.
:type scripts_to_check: dict
:return: List of config files paths within containers
:rtype: list
"""
cmds = sh.execute(
f'sudo jq \'.command\' {" ".join(kolla_jsons)}',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip().split('\n')
LOG.debug(f'{node.name} run containers with commands {cmds}')
config_files = set()
for cmd in cmds:
if cmd in scripts_to_check.keys():
LOG.debug(f'{cmd} is recognized as script to search '
'for config files in')
cmd = sh.execute(
f'sudo cat {scripts_to_check[cmd]}',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip()
cmd = cmd.strip('"')
temp_conf_files = re.findall('--config-file [^ \n]*', cmd)
for conf_file in temp_conf_files:
conf_file = conf_file.split(' ')[1]
if conf_file in conf_ignorelist:
LOG.debug(f'{conf_file} is in ignore list')
continue
config_files.add(conf_file)
LOG.debug(f'There are {config_files} on {node.name}')
return config_files
def get_node_neutron_containers(node):
"""Return list of all neutron containers are available on the node
:param node: Node to search containers on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:return: List of neutron containers names
:rtype: list of strings
"""
neutron_containers = [
'neutron_((ovs|metadata|l3)_agent|dhcp|api)',
'ovn_(controller|metadata_agent)',
r'ovn-dbs-bundle-(podman|docker)-\d*']
return get_filtered_node_containers(node, neutron_containers)
def get_node_neutron_config_files(node):
"""Return all relevant neutron config files
:param node: Overcloud node to search log files on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:return: List of config files paths within neutron containers
:rtype: list of strings
"""
kolla_jsons = ['/var/lib/kolla/config_files/neutron*',
'/var/lib/kolla/config_files/ovn*']
conf_ignorelist = ['/usr/share/neutron/neutron-dist.conf']
scripts_to_check = {'"/neutron_ovs_agent_launcher.sh"':
'/var/lib/container-config-scripts/'
'neutron_ovs_agent_launcher.sh'}
config_files = get_config_files(node,
kolla_jsons,
conf_ignorelist,
scripts_to_check)
return config_files
def get_container_logfile(node, container):
""" Return the logfile of the process that is executed on the container
:param node: Node the container is running on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:param container: Name of the container
:type container: string
:return: Path to the logfiles on the container
:rtype: string
"""
cmd = sh.execute(
f'sudo podman exec -it -u root {container} cat /run_command',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip()
if ' ' not in cmd: # probably script as no space in the command
cmd = sh.execute(
f'sudo podman exec -it -u root {container} cat {cmd}',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip()
LOG.debug(f'The following command is executed in {container} container '
f'on {node.name} node:\n{cmd}')
log_file = re.findall('--log-file=[^ \n]*', cmd)
if log_file:
log_file = log_file[0].split('=')[1]
return log_file
def log_random_msg(node, container, logfile):
"""Print random message to the container log file
:param node: Node the container is running on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:param container: Name of the container
:type container: string
:param logfile: Path to the logfile on the container
:type logfile: string
:return: Message that has been printed to container log file
:rtype: string
"""
symbols = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
random_msg = ''.join(random.choice(symbols) for i in range(30))
LOG.debug(f'Trying to print {random_msg} string to {logfile} log file '
f'in {container} container on {node.name} node')
cmd = f"sh -c 'echo {random_msg} >> {logfile}'"
error = sh.execute(f'sudo podman exec -it -u root {container} {cmd}',
ssh_client=node.ssh_client,
expect_exit_status=None).stderr
if error:
fail(f'Cannot edit {logfile} in {container} on {node.name} '
f'got the following error:\n{error}')
return random_msg
def find_msg_in_file(node, logfile, message):
"""Search for the message in the logfile
:param node: Node the container is running on
:type node: class: tobiko.openstack.topology.OpenStackTopologyNode
:param logfile: Path of the logfile
:type logfile: string
:param message: Message to search for
:type message: string
:return: True if message exists in file or False otherwise
:rtype: bool
"""
LOG.debug(f'Searching for {message} in {logfile} on {node.name}')
result = sh.execute(f'sudo grep -h {message} {logfile}{{,.1}}',
ssh_client=node.ssh_client,
expect_exit_status=None)
if result.stderr:
fail(f'Failed reading {logfile} on {node.name}:\n{result.stderr}')
elif result.stdout.strip() == message:
return True
else:
return False

94
tobiko/tests/faults/containers/test_container_config.py
View File

@ -14,13 +14,11 @@
# under the License.
from __future__ import absolute_import
import re
from oslo_log import log
import testtools
from tobiko.openstack import topology
from tobiko.shell import sh
from tobiko.tests.faults.containers import container_ops
LOG = log.getLogger(__name__)
@ -28,27 +26,7 @@ LOG = log.getLogger(__name__)
class ConfigurationFilesTest(testtools.TestCase):
def get_filtered_node_containers(self, node, containers_regex):
"""Search all containers are matched with containers_regex list
"""
filtered_containers = []
all_node_containers = sh.execute(
'sudo podman ps --format "{{.Names}}"',
ssh_client=node.ssh_client,
expect_exit_status=None
).stdout.strip().split('\n')
for container in all_node_containers:
container = container.strip('"')
if any(re.fullmatch(reg, container) for reg in containers_regex):
filtered_containers.append(container)
# The following code causes test to take 4 times more time
# podman = node.podman_client.setup_client()
# for container in podman.containers.list():
# if any(re.fullmatch(reg, container) for reg in containers_regex):
# filtered_containers.append(container.names)
return filtered_containers
def verify_config_files(self, node, containers, file_list, service):
def check_config(self, node, containers, file_list, service):
"""Verify that files on the node are equal to the file in container
It calculates MD5 sum of the configuration file in the changed
@ -94,69 +72,11 @@ class ConfigurationFilesTest(testtools.TestCase):
verified = False
return verified
def get_neutron_nodes(self):
nodes = []
supported_groups = ['controller', 'compute', 'networker']
for node in topology.list_openstack_nodes():
for group in node.groups:
if group in supported_groups:
nodes.append(node)
return(nodes)
def get_node_neutron_containers(self, node):
neutron_containers = [
'neutron_((ovs|metadata|l3)_agent|dhcp|api)',
'ovn_(controller|metadata_agent)',
r'ovn-dbs-bundle-(podman|docker)-\d*']
return self.get_filtered_node_containers(node, neutron_containers)
def get_node_neutron_config_files(self, node):
"""Return the list of configuration files according to kolla JSONs
Kolla has the container execution commands in the matched JSON files.
Need to get all the '--config-file <file_name>' parameters from those
JSON files.
It is possible that the container is executed with the set of commands
are written in a separate script instead of a single command so need
to follow those scripts to get configuration files from there too.
"""
neutron_cmds = sh.execute(
'sudo jq \'.command\' /var/lib/kolla/config_files/neutron* '
'/var/lib/kolla/config_files/ovn*',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip().split('\n')
LOG.debug(f'{node.name} run containers with commands {neutron_cmds}')
conf_ignorelist = ['/usr/share/neutron/neutron-dist.conf']
scripts_to_check = {'"/neutron_ovs_agent_launcher.sh"':
'/var/lib/container-config-scripts/'
'neutron_ovs_agent_launcher.sh'}
neutron_config_files = set()
for cmd in neutron_cmds:
if cmd in scripts_to_check.keys():
LOG.debug(f'{cmd} is recognized as script to search '
'for config files in')
cmd = sh.execute(
f'sudo cat {scripts_to_check[cmd]}',
ssh_client=node.ssh_client,
expect_exit_status=None).stdout.strip()
cmd = cmd.strip('"')
conf_files = re.findall('--config-file [^ ]*', cmd)
for conf_file in conf_files:
conf_file = conf_file.split(' ')[1]
if conf_file in conf_ignorelist:
LOG.debug(f'{conf_file} is in ignore list')
continue
neutron_config_files.add(conf_file)
LOG.debug(f'There are {neutron_config_files} on {node.name}')
return neutron_config_files
def test_neutron_config_files(self):
neutron_nodes = self.get_neutron_nodes()
groups = ['controller', 'compute', 'networker']
neutron_nodes = container_ops.get_nodes_for_groups(groups)
for node in neutron_nodes:
neutron_containers = self.get_node_neutron_containers(node)
neutron_config_files = self.get_node_neutron_config_files(node)
containers = container_ops.get_node_neutron_containers(node)
config = container_ops.get_node_neutron_config_files(node)
self.assertTrue(
self.verify_config_files(node,
neutron_containers,
neutron_config_files,
'neutron'))
self.check_config(node, containers, config, 'neutron'))

43
tobiko/tests/faults/containers/test_container_log.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2020 Red Hat
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
from oslo_log import log
import testtools
from tobiko.tests.faults.containers import container_ops
LOG = log.getLogger(__name__)
class LogFilesTest(testtools.TestCase):
def test_neutron_logs_exist(self):
groups = ['controller', 'compute', 'networker']
neutron_nodes = container_ops.get_nodes_for_groups(groups)
for node in neutron_nodes:
containers = container_ops.get_node_neutron_containers(node)
for container in containers:
l_file = container_ops.get_container_logfile(node, container)
if not l_file:
LOG.warning(f'No logfile has been found in {container} '
f'container of {node.name} node')
continue
log_msg = container_ops.log_random_msg(node, container, l_file)
l_node = f'/var/log/containers/neutron/{l_file.split("/")[-1]}'
self.assertTrue(container_ops.find_msg_in_file(node,
l_node,
log_msg))