f08f967085
Because devstack configures 'oslo.log' python based projects to be used with journalctl by default configures logging handlers not to add timestamp in logging messages. This causes tobiko upstream jobs to loose this valueable info in tobiko report files. In order to have this conflicting configuration not affected by DevStack scripts let make tobiko test cases use special options that are not the same as default oslo.log based projects. New configuration options are: [logging] # Default logging line format string line_format = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s - %(message)s # Default logging date format string date_format = %Y-%m-%d %H:%M:%S Change-Id: I1d41a4398c3c1f8667faaccca32f77491cd5305f
397 lines
12 KiB
Python
397 lines
12 KiB
Python
# Copyright 2018 Red Hat
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from __future__ import absolute_import
|
|
|
|
import importlib
|
|
import itertools
|
|
import logging
|
|
import os
|
|
import typing # noqa
|
|
|
|
from oslo_config import cfg
|
|
from oslo_log import log
|
|
|
|
import tobiko
|
|
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
CONFIG_MODULES = ['tobiko.openstack.glance.config',
|
|
'tobiko.openstack.keystone.config',
|
|
'tobiko.openstack.neutron.config',
|
|
'tobiko.openstack.nova.config',
|
|
'tobiko.openstack.octavia.config',
|
|
'tobiko.openstack.topology.config',
|
|
'tobiko.shell.ssh.config',
|
|
'tobiko.shell.ping.config',
|
|
'tobiko.shell.iperf3.config',
|
|
'tobiko.shell.sh.config',
|
|
'tobiko.tripleo.config']
|
|
|
|
|
|
LOGGING_CONF_GROUP_NAME = "logging"
|
|
|
|
LOGGING_OPTIONS = [
|
|
cfg.BoolOpt('capture_log',
|
|
default=True,
|
|
help="Whenever to report debugging log lines"),
|
|
cfg.StrOpt('line_format',
|
|
default=('%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
|
'%(name)s - %(message)s'),
|
|
help='Default logging line format string'),
|
|
cfg.StrOpt('date_format',
|
|
default='%Y-%m-%d %H:%M:%S',
|
|
help='Default logging date format string'),
|
|
]
|
|
|
|
HTTP_CONF_GROUP_NAME = "http"
|
|
|
|
HTTP_OPTIONS = [
|
|
cfg.StrOpt('http_proxy',
|
|
help="HTTP proxy URL for Rest APIs"),
|
|
cfg.StrOpt('https_proxy',
|
|
help="HTTPS proxy URL for Rest APIs"),
|
|
cfg.StrOpt('no_proxy',
|
|
help="Don't use proxy server to connect to listed hosts")]
|
|
|
|
|
|
TESTCASE_CONF_GROUP_NAME = "testcase"
|
|
|
|
TESTCASE_OPTIONS = [
|
|
cfg.FloatOpt('timeout',
|
|
default=None,
|
|
help=("Timeout (in seconds) used for interrupting test case "
|
|
"execution")),
|
|
cfg.FloatOpt('test_runner_timeout',
|
|
default=None,
|
|
help=("Timeout (in seconds) used for interrupting test "
|
|
"runner execution"))]
|
|
|
|
|
|
def workspace_config_files(project=None, prog=None):
|
|
project = project or 'tobiko'
|
|
filenames = []
|
|
if prog is not None:
|
|
filenames.append(prog + '.conf')
|
|
filenames.append(project + '.conf')
|
|
root_dir = os.path.realpath("/")
|
|
current_dir = os.path.realpath(os.getcwd())
|
|
config_files = []
|
|
while current_dir != root_dir:
|
|
for filename in filenames:
|
|
filename = os.path.join(current_dir, filename)
|
|
if os.path.isfile(filename):
|
|
config_files.append(filename)
|
|
current_dir = os.path.dirname(current_dir)
|
|
return config_files
|
|
|
|
|
|
class GlobalConfig(object):
|
|
|
|
# this is a singletone
|
|
_instance = None
|
|
_sources = {} # type: typing.Dict[str, typing.Any]
|
|
|
|
def __new__(cls):
|
|
if cls._instance is None:
|
|
cls._instance = object.__new__(cls)
|
|
return cls._instance
|
|
|
|
def set_source(self, name, source_conf):
|
|
if source_conf is None:
|
|
raise TypeError("Config source cannot be None")
|
|
actual = self._sources.setdefault(name, source_conf)
|
|
if actual is not source_conf:
|
|
msg = "Config source already registered: {!r}".format(name)
|
|
raise RuntimeError(msg)
|
|
|
|
def __getattr__(self, name):
|
|
sources = self._sources.get(name)
|
|
if sources is None:
|
|
msg = "Config source not registered: {!r}".format(name)
|
|
raise NoSuchConfigSource(msg)
|
|
return sources
|
|
|
|
|
|
CONF = GlobalConfig()
|
|
|
|
|
|
class InitConfigFixture(tobiko.SharedFixture):
|
|
|
|
def setup_fixture(self):
|
|
init_tobiko_config()
|
|
init_environ_config()
|
|
|
|
|
|
def init_config():
|
|
tobiko.setup_fixture(InitConfigFixture)
|
|
|
|
|
|
def get_version():
|
|
from tobiko import version
|
|
return version.release
|
|
|
|
|
|
def init_tobiko_config(default_config_dirs=None, default_config_files=None,
|
|
project=None, prog=None, product_name=None,
|
|
version=None):
|
|
|
|
if project is None:
|
|
project = 'tobiko'
|
|
|
|
if product_name is None:
|
|
product_name = 'tobiko'
|
|
|
|
if version is None:
|
|
version = get_version()
|
|
|
|
if default_config_dirs is None:
|
|
default_config_dirs = cfg.find_config_dirs(project=project, prog=prog)
|
|
if default_config_files is None:
|
|
default_config_files = (cfg.find_config_files(project=project,
|
|
prog=prog) +
|
|
workspace_config_files(project=project,
|
|
prog=prog))
|
|
|
|
# Register configuration options
|
|
conf = cfg.ConfigOpts()
|
|
log.register_options(conf)
|
|
register_tobiko_options(conf=conf)
|
|
|
|
# Initialize Tobiko configuration object
|
|
conf(args=[],
|
|
validate_default_values=True,
|
|
default_config_dirs=default_config_dirs,
|
|
default_config_files=default_config_files)
|
|
CONF.set_source('tobiko', conf)
|
|
|
|
# expand and normalize log_file and log_dir names
|
|
conf.config_dir = os.path.realpath(conf.find_file('.'))
|
|
log_dir = conf.log_dir or conf.config_dir
|
|
log_file = conf.log_file or 'tobiko.log'
|
|
log_path = os.path.realpath(os.path.expanduser(
|
|
os.path.join(log_dir, log_file)))
|
|
conf.log_dir = os.path.dirname(log_path)
|
|
conf.log_file = os.path.basename(log_path)
|
|
|
|
# setup final configuration
|
|
log.setup(conf=conf, product_name=product_name, version=version)
|
|
setup_tobiko_config(conf=conf)
|
|
LOG.debug("Configuration setup using parameters:\n"
|
|
" - product_name: %r\n"
|
|
" - version: %r\n"
|
|
" - default_config_dirs: %r\n"
|
|
" - default_config_files: %r\n"
|
|
" - log_file: %r\n",
|
|
product_name,
|
|
version,
|
|
default_config_dirs,
|
|
default_config_files,
|
|
os.path.join(log_dir, log_file))
|
|
|
|
|
|
def register_tobiko_options(conf):
|
|
|
|
conf.register_opts(
|
|
group=cfg.OptGroup(LOGGING_CONF_GROUP_NAME), opts=LOGGING_OPTIONS)
|
|
|
|
conf.register_opts(
|
|
group=cfg.OptGroup(HTTP_CONF_GROUP_NAME), opts=HTTP_OPTIONS)
|
|
|
|
conf.register_opts(
|
|
group=cfg.OptGroup(TESTCASE_CONF_GROUP_NAME), opts=TESTCASE_OPTIONS)
|
|
|
|
for module_name in CONFIG_MODULES:
|
|
module = importlib.import_module(module_name)
|
|
if hasattr(module, 'register_tobiko_options'):
|
|
module.register_tobiko_options(conf=conf)
|
|
|
|
|
|
def list_http_options():
|
|
return [
|
|
(HTTP_CONF_GROUP_NAME, itertools.chain(HTTP_OPTIONS))
|
|
]
|
|
|
|
|
|
def list_testcase_options():
|
|
return [
|
|
(TESTCASE_CONF_GROUP_NAME, itertools.chain(TESTCASE_OPTIONS))
|
|
]
|
|
|
|
|
|
def list_tobiko_options():
|
|
all_options = (list_http_options() +
|
|
list_testcase_options())
|
|
|
|
for module_name in CONFIG_MODULES:
|
|
module = importlib.import_module(module_name)
|
|
if hasattr(module, 'list_options'):
|
|
all_options += module.list_options()
|
|
return all_options
|
|
|
|
|
|
def setup_tobiko_config(conf):
|
|
# Redirect all warnings to logging library
|
|
logging.captureWarnings(True)
|
|
warnings_logger = log.getLogger('py.warnings')
|
|
if conf.debug:
|
|
if not warnings_logger.isEnabledFor(log.WARNING):
|
|
# Print Python warnings
|
|
warnings_logger.logger.setLevel(log.WARNING)
|
|
elif warnings_logger.isEnabledFor(log.WARNING):
|
|
# Silence Python warnings
|
|
warnings_logger.logger.setLevel(log.ERROR)
|
|
|
|
tobiko.setup_fixture(HttpProxyFixture)
|
|
|
|
for module_name in CONFIG_MODULES:
|
|
module = importlib.import_module(module_name)
|
|
if hasattr(module, 'setup_tobiko_config'):
|
|
module.setup_tobiko_config(conf=conf)
|
|
|
|
|
|
class HttpProxyFixture(tobiko.SharedFixture):
|
|
"""Make sure we have http proxy environment variables defined when required
|
|
"""
|
|
|
|
http_proxy = None
|
|
https_proxy = None
|
|
no_proxy = None
|
|
source = None
|
|
|
|
def setup_fixture(self):
|
|
source = None
|
|
http_proxy = os.environ.get('http_proxy')
|
|
https_proxy = os.environ.get('https_proxy')
|
|
no_proxy = os.environ.get('no_proxy')
|
|
if http_proxy or https_proxy:
|
|
source = 'environment'
|
|
else:
|
|
http_conf = CONF.tobiko.http
|
|
http_proxy = http_conf.http_proxy
|
|
https_proxy = http_conf.https_proxy
|
|
no_proxy = http_conf.no_proxy
|
|
if http_proxy:
|
|
os.environ['http_proxy'] = http_proxy
|
|
if https_proxy:
|
|
os.environ['https_proxy'] = https_proxy
|
|
if no_proxy:
|
|
os.environ['no_proxy'] = no_proxy
|
|
if http_proxy or https_proxy or no_proxy:
|
|
source = 'tobiko.conf'
|
|
|
|
if source:
|
|
LOG.info("Using HTTP proxy configuration defined in %s:\n"
|
|
" http_proxy: %r\n"
|
|
" https_proxy: %r\n"
|
|
" no_proxy: %r",
|
|
source, os.environ.get('http_proxy'),
|
|
os.environ.get('https_proxy'), os.environ.get('no_proxy'))
|
|
else:
|
|
LOG.debug("Connecting to REST API services without a proxy "
|
|
"server")
|
|
|
|
self.source = source
|
|
self.http_proxy = os.environ.get('http_proxy')
|
|
self.https_proxy = os.environ.get('https_proxy')
|
|
self.no_proxy = os.environ.get('no_proxy')
|
|
|
|
|
|
def init_environ_config():
|
|
CONF.set_source('environ', EnvironConfig())
|
|
|
|
|
|
class EnvironConfig(object):
|
|
|
|
def __getattr__(self, name):
|
|
value = get_env(name)
|
|
if value is None:
|
|
msg = "Environment variable not defined: {!r}".format(name)
|
|
raise cfg.NoSuchOptError(msg)
|
|
return value
|
|
|
|
|
|
class NoSuchConfigSource(AttributeError):
|
|
pass
|
|
|
|
|
|
def get_any_option(*sources, **kwargs):
|
|
default = kwargs.get('default', None)
|
|
for source in sources:
|
|
value = CONF
|
|
for name in source.split('.'):
|
|
try:
|
|
value = getattr(value, name)
|
|
except (NoSuchConfigSource,
|
|
cfg.NoSuchOptError,
|
|
cfg.NoSuchGroupError) as ex:
|
|
LOG.debug("No such option value for %r: %s", source, ex)
|
|
break
|
|
else:
|
|
if value != default:
|
|
return value
|
|
return default
|
|
|
|
|
|
def get_env(name):
|
|
value = os.environ.get(name)
|
|
if value:
|
|
return value
|
|
else:
|
|
LOG.debug("Environment variable %r is not defined", name)
|
|
return None
|
|
|
|
|
|
def get_int_env(name):
|
|
value = get_env(name)
|
|
if value:
|
|
try:
|
|
return int(value)
|
|
except TypeError:
|
|
LOG.exception("Environment variable %r is not an integer: %r",
|
|
name, value)
|
|
return None
|
|
|
|
|
|
def get_bool_env(name):
|
|
value = get_env(name)
|
|
if value:
|
|
value = str(value).lower()
|
|
if value in ['true', '1', 'yes']:
|
|
return True
|
|
elif value in ['false', '0', 'no']:
|
|
return False
|
|
else:
|
|
LOG.exception("Environment variable %r is not a boolean: %r",
|
|
name, value)
|
|
return None
|
|
|
|
|
|
def get_list_env(name, separator=','):
|
|
value = get_env(name)
|
|
if value:
|
|
return value.split(separator)
|
|
else:
|
|
return []
|
|
|
|
|
|
def is_prevent_create() -> bool:
|
|
return get_bool_env('TOBIKO_PREVENT_CREATE') is True
|
|
|
|
|
|
def skip_if_prevent_create(reason='TOBIKO_PREVENT_CREATE is True'):
|
|
return tobiko.skip_if(reason=reason,
|
|
predicate=is_prevent_create)
|