Do not hardcode configuration files location

Configuration files of monasca components
defined in mon detection group are hardcoded.
This commits tries to best-guess where those files
are stored.

Change-Id: Id9921012b9242cc367df5ba896aa98dc49e1d572
Story: 1664188
Task: 3941
This commit is contained in:
Tomasz Trębski 2017-03-29 06:28:59 +02:00
parent 37bb11e6a0
commit e8aab3d35f
2 changed files with 283 additions and 117 deletions

View File

@ -8,6 +8,7 @@
"""
import logging
import re
import yaml
from six.moves import configparser
@ -126,8 +127,9 @@ class MonAPI(monasca_setup.detection.Plugin):
if apache_process:
log.info('\tmonasca-api runs under Apache WSGI')
api_process = apache_process
impl_helper = self._init_impl_helper(impl_lang)
impl_helper = self._init_impl_helper(api_process.as_dict()['cmdline'],
impl_lang)
impl_helper.load_configuration()
api_port = impl_helper.get_bound_port()
@ -166,8 +168,7 @@ class MonAPI(monasca_setup.detection.Plugin):
def dependencies_installed(self):
return True
@staticmethod
def _init_impl_helper(impl_lang):
def _init_impl_helper(self, cmdline, impl_lang):
"""Returns appropriate helper implementation.
:param impl_lang: implementation language, either `java` or `python`
@ -177,9 +178,9 @@ class MonAPI(monasca_setup.detection.Plugin):
"""
if impl_lang == 'java':
return _MonAPIJavaHelper()
return _MonAPIJavaHelper(cmdline=cmdline)
else:
return _MonAPIPythonHelper()
return _MonAPIPythonHelper(cmdline=cmdline, args=self.args)
class MonNotification(monasca_setup.detection.Plugin):
@ -216,7 +217,10 @@ class MonPersister(monasca_setup.detection.Plugin):
if process_found:
impl_lang = _get_impl_lang(p_process)
impl_helper = self._init_impl_helper(impl_lang)
impl_helper = self._init_impl_helper(
p_process.as_dict()['cmdline'],
impl_lang
)
if impl_helper is not None:
impl_helper.load_configuration()
@ -252,7 +256,7 @@ class MonPersister(monasca_setup.detection.Plugin):
return True
@staticmethod
def _init_impl_helper(impl_lang):
def _init_impl_helper(cmdline, impl_lang):
"""Returns appropriate helper implementation.
Note:
@ -267,7 +271,7 @@ class MonPersister(monasca_setup.detection.Plugin):
"""
if impl_lang == 'java':
return _MonPersisterJavaHelper()
return _MonPersisterJavaHelper(cmdline=cmdline)
return None
@ -337,16 +341,73 @@ class MonInfluxDB(monasca_setup.detection.Plugin):
return True
class _MonPersisterJavaHelper(object):
class _DropwizardJavaHelper(object):
"""Mixing to locate configuration file for DropWizard app
Class utilizes process of search the configuartion file
for:
* monasca-api [**Java**]
* monasca-persister [**Java**]
"""
YAML_PATTERN = re.compile('.*\.ya?ml', re.IGNORECASE)
def __init__(self, cmdline=None):
self._cmdline = cmdline
def load_configuration(self):
"""Loads java specific configuration.
Load java specific configuration from:
* :py:attr:`DEFAULT_CONFIG_FILE`
:return: True if both configuration files were successfully loaded
:rtype: bool
"""
try:
config_file = self._get_config_file()
self._read_config_file(config_file)
except Exception as ex:
log.error('Failed to parse %s', config_file)
log.exception(ex)
raise ex
def _find_config_file_in_cmdline(self, cmdline):
# note(trebskit) file should be mentioned
# somewhere in the end of cmdline
for item in cmdline[::-1]:
if self.YAML_PATTERN.match(item):
return item
return None
def _read_config_file(self, config_file):
with open(config_file, 'r') as config:
self._cfg = yaml.safe_load(config.read())
def _get_config_file(self):
if self._cmdline:
config_file = self._find_config_file_in_cmdline(
cmdline=self._cmdline
)
if config_file:
log.debug('\tFound %s for java configuration from CLI',
config_file)
return config_file
config_file = self.DEFAULT_CONFIG_FILE
log.debug('\tAssuming default configuration file=%s', config_file)
return config_file
class _MonPersisterJavaHelper(_DropwizardJavaHelper):
"""Encapsulates Java specific configuration for monasca-persister"""
CONFIG_FILE = '/etc/monasca/persister-config.yml'
DEFAULT_CONFIG_FILE = '/etc/monasca/persister-config.yml'
"""Default location where plugin expects configuration file"""
def __init__(self):
super(_MonPersisterJavaHelper, self).__init__()
self._cfg = None
def build_config(self):
config = monasca_setup.agent_config.Plugins()
metrics = self._collect_metrics()
@ -407,7 +468,7 @@ class _MonPersisterJavaHelper(object):
elif database_type == 'vertica':
self._add_vertica_metrics(whitelist)
else:
log.warn('Failed finding database type in %s', self.CONFIG_FILE)
log.warn('Failed finding database type in %s', self.DEFAULT_CONFIG_FILE)
def _collect_internal_metrics(self, whitelist):
alarm_num_threads = self._cfg['alarmHistoryConfiguration']['numThreads']
@ -497,34 +558,11 @@ class _MonPersisterJavaHelper(object):
admin_endpoint_port),
metrics))
def load_configuration(self):
"""Loads java specific configuration.
Load java specific configuration from:
* :py:attr:`API_CONFIG_YML`
:return: True if both configuration files were successfully loaded
:rtype: bool
"""
try:
with open(self.CONFIG_FILE, 'r') as config:
self._cfg = yaml.safe_load(config.read())
except Exception as ex:
log.error('Failed to parse %s', self.CONFIG_FILE)
log.exception(ex)
raise ex
class _MonAPIJavaHelper(object):
class _MonAPIJavaHelper(_DropwizardJavaHelper):
"""Encapsulates Java specific configuration for monasca-api"""
CONFIG_FILE = '/etc/monasca/api-config.yml'
def __init__(self):
super(_MonAPIJavaHelper, self).__init__()
self._api_config = None
DEFAULT_CONFIG_FILE = '/etc/monasca/api-config.yml'
def build_config(self):
"""Builds monitoring configuration for monasca-api Java flavour.
@ -582,7 +620,7 @@ class _MonAPIJavaHelper(object):
return config
def _monitor_endpoints(self, config, metrics):
admin_connector = self._api_config['server']['adminConnectors'][0]
admin_connector = self._cfg['server']['adminConnectors'][0]
try:
admin_endpoint_type = admin_connector['type']
@ -610,35 +648,16 @@ class _MonAPIJavaHelper(object):
# if not it means that configuration file was not found
# or monasca-api Python implementation is running
api_config = getattr(self, '_api_config', None)
if api_config is None:
cfg = getattr(self, '_cfg', None)
if cfg is None:
return False
hibernate_cfg = self._api_config.get('hibernate', None)
hibernate_cfg = cfg.get('hibernate', None)
if hibernate_cfg is None:
return False
return hibernate_cfg.get('supportEnabled', False)
def load_configuration(self):
"""Loads java specific configuration.
Load java specific configuration from:
* :py:attr:`API_CONFIG_YML`
:return: True if both configuration files were successfully loaded
:rtype: bool
"""
try:
with open(self.CONFIG_FILE, 'r') as config:
self._api_config = yaml.safe_load(config.read())
except Exception as ex:
log.error('Failed to parse %s', self.CONFIG_FILE)
log.exception(ex)
raise ex
def get_bound_port(self):
"""Returns port API is listening on.
@ -655,13 +674,12 @@ class _MonAPIJavaHelper(object):
:rtype: int
"""
if self._api_config is None:
if self._cfg is None:
return _DEFAULT_API_PORT
try:
return self._api_config['server']['applicationConnectors'][0]['port']
return self._cfg['server']['applicationConnectors'][0]['port']
except Exception as ex:
log.error('Failed to read api port from '
'/etc/monasca/api-config.yml')
log.error('Failed to read api port from configuration file')
log.exception(ex)
return _DEFAULT_API_PORT
@ -669,10 +687,15 @@ class _MonAPIJavaHelper(object):
class _MonAPIPythonHelper(object):
"""Encapsulates Python specific configuration for monasca-api"""
CONFIG_FILE = '/etc/monasca/api-config.ini'
DEFAULT_CONFIG_FILE = '/etc/monasca/api-config.ini'
PASTE_CLI_OPTS = '--paste', '--paster',
"""Possible flags passed to gunicorn processed,
pointing at paste file"""
def __init__(self):
def __init__(self, cmdline=None, args=None):
super(_MonAPIPythonHelper, self).__init__()
self._cmdline = cmdline
self._args = args
self._paste_config = None
def build_config(self):
@ -688,12 +711,11 @@ class _MonAPIPythonHelper(object):
and parses it with :py:class:`configparser.RawConfigParser`
"""
cp = configparser.RawConfigParser()
try:
cp.readfp(open(self.CONFIG_FILE, 'r'))
self._paste_config = cp
config_file = self._get_config_file()
self._paste_config = self._read_config_file(config_file)
except Exception as ex:
log.error('Failed to parse %s', self.CONFIG_FILE)
log.error('Failed to parse %s', config_file)
log.exception(ex)
raise ex
@ -716,3 +738,51 @@ class _MonAPIPythonHelper(object):
if not self._paste_config:
return _DEFAULT_API_PORT
return self._paste_config.getint('server:main', 'port')
def _read_config_file(self, config_file):
cp = configparser.RawConfigParser()
return cp.readfp(open(config_file, 'r'))
def _get_config_file(self):
"""Method gets configuration file of Python monasca-api.
Method tries to examine following locations:
* cmdline of process (looking for either
of :py:attr:`_MonAPIPythonHelper.PASTE_CLI_OPTS`)
* this plugin args
loooking for location of configuration file
:param args: plugin arguments
:type args: dict
"""
if self._cmdline:
# we're interested in PASTE_CLI_OPTS
for paste in self.PASTE_CLI_OPTS:
if paste in self._cmdline:
pos = self._cmdline.index(paste)
flag = self._cmdline[pos]
config_file = self._cmdline[pos + 1]
if config_file:
log.debug(('\tFound %s=%s for python configuration '
'from CLI'),
flag, config_file)
return config_file
else:
log.warn(('\tCannot determine neither %s from process'
'cmdline'), self.PASTE_CLI_OPTS)
if self._args and 'paste-file' in self._args:
# check if args mentions config file param
config_file = self._args.get('paste-file', None)
log.debug(('\tFound paste-file=%s for python configuration '
'passed as plugin argument'), config_file)
return config_file
config_file = self.DEFAULT_CONFIG_FILE
log.debug('\tAssuming default paste_file=%s', config_file)
return config_file

View File

@ -20,31 +20,45 @@ import psutil
from monasca_setup.detection.plugins import mon
_PYTHON_CMD_API = ('/opt/monasca-api/bin/gunicorn'
' -n monasca-api'
' -k eventlet --worker-connections=2000 '
'--backlog=1000 '
'--paste /etc/monasca/api-config.ini -w 9')
_JAVA_CMD_API = ('/usr/bin/java '
'-Dfile.encoding=UTF-8 -Xmx128m '
'-cp /opt/monasca/monasca-api.jar '
'monasca.api.MonApiApplication server '
'/etc/monasca/api-config.yml')
_PYTHON_WSGI_CMD_API = ('/usr/sbin/httpd-prefork (wsgi:monasca-api)'
'-f /etc/apache2/httpd.conf'
'-DSYSCONFIG -DSYSTEMD'
'-C PidFile /var/run/httpd.pid'
'-C Include /etc/apache2/sysconfig.d/'
'-DFOREGROUND -k start')
_PYTHON_CMD_PERSISTER = ('/opt/monasca-persister/bin/python '
'/opt/monasca-persister/lib/python2.7/'
'site-packages/monasca_persister/persister.py'
' --config-file /etc/monasca/persister.conf')
_JAVA_CMD_PERSISTER = ('/usr/bin/java -Dfile.encoding=UTF-8 -Xmx128m '
'-cp /opt/monasca/monasca-persister.jar '
'monasca.persister.PersisterApplication server '
'/etc/monasca/persister-config.yml')
def _mock_python_cmd_api(paste_flag='--paste',
paste_file='/etc/monasca/api-config.ini'):
return (('/opt/monasca-api/bin/gunicorn'
' -n monasca-api'
' -k eventlet --worker-connections=2000'
' --backlog=1000'
' %s %s -w 9') % (paste_flag, paste_file)).split(' ')
def _mock_java_cmd_api(config_file='/etc/monasca/api-config.yml'):
return (('/usr/bin/java'
' -Dfile.encoding=UTF-8 -Xmx128m '
' -cp /opt/monasca/monasca-api.jar '
' monasca.api.MonApiApplication server '
' %s') % config_file).split(' ')
def _mock_java_persister(config_file='/etc/monasca/persister-config.yml'):
return (('/usr/bin/java -Dfile.encoding=UTF-8 -Xmx128m '
' -cp /opt/monasca/monasca-persister.jar '
' monasca.persister.PersisterApplication server '
' %s') % config_file).split(' ')
_PYTHON_CMD_API = _mock_python_cmd_api()
_JAVA_CMD_API = _mock_java_cmd_api()
_JAVA_CMD_PERSISTER = _mock_java_persister()
_PYTHON_WSGI_CMD_API = ('/usr/sbin/httpd-prefork (wsgi:monasca-api)'
' -f /etc/apache2/httpd.conf'
' -DSYSCONFIG -DSYSTEMD'
' -C PidFile /var/run/httpd.pid'
' -C Include /etc/apache2/sysconfig.d/'
' -DFOREGROUND -k start').split(' ')
_PYTHON_CMD_PERSISTER = ('/opt/monasca-persister/bin/python'
' /opt/monasca-persister/lib/python2.7/'
' site-packages/monasca_persister/persister.py'
' --config-file /etc/monasca/persister.conf').split(' ')
_JAVA_YML_CFG_BIT_API = '''
hibernate:
@ -146,18 +160,18 @@ class TestMonPersisterDetectionPlugin(unittest.TestCase):
'monasca_setup.detection.plugins.mon._MonPersisterJavaHelper')
def test_should_use_java_helper_if_persister_is_java(self,
impl_helper):
FakeProcesses.cmdLine = [_JAVA_CMD_PERSISTER]
FakeProcesses.cmdLine = _JAVA_CMD_PERSISTER
self._mon_p._init_impl_helper = iih = mock.Mock(
return_value=impl_helper)
self._detect()
iih.assert_called_once_with('java')
iih.assert_called_once_with(_JAVA_CMD_PERSISTER, 'java')
self.assertTrue(impl_helper.load_configuration.called_once)
def test_should_detect_java_persister_has_config(self):
FakeProcesses.cmdLine = [_JAVA_CMD_PERSISTER]
FakeProcesses.cmdLine = _JAVA_CMD_PERSISTER
yml_cfg = _JAVA_YML_CFG_BIT_PERSISTER.format(
ah_threads=5,
@ -176,6 +190,26 @@ class TestMonPersisterDetectionPlugin(unittest.TestCase):
self.assertTrue(self._mon_p.available)
def test_should_use_config_file_from_cli_for_java(self):
extensions = 'yml', 'yaml'
for ext in extensions:
config_file = '/tmp/monasca-persister-abc.%s' % ext
cmdline = _mock_java_persister(config_file)
FakeProcesses.cmdLine = cmdline
helper = mon._MonPersisterJavaHelper(cmdline=cmdline)
helper._read_config_file = rcf = mock.Mock()
self._mon_p._init_impl_helper = iih = mock.Mock(
return_value=helper
)
self._detect()
iih.assert_called_once_with(cmdline, 'java')
rcf.assert_called_once_with(config_file)
self.assertTrue(iih.get_bound_port.called_once)
@mock.patch('six.moves.configparser.RawConfigParser')
def test_should_detect_python_persister_has_config(self, _):
# NOTE(trebskit) this cannot use mocking the read of the file
@ -186,14 +220,14 @@ class TestMonPersisterDetectionPlugin(unittest.TestCase):
#
# to sum it up => ;-(((
FakeProcesses.cmdLine = [_PYTHON_CMD_PERSISTER]
FakeProcesses.cmdLine = _PYTHON_CMD_PERSISTER
self._mon_p._init_impl_helper = mock.Mock(return_value=mock.Mock())
self._detect()
self.assertTrue(self._mon_p.available)
def test_build_java_config(self):
FakeProcesses.cmdLine = [_JAVA_CMD_PERSISTER]
FakeProcesses.cmdLine = _JAVA_CMD_PERSISTER
# note(trebskit) this is always set to 2
jvm_metrics_count = 2
@ -281,7 +315,7 @@ class TestMonPersisterDetectionPlugin(unittest.TestCase):
}, process_instance['dimensions'])
def test_build_python_config(self):
FakeProcesses.cmdLine = [_PYTHON_CMD_PERSISTER]
FakeProcesses.cmdLine = _PYTHON_CMD_PERSISTER
self._detect()
conf = self._build_config()
@ -326,17 +360,57 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
@mock.patch('monasca_setup.detection.plugins.mon._MonAPIPythonHelper')
def test_should_use_python_helper_if_api_is_python(self, impl_helper):
FakeProcesses.cmdLine = [_PYTHON_CMD_API]
FakeProcesses.cmdLine = _PYTHON_CMD_API
self._mon_api._init_impl_helper = iih = mock.Mock(
return_value=impl_helper)
self._detect()
iih.assert_called_once_with('python')
iih.assert_called_once_with(mock.ANY, 'python')
self.assertTrue(impl_helper.load_configuration.called_once)
self.assertTrue(impl_helper.get_bound_port.called_once)
def test_should_use_config_file_from_args_for_python(self):
FakeProcesses.cmdLine = _PYTHON_CMD_API
custom_conf_file = '/tmp/mon.ini'
plugin_args = {'paste-file': custom_conf_file}
helper = mon._MonAPIPythonHelper(args=plugin_args)
helper._read_config_file = rcf = mock.Mock()
helper.get_bound_port = mock.Mock(return_value=123)
self._mon_api._init_impl_helper = iih = mock.Mock(
return_value=helper
)
self._detect(args=plugin_args)
iih.assert_called_once_with(mock.ANY, 'python')
rcf.assert_called_once_with(custom_conf_file)
self.assertTrue(iih.get_bound_port.called_once)
def test_should_use_config_file_from_cmdline_for_python(self):
paste_flags = '--paste', '--paster'
for pf in paste_flags:
paste_file = '/tmp/monasca-api-paste.ini'
cmdline = _mock_python_cmd_api(paste_flag=pf,
paste_file=paste_file)
FakeProcesses.cmdLine = cmdline
helper = mon._MonAPIPythonHelper(cmdline=cmdline)
helper._read_config_file = rcf = mock.Mock()
helper.get_bound_port = mock.Mock(return_value=123)
self._mon_api._init_impl_helper = iih = mock.Mock(
return_value=helper
)
self._detect()
iih.assert_called_once_with(cmdline, 'python')
rcf.assert_called_once_with(paste_file)
self.assertTrue(iih.get_bound_port.called_once)
@mock.patch('monasca_setup.detection.plugins.mon._MonAPIPythonHelper')
def test_should_use_python_helper_if_api_is_wsgi(self, impl_helper):
self._mon_api._init_impl_helper = iih = mock.Mock(
@ -344,23 +418,44 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
self._detect([FakeWSGIWorkers([_PYTHON_WSGI_CMD_API])])
iih.assert_called_once_with('python')
iih.assert_called_once_with(mock.ANY, 'python')
self.assertTrue(impl_helper.load_configuration.called_once)
self.assertTrue(impl_helper.get_bound_port.called_once)
@mock.patch('monasca_setup.detection.plugins.mon._MonAPIJavaHelper')
def test_should_use_java_helper_if_api_is_java(self, impl_helper):
FakeProcesses.cmdLine = [_JAVA_CMD_API]
FakeProcesses.cmdLine = _JAVA_CMD_API
self._mon_api._init_impl_helper = iih = mock.Mock(
return_value=impl_helper)
self._detect()
iih.assert_called_once_with('java')
iih.assert_called_once_with(mock.ANY, 'java')
self.assertTrue(impl_helper.load_configuration.called_once)
self.assertTrue(impl_helper.get_bound_port.called_once)
def test_should_use_config_file_from_cli_for_java(self):
extensions = 'yml', 'yaml'
for ext in extensions:
config_file = '/tmp/monasca-api-foo-bar.%s' % ext
cmdline = _mock_java_cmd_api(config_file)
FakeProcesses.cmdLine = cmdline
helper = mon._MonAPIJavaHelper(cmdline=cmdline)
helper._read_config_file = rcf = mock.Mock()
helper.get_bound_port = mock.Mock(return_value=123)
self._mon_api._init_impl_helper = iih = mock.Mock(
return_value=helper
)
self._detect()
iih.assert_called_once_with(cmdline, 'java')
rcf.assert_called_once_with(config_file)
self.assertTrue(iih.get_bound_port.called_once)
def test_should_detect_java_api_has_config(self):
app_port = random.randint(1000, 10000)
admin_port = random.randint(1000, 10000)
@ -368,7 +463,7 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
if admin_port % 2 != 0:
admin_type += 's'
FakeProcesses.cmdLine = [_JAVA_CMD_API]
FakeProcesses.cmdLine = _JAVA_CMD_API
FakeProcesses.inetConnections = [FakeInetConnection(app_port)]
yml_cfg = _JAVA_YML_CFG_BIT_API.format(
@ -392,7 +487,7 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
expected_port = 6666
actual_port = 6666
FakeProcesses.cmdLine = [_PYTHON_CMD_API]
FakeProcesses.cmdLine = _PYTHON_CMD_API
FakeProcesses.inetConnections = [FakeInetConnection(actual_port)]
# make sure we return the port as we would read from the cfg
@ -414,7 +509,7 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
actual_port = 8070
# assume having python implementation
FakeProcesses.cmdLine = [_PYTHON_CMD_API]
FakeProcesses.cmdLine = _PYTHON_CMD_API
FakeProcesses.inetConnections = [FakeInetConnection(actual_port)]
# make sure we return the port as we would read from the cfg
@ -436,16 +531,16 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
'be configured.' % expected_port)
def test_build_java_config_no_hibernate(self):
self._run_java_build_config(False)
self._run_java_build_config(hibernate_enabled=False)
def test_build_java_config_with_hibernate(self):
self._run_java_build_config(True)
self._run_java_build_config(hibernate_enabled=True)
@mock.patch('six.moves.configparser.RawConfigParser')
def test_build_python_config(self, rcp):
expected_port = 8070
FakeProcesses.cmdLine = [_PYTHON_CMD_API]
FakeProcesses.cmdLine = _PYTHON_CMD_API
FakeProcesses.inetConnections = [FakeInetConnection(expected_port)]
rcp.getint.return_value = expected_port
@ -466,14 +561,14 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
self.assertNotEqual({}, bit)
def _run_java_build_config(self, hibernate_enabled):
FakeProcesses.cmdLine = [_JAVA_CMD_API]
FakeProcesses.cmdLine = _JAVA_CMD_API
app_port = random.randint(1000, 10000)
admin_port = random.randint(1000, 10000)
admin_type = 'http'
if admin_port % 2 != 0:
admin_type += 's'
FakeProcesses.cmdLine = [_JAVA_CMD_API]
FakeProcesses.cmdLine = _JAVA_CMD_API
FakeProcesses.inetConnections = [FakeInetConnection(app_port)]
# note(trebskit) this is always set to 2
@ -550,8 +645,9 @@ class TestMonAPIDetectionPlugin(unittest.TestCase):
self.assertNotEqual({}, conf)
return conf
def _detect(self, retval=[FakeProcesses()]):
def _detect(self, args=None, retval=[FakeProcesses()]):
self._mon_api.available = False
self.args = args if args else {}
process_iter = mock.patch.object(psutil, 'process_iter',
return_value=retval)