Various fixes to make it easier to run in diverse environments

Removed all the lsb helper utils from the sysv init script
Added a better error message for template missing.
Added the ability to skip the definition of the monasca_api_url
Made check frequency configurable and added an option to monasca setup
Also set the forwarder flush interval to be calculated based on check
frequency.
Modified monasca-setup so it can work with non-standard paths.

Change-Id: If8916e17bf42ffb804fa6c79166149a9fd33c553
This commit is contained in:
Tim Kuhlman 2015-02-18 09:43:37 -07:00
parent 51b4f9b221
commit 721b013d11
11 changed files with 174 additions and 144 deletions

View File

@ -6,8 +6,8 @@
# Set username, password and project id.
# Set username, password, project name and (domain id or domain name).
#
# Monitoring API URL: URL for the monitoring API
# Example: https://region-a.geo-1.monitoring.hpcloudsvc.com/v1.1/metrics
# Monitoring API URL: URL for the monitoring API, if undefined it will be pulled from the keystone service catalog
# Example: https://region-a.geo-1.monitoring.hpcloudsvc.com:8080/v2.0
url: {args.monasca_url}
# Keystone Username
username: {args.username}
@ -62,6 +62,9 @@ dimensions: {args.dimensions}
# Defaults to 30 seconds if no value is provided
#recent_point_threshold: 30
# time to wait between collection runs
check_freq: {args.check_frequency}
# Use mount points instead of volumes to track disk and fs metrics
use_mount: no

View File

@ -16,14 +16,12 @@ except ImportError:
import monasca_agent.common.singleton as singleton
DEFAULT_CONFIG_FILE = '/etc/monasca/agent/agent.conf'
DEFAULT_CHECK_FREQUENCY = 15 # seconds
DEFAULT_LOG_DIR = '/var/log/monasca/agent'
DEFAULT_STATSD_FREQUENCY = 20 # seconds
DEFAULT_STATSD_INTERVAL = 10 # seconds
LOGGING_MAX_BYTES = 5 * 1024 * 1024
log = logging.getLogger(__name__)
class Config(object):
# Make this a singleton class so we don't get the config every time
# the class is created
@ -43,7 +41,7 @@ class Config(object):
self._read_config()
def get_config(self, sections='Main'):
'''Get the config info.'''
"""Get the config info."""
section_list = []
if isinstance(sections, six.string_types):
section_list.append(sections)
@ -63,7 +61,7 @@ class Config(object):
return pkg_resources.require("monasca-agent")[0].version
def _read_config(self):
'''Read in the config file.'''
"""Read in the config file."""
file_config = parser.SafeConfigParser()
log.debug("Loading config file from {0}".format(self._configFile))
@ -74,61 +72,55 @@ class Config(object):
self._parse_config()
def _retrieve_sections(self, config):
'''Get the section values from the config file.'''
"""Get the section values from the config file."""
# Define default values for the possible config items
the_config = { 'Main': {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'forwarder_url': 'http://localhost:17123',
'hostname': None,
'dimensions': None,
'listen_port': None,
'version': self.get_version(),
'additional_checksd': os.path.join(os.path.dirname(self._configFile), '/checks_d/'),
'system_metrics': None,
'ignore_filesystem_types': None,
'device_blacklist_re': None,
'limit_memory_consumption': None,
'skip_ssl_validation': False,
'watchdog': True,
'use_mount': False,
'autorestart': False,
'non_local_traffic': False
}, 'Api': {
'is_enabled': False,
'url': '',
'project_name': '',
'project_id': '',
'project_domain_name': '',
'project_domain_id': '',
'ca_file': '',
'insecure': '',
'username': '',
'password': '',
'use_keystone': True,
'keystone_url': '',
'max_buffer_size': 1000,
'backlog_send_rate': 5
}, 'Statsd': {
'recent_point_threshold': None,
'monasca_statsd_interval': DEFAULT_STATSD_FREQUENCY,
'monasca_statsd_agregator_interval': DEFAULT_STATSD_INTERVAL,
'monasca_statsd_forward_host': None,
'monasca_statsd_forward_port': 8125,
'monasca_statsd_port': 8125
}, 'Logging': {
'disable_file_logging': False,
'log_level': None,
'collector_log_file': DEFAULT_LOG_DIR + '/collector.log',
'forwarder_log_file': DEFAULT_LOG_DIR + '/forwarder.log',
'statsd_log_file': DEFAULT_LOG_DIR + '/statsd.log',
'jmxfetch_log_file': DEFAULT_LOG_DIR + '/jmxfetch.log',
'log_to_event_viewer': False,
'log_to_syslog': False,
'syslog_host': None,
'syslog_port': None
}
}
the_config = {'Main': {'check_freq': 15,
'forwarder_url': 'http://localhost:17123',
'hostname': None,
'dimensions': None,
'listen_port': None,
'version': self.get_version(),
'additional_checksd': os.path.join(os.path.dirname(self._configFile), '/checks_d/'),
'system_metrics': None,
'ignore_filesystem_types': None,
'device_blacklist_re': None,
'limit_memory_consumption': None,
'skip_ssl_validation': False,
'watchdog': True,
'use_mount': False,
'autorestart': False,
'non_local_traffic': False},
'Api': {'is_enabled': False,
'url': '',
'project_name': '',
'project_id': '',
'project_domain_name': '',
'project_domain_id': '',
'ca_file': '',
'insecure': '',
'username': '',
'password': '',
'use_keystone': True,
'keystone_url': '',
'max_buffer_size': 1000,
'backlog_send_rate': 5},
'Statsd': {'recent_point_threshold': None,
'monasca_statsd_interval': 20,
'monasca_statsd_agregator_interval': 10,
'monasca_statsd_forward_host': None,
'monasca_statsd_forward_port': 8125,
'monasca_statsd_port': 8125},
'Logging': {'disable_file_logging': False,
'log_level': None,
'collector_log_file': DEFAULT_LOG_DIR + '/collector.log',
'forwarder_log_file': DEFAULT_LOG_DIR + '/forwarder.log',
'statsd_log_file': DEFAULT_LOG_DIR + '/statsd.log',
'jmxfetch_log_file': DEFAULT_LOG_DIR + '/jmxfetch.log',
'log_to_event_viewer': False,
'log_to_syslog': False,
'syslog_host': None,
'syslog_port': None}}
# Load values from configuration file into config file dictionary
for section in config.sections():
@ -137,7 +129,8 @@ class Config(object):
option_value = config.get(section, option)
if option_value == -1:
log.debug("Config option missing: {0}, using default value of {1}".format(option,
the_config[section][option]))
the_config[section][
option]))
else:
the_config[section][option] = option_value
@ -147,7 +140,7 @@ class Config(object):
return the_config
def _skip_leading_wsp(self, file):
'''Works on a file, returns a file-like object'''
"""Works on a file, returns a file-like object"""
return cstringio.StringIO("\n".join(map(string.strip, file.readlines())))
def _parse_config(self):
@ -228,5 +221,6 @@ def main():
print "Main Configuration: \n {}".format(config)
print "\nApi Configuration: \n {}".format(api_config)
if __name__ == "__main__":
main()

View File

@ -1,11 +1,12 @@
import logging
from keystoneclient.v3 import client as ksclient
from monascaclient import ksclient
import monasca_agent.common.singleton as singleton
log = logging.getLogger(__name__)
class Keystone(object):
# Make this a singleton class so we don't get the token every time
# the class is created
@ -38,17 +39,26 @@ class Keystone(object):
kc_args.update({'insecure': insecure})
else:
if cacert:
kc_args.update({'cacert': cacert})
kc_args.update({'os_cacert': cacert})
if project_id:
kc_args.update({'project_id': project_id})
elif project_name:
kc_args.update({'project_name': project_name})
if project_domain_name:
kc_args.update({'project_domain_name': project_domain_name})
kc_args.update({'domain_name': project_domain_name})
if project_domain_id:
kc_args.update({'project_domain_id': project_domain_id})
kc_args.update({'domain_id': project_domain_id})
return ksclient.Client(**kc_args)
return ksclient.KSClient(**kc_args)
def get_monasca_url(self):
if not self._keystone_client:
self.get_token()
if self._keystone_client:
return self._keystone_client.monasca_url
else:
return None
def get_token(self):
"""Validate token is project scoped and return it if it is
@ -64,16 +74,9 @@ class Keystone(object):
log.error("Unable to create the Keystone Client. " +
"Error was {}".format(repr(exc)))
return None
if self._keystone_client.project_id:
self._token = self._keystone_client.auth_token
else:
self._token = None
self._keystone_client = None
raise exc.CommandError("User does not have a default project. "
"You must provide the following parameters "
"in the agent config file: "
"project_id OR (project_name AND "
"(project_domain OR project_domain_name)).")
self._token = self._keystone_client.token
return self._token
def refresh_token(self):

View File

@ -115,6 +115,9 @@ class MonAPI(object):
'token': token
}
if not self.url:
self.url = self.keystone.get_monasca_url()
return monascaclient.client.Client(self.api_version, self.url, **kwargs)
return None

View File

@ -38,7 +38,6 @@ import monasca_agent.forwarder.transaction as transaction
log = logging.getLogger('forwarder')
TRANSACTION_FLUSH_INTERVAL = 5000 # Every 5 seconds
WATCHDOG_INTERVAL_MULTIPLIER = 10 # 10x flush interval
# Maximum delay before replaying a transaction
@ -103,6 +102,7 @@ class Forwarder(tornado.web.Application):
use_simple_http_client=False):
self._port = int(port)
self._agent_config = agent_config
self.flush_interval = int(agent_config.get('check_freq'))/2
self._metrics = {}
transaction.MetricTransaction.set_application(self)
transaction.MetricTransaction.set_endpoints(mon.MonAPI(agent_config))
@ -120,7 +120,7 @@ class Forwarder(tornado.web.Application):
log.info("Skipping SSL hostname validation, useful when using a transparent proxy")
if watchdog:
watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER
watchdog_timeout = self.flush_interval * WATCHDOG_INTERVAL_MULTIPLIER
self._watchdog = util.Watchdog(
watchdog_timeout, max_mem_mb=agent_config.get('limit_memory_consumption', None))
@ -204,7 +204,7 @@ class Forwarder(tornado.web.Application):
self._tr_manager.flush()
tr_sched = tornado.ioloop.PeriodicCallback(
flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop)
flush_trs, self.flush_interval, io_loop=self.mloop)
# Start everything
if self._watchdog:

View File

@ -24,6 +24,27 @@ OS_SERVICE_MAP = {'Linux': sysv.SysV}
log = logging.getLogger(__name__)
# dirname is called twice to get the dir 1 above the location of the script
PREFIX_DIR = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
def write_template(template_path, out_path, variables, group):
""" Write a file using a simple python string template.
Assumes 640 for the permissions and root:group for ownership.
:param template_path: Location of the Template to use
:param out_path: Location of the file to write
:param variables: dictionary with key/value pairs to use in writing the template
:return: None
"""
if not os.path.exists(template_path):
print("Error no template found at {0}".format(template_path))
sys.exit(1)
with open(template_path, 'r') as template:
with open(out_path, 'w') as conf:
conf.write(template.read().format(**variables))
os.chown(out_path, 0, group)
os.chmod(out_path, 0640)
def main(argv=None):
parser = argparse.ArgumentParser(description='Detect running daemons then configure and start the agent.',
@ -33,7 +54,7 @@ def main(argv=None):
parser.add_argument(
'-p', '--password', help="Password used for keystone authentication", required=True)
parser.add_argument('--keystone_url', help="Keystone url", required=True)
parser.add_argument('--monasca_url', help="Monasca API url", required=True)
parser.add_argument('--monasca_url', help="Monasca API url, if not defined the url is pulled from keystone", required=False, default='')
parser.add_argument('--insecure', help="Set whether certificates are used for Keystone authentication", required=False, default=False)
parser.add_argument('--project_name', help="Project name for keystone authentication", required=False, default='')
parser.add_argument('--project_domain_id', help="Project domain id for keystone authentication", required=False, default='')
@ -41,13 +62,14 @@ def main(argv=None):
parser.add_argument('--project_id', help="Keystone project id for keystone authentication", required=False, default='')
parser.add_argument('--ca_file', help="Sets the path to the ca certs file if using certificates. " +
"Required only if insecure is set to False", required=False, default='')
parser.add_argument('--check_frequency', help="How often to run metric collection in seconds", type=int, default=15)
parser.add_argument('--config_dir', help="Configuration directory", default='/etc/monasca/agent')
parser.add_argument('--dimensions', help="Additional dimensions to set for all metrics. A comma seperated list " +
"of name/value pairs, 'name:value,name2:value2'")
parser.add_argument('--log_dir', help="monasca-agent log directory", default='/var/log/monasca/agent')
parser.add_argument('--log_level', help="monasca-agent logging level (ERROR, WARNING, INFO, DEBUG)", required=False, default='INFO')
parser.add_argument(
'--template_dir', help="Alternative template directory", default='/usr/local/share/monasca/agent')
'--template_dir', help="Alternative template directory", default=os.path.join(PREFIX_DIR, 'share/monasca/agent'))
parser.add_argument('--headless', help="Run in a non-interactive mode", action="store_true")
parser.add_argument('--overwrite',
help="Overwrite existing plugin configuration. " +
@ -75,11 +97,11 @@ def main(argv=None):
linux_flavor = platform.linux_distribution()[0]
if 'Ubuntu' or 'debian' in linux_flavor:
for package in ['coreutils', 'sysstat']:
#Check for required dependencies for system checks
# Check for required dependencies for system checks
try:
output = check_output('dpkg -s {0}'.format(package),
stderr=subprocess.STDOUT,
shell=True)
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError:
log.warn("*** {0} package is not installed! ***".format(package) +
"\nNOTE: If you do not install the {0} ".format(package) +
@ -98,32 +120,33 @@ def main(argv=None):
# Service enable, includes setup of users/config directories so must be
# done before configuration
agent_service = OS_SERVICE_MAP[detected_os](os.path.join(args.template_dir, 'monasca-agent.init'),
agent_service = OS_SERVICE_MAP[detected_os](PREFIX_DIR,
args.config_dir,
args.log_dir, username=args.user)
args.log_dir,
args.template_dir,
username=args.user)
if not args.skip_enable:
agent_service.enable()
gid = pwd.getpwnam(args.user).pw_gid
# Write the main agent.conf - Note this is always overwritten
log.info('Configuring base Agent settings.')
agent_conf_path = os.path.join(args.config_dir, 'agent.conf')
with open(os.path.join(args.template_dir, 'agent.conf.template'), 'r') as agent_template:
with open(agent_conf_path, 'w') as agent_conf:
# Join service in with the dimensions
if args.service:
if args.dimensions is None:
args.dimensions = 'service:' + args.service
else:
args.dimensions = ','.join([args.dimensions, 'service:' + args.service])
agent_conf.write(agent_template.read().format(args=args, hostname=socket.getfqdn()))
os.chown(agent_conf_path, 0, gid)
os.chmod(agent_conf_path, 0o640)
# Link the supervisor.conf
supervisor_path = os.path.join(args.config_dir, 'supervisor.conf')
if os.path.exists(supervisor_path):
os.remove(supervisor_path)
os.symlink(os.path.join(args.template_dir, 'supervisor.conf'), supervisor_path)
# Join service in with the dimensions
if args.service:
if args.dimensions is None:
args.dimensions = 'service:' + args.service
else:
args.dimensions = ','.join([args.dimensions, 'service:' + args.service])
write_template(os.path.join(args.template_dir, 'agent.conf.template'),
os.path.join(args.config_dir, 'agent.conf'),
{'args': args, 'hostname': socket.getfqdn() },
gid)
# Write the supervisor.conf
write_template(os.path.join(args.template_dir, 'supervisor.conf.template'),
os.path.join(args.config_dir, 'supervisor.conf'),
{'prefix': PREFIX_DIR, 'log_dir': args.log_dir},
gid)
# Run through detection and config building for the plugins
plugin_config = agent_config.Plugins()

View File

@ -10,7 +10,8 @@ class Service(object):
"""
def __init__(self, config_dir, log_dir, name='monasca-agent'):
def __init__(self, prefix_dir, config_dir, log_dir, name='monasca-agent'):
self.prefix_dir = prefix_dir
self.config_dir = config_dir
self.log_dir = log_dir
self.name = name

View File

@ -14,13 +14,13 @@ log = logging.getLogger(__name__)
class SysV(service.Service):
def __init__(self, init_template, config_dir, log_dir, name='monasca-agent', username='monasca-agent'):
def __init__(self, prefix_dir, config_dir, log_dir, template_dir, name='monasca-agent', username='monasca-agent'):
"""Setup this service with the given init template.
"""
super(SysV, self).__init__(config_dir, log_dir, name)
super(SysV, self).__init__(prefix_dir, config_dir, log_dir, name)
self.init_script = '/etc/init.d/%s' % self.name
self.init_template = init_template
self.init_template = os.path.join(template_dir, 'monasca-agent.init.template')
self.username = username
def enable(self):
@ -39,15 +39,17 @@ class SysV(service.Service):
# todo log dir is hardcoded
for path in (self.log_dir, self.config_dir, '%s/conf.d' % self.config_dir):
if not os.path.exists(path):
os.makedirs(path, 0o755)
os.makedirs(path, 0755)
os.chown(path, 0, user.pw_gid)
# the log dir needs to be writable by the user
os.chown(self.log_dir, user.pw_uid, user.pw_gid)
# link the init script, then enable
if not os.path.exists(self.init_script):
os.symlink(self.init_template, self.init_script)
os.chmod(self.init_script, 0o755)
# Write the init script and enable.
with open(self.init_template, 'r') as template:
with open(self.init_script, 'w') as conf:
conf.write(template.read().format(prefix=self.prefix_dir, config_dir=self.config_dir))
os.chown(self.init_script, 0, 0)
os.chmod(self.init_script, 0755)
for runlevel in ['2', '3', '4', '5']:
link_path = '/etc/rc%s.d/S10monasca-agent' % runlevel

View File

@ -10,20 +10,19 @@
# Default-Stop: 0 1 6
### END INIT INFO
. /lib/lsb/init-functions
PATH=$PATH:/usr/local/bin # supervisord might live here
PATH=$PATH:/sbin # add the location of start-stop-daemon on Debian
PATH=$PATH:/sbin
AGENTPATH="/usr/local/bin/monasca-collector"
AGENTCONF="/etc/monasca/agent/agent.conf"
MONASCASTATSDPATH="/usr/local/bin/monasca-statsd"
AGENTPATH="{prefix}/bin/monasca-collector"
AGENTCONF="{config_dir}/agent.conf"
MONASCASTATSDPATH="{prefix}/bin/monasca-statsd"
AGENTUSER="monasca-agent"
FORWARDERPATH="/usr/local/bin/monasca-forwarder"
FORWARDERPATH="{prefix}/bin/monasca-forwarder"
NAME="monasca-agent"
DESC="Monasca Monitoring Agent"
AGENT_PID_PATH="/var/tmp/monasca-agent.pid"
SUPERVISOR_PIDFILE="/var/tmp/monasca-agent-supervisord.pid"
SUPERVISOR_FILE="/etc/monasca/agent/supervisor.conf"
SUPERVISOR_FILE="{config_dir}/supervisor.conf"
SUPERVISOR_SOCK="/var/tmp/monasca-agent-supervisor.sock"
SUPERVISORD=$(which supervisord)
@ -35,7 +34,7 @@ if [ ! -x $AGENTPATH ]; then
exit 0
fi
check_status() {
check_status() {{
# If the socket exists, we can use supervisorctl
if [ -e $SUPERVISOR_SOCK ]; then
# If we're using supervisor, check the number of processes
@ -63,7 +62,7 @@ check_status() {
echo "$DESC (supervisor) is not running"
return 1
fi
}
}}
# Action to take
case "$1" in
@ -81,23 +80,22 @@ case "$1" in
su $AGENTUSER -c "$AGENTPATH configcheck" > /dev/null
if [ $? -ne 0 ]; then
log_daemon_msg "Invalid check configuration. Please run sudo /etc/init.d/monasca-agent configtest for more details."
log_daemon_msg "Resuming starting process."
echo "Invalid check configuration. Please run sudo /etc/init.d/monasca-agent configtest for more details."
echo "Resuming starting process."
fi
log_daemon_msg "Starting $DESC (using supervisord)" "$NAME"
start-stop-daemon --start --quiet --oknodo --exec $SUPERVISORD -- -c $SUPERVISOR_FILE -u $AGENTUSER --pidfile $SUPERVISOR_PIDFILE
echo "Starting $DESC (using supervisord)" "$NAME"
$SUPERVISORD -c $SUPERVISOR_FILE -u $AGENTUSER --pidfile $SUPERVISOR_PIDFILE
if [ $? -ne 0 ]; then
log_end_msg 1
exit $?
fi
# check if the agent is running once per second for 10 seconds
retries=10
while [ $retries -gt 1 ]; do
if check_status > /dev/null; then
# We've started up successfully. Exit cleanly
log_end_msg 0
exit 0
else
retries=$(($retries - 1))
@ -105,17 +103,21 @@ case "$1" in
fi
done
# After 10 tries the agent didn't start. Report an error
log_end_msg 1
exit 1
check_status # report what went wrong
$0 stop
exit 1
;;
stop)
log_daemon_msg "Stopping $DESC (stopping supervisord)" "$NAME"
start-stop-daemon --stop --retry 30 --quiet --oknodo --pidfile $SUPERVISOR_PIDFILE
log_end_msg $?
echo "Stopping $DESC (stopping supervisord)" "$NAME"
if [ -e $SUPERVISOR_PIDFILE ]; then
kill `cat $SUPERVISOR_PIDFILE`
else
echo "Pid file $SUPERVISOR_PIDFILE not found, nothing to stop"
fi
exit $?
;;
@ -158,8 +160,7 @@ case "$1" in
;;
*)
N=/etc/init.d/$NAME
echo "Usage: $N {start|stop|restart|info|status|configcheck|configtest|jmx}"
echo "Usage: /etc/init.d/$NAME {{start|stop|restart|info|status|configcheck|configtest|jmx}}"
exit 1
;;
esac

View File

@ -11,14 +11,14 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
minfds = 1024
minprocs = 200
loglevel = info
logfile = /var/log/monasca/agent/supervisord.log
logfile = {log_dir}/supervisord.log
logfile_maxbytes = 50MB
nodaemon = false
pidfile = /var/run/monasca-agent-supervisord.pid
logfile_backups = 10
[program:collector]
command=/usr/local/bin/monasca-collector foreground
command={prefix}/bin/monasca-collector foreground
stdout_logfile=NONE
stderr_logfile=NONE
priority=999
@ -26,7 +26,7 @@ startsecs=2
user=monasca-agent
[program:forwarder]
command=/usr/local/bin/monasca-forwarder
command={prefix}/bin/monasca-forwarder
stdout_logfile=NONE
stderr_logfile=NONE
startsecs=3
@ -34,7 +34,7 @@ priority=998
user=monasca-agent
[program:statsd]
command=/usr/local/bin/monasca-statsd
command={prefix}/bin/monasca-statsd
stdout_logfile=NONE
stderr_logfile=NONE
startsecs=3

View File

@ -22,8 +22,8 @@ packages =
data_files=
share/monasca/agent =
agent.conf.template
packaging/supervisor.conf
packaging/monasca-agent.init
packaging/supervisor.conf.template
packaging/monasca-agent.init.template
share/monasca/agent/conf.d = conf.d/*
[entry_points]