Add sample config and it's checker

* Add scripts for generation sample configs
* Fix name in setup.cfg, otherwise module is not recognized
* Add check to pep8 to follow new config parameters

Change-Id: I42918abc7fdcf9c6779c4e97ba7f2b605b9057e8
This commit is contained in:
Ekaterina Fedorova 2014-08-20 20:08:14 +04:00
parent 7cef63f300
commit 334106047c
13 changed files with 664 additions and 30 deletions

View File

@ -0,0 +1,151 @@
[DEFAULT]
#
# Options defined in muranoagent.openstack.common.eventlet_backdoor
#
# Enable eventlet backdoor. Acceptable values are 0, <port>,
# and <start>:<end>, where 0 results in listening on a random
# tcp port number; <port> results in listening on the
# specified port number (and not enabling backdoor if that
# port is in use); and <start>:<end> results in listening on
# the smallest unused port number within the specified range
# of port numbers. The chosen port is displayed in the
# service's log file. (string value)
#backdoor_port=<None>
#
# Options defined in muranoagent.openstack.common.log
#
# Print debugging output (set logging level to DEBUG instead
# of default WARNING level). (boolean value)
#debug=false
# Print more verbose output (set logging level to INFO instead
# of default WARNING level). (boolean value)
#verbose=false
# Log output to standard error. (boolean value)
#use_stderr=true
# Format string to use for log messages with context. (string
# value)
#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages without context.
# (string value)
#logging_default_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
# Data to append to log format when level is DEBUG. (string
# value)
#logging_debug_format_suffix=%(funcName)s %(pathname)s:%(lineno)d
# Prefix each line of exception output with this format.
# (string value)
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# List of logger=LEVEL pairs. (list value)
#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
# Enables or disables publication of error events. (boolean
# value)
#publish_errors=false
# Enables or disables fatal status of deprecations. (boolean
# value)
#fatal_deprecations=false
# The format for an instance that is passed with the log
# message. (string value)
#instance_format="[instance: %(uuid)s] "
# The format for an instance UUID that is passed with the log
# message. (string value)
#instance_uuid_format="[instance: %(uuid)s] "
# The name of a logging configuration file. This file is
# appended to any existing logging configuration files. For
# details about logging configuration files, see the Python
# logging module documentation. (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append=<None>
# DEPRECATED. A logging.Formatter log message format string
# which may use any of the available logging.LogRecord
# attributes. This option is deprecated. Please use
# logging_context_format_string and
# logging_default_format_string instead. (string value)
#log_format=<None>
# Format string for %%(asctime)s in log records. Default:
# %(default)s . (string value)
#log_date_format=%Y-%m-%d %H:%M:%S
# (Optional) Name of log file to output to. If no default is
# set, logging will go to stdout. (string value)
# Deprecated group/name - [DEFAULT]/logfile
#log_file=<None>
# (Optional) The base directory used for relative --log-file
# paths. (string value)
# Deprecated group/name - [DEFAULT]/logdir
#log_dir=<None>
# Use syslog for logging. Existing syslog format is DEPRECATED
# during I, and will change in J to honor RFC5424. (boolean
# value)
#use_syslog=false
# (Optional) Enables or disables syslog rfc5424 format for
# logging. If enabled, prefixes the MSG part of the syslog
# message with APP-NAME (RFC5424). The format without the APP-
# NAME is deprecated in I, and will be removed in J. (boolean
# value)
#use_syslog_rfc_format=false
# Syslog facility to receive log lines. (string value)
#syslog_log_facility=LOG_USER
[rabbitmq]
#
# Options defined in muranoagent.common.config
#
# The RabbitMQ broker address which used for communication
# with Murano guest agents. (string value)
#host=localhost
# The RabbitMQ broker port. (integer value)
#port=5672
# The RabbitMQ login. (string value)
#login=guest
# The RabbitMQ password. (string value)
#password=guest
# The RabbitMQ virtual host. (string value)
#virtual_host=/
# Boolean flag to enable SSL communication through the
# RabbitMQ broker between murano-engine and guest agents.
# (boolean value)
#ssl=false
# SSL cert file (valid only if SSL enabled). (string value)
#ca_certs=
# This value should be obtained from API (string value)
#result_routing_key=<None>
# This value must be obtained from API (string value)
#result_exchange=
# This value must be obtained from API (string value)
#input_queue=

View File

@ -21,7 +21,7 @@ from execution_plan_queue import ExecutionPlanQueue
from execution_result import ExecutionResult
from openstack.common import log as logging
from openstack.common import service
from config import CONF
from muranoagent.common.config import CONF
from muranoagent.common.messaging import MqClient, Message
from exceptions import AgentException
from time import sleep

View File

@ -27,7 +27,7 @@ if os.path.exists(os.path.join(possible_topdir,
'__init__.py')):
sys.path.insert(0, possible_topdir)
from muranoagent import config
from muranoagent.common import config
from muranoagent.openstack.common import log
from muranoagent.openstack.common import service
from muranoagent.app import MuranoAgent

View File

@ -29,36 +29,46 @@ from oslo.config import cfg
CONF = cfg.CONF
CONF.register_cli_opt(cfg.StrOpt('storage'))
CONF.register_cli_opt(cfg.StrOpt('storage',
default='/var/murano/plans',
help='Directory to store execution plans'))
rabbit_opts = [
cfg.StrOpt('host', default='localhost'),
cfg.IntOpt('port', default=5672),
cfg.StrOpt('login', default='guest'),
cfg.StrOpt('password', default='guest'),
cfg.StrOpt('virtual_host', default='/'),
cfg.BoolOpt('ssl', default=False),
cfg.StrOpt('ca_certs', default=''),
cfg.StrOpt('result_routing_key'),
cfg.StrOpt('result_exchange', default=''),
cfg.StrOpt('input_queue', default='')
cfg.StrOpt('host',
help='The RabbitMQ broker address which used for communication '
'with Murano guest agents.',
default='localhost'),
cfg.IntOpt('port', help='The RabbitMQ broker port.', default=5672),
cfg.StrOpt('login',
help='The RabbitMQ login.',
default='guest'),
cfg.StrOpt('password',
help='The RabbitMQ password.',
default='guest'),
cfg.StrOpt('virtual_host',
help='The RabbitMQ virtual host.',
default='/'),
cfg.BoolOpt('ssl',
help='Boolean flag to enable SSL communication through the '
'RabbitMQ broker between murano-engine and guest agents.',
default=False),
cfg.StrOpt('ca_certs',
help='SSL cert file (valid only if SSL enabled).',
default=''),
cfg.StrOpt('result_routing_key',
help='This value should be obtained from API'),
cfg.StrOpt('result_exchange',
help='This value must be obtained from API',
default=''),
cfg.StrOpt('input_queue',
help='This value must be obtained from API',
default='')
]
CONF.register_opts(rabbit_opts, group='rabbitmq')
CONF.import_opt('verbose', 'muranoagent.openstack.common.log')
CONF.import_opt('debug', 'muranoagent.openstack.common.log')
CONF.import_opt('log_dir', 'muranoagent.openstack.common.log')
CONF.import_opt('log_file', 'muranoagent.openstack.common.log')
CONF.import_opt('log_config', 'muranoagent.openstack.common.log')
CONF.import_opt('log_format', 'muranoagent.openstack.common.log')
CONF.import_opt('log_date_format', 'muranoagent.openstack.common.log')
CONF.import_opt('use_syslog', 'muranoagent.openstack.common.log')
CONF.import_opt('syslog_log_facility', 'muranoagent.openstack.common.log')
def parse_args(args=None, usage=None, default_config_files=None):
CONF(args=args,
project='muranoagent',

View File

@ -18,7 +18,7 @@ import os
import shutil
import time
from bunch import Bunch
from config import CONF
from muranoagent.common.config import CONF
class ExecutionPlanQueue(object):

View File

@ -16,7 +16,7 @@
import os
import base64
import shutil
from config import CONF
from muranoagent.common.config import CONF
class FilesManager(object):

View File

@ -0,0 +1,295 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import math
import re
import sys
import unicodedata
import six
from muranoagent.openstack.common.gettextutils import _
UNIT_PREFIX_EXPONENT = {
'k': 1,
'K': 1,
'Ki': 1,
'M': 2,
'Mi': 2,
'G': 3,
'Gi': 3,
'T': 4,
'Ti': 4,
}
UNIT_SYSTEM_INFO = {
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
}
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
# NOTE(flaper87): The following 3 globals are used by `mask_password`
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
'.*?([\'"])',
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
def int_from_bool_as_string(subject):
"""Interpret a string as a boolean and return either 1 or 0.
Any string value in:
('True', 'true', 'On', 'on', '1')
is interpreted as a boolean True.
Useful for JSON-decoded stuff and config file parsing
"""
return bool_from_string(subject) and 1 or 0
def bool_from_string(subject, strict=False, default=False):
"""Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else returns the value specified by 'default'.
Useful for JSON-decoded stuff and config file parsing.
If `strict=True`, unrecognized values, including None, will raise a
ValueError which is useful when parsing values passed in from an API call.
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, six.string_types):
subject = six.text_type(subject)
lowered = subject.strip().lower()
if lowered in TRUE_STRINGS:
return True
elif lowered in FALSE_STRINGS:
return False
elif strict:
acceptable = ', '.join(
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
msg = _("Unrecognized value '%(val)s', acceptable values are:"
" %(acceptable)s") % {'val': subject,
'acceptable': acceptable}
raise ValueError(msg)
else:
return default
def safe_decode(text, incoming=None, errors='strict'):
"""Decodes incoming text/bytes string using `incoming` if they're not
already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a unicode `incoming` encoded
representation of it.
:raises TypeError: If text is not an instance of str
"""
if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be decoded" % type(text))
if isinstance(text, six.text_type):
return text
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
try:
return text.decode(incoming, errors)
except UnicodeDecodeError:
# Note(flaper87) If we get here, it means that
# sys.stdin.encoding / sys.getdefaultencoding
# didn't return a suitable encoding to decode
# text. This happens mostly when global LANG
# var is not set correctly and there's no
# default encoding. In this case, most likely
# python will use ASCII or ANSI encoders as
# default encodings but they won't be capable
# of decoding non-ASCII characters.
#
# Also, UTF-8 is being used since it's an ASCII
# extension.
return text.decode('utf-8', errors)
def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'):
"""Encodes incoming text/bytes string using `encoding`.
If incoming is not specified, text is expected to be encoded with
current python's default encoding. (`sys.getdefaultencoding`)
:param incoming: Text's current encoding
:param encoding: Expected encoding for text (Default UTF-8)
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a bytestring `encoding` encoded
representation of it.
:raises TypeError: If text is not an instance of str
"""
if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be encoded" % type(text))
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
if isinstance(text, six.text_type):
return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = safe_decode(text, incoming, errors)
return text.encode(encoding, errors)
else:
return text
def string_to_bytes(text, unit_system='IEC', return_int=False):
"""Converts a string into an float representation of bytes.
The units supported for IEC ::
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
KB, KiB, MB, MiB, GB, GiB, TB, TiB
The units supported for SI ::
kb(it), Mb(it), Gb(it), Tb(it)
kB, MB, GB, TB
Note that the SI unit system does not support capital letter 'K'
:param text: String input for bytes size conversion.
:param unit_system: Unit system for byte size conversion.
:param return_int: If True, returns integer representation of text
in bytes. (default: decimal)
:returns: Numerical representation of text in bytes.
:raises ValueError: If text has an invalid value.
"""
try:
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
except KeyError:
msg = _('Invalid unit system: "%s"') % unit_system
raise ValueError(msg)
match = reg_ex.match(text)
if match:
magnitude = float(match.group(1))
unit_prefix = match.group(2)
if match.group(3) in ['b', 'bit']:
magnitude /= 8
else:
msg = _('Invalid string format: %s') % text
raise ValueError(msg)
if not unit_prefix:
res = magnitude
else:
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
if return_int:
return int(math.ceil(res))
return res
def to_slug(value, incoming=None, errors="strict"):
"""Normalize string.
Convert to lowercase, remove non-word characters, and convert spaces
to hyphens.
Inspired by Django's `slugify` filter.
:param value: Text to slugify
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: slugified unicode representation of `value`
:raises TypeError: If text is not an instance of str
"""
value = safe_decode(value, incoming, errors)
# NOTE(aababilov): no need to use safe_(encode|decode) here:
# encodings are always "ascii", error handling is always "ignore"
# and types are always known (first: unicode; second: str)
value = unicodedata.normalize("NFKD", value).encode(
"ascii", "ignore").decode("ascii")
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
return SLUGIFY_HYPHENATE_RE.sub("-", value)
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message

View File

@ -0,0 +1,106 @@
# Copyright 2012-2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Helper module for systemd service readiness notification.
"""
import os
import socket
import sys
from muranoagent.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def _abstractify(socket_name):
if socket_name.startswith('@'):
# abstract namespace socket
socket_name = '\0%s' % socket_name[1:]
return socket_name
def _sd_notify(unset_env, msg):
notify_socket = os.getenv('NOTIFY_SOCKET')
if notify_socket:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
try:
sock.connect(_abstractify(notify_socket))
sock.sendall(msg)
if unset_env:
del os.environ['NOTIFY_SOCKET']
except EnvironmentError:
LOG.debug("Systemd notification failed", exc_info=True)
finally:
sock.close()
def notify():
"""Send notification to Systemd that service is ready.
For details see
http://www.freedesktop.org/software/systemd/man/sd_notify.html
"""
_sd_notify(False, 'READY=1')
def notify_once():
"""Send notification once to Systemd that service is ready.
Systemd sets NOTIFY_SOCKET environment variable with the name of the
socket listening for notifications from services.
This method removes the NOTIFY_SOCKET environment variable to ensure
notification is sent only once.
"""
_sd_notify(True, 'READY=1')
def onready(notify_socket, timeout):
"""Wait for systemd style notification on the socket.
:param notify_socket: local socket address
:type notify_socket: string
:param timeout: socket timeout
:type timeout: float
:returns: 0 service ready
1 service not ready
2 timeout occurred
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sock.settimeout(timeout)
sock.bind(_abstractify(notify_socket))
try:
msg = sock.recv(512)
except socket.timeout:
return 2
finally:
sock.close()
if 'READY=1' in msg:
return 0
else:
return 1
if __name__ == '__main__':
# simple CLI for testing
if len(sys.argv) == 1:
notify()
elif len(sys.argv) >= 2:
timeout = float(sys.argv[1])
notify_socket = os.getenv('NOTIFY_SOCKET')
if notify_socket:
retval = onready(notify_socket, timeout)
sys.exit(retval)

View File

@ -2,13 +2,15 @@
# The list of modules to copy from openstack-common
module=exception
module=eventlet_backdoor
module=importutils
module=jsonutils
module=log
module=install_venv_common
module=config
module=service
module=strutils
# The base module to hold the copy of openstack.common
base=muranoagent
base=muranoagent

View File

@ -1,9 +1,11 @@
[metadata]
name = murano-agent
name = muranoagent
summary = Python Murano Agent
description-file =
README.rst
license = Apache License, Version 2.0
author = Mirantis, Inc.
author-email = murano-all@lists.openstack.org
author-email = openstack-dev@lists.openstack.org
home-page = htts://launchpad.net/murano
classifier =
Development Status :: 5 - Production/Stable
@ -20,7 +22,7 @@ packages =
[global]
setup-hooks =
pbr.hooks.setup_hook
pbr.hooks.setup_hook
[entry_points]
console_scripts =

38
tools/config/README Normal file
View File

@ -0,0 +1,38 @@
This generate_sample.sh tool is used to generate sample config files
from OpenStack project source trees.
Run it by passing the base directory and package name i.e.
$> generate_sample.sh --base-dir /opt/stack/nova --package-name nova \
--output-dir /opt/stack/nova/etc
$> generate_sample.sh -b /opt/stack/neutron -p nova -o /opt/stack/neutron/etc
Optionally, include libraries that register entry points for option
discovery, such as oslo.messaging:
$> generate_sample.sh -b /opt/stack/ceilometer -p ceilometer \
-o /opt/stack/ceilometer/etc -l oslo.messaging
Watch out for warnings about modules like libvirt, qpid and zmq not
being found - these warnings are significant because they result
in options not appearing in the generated config file.
This check_uptodate.sh tool is used to ensure that the generated sample
config file in the OpenStack project source tree is continually kept up
to date with the code itself.
This can be done by adding a hook to tox.ini. For example, if a project
already had flake8 enabled in a section like this:
[testenv.pep8]
commands =
flake8 {posargs}
This section would be changed to:
[testenv.pep8]
commands =
flake8 {posargs}
{toxinidir}/tools/config/check_uptodate.sh

29
tools/config/check_uptodate.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
PROJECT_NAME=${PROJECT_NAME:-muranoagent}
CFGFILE_NAME=${PROJECT_NAME}.conf.sample
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
elif [ -e etc/${CFGFILE_NAME} ]; then
CFGFILE=etc/${CFGFILE_NAME}
else
echo "${0##*/}: can not find config file"
exit 1
fi
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
trap "rm -rf $TEMPDIR" EXIT
tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
if [ $? != 0 ]
then
exit 1
fi
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
then
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
exit 1
fi

View File

@ -17,6 +17,7 @@ commands = nosetests
[testenv:pep8]
commands =
flake8 {posargs}
{toxinidir}/tools/config/check_uptodate.sh
[testenv:venv]
commands = {posargs}