x
/
kwapi
Code Issues Proposed changes

PEP 8 validation (version 1.4). 

Update openstack.common.
License in each file.
This commit is contained in:
François Rossigneux 2013-01-10 13:35:18 +01:00
parent 7d294500da
commit f925e6a8dc
76 changed files with 5048 additions and 802 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/kwapi-api
View File

@ -1,5 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys import sys

14
bin/kwapi-drivers
View File

@ -1,5 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys import sys
import signal import signal

14
bin/kwapi-rrd
View File

@ -1,5 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys import sys

16
kwapi/drivers/__init__.py
View File

@ -1,2 +1,18 @@
# -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from wattsup import Wattsup from wattsup import Wattsup
from dummy import Dummy from dummy import Dummy

23
kwapi/drivers/driver.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json import json
from threading import Thread, Event from threading import Thread, Event
@ -21,12 +35,14 @@ driver_opts = [
cfg.CONF.register_opts(driver_opts) cfg.CONF.register_opts(driver_opts)
class Driver(Thread): class Driver(Thread):
"""Generic driver class, derived from Thread.""" """Generic driver class, derived from Thread."""
def __init__(self, probe_ids, kwargs): def __init__(self, probe_ids, kwargs):
"""Initializes driver.""" """Initializes driver."""
LOG.info('Loading driver %s(probe_ids=%s, kwargs=%s)' % (self.__class__.__name__, probe_ids, kwargs)) LOG.info('Loading driver %s(probe_ids=%s, kwargs=%s)'
% (self.__class__.__name__, probe_ids, kwargs))
Thread.__init__(self) Thread.__init__(self)
self.probe_ids = probe_ids self.probe_ids = probe_ids
self.kwargs = kwargs self.kwargs = kwargs
@ -36,7 +52,10 @@ class Driver(Thread):
self.publisher.connect('inproc://drivers') self.publisher.connect('inproc://drivers')
def run(self): def run(self):
"""Run the driver thread. Needs to be implemented in a derived class.""" """Runs the driver thread. Needs to be implemented in a derived
class.
"""
raise NotImplementedError raise NotImplementedError
def join(self): def join(self):

30
kwapi/drivers/driver_manager.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Loads and checks driver threads.""" """Loads and checks driver threads."""
@ -27,6 +41,7 @@ cfg.CONF.register_opts(driver_manager_opts)
threads = [] threads = []
def load_all_drivers(): def load_all_drivers():
"""Loads all drivers from config file.""" """Loads all drivers from config file."""
parser = cfg.ConfigParser(cfg.CONF.config_file[0], {}) parser = cfg.ConfigParser(cfg.CONF.config_file[0], {})
@ -42,6 +57,7 @@ def load_all_drivers():
if thread is not None: if thread is not None:
threads.append(thread) threads.append(thread)
def load_driver(class_name, probe_ids, kwargs): def load_driver(class_name, probe_ids, kwargs):
"""Starts a probe thread.""" """Starts a probe thread."""
try: try:
@ -51,11 +67,13 @@ def load_driver(class_name, probe_ids, kwargs):
try: try:
probeObject = probeClass(probe_ids, **kwargs) probeObject = probeClass(probe_ids, **kwargs)
except Exception as exception: except Exception as exception:
LOG.error('Exception occurred while initializing %s(%s, %s): %s' % (class_name, probe_ids, kwargs, exception)) LOG.error('Exception occurred while initializing %s(%s, %s): %s'
% (class_name, probe_ids, kwargs, exception))
else: else:
probeObject.start() probeObject.start()
return probeObject return probeObject
def check_drivers_alive(): def check_drivers_alive():
"""Checks all drivers and reloads those that crashed. """Checks all drivers and reloads those that crashed.
This method is executed automatically at the given interval. This method is executed automatically at the given interval.
@ -64,8 +82,11 @@ def check_drivers_alive():
LOG.info('Checks driver threads') LOG.info('Checks driver threads')
for index, thread in enumerate(threads): for index, thread in enumerate(threads):
if not thread.is_alive(): if not thread.is_alive():
LOG.warning('%s(probe_ids=%s, kwargs=%s) is crashed' % (thread.__class__.__name__, thread.probe_ids, thread.kwargs)) LOG.warning('%s(probe_ids=%s, kwargs=%s) is crashed'
new_thread = load_driver(thread.__class__.__name__, thread.probe_ids, thread.kwargs) % (thread.__class__.__name__,
thread.probe_ids, thread.kwargs))
new_thread = load_driver(thread.__class__.__name__,
thread.probe_ids, thread.kwargs)
if new_thread is not None: if new_thread is not None:
threads[index] = new_thread threads[index] = new_thread
@ -75,6 +96,7 @@ def check_drivers_alive():
timer.daemon = True timer.daemon = True
timer.start() timer.start()
def start_zmq_server(): def start_zmq_server():
"""Forwards probe values to the probes_endpoint.""" """Forwards probe values to the probes_endpoint."""
context = zmq.Context.instance() context = zmq.Context.instance()
@ -85,11 +107,13 @@ def start_zmq_server():
backend.bind(cfg.CONF.probes_endpoint) backend.bind(cfg.CONF.probes_endpoint)
thread.start_new_thread(zmq.device, (zmq.FORWARDER, frontend, backend)) thread.start_new_thread(zmq.device, (zmq.FORWARDER, frontend, backend))
def signal_handler(signum, frame): def signal_handler(signum, frame):
"""Intercepts TERM signal and properly terminates probe threads.""" """Intercepts TERM signal and properly terminates probe threads."""
if signum is signal.SIGTERM: if signum is signal.SIGTERM:
terminate() terminate()
def terminate(): def terminate():
"""Terminates driver threads.""" """Terminates driver threads."""
for driver in threads: for driver in threads:

21
kwapi/drivers/dummy.py
View File

@ -1,10 +1,25 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from random import randrange from random import randrange
import time import time
from driver import Driver from driver import Driver
class Dummy(Driver): class Dummy(Driver):
"""Dummy driver derived from Driver class. Usefull for tests.""" """Dummy driver derived from Driver class. Usefull for tests."""
@ -12,8 +27,10 @@ class Dummy(Driver):
"""Initializes the dummy driver. """Initializes the dummy driver.
Keyword arguments: Keyword arguments:
probe_ids -- list containing the probes IDs (a wattmeter monitor sometimes several probes probe_ids -- list containing the probes IDs
kwargs -- keywords (min_value and max_value) defining the random value interval (a wattmeter monitor sometimes several probes)
kwargs -- keywords (min_value and max_value)
defining the random value interval
""" """
Driver.__init__(self, probe_ids, kwargs) Driver.__init__(self, probe_ids, kwargs)

18
kwapi/drivers/wattsup.py
View File

@ -1,10 +1,25 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import serial import serial
from serial.serialutil import SerialException from serial.serialutil import SerialException
from driver import Driver from driver import Driver
class Wattsup(Driver): class Wattsup(Driver):
"""Driver for Wattsup wattmeters.""" """Driver for Wattsup wattmeters."""
@ -12,7 +27,8 @@ class Wattsup(Driver):
"""Initializes the Wattsup driver. """Initializes the Wattsup driver.
Keyword arguments: Keyword arguments:
probe_ids -- list containing the probes IDs (a wattmeter monitor sometimes several probes probe_ids -- list containing the probes IDs
(a wattmeter monitor sometimes several probes)
kwargs -- keyword (device) defining the device to read (/dev/ttyUSB0) kwargs -- keyword (device) defining the device to read (/dev/ttyUSB0)
""" """

24
kwapi/openstack/__init__.py
View File

@ -0,0 +1,24 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This ensures the openstack namespace is defined
try:
import pkg_resources
pkg_resources.declare_namespace(__name__)
except ImportError:
import pkgutil
__path__ = pkgutil.extend_path(__path__, __name__)

19
kwapi/openstack/common/__init__.py
View File

@ -0,0 +1,19 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# TODO(jaypipes) Code in this module is intended to be ported to the eventual
# openstack-common library

44
kwapi/openstack/common/authutils.py Normal file
View File

@ -0,0 +1,44 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Authentication related utilities and helper functions.
"""
def auth_str_equal(provided, known):
"""Constant-time string comparison.
:params provided: the first string
:params known: the second string
:return: True if the strings are equal.
This function takes two strings and compares them. It is intended to be
used when doing a comparison for authentication purposes to help guard
against timing attacks. When using the function for this purpose, always
provide the user-provided password as the first argument. The time this
function will take is always a factor of the length of this string.
"""
result = 0
p_len = len(provided)
k_len = len(known)
for i in xrange(p_len):
a = ord(provided[i]) if i < p_len else 0
b = ord(known[i]) if i < k_len else 0
result |= a ^ b
return (p_len == k_len) & (result == 0)

432
kwapi/openstack/common/cfg.py
View File

@ -205,27 +205,11 @@ Option values may reference other values using PEP 292 string substitution::
Note that interpolation can be avoided by using '$$'. Note that interpolation can be avoided by using '$$'.
For command line utilities that dispatch to other command line utilities, the
disable_interspersed_args() method is available. If this this method is called,
then parsing e.g.::
script --verbose cmd --debug /tmp/mything
will no longer return::
['cmd', '/tmp/mything']
as the leftover arguments, but will instead return::
['cmd', '--debug', '/tmp/mything']
i.e. argument parsing is stopped at the first non-option argument.
Options may be declared as required so that an error is raised if the user Options may be declared as required so that an error is raised if the user
does not supply a value for the option. does not supply a value for the option.
Options may be declared as secret so that their values are not leaked into Options may be declared as secret so that their values are not leaked into
log files: log files::
opts = [ opts = [
cfg.StrOpt('s3_store_access_key', secret=True), cfg.StrOpt('s3_store_access_key', secret=True),
@ -234,7 +218,7 @@ log files:
] ]
This module also contains a global instance of the CommonConfigOpts class This module also contains a global instance of the CommonConfigOpts class
in order to support a common usage pattern in OpenStack: in order to support a common usage pattern in OpenStack::
from kwapi.openstack.common import cfg from kwapi.openstack.common import cfg
@ -249,13 +233,35 @@ in order to support a common usage pattern in OpenStack:
def start(server, app): def start(server, app):
server.start(app, CONF.bind_port, CONF.bind_host) server.start(app, CONF.bind_port, CONF.bind_host)
Positional command line arguments are supported via a 'positional' Opt
constructor argument::
>>> CONF.register_cli_opt(MultiStrOpt('bar', positional=True))
True
>>> CONF(['a', 'b'])
>>> CONF.bar
['a', 'b']
It is also possible to use argparse "sub-parsers" to parse additional
command line arguments using the SubCommandOpt class:
>>> def add_parsers(subparsers):
... list_action = subparsers.add_parser('list')
... list_action.add_argument('id')
...
>>> CONF.register_cli_opt(SubCommandOpt('action', handler=add_parsers))
True
>>> CONF(['list', '10'])
>>> CONF.action.name, CONF.action.id
('list', '10')
""" """
import argparse
import collections import collections
import copy import copy
import functools import functools
import glob import glob
import optparse
import os import os
import string import string
import sys import sys
@ -474,6 +480,13 @@ def _is_opt_registered(opts, opt):
return False return False
def set_defaults(opts, **kwargs):
for opt in opts:
if opt.dest in kwargs:
opt.default = kwargs[opt.dest]
break
class Opt(object): class Opt(object):
"""Base class for all configuration options. """Base class for all configuration options.
@ -489,6 +502,8 @@ class Opt(object):
a single character CLI option name a single character CLI option name
default: default:
the default value of the option the default value of the option
positional:
True if the option is a positional CLI argument
metavar: metavar:
the name shown as the argument to a CLI option in --help output the name shown as the argument to a CLI option in --help output
help: help:
@ -497,8 +512,8 @@ class Opt(object):
multi = False multi = False
def __init__(self, name, dest=None, short=None, default=None, def __init__(self, name, dest=None, short=None, default=None,
metavar=None, help=None, secret=False, required=False, positional=False, metavar=None, help=None,
deprecated_name=None): secret=False, required=False, deprecated_name=None):
"""Construct an Opt object. """Construct an Opt object.
The only required parameter is the option's name. However, it is The only required parameter is the option's name. However, it is
@ -508,6 +523,7 @@ class Opt(object):
:param dest: the name of the corresponding ConfigOpts property :param dest: the name of the corresponding ConfigOpts property
:param short: a single character CLI option name :param short: a single character CLI option name
:param default: the default value of the option :param default: the default value of the option
:param positional: True if the option is a positional CLI argument
:param metavar: the option argument to show in --help :param metavar: the option argument to show in --help
:param help: an explanation of how the option is used :param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output :param secret: true iff the value should be obfuscated in log output
@ -521,6 +537,7 @@ class Opt(object):
self.dest = dest self.dest = dest
self.short = short self.short = short
self.default = default self.default = default
self.positional = positional
self.metavar = metavar self.metavar = metavar
self.help = help self.help = help
self.secret = secret self.secret = secret
@ -561,64 +578,73 @@ class Opt(object):
:param parser: the CLI option parser :param parser: the CLI option parser
:param group: an optional OptGroup object :param group: an optional OptGroup object
""" """
container = self._get_optparse_container(parser, group) container = self._get_argparse_container(parser, group)
kwargs = self._get_optparse_kwargs(group) kwargs = self._get_argparse_kwargs(group)
prefix = self._get_optparse_prefix('', group) prefix = self._get_argparse_prefix('', group)
self._add_to_optparse(container, self.name, self.short, kwargs, prefix, self._add_to_argparse(container, self.name, self.short, kwargs, prefix,
self.deprecated_name) self.positional, self.deprecated_name)
def _add_to_optparse(self, container, name, short, kwargs, prefix='', def _add_to_argparse(self, container, name, short, kwargs, prefix='',
deprecated_name=None): positional=False, deprecated_name=None):
"""Add an option to an optparse parser or group. """Add an option to an argparse parser or group.
:param container: an optparse.OptionContainer object :param container: an argparse._ArgumentGroup object
:param name: the opt name :param name: the opt name
:param short: the short opt name :param short: the short opt name
:param kwargs: the keyword arguments for add_option() :param kwargs: the keyword arguments for add_argument()
:param prefix: an optional prefix to prepend to the opt name :param prefix: an optional prefix to prepend to the opt name
:param position: whether the optional is a positional CLI argument
:raises: DuplicateOptError if a naming confict is detected :raises: DuplicateOptError if a naming confict is detected
""" """
args = ['--' + prefix + name] def hyphen(arg):
return arg if not positional else ''
args = [hyphen('--') + prefix + name]
if short: if short:
args += ['-' + short] args.append(hyphen('-') + short)
if deprecated_name: if deprecated_name:
args += ['--' + prefix + deprecated_name] args.append(hyphen('--') + prefix + deprecated_name)
for a in args:
if container.has_option(a):
raise DuplicateOptError(a)
container.add_option(*args, **kwargs)
def _get_optparse_container(self, parser, group): try:
"""Returns an optparse.OptionContainer. container.add_argument(*args, **kwargs)
except argparse.ArgumentError as e:
raise DuplicateOptError(e)
:param parser: an optparse.OptionParser def _get_argparse_container(self, parser, group):
"""Returns an argparse._ArgumentGroup.
:param parser: an argparse.ArgumentParser
:param group: an (optional) OptGroup object :param group: an (optional) OptGroup object
:returns: an optparse.OptionGroup if a group is given, else the parser :returns: an argparse._ArgumentGroup if group is given, else parser
""" """
if group is not None: if group is not None:
return group._get_optparse_group(parser) return group._get_argparse_group(parser)
else: else:
return parser return parser
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Build a dict of keyword arguments for optparse's add_option(). """Build a dict of keyword arguments for argparse's add_argument().
Most opt types extend this method to customize the behaviour of the Most opt types extend this method to customize the behaviour of the
options added to optparse. options added to argparse.
:param group: an optional group :param group: an optional group
:param kwargs: optional keyword arguments to add to :param kwargs: optional keyword arguments to add to
:returns: a dict of keyword arguments :returns: a dict of keyword arguments
""" """
if not self.positional:
dest = self.dest dest = self.dest
if group is not None: if group is not None:
dest = group.name + '_' + dest dest = group.name + '_' + dest
kwargs.update({'dest': dest, kwargs['dest'] = dest
else:
kwargs['nargs'] = '?'
kwargs.update({'default': None,
'metavar': self.metavar, 'metavar': self.metavar,
'help': self.help, }) 'help': self.help, })
return kwargs return kwargs
def _get_optparse_prefix(self, prefix, group): def _get_argparse_prefix(self, prefix, group):
"""Build a prefix for the CLI option name, if required. """Build a prefix for the CLI option name, if required.
CLI options in a group are prefixed with the group's name in order CLI options in a group are prefixed with the group's name in order
@ -656,6 +682,11 @@ class BoolOpt(Opt):
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False} '0': False, 'no': False, 'false': False, 'off': False}
def __init__(self, *args, **kwargs):
if 'positional' in kwargs:
raise ValueError('positional boolean args not supported')
super(BoolOpt, self).__init__(*args, **kwargs)
def _get_from_config_parser(self, cparser, section): def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a boolean from ConfigParser.""" """Retrieve the opt value as a boolean from ConfigParser."""
def convert_bool(v): def convert_bool(v):
@ -671,21 +702,32 @@ class BoolOpt(Opt):
def _add_to_cli(self, parser, group=None): def _add_to_cli(self, parser, group=None):
"""Extends the base class method to add the --nooptname option.""" """Extends the base class method to add the --nooptname option."""
super(BoolOpt, self)._add_to_cli(parser, group) super(BoolOpt, self)._add_to_cli(parser, group)
self._add_inverse_to_optparse(parser, group) self._add_inverse_to_argparse(parser, group)
def _add_inverse_to_optparse(self, parser, group): def _add_inverse_to_argparse(self, parser, group):
"""Add the --nooptname option to the option parser.""" """Add the --nooptname option to the option parser."""
container = self._get_optparse_container(parser, group) container = self._get_argparse_container(parser, group)
kwargs = self._get_optparse_kwargs(group, action='store_false') kwargs = self._get_argparse_kwargs(group, action='store_false')
prefix = self._get_optparse_prefix('no', group) prefix = self._get_argparse_prefix('no', group)
kwargs["help"] = "The inverse of --" + self.name kwargs["help"] = "The inverse of --" + self.name
self._add_to_optparse(container, self.name, None, kwargs, prefix, self._add_to_argparse(container, self.name, None, kwargs, prefix,
self.deprecated_name) self.positional, self.deprecated_name)
def _get_optparse_kwargs(self, group, action='store_true', **kwargs): def _get_argparse_kwargs(self, group, action='store_true', **kwargs):
"""Extends the base optparse keyword dict for boolean options.""" """Extends the base argparse keyword dict for boolean options."""
return super(BoolOpt,
self)._get_optparse_kwargs(group, action=action, **kwargs) kwargs = super(BoolOpt, self)._get_argparse_kwargs(group, **kwargs)
# metavar has no effect for BoolOpt
if 'metavar' in kwargs:
del kwargs['metavar']
if action != 'store_true':
action = 'store_false'
kwargs['action'] = action
return kwargs
class IntOpt(Opt): class IntOpt(Opt):
@ -697,10 +739,10 @@ class IntOpt(Opt):
return [int(v) for v in self._cparser_get_with_deprecated(cparser, return [int(v) for v in self._cparser_get_with_deprecated(cparser,
section)] section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for integer options.""" """Extends the base argparse keyword dict for integer options."""
return super(IntOpt, return super(IntOpt,
self)._get_optparse_kwargs(group, type='int', **kwargs) self)._get_argparse_kwargs(group, type=int, **kwargs)
class FloatOpt(Opt): class FloatOpt(Opt):
@ -712,10 +754,10 @@ class FloatOpt(Opt):
return [float(v) for v in return [float(v) for v in
self._cparser_get_with_deprecated(cparser, section)] self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for float options.""" """Extends the base argparse keyword dict for float options."""
return super(FloatOpt, return super(FloatOpt, self)._get_argparse_kwargs(group,
self)._get_optparse_kwargs(group, type='float', **kwargs) type=float, **kwargs)
class ListOpt(Opt): class ListOpt(Opt):
@ -725,24 +767,27 @@ class ListOpt(Opt):
is a list containing these strings. is a list containing these strings.
""" """
class _StoreListAction(argparse.Action):
"""
An argparse action for parsing an option value into a list.
"""
def __call__(self, parser, namespace, values, option_string=None):
if values is not None:
values = [a.strip() for a in values.split(',')]
setattr(namespace, self.dest, values)
def _get_from_config_parser(self, cparser, section): def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a list from ConfigParser.""" """Retrieve the opt value as a list from ConfigParser."""
return [v.split(',') for v in return [[a.strip() for a in v.split(',')] for v in
self._cparser_get_with_deprecated(cparser, section)] self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for list options.""" """Extends the base argparse keyword dict for list options."""
return super(ListOpt, return Opt._get_argparse_kwargs(self,
self)._get_optparse_kwargs(group, group,
type='string', action=ListOpt._StoreListAction,
action='callback',
callback=self._parse_list,
**kwargs) **kwargs)
def _parse_list(self, option, opt, value, parser):
"""An optparse callback for parsing an option value into a list."""
setattr(parser.values, self.dest, value.split(','))
class MultiStrOpt(Opt): class MultiStrOpt(Opt):
@ -752,10 +797,14 @@ class MultiStrOpt(Opt):
""" """
multi = True multi = True
def _get_optparse_kwargs(self, group, **kwargs): def _get_argparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for multi str options.""" """Extends the base argparse keyword dict for multi str options."""
return super(MultiStrOpt, kwargs = super(MultiStrOpt, self)._get_argparse_kwargs(group)
self)._get_optparse_kwargs(group, action='append') if not self.positional:
kwargs['action'] = 'append'
else:
kwargs['nargs'] = '*'
return kwargs
def _cparser_get_with_deprecated(self, cparser, section): def _cparser_get_with_deprecated(self, cparser, section):
"""If cannot find option as dest try deprecated_name alias.""" """If cannot find option as dest try deprecated_name alias."""
@ -765,6 +814,57 @@ class MultiStrOpt(Opt):
return cparser.get(section, [self.dest], multi=True) return cparser.get(section, [self.dest], multi=True)
class SubCommandOpt(Opt):
"""
Sub-command options allow argparse sub-parsers to be used to parse
additional command line arguments.
The handler argument to the SubCommandOpt contructor is a callable
which is supplied an argparse subparsers object. Use this handler
callable to add sub-parsers.
The opt value is SubCommandAttr object with the name of the chosen
sub-parser stored in the 'name' attribute and the values of other
sub-parser arguments available as additional attributes.
"""
def __init__(self, name, dest=None, handler=None,
title=None, description=None, help=None):
"""Construct an sub-command parsing option.
This behaves similarly to other Opt sub-classes but adds a
'handler' argument. The handler is a callable which is supplied
an subparsers object when invoked. The add_parser() method on
this subparsers object can be used to register parsers for
sub-commands.
:param name: the option's name
:param dest: the name of the corresponding ConfigOpts property
:param title: title of the sub-commands group in help output
:param description: description of the group in help output
:param help: a help string giving an overview of available sub-commands
"""
super(SubCommandOpt, self).__init__(name, dest=dest, help=help)
self.handler = handler
self.title = title
self.description = description
def _add_to_cli(self, parser, group=None):
"""Add argparse sub-parsers and invoke the handler method."""
dest = self.dest
if group is not None:
dest = group.name + '_' + dest
subparsers = parser.add_subparsers(dest=dest,
title=self.title,
description=self.description,
help=self.help)
if not self.handler is None:
self.handler(subparsers)
class OptGroup(object): class OptGroup(object):
""" """
@ -800,19 +900,20 @@ class OptGroup(object):
self.help = help self.help = help
self._opts = {} # dict of dicts of (opt:, override:, default:) self._opts = {} # dict of dicts of (opt:, override:, default:)
self._optparse_group = None self._argparse_group = None
def _register_opt(self, opt): def _register_opt(self, opt, cli=False):
"""Add an opt to this group. """Add an opt to this group.
:param opt: an Opt object :param opt: an Opt object
:param cli: whether this is a CLI option
:returns: False if previously registered, True otherwise :returns: False if previously registered, True otherwise
:raises: DuplicateOptError if a naming conflict is detected :raises: DuplicateOptError if a naming conflict is detected
""" """
if _is_opt_registered(self._opts, opt): if _is_opt_registered(self._opts, opt):
return False return False
self._opts[opt.dest] = {'opt': opt} self._opts[opt.dest] = {'opt': opt, 'cli': cli}
return True return True
@ -824,16 +925,16 @@ class OptGroup(object):
if opt.dest in self._opts: if opt.dest in self._opts:
del self._opts[opt.dest] del self._opts[opt.dest]
def _get_optparse_group(self, parser): def _get_argparse_group(self, parser):
"""Build an optparse.OptionGroup for this group.""" if self._argparse_group is None:
if self._optparse_group is None: """Build an argparse._ArgumentGroup for this group."""
self._optparse_group = optparse.OptionGroup(parser, self.title, self._argparse_group = parser.add_argument_group(self.title,
self.help) self.help)
return self._optparse_group return self._argparse_group
def _clear(self): def _clear(self):
"""Clear this group's option parsing state.""" """Clear this group's option parsing state."""
self._optparse_group = None self._argparse_group = None
class ParseError(iniparser.ParseError): class ParseError(iniparser.ParseError):
@ -928,26 +1029,31 @@ class ConfigOpts(collections.Mapping):
self._groups = {} self._groups = {}
self._args = None self._args = None
self._oparser = None self._oparser = None
self._cparser = None self._cparser = None
self._cli_values = {} self._cli_values = {}
self.__cache = {} self.__cache = {}
self._config_opts = [] self._config_opts = []
self._disable_interspersed_args = False
def _setup(self, project, prog, version, usage, default_config_files): def _pre_setup(self, project, prog, version, usage, default_config_files):
"""Initialize a ConfigOpts object for option parsing.""" """Initialize a ConfigCliParser object for option parsing."""
if prog is None: if prog is None:
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
if default_config_files is None: if default_config_files is None:
default_config_files = find_config_files(project, prog) default_config_files = find_config_files(project, prog)
self._oparser = optparse.OptionParser(prog=prog, self._oparser = argparse.ArgumentParser(prog=prog, usage=usage)
version=version, self._oparser.add_argument('--version',
usage=usage) action='version',
if self._disable_interspersed_args: version=version)
self._oparser.disable_interspersed_args()
return prog, default_config_files
def _setup(self, project, prog, version, usage, default_config_files):
"""Initialize a ConfigOpts object for option parsing."""
self._config_opts = [ self._config_opts = [
MultiStrOpt('config-file', MultiStrOpt('config-file',
@ -1017,18 +1123,23 @@ class ConfigOpts(collections.Mapping):
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
RequiredOptError, DuplicateOptError RequiredOptError, DuplicateOptError
""" """
self.clear() self.clear()
prog, default_config_files = self._pre_setup(project,
prog,
version,
usage,
default_config_files)
self._setup(project, prog, version, usage, default_config_files) self._setup(project, prog, version, usage, default_config_files)
self._cli_values, leftovers = self._parse_cli_opts(args) self._cli_values = self._parse_cli_opts(args)
self._parse_config_files() self._parse_config_files()
self._check_required_opts() self._check_required_opts()
return leftovers
def __getattr__(self, name): def __getattr__(self, name):
"""Look up an option value and perform string substitution. """Look up an option value and perform string substitution.
@ -1062,17 +1173,21 @@ class ConfigOpts(collections.Mapping):
@__clear_cache @__clear_cache
def clear(self): def clear(self):
"""Clear the state of the object to before it was called.""" """Clear the state of the object to before it was called.
Any subparsers added using the add_cli_subparsers() will also be
removed as a side-effect of this method.
"""
self._args = None self._args = None
self._cli_values.clear() self._cli_values.clear()
self._oparser = None self._oparser = argparse.ArgumentParser()
self._cparser = None self._cparser = None
self.unregister_opts(self._config_opts) self.unregister_opts(self._config_opts)
for group in self._groups.values(): for group in self._groups.values():
group._clear() group._clear()
@__clear_cache @__clear_cache
def register_opt(self, opt, group=None): def register_opt(self, opt, group=None, cli=False):
"""Register an option schema. """Register an option schema.
Registering an option schema makes any option value which is previously Registering an option schema makes any option value which is previously
@ -1080,17 +1195,19 @@ class ConfigOpts(collections.Mapping):
as an attribute of this object. as an attribute of this object.
:param opt: an instance of an Opt sub-class :param opt: an instance of an Opt sub-class
:param cli: whether this is a CLI option
:param group: an optional OptGroup object or group name :param group: an optional OptGroup object or group name
:return: False if the opt was already register, True otherwise :return: False if the opt was already register, True otherwise
:raises: DuplicateOptError :raises: DuplicateOptError
""" """
if group is not None: if group is not None:
return self._get_group(group, autocreate=True)._register_opt(opt) group = self._get_group(group, autocreate=True)
return group._register_opt(opt, cli)
if _is_opt_registered(self._opts, opt): if _is_opt_registered(self._opts, opt):
return False return False
self._opts[opt.dest] = {'opt': opt} self._opts[opt.dest] = {'opt': opt, 'cli': cli}
return True return True
@ -1116,7 +1233,7 @@ class ConfigOpts(collections.Mapping):
if self._args is not None: if self._args is not None:
raise ArgsAlreadyParsedError("cannot register CLI option") raise ArgsAlreadyParsedError("cannot register CLI option")
return self.register_opt(opt, group, clear_cache=False) return self.register_opt(opt, group, cli=True, clear_cache=False)
@__clear_cache @__clear_cache
def register_cli_opts(self, opts, group=None): def register_cli_opts(self, opts, group=None):
@ -1243,9 +1360,10 @@ class ConfigOpts(collections.Mapping):
for info in group._opts.values(): for info in group._opts.values():
yield info, group yield info, group
def _all_opts(self): def _all_cli_opts(self):
"""A generator function for iteration opts.""" """A generator function for iterating CLI opts."""
for info, group in self._all_opt_infos(): for info, group in self._all_opt_infos():
if info['cli']:
yield info['opt'], group yield info['opt'], group
def _unset_defaults_and_overrides(self): def _unset_defaults_and_overrides(self):
@ -1254,31 +1372,6 @@ class ConfigOpts(collections.Mapping):
info.pop('default', None) info.pop('default', None)
info.pop('override', None) info.pop('override', None)
def disable_interspersed_args(self):
"""Set parsing to stop on the first non-option.
If this this method is called, then parsing e.g.
script --verbose cmd --debug /tmp/mything
will no longer return:
['cmd', '/tmp/mything']
as the leftover arguments, but will instead return:
['cmd', '--debug', '/tmp/mything']
i.e. argument parsing is stopped at the first non-option argument.
"""
self._disable_interspersed_args = True
def enable_interspersed_args(self):
"""Set parsing to not stop on the first non-option.
This it the default behaviour."""
self._disable_interspersed_args = False
def find_file(self, name): def find_file(self, name):
"""Locate a file located alongside the config files. """Locate a file located alongside the config files.
@ -1377,6 +1470,9 @@ class ConfigOpts(collections.Mapping):
info = self._get_opt_info(name, group) info = self._get_opt_info(name, group)
opt = info['opt'] opt = info['opt']
if isinstance(opt, SubCommandOpt):
return self.SubCommandAttr(self, group, opt.dest)
if 'override' in info: if 'override' in info:
return info['override'] return info['override']
@ -1401,6 +1497,10 @@ class ConfigOpts(collections.Mapping):
if not opt.multi: if not opt.multi:
return value return value
# argparse ignores default=None for nargs='*'
if opt.positional and not value:
value = opt.default
return value + values return value + values
if values: if values:
@ -1523,12 +1623,10 @@ class ConfigOpts(collections.Mapping):
""" """
self._args = args self._args = args
for opt, group in self._all_opts(): for opt, group in self._all_cli_opts():
opt._add_to_cli(self._oparser, group) opt._add_to_cli(self._oparser, group)
values, leftovers = self._oparser.parse_args(args) return vars(self._oparser.parse_args(args))
return vars(values), leftovers
class GroupAttr(collections.Mapping): class GroupAttr(collections.Mapping):
@ -1543,12 +1641,12 @@ class ConfigOpts(collections.Mapping):
:param conf: a ConfigOpts object :param conf: a ConfigOpts object
:param group: an OptGroup object :param group: an OptGroup object
""" """
self.conf = conf self._conf = conf
self.group = group self._group = group
def __getattr__(self, name): def __getattr__(self, name):
"""Look up an option value and perform template substitution.""" """Look up an option value and perform template substitution."""
return self.conf._get(name, self.group) return self._conf._get(name, self._group)
def __getitem__(self, key): def __getitem__(self, key):
"""Look up an option value and perform string substitution.""" """Look up an option value and perform string substitution."""
@ -1556,16 +1654,50 @@ class ConfigOpts(collections.Mapping):
def __contains__(self, key): def __contains__(self, key):
"""Return True if key is the name of a registered opt or group.""" """Return True if key is the name of a registered opt or group."""
return key in self.group._opts return key in self._group._opts
def __iter__(self): def __iter__(self):
"""Iterate over all registered opt and group names.""" """Iterate over all registered opt and group names."""
for key in self.group._opts.keys(): for key in self._group._opts.keys():
yield key yield key
def __len__(self): def __len__(self):
"""Return the number of options and option groups.""" """Return the number of options and option groups."""
return len(self.group._opts) return len(self._group._opts)
class SubCommandAttr(object):
"""
A helper class representing the name and arguments of an argparse
sub-parser.
"""
def __init__(self, conf, group, dest):
"""Construct a SubCommandAttr object.
:param conf: a ConfigOpts object
:param group: an OptGroup object
:param dest: the name of the sub-parser
"""
self._conf = conf
self._group = group
self._dest = dest
def __getattr__(self, name):
"""Look up a sub-parser name or argument value."""
if name == 'name':
name = self._dest
if self._group is not None:
name = self._group.name + '_' + name
return self._conf._cli_values[name]
if name in self._conf:
raise DuplicateOptError(name)
try:
return self._conf._cli_values[name]
except KeyError:
raise NoSuchOptError(name)
class StrSubWrapper(object): class StrSubWrapper(object):
@ -1623,19 +1755,21 @@ class CommonConfigOpts(ConfigOpts):
metavar='FORMAT', metavar='FORMAT',
help='A logging.Formatter log message format string which may ' help='A logging.Formatter log message format string which may '
'use any of the available logging.LogRecord attributes. ' 'use any of the available logging.LogRecord attributes. '
'Default: %default'), 'Default: %(default)s'),
StrOpt('log-date-format', StrOpt('log-date-format',
default=DEFAULT_LOG_DATE_FORMAT, default=DEFAULT_LOG_DATE_FORMAT,
metavar='DATE_FORMAT', metavar='DATE_FORMAT',
help='Format string for %(asctime)s in log records. ' help='Format string for %%(asctime)s in log records. '
'Default: %default'), 'Default: %(default)s'),
StrOpt('log-file', StrOpt('log-file',
metavar='PATH', metavar='PATH',
deprecated_name='logfile',
help='(Optional) Name of log file to output to. ' help='(Optional) Name of log file to output to. '
'If not set, logging will go to stdout.'), 'If not set, logging will go to stdout.'),
StrOpt('log-dir', StrOpt('log-dir',
deprecated_name='logdir',
help='(Optional) The directory to keep log files in ' help='(Optional) The directory to keep log files in '
'(will be prepended to --logfile)'), '(will be prepended to --log-file)'),
BoolOpt('use-syslog', BoolOpt('use-syslog',
default=False, default=False,
help='Use syslog for logging.'), help='Use syslog for logging.'),

63
kwapi/openstack/common/cliutils.py Normal file
View File

@ -0,0 +1,63 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
class MissingArgs(Exception):
def __init__(self, missing):
self.missing = missing
def __str__(self):
if len(self.missing) == 1:
return "An argument is missing"
else:
return ("%(num)d arguments are missing" %
dict(num=len(self.missing)))
def validate_args(fn, *args, **kwargs):
"""Check that the supplied args are sufficient for calling a function.
>>> validate_args(lambda a: None)
Traceback (most recent call last):
...
MissingArgs: An argument is missing: a
>>> validate_args(lambda a, b, c, d: None, 0, c=1)
Traceback (most recent call last):
...
MissingArgs: 2 arguments are missing: b, d
:param fn: the function to check
:param arg: the positional arguments supplied
:param kwargs: the keyword arguments supplied
"""
argspec = inspect.getargspec(fn)
num_defaults = len(argspec.defaults or [])
required_args = argspec.args[:len(argspec.args) - num_defaults]
def isbound(method):
return getattr(method, 'im_self', None) is not None
if isbound(fn):
required_args.pop(0)
missing = [arg for arg in required_args if arg not in kwargs]
missing = missing[len(args):]
if missing:
raise MissingArgs(missing)

2
kwapi/openstack/common/eventlet_backdoor.py
View File

@ -46,7 +46,7 @@ def _find_objects(t):
def _print_greenthreads(): def _print_greenthreads():
for i, gt in enumerate(find_objects(greenlet.greenlet)): for i, gt in enumerate(_find_objects(greenlet.greenlet)):
print i, gt print i, gt
traceback.print_stack(gt.gr_frame) traceback.print_stack(gt.gr_frame)
print print

137
kwapi/openstack/common/exception.py Normal file
View File

@ -0,0 +1,137 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Exceptions common to OpenStack projects
"""
import logging
from kwapi.openstack.common.gettextutils import _
class Error(Exception):
def __init__(self, message=None):
super(Error, self).__init__(message)
class ApiError(Error):
def __init__(self, message='Unknown', code='Unknown'):
self.message = message
self.code = code
super(ApiError, self).__init__('%s: %s' % (code, message))
class NotFound(Error):
pass
class UnknownScheme(Error):
msg = "Unknown scheme '%s' found in URI"
def __init__(self, scheme):
msg = self.__class__.msg % scheme
super(UnknownScheme, self).__init__(msg)
class BadStoreUri(Error):
msg = "The Store URI %s was malformed. Reason: %s"
def __init__(self, uri, reason):
msg = self.__class__.msg % (uri, reason)
super(BadStoreUri, self).__init__(msg)
class Duplicate(Error):
pass
class NotAuthorized(Error):
pass
class NotEmpty(Error):
pass
class Invalid(Error):
pass
class BadInputError(Exception):
"""Error resulting from a client sending bad input to a server"""
pass
class MissingArgumentError(Error):
pass
class DatabaseMigrationError(Error):
pass
class ClientConnectionError(Exception):
"""Error resulting from a client connecting to a server"""
pass
def wrap_exception(f):
def _wrap(*args, **kw):
try:
return f(*args, **kw)
except Exception, e:
if not isinstance(e, Error):
#exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception(_('Uncaught exception'))
#logging.error(traceback.extract_stack(exc_traceback))
raise Error(str(e))
raise
_wrap.func_name = f.func_name
return _wrap
class OpenstackException(Exception):
"""
Base Exception
To correctly use this class, inherit from it and define
a 'message' property. That message will get printf'd
with the keyword arguments provided to the constructor.
"""
message = "An unknown exception occurred"
def __init__(self, **kwargs):
try:
self._error_string = self.message % kwargs
except Exception:
# at least get the core message out if something happened
self._error_string = self.message
def __str__(self):
return self._error_string
class MalformedRequestBody(OpenstackException):
message = "Malformed message body: %(reason)s"
class InvalidContentType(OpenstackException):
message = "Invalid content type %(content_type)s"

6
kwapi/openstack/common/excutils.py
View File

@ -24,6 +24,8 @@ import logging
import sys import sys
import traceback import traceback
from kwapi.openstack.common.gettextutils import _
@contextlib.contextmanager @contextlib.contextmanager
def save_and_reraise_exception(): def save_and_reraise_exception():
@ -43,7 +45,7 @@ def save_and_reraise_exception():
try: try:
yield yield
except Exception: except Exception:
logging.error('Original exception being dropped: %s' % logging.error(_('Original exception being dropped: %s'),
(traceback.format_exception(type_, value, tb))) traceback.format_exception(type_, value, tb))
raise raise
raise type_, value, tb raise type_, value, tb

35
kwapi/openstack/common/fileutils.py Normal file
View File

@ -0,0 +1,35 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import errno
import os
def ensure_tree(path):
"""Create a directory (and any ancestor directories required)
:param path: Directory to create
"""
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST:
if not os.path.isdir(path):
raise
else:
raise

2
kwapi/openstack/common/importutils.py
View File

@ -29,7 +29,7 @@ def import_class(import_str):
try: try:
__import__(mod_str) __import__(mod_str)
return getattr(sys.modules[mod_str], class_str) return getattr(sys.modules[mod_str], class_str)
except (ValueError, AttributeError), exc: except (ValueError, AttributeError):
raise ImportError('Class %s cannot be found (%s)' % raise ImportError('Class %s cannot be found (%s)' %
(class_str, (class_str,
traceback.format_exception(*sys.exc_info()))) traceback.format_exception(*sys.exc_info())))

2
kwapi/openstack/common/jsonutils.py
View File

@ -120,7 +120,7 @@ def to_primitive(value, convert_instances=False, level=0):
level=level + 1) level=level + 1)
else: else:
return value return value
except TypeError, e: except TypeError:
# Class objects are tricky since they may define something like # Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list(). # __iter__ defined but it isn't callable as list().
return unicode(value) return unicode(value)

233
kwapi/openstack/common/lockutils.py Normal file
View File

@ -0,0 +1,233 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import errno
import functools
import os
import shutil
import tempfile
import time
import weakref
from eventlet import semaphore
from kwapi.openstack.common import cfg
from kwapi.openstack.common import fileutils
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging
LOG = logging.getLogger(__name__)
util_opts = [
cfg.BoolOpt('disable_process_locking', default=False,
help='Whether to disable inter-process locks'),
cfg.StrOpt('lock_path',
default=os.path.abspath(os.path.join(os.path.dirname(__file__),
'../')),
help='Directory to use for lock files')
]
CONF = cfg.CONF
CONF.register_opts(util_opts)
class _InterProcessLock(object):
"""Lock implementation which allows multiple locks, working around
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
not require any cleanup. Since the lock is always held on a file
descriptor rather than outside of the process, the lock gets dropped
automatically if the process crashes, even if __exit__ is not executed.
There are no guarantees regarding usage by multiple green threads in a
single process here. This lock works only between processes. Exclusive
access between local threads should be achieved using the semaphores
in the @synchronized decorator.
Note these locks are released when the descriptor is closed, so it's not
safe to close the file descriptor while another green thread holds the
lock. Just opening and closing the lock file can break synchronisation,
so lock files must be accessed only using this abstraction.
"""
def __init__(self, name):
self.lockfile = None
self.fname = name
def __enter__(self):
self.lockfile = open(self.fname, 'w')
while True:
try:
# Using non-blocking locks since green threads are not
# patched to deal with blocking locking calls.
# Also upon reading the MSDN docs for locking(), it seems
# to have a laughable 10 attempts "blocking" mechanism.
self.trylock()
return self
except IOError, e:
if e.errno in (errno.EACCES, errno.EAGAIN):
# external locks synchronise things like iptables
# updates - give it some time to prevent busy spinning
time.sleep(0.01)
else:
raise
def __exit__(self, exc_type, exc_val, exc_tb):
try:
self.unlock()
self.lockfile.close()
except IOError:
LOG.exception(_("Could not release the acquired lock `%s`"),
self.fname)
def trylock(self):
raise NotImplementedError()
def unlock(self):
raise NotImplementedError()
class _WindowsLock(_InterProcessLock):
def trylock(self):
msvcrt.locking(self.lockfile, msvcrt.LK_NBLCK, 1)
def unlock(self):
msvcrt.locking(self.lockfile, msvcrt.LK_UNLCK, 1)
class _PosixLock(_InterProcessLock):
def trylock(self):
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
def unlock(self):
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
if os.name == 'nt':
import msvcrt
InterProcessLock = _WindowsLock
else:
import fcntl
InterProcessLock = _PosixLock
_semaphores = weakref.WeakValueDictionary()
def synchronized(name, lock_file_prefix, external=False, lock_path=None):
"""Synchronization decorator.
Decorating a method like so::
@synchronized('mylock')
def foo(self, *args):
...
ensures that only one thread will execute the bar method at a time.
Different methods can share the same lock::
@synchronized('mylock')
def foo(self, *args):
...
@synchronized('mylock')
def bar(self, *args):
...
This way only one of either foo or bar can be executing at a time.
The lock_file_prefix argument is used to provide lock files on disk with a
meaningful prefix. The prefix should end with a hyphen ('-') if specified.
The external keyword argument denotes whether this lock should work across
multiple processes. This means that if two different workers both run a
a method decorated with @synchronized('mylock', external=True), only one
of them will execute at a time.
The lock_path keyword argument is used to specify a special location for
external lock files to live. If nothing is set, then CONF.lock_path is
used as a default.
"""
def wrap(f):
@functools.wraps(f)
def inner(*args, **kwargs):
# NOTE(soren): If we ever go natively threaded, this will be racy.
# See http://stackoverflow.com/questions/5390569/dyn
# amically-allocating-and-destroying-mutexes
sem = _semaphores.get(name, semaphore.Semaphore())
if name not in _semaphores:
# this check is not racy - we're already holding ref locally
# so GC won't remove the item and there was no IO switch
# (only valid in greenthreads)
_semaphores[name] = sem
with sem:
LOG.debug(_('Got semaphore "%(lock)s" for method '
'"%(method)s"...'), {'lock': name,
'method': f.__name__})
if external and not CONF.disable_process_locking:
LOG.debug(_('Attempting to grab file lock "%(lock)s" for '
'method "%(method)s"...'),
{'lock': name, 'method': f.__name__})
cleanup_dir = False
# We need a copy of lock_path because it is non-local
local_lock_path = lock_path
if not local_lock_path:
local_lock_path = CONF.lock_path
if not local_lock_path:
cleanup_dir = True
local_lock_path = tempfile.mkdtemp()
if not os.path.exists(local_lock_path):
cleanup_dir = True
fileutils.ensure_tree(local_lock_path)
# NOTE(mikal): the lock name cannot contain directory
# separators
safe_name = name.replace(os.sep, '_')
lock_file_name = '%s%s' % (lock_file_prefix, safe_name)
lock_file_path = os.path.join(local_lock_path,
lock_file_name)
try:
lock = InterProcessLock(lock_file_path)
with lock:
LOG.debug(_('Got file lock "%(lock)s" at %(path)s '
'for method "%(method)s"...'),
{'lock': name,
'path': lock_file_path,
'method': f.__name__})
retval = f(*args, **kwargs)
finally:
# NOTE(vish): This removes the tempdir if we needed
# to create one. This is used to cleanup
# the locks left behind by unit tests.
if cleanup_dir:
shutil.rmtree(local_lock_path)
else:
retval = f(*args, **kwargs)
return retval
return inner
return wrap

21
kwapi/openstack/common/log.py
View File

@ -49,19 +49,20 @@ from kwapi.openstack.common import notifier
log_opts = [ log_opts = [
cfg.StrOpt('logging_context_format_string', cfg.StrOpt('logging_context_format_string',
default='%(asctime)s %(levelname)s %(name)s [%(request_id)s ' default='%(asctime)s.%(msecs)d %(levelname)s %(name)s '
'%(user)s %(tenant)s] %(instance)s' '[%(request_id)s %(user)s %(tenant)s] %(instance)s'
'%(message)s', '%(message)s',
help='format string to use for log messages with context'), help='format string to use for log messages with context'),
cfg.StrOpt('logging_default_format_string', cfg.StrOpt('logging_default_format_string',
default='%(asctime)s %(process)d %(levelname)s %(name)s [-]' default='%(asctime)s.%(msecs)d %(process)d %(levelname)s '
' %(instance)s%(message)s', '%(name)s [-] %(instance)s%(message)s',
help='format string to use for log messages without context'), help='format string to use for log messages without context'),
cfg.StrOpt('logging_debug_format_suffix', cfg.StrOpt('logging_debug_format_suffix',
default='%(funcName)s %(pathname)s:%(lineno)d', default='%(funcName)s %(pathname)s:%(lineno)d',
help='data to append to log format when level is DEBUG'), help='data to append to log format when level is DEBUG'),
cfg.StrOpt('logging_exception_prefix', cfg.StrOpt('logging_exception_prefix',
default='%(asctime)s %(process)d TRACE %(name)s %(instance)s', default='%(asctime)s.%(msecs)d %(process)d TRACE %(name)s '
'%(instance)s',
help='prefix each line of exception output with this format'), help='prefix each line of exception output with this format'),
cfg.ListOpt('default_log_levels', cfg.ListOpt('default_log_levels',
default=[ default=[
@ -174,7 +175,7 @@ class ContextAdapter(logging.LoggerAdapter):
self.log(logging.AUDIT, msg, *args, **kwargs) self.log(logging.AUDIT, msg, *args, **kwargs)
def deprecated(self, msg, *args, **kwargs): def deprecated(self, msg, *args, **kwargs):
stdmsg = _("Deprecated Config: %s") % msg stdmsg = _("Deprecated: %s") % msg
if CONF.fatal_deprecations: if CONF.fatal_deprecations:
self.critical(stdmsg, *args, **kwargs) self.critical(stdmsg, *args, **kwargs)
raise DeprecatedConfig(msg=stdmsg) raise DeprecatedConfig(msg=stdmsg)
@ -257,7 +258,7 @@ class JSONFormatter(logging.Formatter):
class PublishErrorsHandler(logging.Handler): class PublishErrorsHandler(logging.Handler):
def emit(self, record): def emit(self, record):
if ('kwapi.openstack.common.notifier.log_notifier' in if ('openstack.common.notifier.log_notifier' in
CONF.notification_driver): CONF.notification_driver):
return return
notifier.api.notify(None, 'error.publisher', notifier.api.notify(None, 'error.publisher',
@ -289,6 +290,12 @@ def setup(product_name):
_setup_logging_from_conf(product_name) _setup_logging_from_conf(product_name)
def set_defaults(logging_context_format_string):
cfg.set_defaults(log_opts,
logging_context_format_string=
logging_context_format_string)
def _find_facility_from_conf(): def _find_facility_from_conf():
facility_names = logging.handlers.SysLogHandler.facility_names facility_names = logging.handlers.SysLogHandler.facility_names
facility = getattr(logging.handlers.SysLogHandler, facility = getattr(logging.handlers.SysLogHandler,

9
kwapi/openstack/common/loopingcall.py
View File

@ -24,6 +24,7 @@ from eventlet import greenthread
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging from kwapi.openstack.common import log as logging
from kwapi.openstack.common import timeutils
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -62,10 +63,16 @@ class LoopingCall(object):
try: try:
while self._running: while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw) self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running: if not self._running:
break break
greenthread.sleep(interval) delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone, e: except LoopingCallDone, e:
self.stop() self.stop()
done.send(e.retvalue) done.send(e.retvalue)

0
kwapi/openstack/common/middleware/__init__.py Normal file
View File

64
kwapi/openstack/common/middleware/context.py Normal file
View File

@ -0,0 +1,64 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Middleware that attaches a context to the WSGI request
"""
from kwapi.openstack.common import context
from kwapi.openstack.common import importutils
from kwapi.openstack.common import wsgi
class ContextMiddleware(wsgi.Middleware):
def __init__(self, app, options):
self.options = options
super(ContextMiddleware, self).__init__(app)
def make_context(self, *args, **kwargs):
"""
Create a context with the given arguments.
"""
# Determine the context class to use
ctxcls = context.RequestContext
if 'context_class' in self.options:
ctxcls = importutils.import_class(self.options['context_class'])
return ctxcls(*args, **kwargs)
def process_request(self, req):
"""
Extract any authentication information in the request and
construct an appropriate context from it.
"""
# Use the default empty context, with admin turned on for
# backwards compatibility
req.context = self.make_context(is_admin=True)
def filter_factory(global_conf, **local_conf):
"""
Factory method for paste.deploy
"""
conf = global_conf.copy()
conf.update(local_conf)
def filter(app):
return ContextMiddleware(app, conf)
return filter

7
kwapi/openstack/common/notifier/api.py
View File

@ -137,10 +137,11 @@ def notify(context, publisher_id, event_type, priority, payload):
for driver in _get_drivers(): for driver in _get_drivers():
try: try:
driver.notify(context, msg) driver.notify(context, msg)
except Exception, e: except Exception as e:
LOG.exception(_("Problem '%(e)s' attempting to " LOG.exception(_("Problem '%(e)s' attempting to "
"send to notification system. " "send to notification system. "
"Payload=%(payload)s") % locals()) "Payload=%(payload)s")
% dict(e=e, payload=payload))
_drivers = None _drivers = None
@ -166,7 +167,7 @@ def add_driver(notification_driver):
try: try:
driver = importutils.import_module(notification_driver) driver = importutils.import_module(notification_driver)
_drivers[notification_driver] = driver _drivers[notification_driver] = driver
except ImportError as e: except ImportError:
LOG.exception(_("Failed to load notifier %s. " LOG.exception(_("Failed to load notifier %s. "
"These notifications will not be sent.") % "These notifications will not be sent.") %
notification_driver) notification_driver)

118
kwapi/openstack/common/notifier/list_notifier.py
View File

@ -1,118 +0,0 @@
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common import cfg
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import importutils
from kwapi.openstack.common import log as logging
list_notifier_drivers_opt = cfg.MultiStrOpt(
'list_notifier_drivers',
default=['kwapi.openstack.common.notifier.no_op_notifier'],
help='List of drivers to send notifications')
CONF = cfg.CONF
CONF.register_opt(list_notifier_drivers_opt)
LOG = logging.getLogger(__name__)
drivers = None
class ImportFailureNotifier(object):
"""Noisily re-raises some exception over-and-over when notify is called."""
def __init__(self, exception):
self.exception = exception
def notify(self, context, message):
raise self.exception
def _get_drivers():
"""Instantiates and returns drivers based on the flag values."""
global drivers
if drivers is None:
drivers = []
for notification_driver in CONF.list_notifier_drivers:
try:
drivers.append(importutils.import_module(notification_driver))
except ImportError as e:
drivers.append(ImportFailureNotifier(e))
return drivers
def add_driver(notification_driver):
"""Add a notification driver at runtime."""
# Make sure the driver list is initialized.
_get_drivers()
if isinstance(notification_driver, basestring):
# Load and add
try:
drivers.append(importutils.import_module(notification_driver))
except ImportError as e:
drivers.append(ImportFailureNotifier(e))
else:
# Driver is already loaded; just add the object.
drivers.append(notification_driver)
def _object_name(obj):
name = []
if hasattr(obj, '__module__'):
name.append(obj.__module__)
if hasattr(obj, '__name__'):
name.append(obj.__name__)
else:
name.append(obj.__class__.__name__)
return '.'.join(name)
def remove_driver(notification_driver):
"""Remove a notification driver at runtime."""
# Make sure the driver list is initialized.
_get_drivers()
removed = False
if notification_driver in drivers:
# We're removing an object. Easy.
drivers.remove(notification_driver)
removed = True
else:
# We're removing a driver by name. Search for it.
for driver in drivers:
if _object_name(driver) == notification_driver:
drivers.remove(driver)
removed = True
if not removed:
raise ValueError("Cannot remove; %s is not in list" %
notification_driver)
def notify(context, message):
"""Passes notification to multiple notifiers in a list."""
for driver in _get_drivers():
try:
driver.notify(context, message)
except Exception as e:
LOG.exception(_("Problem '%(e)s' attempting to send to "
"notification driver %(driver)s."), locals())
def _reset_drivers():
"""Used by unit tests to reset the drivers."""
global drivers
drivers = None

2
kwapi/openstack/common/notifier/log_notifier.py
View File

@ -30,6 +30,6 @@ def notify(_context, message):
CONF.default_notification_level) CONF.default_notification_level)
priority = priority.lower() priority = priority.lower()
logger = logging.getLogger( logger = logging.getLogger(
'kwapi.openstack.common.notification.%s' % 'openstack.common.notification.%s' %
message['event_type']) message['event_type'])
getattr(logger, priority)(jsonutils.dumps(message)) getattr(logger, priority)(jsonutils.dumps(message))

2
kwapi/openstack/common/notifier/rpc_notifier.py
View File

@ -41,6 +41,6 @@ def notify(context, message):
topic = '%s.%s' % (topic, priority) topic = '%s.%s' % (topic, priority)
try: try:
rpc.notify(context, topic, message) rpc.notify(context, topic, message)
except Exception, e: except Exception:
LOG.exception(_("Could not send notification to %(topic)s. " LOG.exception(_("Could not send notification to %(topic)s. "
"Payload=%(message)s"), locals()) "Payload=%(message)s"), locals())

51
kwapi/openstack/common/notifier/rpc_notifier2.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''messaging based notification driver, with message envelopes'''
from kwapi.openstack.common import cfg
from kwapi.openstack.common import context as req_context
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging
from kwapi.openstack.common import rpc
LOG = logging.getLogger(__name__)
notification_topic_opt = cfg.ListOpt(
'topics', default=['notifications', ],
help='AMQP topic(s) used for openstack notifications')
opt_group = cfg.OptGroup(name='rpc_notifier2',
title='Options for rpc_notifier2')
CONF = cfg.CONF
CONF.register_group(opt_group)
CONF.register_opt(notification_topic_opt, opt_group)
def notify(context, message):
"""Sends a notification via RPC"""
if not context:
context = req_context.get_admin_context()
priority = message.get('priority',
CONF.default_notification_level)
priority = priority.lower()
for topic in CONF.rpc_notifier2.topics:
topic = '%s.%s' % (topic, priority)
try:
rpc.notify(context, topic, message, envelope=True)
except Exception:
LOG.exception(_("Could not send notification to %(topic)s. "
"Payload=%(message)s"), locals())

164
kwapi/openstack/common/pastedeploy.py Normal file
View File

@ -0,0 +1,164 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from paste import deploy
from kwapi.openstack.common import local
class BasePasteFactory(object):
"""A base class for paste app and filter factories.
Sub-classes must override the KEY class attribute and provide
a __call__ method.
"""
KEY = None
def __init__(self, data):
self.data = data
def _import_factory(self, local_conf):
"""Import an app/filter class.
Lookup the KEY from the PasteDeploy local conf and import the
class named there. This class can then be used as an app or
filter factory.
Note we support the <module>:<class> format.
Note also that if you do e.g.
key =
value
then ConfigParser returns a value with a leading newline, so
we strip() the value before using it.
"""
mod_str, _sep, class_str = local_conf[self.KEY].strip().rpartition(':')
del local_conf[self.KEY]
__import__(mod_str)
return getattr(sys.modules[mod_str], class_str)
class AppFactory(BasePasteFactory):
"""A Generic paste.deploy app factory.
This requires openstack.app_factory to be set to a callable which returns a
WSGI app when invoked. The format of the name is <module>:<callable> e.g.
[app:myfooapp]
paste.app_factory = openstack.common.pastedeploy:app_factory
openstack.app_factory = myapp:Foo
The WSGI app constructor must accept a data object and a local config
dict as its two arguments.
"""
KEY = 'openstack.app_factory'
def __call__(self, global_conf, **local_conf):
"""The actual paste.app_factory protocol method."""
factory = self._import_factory(local_conf)
return factory(self.data, **local_conf)
class FilterFactory(AppFactory):
"""A Generic paste.deploy filter factory.
This requires openstack.filter_factory to be set to a callable which
returns a WSGI filter when invoked. The format is <module>:<callable> e.g.
[filter:myfoofilter]
paste.filter_factory = openstack.common.pastedeploy:filter_factory
openstack.filter_factory = myfilter:Foo
The WSGI filter constructor must accept a WSGI app, a data object and
a local config dict as its three arguments.
"""
KEY = 'openstack.filter_factory'
def __call__(self, global_conf, **local_conf):
"""The actual paste.filter_factory protocol method."""
factory = self._import_factory(local_conf)
def filter(app):
return factory(app, self.data, **local_conf)
return filter
def app_factory(global_conf, **local_conf):
"""A paste app factory used with paste_deploy_app()."""
return local.store.app_factory(global_conf, **local_conf)
def filter_factory(global_conf, **local_conf):
"""A paste filter factory used with paste_deploy_app()."""
return local.store.filter_factory(global_conf, **local_conf)
def paste_deploy_app(paste_config_file, app_name, data):
"""Load a WSGI app from a PasteDeploy configuration.
Use deploy.loadapp() to load the app from the PasteDeploy configuration,
ensuring that the supplied data object is passed to the app and filter
factories defined in this module.
To use these factories and the data object, the configuration should look
like this:
[app:myapp]
paste.app_factory = openstack.common.pastedeploy:app_factory
openstack.app_factory = myapp:App
...
[filter:myfilter]
paste.filter_factory = openstack.common.pastedeploy:filter_factory
openstack.filter_factory = myapp:Filter
and then:
myapp.py:
class App(object):
def __init__(self, data):
...
class Filter(object):
def __init__(self, app, data):
...
:param paste_config_file: a PasteDeploy config file
:param app_name: the name of the app/pipeline to load from the file
:param data: a data object to supply to the app and its filters
:returns: the WSGI app
"""
(af, ff) = (AppFactory(data), FilterFactory(data))
local.store.app_factory = af
local.store.filter_factory = ff
try:
return deploy.loadapp("config:%s" % paste_config_file, name=app_name)
finally:
del local.store.app_factory
del local.store.filter_factory

115
kwapi/openstack/common/periodic_task.py Normal file
View File

@ -0,0 +1,115 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def periodic_task(*args, **kwargs):
"""Decorator to indicate that a method is a periodic task.
This decorator can be used in two ways:
1. Without arguments '@periodic_task', this will be run on every tick
of the periodic scheduler.
2. With arguments, @periodic_task(ticks_between_runs=N), this will be
run on every N ticks of the periodic scheduler.
"""
def decorator(f):
f._periodic_task = True
f._ticks_between_runs = kwargs.pop('ticks_between_runs', 0)
return f
# NOTE(sirp): The `if` is necessary to allow the decorator to be used with
# and without parens.
#
# In the 'with-parens' case (with kwargs present), this function needs to
# return a decorator function since the interpreter will invoke it like:
#
# periodic_task(*args, **kwargs)(f)
#
# In the 'without-parens' case, the original function will be passed
# in as the first argument, like:
#
# periodic_task(f)
if kwargs:
return decorator
else:
return decorator(args[0])
class _PeriodicTasksMeta(type):
def __init__(cls, names, bases, dict_):
"""Metaclass that allows us to collect decorated periodic tasks."""
super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_)
# NOTE(sirp): if the attribute is not present then we must be the base
# class, so, go ahead and initialize it. If the attribute is present,
# then we're a subclass so make a copy of it so we don't step on our
# parent's toes.
try:
cls._periodic_tasks = cls._periodic_tasks[:]
except AttributeError:
cls._periodic_tasks = []
try:
cls._ticks_to_skip = cls._ticks_to_skip.copy()
except AttributeError:
cls._ticks_to_skip = {}
# This uses __dict__ instead of
# inspect.getmembers(cls, inspect.ismethod) so only the methods of the
# current class are added when this class is scanned, and base classes
# are not added redundantly.
for value in cls.__dict__.values():
if getattr(value, '_periodic_task', False):
task = value
name = task.__name__
cls._periodic_tasks.append((name, task))
cls._ticks_to_skip[name] = task._ticks_between_runs
class PeriodicTasks(object):
__metaclass__ = _PeriodicTasksMeta
def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
for task_name, task in self._periodic_tasks:
full_task_name = '.'.join([self.__class__.__name__, task_name])
ticks_to_skip = self._ticks_to_skip[task_name]
if ticks_to_skip > 0:
LOG.debug(_("Skipping %(full_task_name)s, %(ticks_to_skip)s"
" ticks left until next run"),
dict(full_task_name=full_task_name,
ticks_to_skip=ticks_to_skip))
self._ticks_to_skip[task_name] -= 1
continue
self._ticks_to_skip[task_name] = task._ticks_between_runs
LOG.debug(_("Running periodic task %(full_task_name)s"),
dict(full_task_name=full_task_name))
try:
task(self, context)
except Exception as e:
if raise_on_error:
raise
LOG.exception(_("Error during %(full_task_name)s:"
" %(e)s"),
dict(e=e, full_task_name=full_task_name))

14
kwapi/openstack/common/plugin/__init__.py Normal file
View File

@ -0,0 +1,14 @@
# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

93
kwapi/openstack/common/plugin/callbackplugin.py Normal file
View File

@ -0,0 +1,93 @@
# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.plugin import plugin
LOG = logging.getLogger(__name__)
class _CallbackNotifier(object):
"""Manages plugin-defined notification callbacks.
For each Plugin, a CallbackNotifier will be added to the
notification driver list. Calls to notify() with appropriate
messages will be hooked and prompt callbacks.
A callback should look like this:
def callback(context, message, user_data)
"""
def __init__(self):
self._callback_dict = {}
def _add_callback(self, event_type, callback, user_data):
callback_list = self._callback_dict.get(event_type, [])
callback_list.append({'function': callback,
'user_data': user_data})
self._callback_dict[event_type] = callback_list
def _remove_callback(self, callback):
for callback_list in self._callback_dict.values():
for entry in callback_list:
if entry['function'] == callback:
callback_list.remove(entry)
def notify(self, context, message):
if message.get('event_type') not in self._callback_dict:
return
for entry in self._callback_dict[message.get('event_type')]:
entry['function'](context, message, entry.get('user_data'))
def callbacks(self):
return self._callback_dict
class CallbackPlugin(plugin.Plugin):
""" Plugin with a simple callback interface.
This class is provided as a convenience for producing a simple
plugin that only watches a couple of events. For example, here's
a subclass which prints a line the first time an instance is created.
class HookInstanceCreation(CallbackPlugin):
def __init__(self, _service_name):
super(HookInstanceCreation, self).__init__()
self._add_callback(self.magic, 'compute.instance.create.start')
def magic(self):
print "An instance was created!"
self._remove_callback(self, self.magic)
"""
def __init__(self, service_name):
super(CallbackPlugin, self).__init__(service_name)
self._callback_notifier = _CallbackNotifier()
self._add_notifier(self._callback_notifier)
def _add_callback(self, callback, event_type, user_data=None):
"""Add callback for a given event notification.
Subclasses can call this as an alternative to implementing
a fullblown notify notifier.
"""
self._callback_notifier._add_callback(event_type, callback, user_data)
def _remove_callback(self, callback):
"""Remove all notification callbacks to specified function."""
self._callback_notifier._remove_callback(callback)

86
kwapi/openstack/common/plugin/plugin.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class Plugin(object):
"""Defines an interface for adding functionality to an OpenStack service.
A plugin interacts with a service via the following pathways:
- An optional set of notifiers, managed by calling add_notifier()
or by overriding _notifiers()
- A set of api extensions, managed via add_api_extension_descriptor()
- Direct calls to service functions.
- Whatever else the plugin wants to do on its own.
This is the reference implementation.
"""
# The following functions are provided as convenience methods
# for subclasses. Subclasses should call them but probably not
# override them.
def _add_api_extension_descriptor(self, descriptor):
"""Subclass convenience method which adds an extension descriptor.
Subclass constructors should call this method when
extending a project's REST interface.
Note that once the api service has loaded, the
API extension set is more-or-less fixed, so
this should mainly be called by subclass constructors.
"""
self._api_extension_descriptors.append(descriptor)
def _add_notifier(self, notifier):
"""Subclass convenience method which adds a notifier.
Notifier objects should implement the function notify(message).
Each notifier receives a notify() call whenever an openstack
service broadcasts a notification.
Best to call this during construction. Notifiers are enumerated
and registered by the pluginmanager at plugin load time.
"""
self._notifiers.append(notifier)
# The following methods are called by OpenStack services to query
# plugin features. Subclasses should probably not override these.
def _notifiers(self):
"""Returns list of notifiers for this plugin."""
return self._notifiers
notifiers = property(_notifiers)
def _api_extension_descriptors(self):
"""Return a list of API extension descriptors.
Called by a project API during its load sequence.
"""
return self._api_extension_descriptors
api_extension_descriptors = property(_api_extension_descriptors)
# Most plugins will override this:
def __init__(self, service_name):
self._notifiers = []
self._api_extension_descriptors = []

77
kwapi/openstack/common/plugin/pluginmanager.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pkg_resources
from kwapi.openstack.common import cfg
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.notifier import api as notifier_api
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PluginManager(object):
"""Manages plugin entrypoints and loading.
For a service to implement this plugin interface for callback purposes:
- Make use of the openstack-common notifier system
- Instantiate this manager in each process (passing in
project and service name)
For an API service to extend itself using this plugin interface,
it needs to query the plugin_extension_factory provided by
the already-instantiated PluginManager.
"""
def __init__(self, project_name, service_name):
""" Construct Plugin Manager; load and initialize plugins.
project_name (e.g. 'nova' or 'glance') is used
to construct the entry point that identifies plugins.
The service_name (e.g. 'compute') is passed on to
each plugin as a raw string for it to do what it will.
"""
self._project_name = project_name
self._service_name = service_name
self.plugins = []
def load_plugins(self):
self.plugins = []
for entrypoint in pkg_resources.iter_entry_points('%s.plugin' %
self._project_name):
try:
pluginclass = entrypoint.load()
plugin = pluginclass(self._service_name)
self.plugins.append(plugin)
except Exception, exc:
LOG.error(_("Failed to load plugin %(plug)s: %(exc)s") %
{'plug': entrypoint, 'exc': exc})
# Register individual notifiers.
for plugin in self.plugins:
for notifier in plugin.notifiers:
notifier_api.add_driver(notifier)
def plugin_extension_factory(self, ext_mgr):
for plugin in self.plugins:
descriptors = plugin.api_extension_descriptors
for descriptor in descriptors:
ext_mgr.load_extension(descriptor)

890
kwapi/openstack/common/policy.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 OpenStack, LLC. # Copyright (c) 2012 OpenStack, LLC.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,10 +15,52 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
"""Common Policy Engine Implementation""" """
Common Policy Engine Implementation
Policies can be expressed in one of two forms: A list of lists, or a
string written in the new policy language.
In the list-of-lists representation, each check inside the innermost
list is combined as with an "and" conjunction--for that check to pass,
all the specified checks must pass. These innermost lists are then
combined as with an "or" conjunction. This is the original way of
expressing policies, but there now exists a new way: the policy
language.
In the policy language, each check is specified the same way as in the
list-of-lists representation: a simple "a:b" pair that is matched to
the correct code to perform that check. However, conjunction
operators are available, allowing for more expressiveness in crafting
policies.
As an example, take the following rule, expressed in the list-of-lists
representation::
[["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]]
In the policy language, this becomes::
role:admin or (project_id:%(project_id)s and role:projectadmin)
The policy language also has the "not" operator, allowing a richer
policy rule::
project_id:%(project_id)s and not role:dunce
Finally, two special policy checks should be mentioned; the policy
check "@" will always accept an access, and the policy check "!" will
always reject an access. (Note that if a rule is either the empty
list ("[]") or the empty string, this is equivalent to the "@" policy
check.) Of these, the "!" policy check is probably the most useful,
as it allows particular rules to be explicitly disabled.
"""
import abc
import logging import logging
import re
import urllib import urllib
import urllib2 import urllib2
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
@ -28,218 +70,650 @@ from kwapi.openstack.common import jsonutils
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
_BRAIN = None _rules = None
def set_brain(brain):
"""Set the brain used by enforce().
Defaults use Brain() if not set.
"""
global _BRAIN
_BRAIN = brain
def reset():
"""Clear the brain used by enforce()."""
global _BRAIN
_BRAIN = None
def enforce(match_list, target_dict, credentials_dict, exc=None,
*args, **kwargs):
"""Enforces authorization of some rules against credentials.
:param match_list: nested tuples of data to match against
The basic brain supports three types of match lists:
1) rules
looks like: ``('rule:compute:get_instance',)``
Retrieves the named rule from the rules dict and recursively
checks against the contents of the rule.
2) roles
looks like: ``('role:compute:admin',)``
Matches if the specified role is in credentials_dict['roles'].
3) generic
looks like: ``('tenant_id:%(tenant_id)s',)``
Substitutes values from the target dict into the match using
the % operator and matches them against the creds dict.
Combining rules:
The brain returns True if any of the outer tuple of rules
match and also True if all of the inner tuples match. You
can use this to perform simple boolean logic. For
example, the following rule would return True if the creds
contain the role 'admin' OR the if the tenant_id matches
the target dict AND the the creds contains the role
'compute_sysadmin':
::
{
"rule:combined": (
'role:admin',
('tenant_id:%(tenant_id)s', 'role:compute_sysadmin')
)
}
Note that rule and role are reserved words in the credentials match, so
you can't match against properties with those names. Custom brains may
also add new reserved words. For example, the HttpBrain adds http as a
reserved word.
:param target_dict: dict of object properties
Target dicts contain as much information as we can about the object being
operated on.
:param credentials_dict: dict of actor properties
Credentials dicts contain as much information as we can about the user
performing the action.
:param exc: exception to raise
Class of the exception to raise if the check fails. Any remaining
arguments passed to enforce() (both positional and keyword arguments)
will be passed to the exception class. If exc is not provided, returns
False.
:return: True if the policy allows the action
:return: False if the policy does not allow the action and exc is not set
"""
global _BRAIN
if not _BRAIN:
_BRAIN = Brain()
if not _BRAIN.check(match_list, target_dict, credentials_dict):
if exc:
raise exc(*args, **kwargs)
return False
return True
class Brain(object):
"""Implements policy checking."""
_checks = {} _checks = {}
@classmethod
def _register(cls, name, func): class Rules(dict):
cls._checks[name] = func """
A store for rules. Handles the default_rule setting directly.
"""
@classmethod @classmethod
def load_json(cls, data, default_rule=None): def load_json(cls, data, default_rule=None):
"""Init a brain using json instead of a rules dictionary.""" """
rules_dict = jsonutils.loads(data) Allow loading of JSON rule data.
return cls(rules=rules_dict, default_rule=default_rule) """
# Suck in the JSON data and parse the rules
rules = dict((k, parse_rule(v)) for k, v in
jsonutils.loads(data).items())
return cls(rules, default_rule)
def __init__(self, rules=None, default_rule=None): def __init__(self, rules=None, default_rule=None):
if self.__class__ != Brain: """Initialize the Rules store."""
LOG.warning(_("Inheritance-based rules are deprecated; use "
"the default brain instead of %s.") %
self.__class__.__name__)
self.rules = rules or {} super(Rules, self).__init__(rules or {})
self.default_rule = default_rule self.default_rule = default_rule
def add_rule(self, key, match): def __missing__(self, key):
self.rules[key] = match """Implements the default rule handling."""
def _check(self, match, target_dict, cred_dict): # If the default rule isn't actually defined, do something
try: # reasonably intelligent
match_kind, match_value = match.split(':', 1) if not self.default_rule or self.default_rule not in self:
except Exception: raise KeyError(key)
LOG.exception(_("Failed to understand rule %(match)r") % locals())
# If the rule is invalid, fail closed
return False
func = None return self[self.default_rule]
try:
old_func = getattr(self, '_check_%s' % match_kind) def __str__(self):
except AttributeError: """Dumps a string representation of the rules."""
func = self._checks.get(match_kind, self._checks.get(None, None))
# Start by building the canonical strings for the rules
out_rules = {}
for key, value in self.items():
# Use empty string for singleton TrueCheck instances
if isinstance(value, TrueCheck):
out_rules[key] = ''
else: else:
LOG.warning(_("Inheritance-based rules are deprecated; update " out_rules[key] = str(value)
"_check_%s") % match_kind)
func = lambda brain, kind, value, target, cred: old_func(value,
target,
cred)
if not func: # Dump a pretty-printed JSON representation
LOG.error(_("No handler for matches of kind %s") % match_kind) return jsonutils.dumps(out_rules, indent=4)
# Fail closed
return False
return func(self, match_kind, match_value, target_dict, cred_dict)
def check(self, match_list, target_dict, cred_dict): # Really have to figure out a way to deprecate this
"""Checks authorization of some rules against credentials. def set_rules(rules):
"""Set the rules in use for policy checks."""
Detailed description of the check with examples in policy.enforce(). global _rules
:param match_list: nested tuples of data to match against _rules = rules
:param target_dict: dict of object properties
:param credentials_dict: dict of actor properties
:returns: True if the check passes
# Ditto
def reset():
"""Clear the rules used for policy checks."""
global _rules
_rules = None
def check(rule, target, creds, exc=None, *args, **kwargs):
""" """
if not match_list: Checks authorization of a rule against the target and credentials.
return True
for and_list in match_list: :param rule: The rule to evaluate.
if isinstance(and_list, basestring): :param target: As much information about the object being operated
and_list = (and_list,) on as possible, as a dictionary.
if all([self._check(item, target_dict, cred_dict) :param creds: As much information about the user performing the
for item in and_list]): action as possible, as a dictionary.
return True :param exc: Class of the exception to raise if the check fails.
return False Any remaining arguments passed to check() (both
positional and keyword arguments) will be passed to
the exception class. If exc is not provided, returns
False.
:return: Returns False if the policy does not allow the action and
exc is not provided; otherwise, returns a value that
evaluates to True. Note: for rules using the "case"
expression, this True value will be the specified string
from the expression.
"""
# Allow the rule to be a Check tree
if isinstance(rule, BaseCheck):
result = rule(target, creds)
elif not _rules:
# No rules to reference means we're going to fail closed
result = False
else:
try:
# Evaluate the rule
result = _rules[rule](target, creds)
except KeyError:
# If the rule doesn't exist, fail closed
result = False
# If it is False, raise the exception if requested
if exc and result is False:
raise exc(*args, **kwargs)
return result
class HttpBrain(Brain): class BaseCheck(object):
"""A brain that can check external urls for policy. """
Abstract base class for Check classes.
"""
Posts json blobs for target and credentials. __metaclass__ = abc.ABCMeta
Note that this brain is deprecated; the http check is registered @abc.abstractmethod
by default. def __str__(self):
"""
Retrieve a string representation of the Check tree rooted at
this node.
"""
pass
@abc.abstractmethod
def __call__(self, target, cred):
"""
Perform the check. Returns False to reject the access or a
true value (not necessary True) to accept the access.
""" """
pass pass
def register(name, func=None): class FalseCheck(BaseCheck):
""" """
Register a function as a policy check. A policy check that always returns False (disallow).
"""
:param name: Gives the name of the check type, e.g., 'rule',
'role', etc. If name is None, a default function def __str__(self):
will be registered. """Return a string representation of this check."""
:param func: If given, provides the function to register. If not
given, returns a function taking one argument to return "!"
specify the function to register, allowing use as a
decorator. def __call__(self, target, cred):
"""Check the policy."""
return False
class TrueCheck(BaseCheck):
"""
A policy check that always returns True (allow).
"""
def __str__(self):
"""Return a string representation of this check."""
return "@"
def __call__(self, target, cred):
"""Check the policy."""
return True
class Check(BaseCheck):
"""
A base class to allow for user-defined policy checks.
"""
def __init__(self, kind, match):
"""
:param kind: The kind of the check, i.e., the field before the
':'.
:param match: The match of the check, i.e., the field after
the ':'.
"""
self.kind = kind
self.match = match
def __str__(self):
"""Return a string representation of this check."""
return "%s:%s" % (self.kind, self.match)
class NotCheck(BaseCheck):
"""
A policy check that inverts the result of another policy check.
Implements the "not" operator.
"""
def __init__(self, rule):
"""
Initialize the 'not' check.
:param rule: The rule to negate. Must be a Check.
"""
self.rule = rule
def __str__(self):
"""Return a string representation of this check."""
return "not %s" % self.rule
def __call__(self, target, cred):
"""
Check the policy. Returns the logical inverse of the wrapped
check.
"""
return not self.rule(target, cred)
class AndCheck(BaseCheck):
"""
A policy check that requires that a list of other checks all
return True. Implements the "and" operator.
"""
def __init__(self, rules):
"""
Initialize the 'and' check.
:param rules: A list of rules that will be tested.
"""
self.rules = rules
def __str__(self):
"""Return a string representation of this check."""
return "(%s)" % ' and '.join(str(r) for r in self.rules)
def __call__(self, target, cred):
"""
Check the policy. Requires that all rules accept in order to
return True.
"""
for rule in self.rules:
if not rule(target, cred):
return False
return True
def add_check(self, rule):
"""
Allows addition of another rule to the list of rules that will
be tested. Returns the AndCheck object for convenience.
"""
self.rules.append(rule)
return self
class OrCheck(BaseCheck):
"""
A policy check that requires that at least one of a list of other
checks returns True. Implements the "or" operator.
"""
def __init__(self, rules):
"""
Initialize the 'or' check.
:param rules: A list of rules that will be tested.
"""
self.rules = rules
def __str__(self):
"""Return a string representation of this check."""
return "(%s)" % ' or '.join(str(r) for r in self.rules)
def __call__(self, target, cred):
"""
Check the policy. Requires that at least one rule accept in
order to return True.
"""
for rule in self.rules:
if rule(target, cred):
return True
return False
def add_check(self, rule):
"""
Allows addition of another rule to the list of rules that will
be tested. Returns the OrCheck object for convenience.
"""
self.rules.append(rule)
return self
def _parse_check(rule):
"""
Parse a single base check rule into an appropriate Check object.
"""
# Handle the special checks
if rule == '!':
return FalseCheck()
elif rule == '@':
return TrueCheck()
try:
kind, match = rule.split(':', 1)
except Exception:
LOG.exception(_("Failed to understand rule %(rule)s") % locals())
# If the rule is invalid, we'll fail closed
return FalseCheck()
# Find what implements the check
if kind in _checks:
return _checks[kind](kind, match)
elif None in _checks:
return _checks[None](kind, match)
else:
LOG.error(_("No handler for matches of kind %s") % kind)
return FalseCheck()
def _parse_list_rule(rule):
"""
Provided for backwards compatibility. Translates the old
list-of-lists syntax into a tree of Check objects.
"""
# Empty rule defaults to True
if not rule:
return TrueCheck()
# Outer list is joined by "or"; inner list by "and"
or_list = []
for inner_rule in rule:
# Elide empty inner lists
if not inner_rule:
continue
# Handle bare strings
if isinstance(inner_rule, basestring):
inner_rule = [inner_rule]
# Parse the inner rules into Check objects
and_list = [_parse_check(r) for r in inner_rule]
# Append the appropriate check to the or_list
if len(and_list) == 1:
or_list.append(and_list[0])
else:
or_list.append(AndCheck(and_list))
# If we have only one check, omit the "or"
if len(or_list) == 0:
return FalseCheck()
elif len(or_list) == 1:
return or_list[0]
return OrCheck(or_list)
# Used for tokenizing the policy language
_tokenize_re = re.compile(r'\s+')
def _parse_tokenize(rule):
"""
Tokenizer for the policy language.
Most of the single-character tokens are specified in the
_tokenize_re; however, parentheses need to be handled specially,
because they can appear inside a check string. Thankfully, those
parentheses that appear inside a check string can never occur at
the very beginning or end ("%(variable)s" is the correct syntax).
"""
for tok in _tokenize_re.split(rule):
# Skip empty tokens
if not tok or tok.isspace():
continue
# Handle leading parens on the token
clean = tok.lstrip('(')
for i in range(len(tok) - len(clean)):
yield '(', '('
# If it was only parentheses, continue
if not clean:
continue
else:
tok = clean
# Handle trailing parens on the token
clean = tok.rstrip(')')
trail = len(tok) - len(clean)
# Yield the cleaned token
lowered = clean.lower()
if lowered in ('and', 'or', 'not'):
# Special tokens
yield lowered, clean
elif clean:
# Not a special token, but not composed solely of ')'
if len(tok) >= 2 and ((tok[0], tok[-1]) in
[('"', '"'), ("'", "'")]):
# It's a quoted string
yield 'string', tok[1:-1]
else:
yield 'check', _parse_check(clean)
# Yield the trailing parens
for i in range(trail):
yield ')', ')'
class ParseStateMeta(type):
"""
Metaclass for the ParseState class. Facilitates identifying
reduction methods.
"""
def __new__(mcs, name, bases, cls_dict):
"""
Create the class. Injects the 'reducers' list, a list of
tuples matching token sequences to the names of the
corresponding reduction methods.
"""
reducers = []
for key, value in cls_dict.items():
if not hasattr(value, 'reducers'):
continue
for reduction in value.reducers:
reducers.append((reduction, key))
cls_dict['reducers'] = reducers
return super(ParseStateMeta, mcs).__new__(mcs, name, bases, cls_dict)
def reducer(*tokens):
"""
Decorator for reduction methods. Arguments are a sequence of
tokens, in order, which should trigger running this reduction
method.
""" """
# Perform the actual decoration by registering the function.
# Returns the function for compliance with the decorator
# interface.
def decorator(func): def decorator(func):
# Register the function # Make sure we have a list of reducer sequences
Brain._register(name, func) if not hasattr(func, 'reducers'):
func.reducers = []
# Add the tokens to the list of reducer sequences
func.reducers.append(list(tokens))
return func return func
# If the function is given, do the registration return decorator
class ParseState(object):
"""
Implement the core of parsing the policy language. Uses a greedy
reduction algorithm to reduce a sequence of tokens into a single
terminal, the value of which will be the root of the Check tree.
Note: error reporting is rather lacking. The best we can get with
this parser formulation is an overall "parse failed" error.
Fortunately, the policy language is simple enough that this
shouldn't be that big a problem.
"""
__metaclass__ = ParseStateMeta
def __init__(self):
"""Initialize the ParseState."""
self.tokens = []
self.values = []
def reduce(self):
"""
Perform a greedy reduction of the token stream. If a reducer
method matches, it will be executed, then the reduce() method
will be called recursively to search for any more possible
reductions.
"""
for reduction, methname in self.reducers:
if (len(self.tokens) >= len(reduction) and
self.tokens[-len(reduction):] == reduction):
# Get the reduction method
meth = getattr(self, methname)
# Reduce the token stream
results = meth(*self.values[-len(reduction):])
# Update the tokens and values
self.tokens[-len(reduction):] = [r[0] for r in results]
self.values[-len(reduction):] = [r[1] for r in results]
# Check for any more reductions
return self.reduce()
def shift(self, tok, value):
"""Adds one more token to the state. Calls reduce()."""
self.tokens.append(tok)
self.values.append(value)
# Do a greedy reduce...
self.reduce()
@property
def result(self):
"""
Obtain the final result of the parse. Raises ValueError if
the parse failed to reduce to a single result.
"""
if len(self.values) != 1:
raise ValueError("Could not parse rule")
return self.values[0]
@reducer('(', 'check', ')')
@reducer('(', 'and_expr', ')')
@reducer('(', 'or_expr', ')')
def _wrap_check(self, _p1, check, _p2):
"""Turn parenthesized expressions into a 'check' token."""
return [('check', check)]
@reducer('check', 'and', 'check')
def _make_and_expr(self, check1, _and, check2):
"""
Create an 'and_expr' from two checks joined by the 'and'
operator.
"""
return [('and_expr', AndCheck([check1, check2]))]
@reducer('and_expr', 'and', 'check')
def _extend_and_expr(self, and_expr, _and, check):
"""
Extend an 'and_expr' by adding one more check.
"""
return [('and_expr', and_expr.add_check(check))]
@reducer('check', 'or', 'check')
def _make_or_expr(self, check1, _or, check2):
"""
Create an 'or_expr' from two checks joined by the 'or'
operator.
"""
return [('or_expr', OrCheck([check1, check2]))]
@reducer('or_expr', 'or', 'check')
def _extend_or_expr(self, or_expr, _or, check):
"""
Extend an 'or_expr' by adding one more check.
"""
return [('or_expr', or_expr.add_check(check))]
@reducer('not', 'check')
def _make_not_expr(self, _not, check):
"""Invert the result of another check."""
return [('check', NotCheck(check))]
def _parse_text_rule(rule):
"""
Translates a policy written in the policy language into a tree of
Check objects.
"""
# Empty rule means always accept
if not rule:
return TrueCheck()
# Parse the token stream
state = ParseState()
for tok, value in _parse_tokenize(rule):
state.shift(tok, value)
try:
return state.result
except ValueError:
# Couldn't parse the rule
LOG.exception(_("Failed to understand rule %(rule)r") % locals())
# Fail closed
return FalseCheck()
def parse_rule(rule):
"""
Parses a policy rule into a tree of Check objects.
"""
# If the rule is a string, it's in the policy language
if isinstance(rule, basestring):
return _parse_text_rule(rule)
return _parse_list_rule(rule)
def register(name, func=None):
"""
Register a function or Check class as a policy check.
:param name: Gives the name of the check type, e.g., 'rule',
'role', etc. If name is None, a default check type
will be registered.
:param func: If given, provides the function or class to register.
If not given, returns a function taking one argument
to specify the function or class to register,
allowing use as a decorator.
"""
# Perform the actual decoration by registering the function or
# class. Returns the function or class for compliance with the
# decorator interface.
def decorator(func):
_checks[name] = func
return func
# If the function or class is given, do the registration
if func: if func:
return decorator(func) return decorator(func)
@ -247,55 +721,59 @@ def register(name, func=None):
@register("rule") @register("rule")
def _check_rule(brain, match_kind, match, target_dict, cred_dict): class RuleCheck(Check):
"""Recursively checks credentials based on the brains rules.""" def __call__(self, target, creds):
try: """
new_match_list = brain.rules[match] Recursively checks credentials based on the defined rules.
except KeyError: """
if brain.default_rule and match != brain.default_rule:
new_match_list = ('rule:%s' % brain.default_rule,)
else:
return False
return brain.check(new_match_list, target_dict, cred_dict) try:
return _rules[self.match](target, creds)
except KeyError:
# We don't have any matching rule; fail closed
return False
@register("role") @register("role")
def _check_role(brain, match_kind, match, target_dict, cred_dict): class RoleCheck(Check):
def __call__(self, target, creds):
"""Check that there is a matching role in the cred dict.""" """Check that there is a matching role in the cred dict."""
return match.lower() in [x.lower() for x in cred_dict['roles']]
return self.match.lower() in [x.lower() for x in creds['roles']]
@register('http') @register('http')
def _check_http(brain, match_kind, match, target_dict, cred_dict): class HttpCheck(Check):
"""Check http: rules by calling to a remote server. def __call__(self, target, creds):
This example implementation simply verifies that the response is
exactly 'True'. A custom brain using response codes could easily
be implemented.
""" """
url = 'http:' + (match % target_dict) Check http: rules by calling to a remote server.
data = {'target': jsonutils.dumps(target_dict),
'credentials': jsonutils.dumps(cred_dict)} This example implementation simply verifies that the response
is exactly 'True'.
"""
url = ('http:' + self.match) % target
data = {'target': jsonutils.dumps(target),
'credentials': jsonutils.dumps(creds)}
post_data = urllib.urlencode(data) post_data = urllib.urlencode(data)
f = urllib2.urlopen(url, post_data) f = urllib2.urlopen(url, post_data)
return f.read() == "True" return f.read() == "True"
@register(None) @register(None)
def _check_generic(brain, match_kind, match, target_dict, cred_dict): class GenericCheck(Check):
"""Check an individual match. def __call__(self, target, creds):
"""
Check an individual match.
Matches look like: Matches look like:
tenant:%(tenant_id)s tenant:%(tenant_id)s
role:compute:admin role:compute:admin
""" """
# TODO(termie): do dict inspection via dot syntax # TODO(termie): do dict inspection via dot syntax
match = match % target_dict match = self.match % target
if match_kind in cred_dict: if self.kind in creds:
return match == unicode(cred_dict[match_kind]) return match == unicode(creds[self.kind])
return False return False

135
kwapi/openstack/common/processutils.py Normal file
View File

@ -0,0 +1,135 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import logging
import random
import shlex
from eventlet.green import subprocess
from eventlet import greenthread
from kwapi.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
class UnknownArgumentError(Exception):
def __init__(self, message=None):
super(UnknownArgumentError, self).__init__(message)
class ProcessExecutionError(Exception):
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
description=None):
if description is None:
description = "Unexpected error while running command."
if exit_code is None:
exit_code = '-'
message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
% (description, cmd, exit_code, stdout, stderr))
super(ProcessExecutionError, self).__init__(message)
def execute(*cmd, **kwargs):
"""
Helper method to shell out and execute a command through subprocess with
optional retry.
:param cmd: Passed to subprocess.Popen.
:type cmd: string
:param process_input: Send to opened process.
:type proces_input: string
:param check_exit_code: Defaults to 0. Will raise
:class:`ProcessExecutionError`
if the command exits without returning this value
as a returncode
:type check_exit_code: int
:param delay_on_retry: True | False. Defaults to True. If set to True,
wait a short amount of time before retrying.
:type delay_on_retry: boolean
:param attempts: How many times to retry cmd.
:type attempts: int
:param run_as_root: True | False. Defaults to False. If set to True,
the command is prefixed by the command specified
in the root_helper kwarg.
:type run_as_root: boolean
:param root_helper: command to prefix all cmd's with
:type root_helper: string
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
:raises: :class:`ProcessExecutionError`
"""
process_input = kwargs.pop('process_input', None)
check_exit_code = kwargs.pop('check_exit_code', 0)
delay_on_retry = kwargs.pop('delay_on_retry', True)
attempts = kwargs.pop('attempts', 1)
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
if len(kwargs):
raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs)
if run_as_root:
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
while attempts > 0:
attempts -= 1
try:
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
_PIPE = subprocess.PIPE # pylint: disable=E1101
obj = subprocess.Popen(cmd,
stdin=_PIPE,
stdout=_PIPE,
stderr=_PIPE,
close_fds=True)
result = None
if process_input is not None:
result = obj.communicate(process_input)
else:
result = obj.communicate()
obj.stdin.close() # pylint: disable=E1101
_returncode = obj.returncode # pylint: disable=E1101
if _returncode:
LOG.debug(_('Result was %s') % _returncode)
if (isinstance(check_exit_code, int) and
not isinstance(check_exit_code, bool) and
_returncode != check_exit_code):
(stdout, stderr) = result
raise ProcessExecutionError(exit_code=_returncode,
stdout=stdout,
stderr=stderr,
cmd=' '.join(cmd))
return result
except ProcessExecutionError:
if not attempts:
raise
else:
LOG.debug(_('%r failed. Retrying.'), cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:
# NOTE(termie): this appears to be necessary to let the subprocess
# call clean something up in between calls, without
# it two execute calls in a row hangs the second one
greenthread.sleep(0)

16
kwapi/openstack/common/rootwrap/__init__.py Normal file
View File

@ -0,0 +1,16 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

180
kwapi/openstack/common/rootwrap/filters.py Normal file
View File

@ -0,0 +1,180 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
class CommandFilter(object):
"""Command filter only checking that the 1st argument matches exec_path"""
def __init__(self, exec_path, run_as, *args):
self.name = ''
self.exec_path = exec_path
self.run_as = run_as
self.args = args
self.real_exec = None
def get_exec(self, exec_dirs=[]):
"""Returns existing executable, or empty string if none found"""
if self.real_exec is not None:
return self.real_exec
self.real_exec = ""
if self.exec_path.startswith('/'):
if os.access(self.exec_path, os.X_OK):
self.real_exec = self.exec_path
else:
for binary_path in exec_dirs:
expanded_path = os.path.join(binary_path, self.exec_path)
if os.access(expanded_path, os.X_OK):
self.real_exec = expanded_path
break
return self.real_exec
def match(self, userargs):
"""Only check that the first argument (command) matches exec_path"""
if (os.path.basename(self.exec_path) == userargs[0]):
return True
return False
def get_command(self, userargs, exec_dirs=[]):
"""Returns command to execute (with sudo -u if run_as != root)."""
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
if (self.run_as != 'root'):
# Used to run commands at lesser privileges
return ['sudo', '-u', self.run_as, to_exec] + userargs[1:]
return [to_exec] + userargs[1:]
def get_environment(self, userargs):
"""Returns specific environment to set, None if none"""
return None
class RegExpFilter(CommandFilter):
"""Command filter doing regexp matching for every argument"""
def match(self, userargs):
# Early skip if command or number of args don't match
if (len(self.args) != len(userargs)):
# DENY: argument numbers don't match
return False
# Compare each arg (anchoring pattern explicitly at end of string)
for (pattern, arg) in zip(self.args, userargs):
try:
if not re.match(pattern + '$', arg):
break
except re.error:
# DENY: Badly-formed filter
return False
else:
# ALLOW: All arguments matched
return True
# DENY: Some arguments did not match
return False
class DnsmasqFilter(CommandFilter):
"""Specific filter for the dnsmasq call (which includes env)"""
CONFIG_FILE_ARG = 'CONFIG_FILE'
def match(self, userargs):
if (userargs[0] == 'env' and
userargs[1].startswith(self.CONFIG_FILE_ARG) and
userargs[2].startswith('NETWORK_ID=') and
userargs[3] == 'dnsmasq'):
return True
return False
def get_command(self, userargs, exec_dirs=[]):
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
dnsmasq_pos = userargs.index('dnsmasq')
return [to_exec] + userargs[dnsmasq_pos + 1:]
def get_environment(self, userargs):
env = os.environ.copy()
env[self.CONFIG_FILE_ARG] = userargs[1].split('=')[-1]
env['NETWORK_ID'] = userargs[2].split('=')[-1]
return env
class DeprecatedDnsmasqFilter(DnsmasqFilter):
"""Variant of dnsmasq filter to support old-style FLAGFILE"""
CONFIG_FILE_ARG = 'FLAGFILE'
class KillFilter(CommandFilter):
"""Specific filter for the kill calls.
1st argument is the user to run /bin/kill under
2nd argument is the location of the affected executable
Subsequent arguments list the accepted signals (if any)
This filter relies on /proc to accurately determine affected
executable, so it will only work on procfs-capable systems (not OSX).
"""
def __init__(self, *args):
super(KillFilter, self).__init__("/bin/kill", *args)
def match(self, userargs):
if userargs[0] != "kill":
return False
args = list(userargs)
if len(args) == 3:
# A specific signal is requested
signal = args.pop(1)
if signal not in self.args[1:]:
# Requested signal not in accepted list
return False
else:
if len(args) != 2:
# Incorrect number of arguments
return False
if len(self.args) > 1:
# No signal requested, but filter requires specific signal
return False
try:
command = os.readlink("/proc/%d/exe" % int(args[1]))
# NOTE(dprince): /proc/PID/exe may have ' (deleted)' on
# the end if an executable is updated or deleted
if command.endswith(" (deleted)"):
command = command[:command.rindex(" ")]
if command != self.args[0]:
# Affected executable does not match
return False
except (ValueError, OSError):
# Incorrect PID
return False
return True
class ReadFileFilter(CommandFilter):
"""Specific filter for the utils.read_file_as_root call"""
def __init__(self, file_path, *args):
self.file_path = file_path
super(ReadFileFilter, self).__init__("/bin/cat", "root", *args)
def match(self, userargs):
if userargs[0] != 'cat':
return False
if userargs[1] != self.file_path:
return False
if len(userargs) != 2:
return False
return True

149
kwapi/openstack/common/rootwrap/wrapper.py Normal file
View File

@ -0,0 +1,149 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import logging
import logging.handlers
import os
import string
from kwapi.openstack.common.rootwrap import filters
class NoFilterMatched(Exception):
"""This exception is raised when no filter matched."""
pass
class FilterMatchNotExecutable(Exception):
"""
This exception is raised when a filter matched but no executable was
found.
"""
def __init__(self, match=None, **kwargs):
self.match = match
class RootwrapConfig(object):
def __init__(self, config):
# filters_path
self.filters_path = config.get("DEFAULT", "filters_path").split(",")
# exec_dirs
if config.has_option("DEFAULT", "exec_dirs"):
self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",")
else:
# Use system PATH if exec_dirs is not specified
self.exec_dirs = os.environ["PATH"].split(':')
# syslog_log_facility
if config.has_option("DEFAULT", "syslog_log_facility"):
v = config.get("DEFAULT", "syslog_log_facility")
facility_names = logging.handlers.SysLogHandler.facility_names
self.syslog_log_facility = getattr(logging.handlers.SysLogHandler,
v, None)
if self.syslog_log_facility is None and v in facility_names:
self.syslog_log_facility = facility_names.get(v)
if self.syslog_log_facility is None:
raise ValueError('Unexpected syslog_log_facility: %s' % v)
else:
default_facility = logging.handlers.SysLogHandler.LOG_SYSLOG
self.syslog_log_facility = default_facility
# syslog_log_level
if config.has_option("DEFAULT", "syslog_log_level"):
v = config.get("DEFAULT", "syslog_log_level")
self.syslog_log_level = logging.getLevelName(v.upper())
if (self.syslog_log_level == "Level %s" % v.upper()):
raise ValueError('Unexepected syslog_log_level: %s' % v)
else:
self.syslog_log_level = logging.ERROR
# use_syslog
if config.has_option("DEFAULT", "use_syslog"):
self.use_syslog = config.getboolean("DEFAULT", "use_syslog")
else:
self.use_syslog = False
def setup_syslog(execname, facility, level):
rootwrap_logger = logging.getLogger()
rootwrap_logger.setLevel(level)
handler = logging.handlers.SysLogHandler(address='/dev/log',
facility=facility)
handler.setFormatter(logging.Formatter(
os.path.basename(execname) + ': %(message)s'))
rootwrap_logger.addHandler(handler)
def build_filter(class_name, *args):
"""Returns a filter object of class class_name"""
if not hasattr(filters, class_name):
logging.warning("Skipping unknown filter class (%s) specified "
"in filter definitions" % class_name)
return None
filterclass = getattr(filters, class_name)
return filterclass(*args)
def load_filters(filters_path):
"""Load filters from a list of directories"""
filterlist = []
for filterdir in filters_path:
if not os.path.isdir(filterdir):
continue
for filterfile in os.listdir(filterdir):
filterconfig = ConfigParser.RawConfigParser()
filterconfig.read(os.path.join(filterdir, filterfile))
for (name, value) in filterconfig.items("Filters"):
filterdefinition = [string.strip(s) for s in value.split(',')]
newfilter = build_filter(*filterdefinition)
if newfilter is None:
continue
newfilter.name = name
filterlist.append(newfilter)
return filterlist
def match_filter(filters, userargs, exec_dirs=[]):
"""
Checks user command and arguments through command filters and
returns the first matching filter.
Raises NoFilterMatched if no filter matched.
Raises FilterMatchNotExecutable if no executable was found for the
best filter match.
"""
first_not_executable_filter = None
for f in filters:
if f.match(userargs):
# Try other filters if executable is absent
if not f.get_exec(exec_dirs=exec_dirs):
if not first_not_executable_filter:
first_not_executable_filter = f
continue
# Otherwise return matching filter for execution
return f
if first_not_executable_filter:
# A filter matched, but no executable was found for it
raise FilterMatchNotExecutable(match=first_not_executable_filter)
# No filter matched
raise NoFilterMatched()

24
kwapi/openstack/common/rpc/__init__.py
View File

@ -47,28 +47,29 @@ rpc_opts = [
help='Seconds to wait before a cast expires (TTL). ' help='Seconds to wait before a cast expires (TTL). '
'Only supported by impl_zmq.'), 'Only supported by impl_zmq.'),
cfg.ListOpt('allowed_rpc_exception_modules', cfg.ListOpt('allowed_rpc_exception_modules',
default=['kwapi.openstack.common.exception', default=['openstack.common.exception',
'nova.exception', 'nova.exception',
'cinder.exception', 'cinder.exception',
'exceptions',
], ],
help='Modules of exceptions that are permitted to be recreated' help='Modules of exceptions that are permitted to be recreated'
'upon receiving exception data from an rpc call.'), 'upon receiving exception data from an rpc call.'),
cfg.BoolOpt('fake_rabbit', cfg.BoolOpt('fake_rabbit',
default=False, default=False,
help='If passed, use a fake RabbitMQ provider'), help='If passed, use a fake RabbitMQ provider'),
# cfg.StrOpt('control_exchange',
# The following options are not registered here, but are expected to be default='openstack',
# present. The project using this library must register these options with help='AMQP exchange to connect to if using RabbitMQ or Qpid'),
# the configuration so that project-specific defaults may be defined.
#
#cfg.StrOpt('control_exchange',
# default='nova',
# help='AMQP exchange to connect to if using RabbitMQ or Qpid'),
] ]
cfg.CONF.register_opts(rpc_opts) cfg.CONF.register_opts(rpc_opts)
def set_defaults(control_exchange):
cfg.set_defaults(rpc_opts,
control_exchange=control_exchange)
def create_connection(new=True): def create_connection(new=True):
"""Create a connection to the message bus used for rpc. """Create a connection to the message bus used for rpc.
@ -177,17 +178,18 @@ def multicall(context, topic, msg, timeout=None):
return _get_impl().multicall(cfg.CONF, context, topic, msg, timeout) return _get_impl().multicall(cfg.CONF, context, topic, msg, timeout)
def notify(context, topic, msg): def notify(context, topic, msg, envelope=False):
"""Send notification event. """Send notification event.
:param context: Information that identifies the user that has made this :param context: Information that identifies the user that has made this
request. request.
:param topic: The topic to send the notification to. :param topic: The topic to send the notification to.
:param msg: This is a dict of content of event. :param msg: This is a dict of content of event.
:param envelope: Set to True to enable message envelope for notifications.
:returns: None :returns: None
""" """
return _get_impl().notify(cfg.CONF, context, topic, msg) return _get_impl().notify(cfg.CONF, context, topic, msg, envelope)
def cleanup(): def cleanup():

54
kwapi/openstack/common/rpc/amqp.py
View File

@ -26,7 +26,6 @@ AMQP, but is deprecated and predates this code.
""" """
import inspect import inspect
import logging
import sys import sys
import uuid import uuid
@ -34,10 +33,10 @@ from eventlet import greenpool
from eventlet import pools from eventlet import pools
from eventlet import semaphore from eventlet import semaphore
from kwapi.openstack.common import cfg
from kwapi.openstack.common import excutils from kwapi.openstack.common import excutils
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import local from kwapi.openstack.common import local
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.rpc import common as rpc_common from kwapi.openstack.common.rpc import common as rpc_common
@ -55,7 +54,7 @@ class Pool(pools.Pool):
# TODO(comstud): Timeout connections not used in a while # TODO(comstud): Timeout connections not used in a while
def create(self): def create(self):
LOG.debug('Pool creating new connection') LOG.debug(_('Pool creating new connection'))
return self.connection_cls(self.conf) return self.connection_cls(self.conf)
def empty(self): def empty(self):
@ -150,7 +149,7 @@ class ConnectionContext(rpc_common.Connection):
def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None, def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
ending=False): ending=False, log_failure=True):
"""Sends a reply or an error on the channel signified by msg_id. """Sends a reply or an error on the channel signified by msg_id.
Failure should be a sys.exc_info() tuple. Failure should be a sys.exc_info() tuple.
@ -158,7 +157,8 @@ def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
""" """
with ConnectionContext(conf, connection_pool) as conn: with ConnectionContext(conf, connection_pool) as conn:
if failure: if failure:
failure = rpc_common.serialize_remote_exception(failure) failure = rpc_common.serialize_remote_exception(failure,
log_failure)
try: try:
msg = {'result': reply, 'failure': failure} msg = {'result': reply, 'failure': failure}
@ -168,7 +168,7 @@ def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
'failure': failure} 'failure': failure}
if ending: if ending:
msg['ending'] = True msg['ending'] = True
conn.direct_send(msg_id, msg) conn.direct_send(msg_id, rpc_common.serialize_msg(msg))
class RpcContext(rpc_common.CommonRpcContext): class RpcContext(rpc_common.CommonRpcContext):
@ -185,10 +185,10 @@ class RpcContext(rpc_common.CommonRpcContext):
return self.__class__(**values) return self.__class__(**values)
def reply(self, reply=None, failure=None, ending=False, def reply(self, reply=None, failure=None, ending=False,
connection_pool=None): connection_pool=None, log_failure=True):
if self.msg_id: if self.msg_id:
msg_reply(self.conf, self.msg_id, connection_pool, reply, failure, msg_reply(self.conf, self.msg_id, connection_pool, reply, failure,
ending) ending, log_failure)
if ending: if ending:
self.msg_id = None self.msg_id = None
@ -282,11 +282,21 @@ class ProxyCallback(object):
ctxt.reply(rval, None, connection_pool=self.connection_pool) ctxt.reply(rval, None, connection_pool=self.connection_pool)
# This final None tells multicall that it is done. # This final None tells multicall that it is done.
ctxt.reply(ending=True, connection_pool=self.connection_pool) ctxt.reply(ending=True, connection_pool=self.connection_pool)
except Exception as e: except rpc_common.ClientException as e:
LOG.exception('Exception during message handling') LOG.debug(_('Expected exception during message handling (%s)') %
e._exc_info[1])
ctxt.reply(None, e._exc_info,
connection_pool=self.connection_pool,
log_failure=False)
except Exception:
LOG.exception(_('Exception during message handling'))
ctxt.reply(None, sys.exc_info(), ctxt.reply(None, sys.exc_info(),
connection_pool=self.connection_pool) connection_pool=self.connection_pool)
def wait(self):
"""Wait for all callback threads to exit."""
self.pool.waitall()
class MulticallWaiter(object): class MulticallWaiter(object):
def __init__(self, conf, connection, timeout): def __init__(self, conf, connection, timeout):
@ -349,7 +359,7 @@ def multicall(conf, context, topic, msg, timeout, connection_pool):
# that will continue to use the connection. When it's done, # that will continue to use the connection. When it's done,
# connection.close() will get called which will put it back into # connection.close() will get called which will put it back into
# the pool # the pool
LOG.debug(_('Making asynchronous call on %s ...'), topic) LOG.debug(_('Making synchronous call on %s ...'), topic)
msg_id = uuid.uuid4().hex msg_id = uuid.uuid4().hex
msg.update({'_msg_id': msg_id}) msg.update({'_msg_id': msg_id})
LOG.debug(_('MSG_ID is %s') % (msg_id)) LOG.debug(_('MSG_ID is %s') % (msg_id))
@ -358,7 +368,7 @@ def multicall(conf, context, topic, msg, timeout, connection_pool):
conn = ConnectionContext(conf, connection_pool) conn = ConnectionContext(conf, connection_pool)
wait_msg = MulticallWaiter(conf, conn, timeout) wait_msg = MulticallWaiter(conf, conn, timeout)
conn.declare_direct_consumer(msg_id, wait_msg) conn.declare_direct_consumer(msg_id, wait_msg)
conn.topic_send(topic, msg) conn.topic_send(topic, rpc_common.serialize_msg(msg))
return wait_msg return wait_msg
@ -377,7 +387,7 @@ def cast(conf, context, topic, msg, connection_pool):
LOG.debug(_('Making asynchronous cast on %s...'), topic) LOG.debug(_('Making asynchronous cast on %s...'), topic)
pack_context(msg, context) pack_context(msg, context)
with ConnectionContext(conf, connection_pool) as conn: with ConnectionContext(conf, connection_pool) as conn:
conn.topic_send(topic, msg) conn.topic_send(topic, rpc_common.serialize_msg(msg))
def fanout_cast(conf, context, topic, msg, connection_pool): def fanout_cast(conf, context, topic, msg, connection_pool):
@ -385,7 +395,7 @@ def fanout_cast(conf, context, topic, msg, connection_pool):
LOG.debug(_('Making asynchronous fanout cast...')) LOG.debug(_('Making asynchronous fanout cast...'))
pack_context(msg, context) pack_context(msg, context)
with ConnectionContext(conf, connection_pool) as conn: with ConnectionContext(conf, connection_pool) as conn:
conn.fanout_send(topic, msg) conn.fanout_send(topic, rpc_common.serialize_msg(msg))
def cast_to_server(conf, context, server_params, topic, msg, connection_pool): def cast_to_server(conf, context, server_params, topic, msg, connection_pool):
@ -393,7 +403,7 @@ def cast_to_server(conf, context, server_params, topic, msg, connection_pool):
pack_context(msg, context) pack_context(msg, context)
with ConnectionContext(conf, connection_pool, pooled=False, with ConnectionContext(conf, connection_pool, pooled=False,
server_params=server_params) as conn: server_params=server_params) as conn:
conn.topic_send(topic, msg) conn.topic_send(topic, rpc_common.serialize_msg(msg))
def fanout_cast_to_server(conf, context, server_params, topic, msg, def fanout_cast_to_server(conf, context, server_params, topic, msg,
@ -402,15 +412,18 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg,
pack_context(msg, context) pack_context(msg, context)
with ConnectionContext(conf, connection_pool, pooled=False, with ConnectionContext(conf, connection_pool, pooled=False,
server_params=server_params) as conn: server_params=server_params) as conn:
conn.fanout_send(topic, msg) conn.fanout_send(topic, rpc_common.serialize_msg(msg))
def notify(conf, context, topic, msg, connection_pool): def notify(conf, context, topic, msg, connection_pool, envelope):
"""Sends a notification event on a topic.""" """Sends a notification event on a topic."""
event_type = msg.get('event_type') LOG.debug(_('Sending %(event_type)s on %(topic)s'),
LOG.debug(_('Sending %(event_type)s on %(topic)s'), locals()) dict(event_type=msg.get('event_type'),
topic=topic))
pack_context(msg, context) pack_context(msg, context)
with ConnectionContext(conf, connection_pool) as conn: with ConnectionContext(conf, connection_pool) as conn:
if envelope:
msg = rpc_common.serialize_msg(msg, force_envelope=True)
conn.notify_send(topic, msg) conn.notify_send(topic, msg)
@ -420,7 +433,4 @@ def cleanup(connection_pool):
def get_control_exchange(conf): def get_control_exchange(conf):
try:
return conf.control_exchange return conf.control_exchange
except cfg.NoSuchOptError:
return 'openstack'

185
kwapi/openstack/common/rpc/common.py
View File

@ -18,18 +18,61 @@
# under the License. # under the License.
import copy import copy
import logging import sys
import traceback import traceback
from kwapi.openstack.common import cfg
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import importutils from kwapi.openstack.common import importutils
from kwapi.openstack.common import jsonutils from kwapi.openstack.common import jsonutils
from kwapi.openstack.common import local from kwapi.openstack.common import local
from kwapi.openstack.common import log as logging
CONF = cfg.CONF
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
'''RPC Envelope Version.
This version number applies to the top level structure of messages sent out.
It does *not* apply to the message payload, which must be versioned
independently. For example, when using rpc APIs, a version number is applied
for changes to the API being exposed over rpc. This version number is handled
in the rpc proxy and dispatcher modules.
This version number applies to the message envelope that is used in the
serialization done inside the rpc layer. See serialize_msg() and
deserialize_msg().
The current message format (version 2.0) is very simple. It is:
{
'oslo.version': <RPC Envelope Version as a String>,
'oslo.message': <Application Message Payload, JSON encoded>
}
Message format version '1.0' is just considered to be the messages we sent
without a message envelope.
So, the current message envelope just includes the envelope version. It may
eventually contain additional information, such as a signature for the message
payload.
We will JSON encode the application message payload. The message envelope,
which includes the JSON encoded application message body, will be passed down
to the messaging libraries as a dict.
'''
_RPC_ENVELOPE_VERSION = '2.0'
_VERSION_KEY = 'oslo.version'
_MESSAGE_KEY = 'oslo.message'
# TODO(russellb) Turn this on after Grizzly.
_SEND_RPC_ENVELOPE = False
class RPCException(Exception): class RPCException(Exception):
message = _("An unknown RPC related exception occurred.") message = _("An unknown RPC related exception occurred.")
@ -40,7 +83,7 @@ class RPCException(Exception):
try: try:
message = self.message % kwargs message = self.message % kwargs
except Exception as e: except Exception:
# kwargs doesn't match a variable in the message # kwargs doesn't match a variable in the message
# log the issue and the kwargs # log the issue and the kwargs
LOG.exception(_('Exception in string format operation')) LOG.exception(_('Exception in string format operation'))
@ -90,6 +133,11 @@ class UnsupportedRpcVersion(RPCException):
"this endpoint.") "this endpoint.")
class UnsupportedRpcEnvelopeVersion(RPCException):
message = _("Specified RPC envelope version, %(version)s, "
"not supported by this endpoint.")
class Connection(object): class Connection(object):
"""A connection, returned by rpc.create_connection(). """A connection, returned by rpc.create_connection().
@ -164,8 +212,12 @@ class Connection(object):
def _safe_log(log_func, msg, msg_data): def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging.""" """Sanitizes the msg_data field before logging."""
SANITIZE = {'set_admin_password': ('new_pass',), SANITIZE = {'set_admin_password': [('args', 'new_pass')],
'run_instance': ('admin_password',), } 'run_instance': [('args', 'admin_password')],
'route_message': [('args', 'message', 'args', 'method_info',
'method_kwargs', 'password'),
('args', 'message', 'args', 'method_info',
'method_kwargs', 'admin_password')]}
has_method = 'method' in msg_data and msg_data['method'] in SANITIZE has_method = 'method' in msg_data and msg_data['method'] in SANITIZE
has_context_token = '_context_auth_token' in msg_data has_context_token = '_context_auth_token' in msg_data
@ -177,14 +229,16 @@ def _safe_log(log_func, msg, msg_data):
msg_data = copy.deepcopy(msg_data) msg_data = copy.deepcopy(msg_data)
if has_method: if has_method:
method = msg_data['method'] for arg in SANITIZE.get(msg_data['method'], []):
if method in SANITIZE:
args_to_sanitize = SANITIZE[method]
for arg in args_to_sanitize:
try: try:
msg_data['args'][arg] = "<SANITIZED>" d = msg_data
except KeyError: for elem in arg[:-1]:
pass d = d[elem]
d[arg[-1]] = '<SANITIZED>'
except KeyError, e:
LOG.info(_('Failed to sanitize %(item)s. Key error %(err)s'),
{'item': arg,
'err': e})
if has_context_token: if has_context_token:
msg_data['_context_auth_token'] = '<SANITIZED>' msg_data['_context_auth_token'] = '<SANITIZED>'
@ -195,7 +249,7 @@ def _safe_log(log_func, msg, msg_data):
return log_func(msg, msg_data) return log_func(msg, msg_data)
def serialize_remote_exception(failure_info): def serialize_remote_exception(failure_info, log_failure=True):
"""Prepares exception data to be sent over rpc. """Prepares exception data to be sent over rpc.
Failure_info should be a sys.exc_info() tuple. Failure_info should be a sys.exc_info() tuple.
@ -203,6 +257,7 @@ def serialize_remote_exception(failure_info):
""" """
tb = traceback.format_exception(*failure_info) tb = traceback.format_exception(*failure_info)
failure = failure_info[1] failure = failure_info[1]
if log_failure:
LOG.error(_("Returning exception %s to caller"), unicode(failure)) LOG.error(_("Returning exception %s to caller"), unicode(failure))
LOG.error(tb) LOG.error(tb)
@ -258,7 +313,7 @@ def deserialize_remote_exception(conf, data):
# we cannot necessarily change an exception message so we must override # we cannot necessarily change an exception message so we must override
# the __str__ method. # the __str__ method.
failure.__class__ = new_ex_type failure.__class__ = new_ex_type
except TypeError as e: except TypeError:
# NOTE(ameade): If a core exception then just add the traceback to the # NOTE(ameade): If a core exception then just add the traceback to the
# first exception argument. # first exception argument.
failure.args = (message,) + failure.args[1:] failure.args = (message,) + failure.args[1:]
@ -309,3 +364,107 @@ class CommonRpcContext(object):
context.values['read_deleted'] = read_deleted context.values['read_deleted'] = read_deleted
return context return context
class ClientException(Exception):
"""This encapsulates some actual exception that is expected to be
hit by an RPC proxy object. Merely instantiating it records the
current exception information, which will be passed back to the
RPC client without exceptional logging."""
def __init__(self):
self._exc_info = sys.exc_info()
def catch_client_exception(exceptions, func, *args, **kwargs):
try:
return func(*args, **kwargs)
except Exception, e:
if type(e) in exceptions:
raise ClientException()
else:
raise
def client_exceptions(*exceptions):
"""Decorator for manager methods that raise expected exceptions.
Marking a Manager method with this decorator allows the declaration
of expected exceptions that the RPC layer should not consider fatal,
and not log as if they were generated in a real error scenario. Note
that this will cause listed exceptions to be wrapped in a
ClientException, which is used internally by the RPC layer."""
def outer(func):
def inner(*args, **kwargs):
return catch_client_exception(exceptions, func, *args, **kwargs)
return inner
return outer
def version_is_compatible(imp_version, version):
"""Determine whether versions are compatible.
:param imp_version: The version implemented
:param version: The version requested by an incoming message.
"""
version_parts = version.split('.')
imp_version_parts = imp_version.split('.')
if int(version_parts[0]) != int(imp_version_parts[0]): # Major
return False
if int(version_parts[1]) > int(imp_version_parts[1]): # Minor
return False
return True
def serialize_msg(raw_msg, force_envelope=False):
if not _SEND_RPC_ENVELOPE and not force_envelope:
return raw_msg
# NOTE(russellb) See the docstring for _RPC_ENVELOPE_VERSION for more
# information about this format.
msg = {_VERSION_KEY: _RPC_ENVELOPE_VERSION,
_MESSAGE_KEY: jsonutils.dumps(raw_msg)}
return msg
def deserialize_msg(msg):
# NOTE(russellb): Hang on to your hats, this road is about to
# get a little bumpy.
#
# Robustness Principle:
# "Be strict in what you send, liberal in what you accept."
#
# At this point we have to do a bit of guessing about what it
# is we just received. Here is the set of possibilities:
#
# 1) We received a dict. This could be 2 things:
#
# a) Inspect it to see if it looks like a standard message envelope.
# If so, great!
#
# b) If it doesn't look like a standard message envelope, it could either
# be a notification, or a message from before we added a message
# envelope (referred to as version 1.0).
# Just return the message as-is.
#
# 2) It's any other non-dict type. Just return it and hope for the best.
# This case covers return values from rpc.call() from before message
# envelopes were used. (messages to call a method were always a dict)
if not isinstance(msg, dict):
# See #2 above.
return msg
base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY)
if not all(map(lambda key: key in msg, base_envelope_keys)):
# See #1.b above.
return msg
# At this point we think we have the message envelope
# format we were expecting. (#1.a above)
if not version_is_compatible(_RPC_ENVELOPE_VERSION, msg[_VERSION_KEY]):
raise UnsupportedRpcEnvelopeVersion(version=msg[_VERSION_KEY])
raw_msg = jsonutils.loads(msg[_MESSAGE_KEY])
return raw_msg

28
kwapi/openstack/common/rpc/dispatcher.py
View File

@ -41,8 +41,8 @@ server side of the API at the same time. However, as the code stands today,
there can be both versioned and unversioned APIs implemented in the same code there can be both versioned and unversioned APIs implemented in the same code
base. base.
EXAMPLES
EXAMPLES: ========
Nova was the first project to use versioned rpc APIs. Consider the compute rpc Nova was the first project to use versioned rpc APIs. Consider the compute rpc
API as an example. The client side is in nova/compute/rpcapi.py and the server API as an example. The client side is in nova/compute/rpcapi.py and the server
@ -50,12 +50,13 @@ side is in nova/compute/manager.py.
Example 1) Adding a new method. Example 1) Adding a new method.
-------------------------------
Adding a new method is a backwards compatible change. It should be added to Adding a new method is a backwards compatible change. It should be added to
nova/compute/manager.py, and RPC_API_VERSION should be bumped from X.Y to nova/compute/manager.py, and RPC_API_VERSION should be bumped from X.Y to
X.Y+1. On the client side, the new method in nova/compute/rpcapi.py should X.Y+1. On the client side, the new method in nova/compute/rpcapi.py should
have a specific version specified to indicate the minimum API version that must have a specific version specified to indicate the minimum API version that must
be implemented for the method to be supported. For example: be implemented for the method to be supported. For example::
def get_host_uptime(self, ctxt, host): def get_host_uptime(self, ctxt, host):
topic = _compute_topic(self.topic, ctxt, host, None) topic = _compute_topic(self.topic, ctxt, host, None)
@ -67,10 +68,11 @@ get_host_uptime() method.
Example 2) Adding a new parameter. Example 2) Adding a new parameter.
----------------------------------
Adding a new parameter to an rpc method can be made backwards compatible. The Adding a new parameter to an rpc method can be made backwards compatible. The
RPC_API_VERSION on the server side (nova/compute/manager.py) should be bumped. RPC_API_VERSION on the server side (nova/compute/manager.py) should be bumped.
The implementation of the method must not expect the parameter to be present. The implementation of the method must not expect the parameter to be present.::
def some_remote_method(self, arg1, arg2, newarg=None): def some_remote_method(self, arg1, arg2, newarg=None):
# The code needs to deal with newarg=None for cases # The code needs to deal with newarg=None for cases
@ -101,21 +103,6 @@ class RpcDispatcher(object):
self.callbacks = callbacks self.callbacks = callbacks
super(RpcDispatcher, self).__init__() super(RpcDispatcher, self).__init__()
@staticmethod
def _is_compatible(mversion, version):
"""Determine whether versions are compatible.
:param mversion: The API version implemented by a callback.
:param version: The API version requested by an incoming message.
"""
version_parts = version.split('.')
mversion_parts = mversion.split('.')
if int(version_parts[0]) != int(mversion_parts[0]): # Major
return False
if int(version_parts[1]) > int(mversion_parts[1]): # Minor
return False
return True
def dispatch(self, ctxt, version, method, **kwargs): def dispatch(self, ctxt, version, method, **kwargs):
"""Dispatch a message based on a requested version. """Dispatch a message based on a requested version.
@ -137,7 +124,8 @@ class RpcDispatcher(object):
rpc_api_version = proxyobj.RPC_API_VERSION rpc_api_version = proxyobj.RPC_API_VERSION
else: else:
rpc_api_version = '1.0' rpc_api_version = '1.0'
is_compatible = self._is_compatible(rpc_api_version, version) is_compatible = rpc_common.version_is_compatible(rpc_api_version,
version)
had_compatible = had_compatible or is_compatible had_compatible = had_compatible or is_compatible
if not hasattr(proxyobj, method): if not hasattr(proxyobj, method):
continue continue

11
kwapi/openstack/common/rpc/impl_fake.py
View File

@ -18,11 +18,15 @@ queues. Casts will block, but this is very useful for tests.
""" """
import inspect import inspect
# NOTE(russellb): We specifically want to use json, not our own jsonutils.
# jsonutils has some extra logic to automatically convert objects to primitive
# types so that they can be serialized. We want to catch all cases where
# non-primitive types make it into this code and treat it as an error.
import json
import time import time
import eventlet import eventlet
from kwapi.openstack.common import jsonutils
from kwapi.openstack.common.rpc import common as rpc_common from kwapi.openstack.common.rpc import common as rpc_common
CONSUMERS = {} CONSUMERS = {}
@ -75,6 +79,8 @@ class Consumer(object):
else: else:
res.append(rval) res.append(rval)
done.send(res) done.send(res)
except rpc_common.ClientException as e:
done.send_exception(e._exc_info[1])
except Exception as e: except Exception as e:
done.send_exception(e) done.send_exception(e)
@ -121,7 +127,7 @@ def create_connection(conf, new=True):
def check_serialize(msg): def check_serialize(msg):
"""Make sure a message intended for rpc can be serialized.""" """Make sure a message intended for rpc can be serialized."""
jsonutils.dumps(msg) json.dumps(msg)
def multicall(conf, context, topic, msg, timeout=None): def multicall(conf, context, topic, msg, timeout=None):
@ -154,6 +160,7 @@ def call(conf, context, topic, msg, timeout=None):
def cast(conf, context, topic, msg): def cast(conf, context, topic, msg):
check_serialize(msg)
try: try:
call(conf, context, topic, msg) call(conf, context, topic, msg)
except Exception: except Exception:

28
kwapi/openstack/common/rpc/impl_kombu.py
View File

@ -162,7 +162,8 @@ class ConsumerBase(object):
def _callback(raw_message): def _callback(raw_message):
message = self.channel.message_to_python(raw_message) message = self.channel.message_to_python(raw_message)
try: try:
callback(message.payload) msg = rpc_common.deserialize_msg(message.payload)
callback(msg)
message.ack() message.ack()
except Exception: except Exception:
LOG.exception(_("Failed to process message... skipping it.")) LOG.exception(_("Failed to process message... skipping it."))
@ -196,7 +197,7 @@ class DirectConsumer(ConsumerBase):
# Default options # Default options
options = {'durable': False, options = {'durable': False,
'auto_delete': True, 'auto_delete': True,
'exclusive': True} 'exclusive': False}
options.update(kwargs) options.update(kwargs)
exchange = kombu.entity.Exchange(name=msg_id, exchange = kombu.entity.Exchange(name=msg_id,
type='direct', type='direct',
@ -269,7 +270,7 @@ class FanoutConsumer(ConsumerBase):
options = {'durable': False, options = {'durable': False,
'queue_arguments': _get_queue_arguments(conf), 'queue_arguments': _get_queue_arguments(conf),
'auto_delete': True, 'auto_delete': True,
'exclusive': True} 'exclusive': False}
options.update(kwargs) options.update(kwargs)
exchange = kombu.entity.Exchange(name=exchange_name, type='fanout', exchange = kombu.entity.Exchange(name=exchange_name, type='fanout',
durable=options['durable'], durable=options['durable'],
@ -316,7 +317,7 @@ class DirectPublisher(Publisher):
options = {'durable': False, options = {'durable': False,
'auto_delete': True, 'auto_delete': True,
'exclusive': True} 'exclusive': False}
options.update(kwargs) options.update(kwargs)
super(DirectPublisher, self).__init__(channel, msg_id, msg_id, super(DirectPublisher, self).__init__(channel, msg_id, msg_id,
type='direct', **options) type='direct', **options)
@ -350,7 +351,7 @@ class FanoutPublisher(Publisher):
""" """
options = {'durable': False, options = {'durable': False,
'auto_delete': True, 'auto_delete': True,
'exclusive': True} 'exclusive': False}
options.update(kwargs) options.update(kwargs)
super(FanoutPublisher, self).__init__(channel, '%s_fanout' % topic, super(FanoutPublisher, self).__init__(channel, '%s_fanout' % topic,
None, type='fanout', **options) None, type='fanout', **options)
@ -387,6 +388,7 @@ class Connection(object):
def __init__(self, conf, server_params=None): def __init__(self, conf, server_params=None):
self.consumers = [] self.consumers = []
self.consumer_thread = None self.consumer_thread = None
self.proxy_callbacks = []
self.conf = conf self.conf = conf
self.max_retries = self.conf.rabbit_max_retries self.max_retries = self.conf.rabbit_max_retries
# Try forever? # Try forever?
@ -469,7 +471,7 @@ class Connection(object):
LOG.info(_("Reconnecting to AMQP server on " LOG.info(_("Reconnecting to AMQP server on "
"%(hostname)s:%(port)d") % params) "%(hostname)s:%(port)d") % params)
try: try:
self.connection.close() self.connection.release()
except self.connection_errors: except self.connection_errors:
pass pass
# Setting this in case the next statement fails, though # Setting this in case the next statement fails, though
@ -573,12 +575,14 @@ class Connection(object):
def close(self): def close(self):
"""Close/release this connection""" """Close/release this connection"""
self.cancel_consumer_thread() self.cancel_consumer_thread()
self.wait_on_proxy_callbacks()
self.connection.release() self.connection.release()
self.connection = None self.connection = None
def reset(self): def reset(self):
"""Reset a connection so it can be used again""" """Reset a connection so it can be used again"""
self.cancel_consumer_thread() self.cancel_consumer_thread()
self.wait_on_proxy_callbacks()
self.channel.close() self.channel.close()
self.channel = self.connection.channel() self.channel = self.connection.channel()
# work around 'memory' transport bug in 1.1.3 # work around 'memory' transport bug in 1.1.3
@ -644,6 +648,11 @@ class Connection(object):
pass pass
self.consumer_thread = None self.consumer_thread = None
def wait_on_proxy_callbacks(self):
"""Wait for all proxy callback threads to exit."""
for proxy_cb in self.proxy_callbacks:
proxy_cb.wait()
def publisher_send(self, cls, topic, msg, **kwargs): def publisher_send(self, cls, topic, msg, **kwargs):
"""Send to a publisher based on the publisher class""" """Send to a publisher based on the publisher class"""
@ -719,6 +728,7 @@ class Connection(object):
proxy_cb = rpc_amqp.ProxyCallback( proxy_cb = rpc_amqp.ProxyCallback(
self.conf, proxy, self.conf, proxy,
rpc_amqp.get_connection_pool(self.conf, Connection)) rpc_amqp.get_connection_pool(self.conf, Connection))
self.proxy_callbacks.append(proxy_cb)
if fanout: if fanout:
self.declare_fanout_consumer(topic, proxy_cb) self.declare_fanout_consumer(topic, proxy_cb)
@ -730,6 +740,7 @@ class Connection(object):
proxy_cb = rpc_amqp.ProxyCallback( proxy_cb = rpc_amqp.ProxyCallback(
self.conf, proxy, self.conf, proxy,
rpc_amqp.get_connection_pool(self.conf, Connection)) rpc_amqp.get_connection_pool(self.conf, Connection))
self.proxy_callbacks.append(proxy_cb)
self.declare_topic_consumer(topic, proxy_cb, pool_name) self.declare_topic_consumer(topic, proxy_cb, pool_name)
@ -782,11 +793,12 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg):
rpc_amqp.get_connection_pool(conf, Connection)) rpc_amqp.get_connection_pool(conf, Connection))
def notify(conf, context, topic, msg): def notify(conf, context, topic, msg, envelope):
"""Sends a notification event on a topic.""" """Sends a notification event on a topic."""
return rpc_amqp.notify( return rpc_amqp.notify(
conf, context, topic, msg, conf, context, topic, msg,
rpc_amqp.get_connection_pool(conf, Connection)) rpc_amqp.get_connection_pool(conf, Connection),
envelope)
def cleanup(): def cleanup():

50
kwapi/openstack/common/rpc/impl_qpid.py
View File

@ -17,7 +17,6 @@
import functools import functools
import itertools import itertools
import logging
import time import time
import uuid import uuid
@ -29,6 +28,7 @@ import qpid.messaging.exceptions
from kwapi.openstack.common import cfg from kwapi.openstack.common import cfg
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import jsonutils from kwapi.openstack.common import jsonutils
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.rpc import amqp as rpc_amqp from kwapi.openstack.common.rpc import amqp as rpc_amqp
from kwapi.openstack.common.rpc import common as rpc_common from kwapi.openstack.common.rpc import common as rpc_common
@ -41,6 +41,9 @@ qpid_opts = [
cfg.StrOpt('qpid_port', cfg.StrOpt('qpid_port',
default='5672', default='5672',
help='Qpid broker port'), help='Qpid broker port'),
cfg.ListOpt('qpid_hosts',
default=['$qpid_hostname:$qpid_port'],
help='Qpid HA cluster host:port pairs'),
cfg.StrOpt('qpid_username', cfg.StrOpt('qpid_username',
default='', default='',
help='Username for qpid connection'), help='Username for qpid connection'),
@ -121,7 +124,8 @@ class ConsumerBase(object):
"""Fetch the message and pass it to the callback object""" """Fetch the message and pass it to the callback object"""
message = self.receiver.fetch() message = self.receiver.fetch()
try: try:
self.callback(message.content) msg = rpc_common.deserialize_msg(message.content)
self.callback(msg)
except Exception: except Exception:
LOG.exception(_("Failed to process message... skipping it.")) LOG.exception(_("Failed to process message... skipping it."))
finally: finally:
@ -274,25 +278,32 @@ class Connection(object):
self.session = None self.session = None
self.consumers = {} self.consumers = {}
self.consumer_thread = None self.consumer_thread = None
self.proxy_callbacks = []
self.conf = conf self.conf = conf
if server_params and 'hostname' in server_params:
# NOTE(russellb) This enables support for cast_to_server.
server_params['qpid_hosts'] = [
'%s:%d' % (server_params['hostname'],
server_params.get('port', 5672))
]
params = { params = {
'hostname': self.conf.qpid_hostname, 'qpid_hosts': self.conf.qpid_hosts,
'port': self.conf.qpid_port,
'username': self.conf.qpid_username, 'username': self.conf.qpid_username,
'password': self.conf.qpid_password, 'password': self.conf.qpid_password,
} }
params.update(server_params or {}) params.update(server_params or {})
self.broker = params['hostname'] + ":" + str(params['port']) self.brokers = params['qpid_hosts']
self.username = params['username'] self.username = params['username']
self.password = params['password'] self.password = params['password']
self.connection_create() self.connection_create(self.brokers[0])
self.reconnect() self.reconnect()
def connection_create(self): def connection_create(self, broker):
# Create the connection - this does not open the connection # Create the connection - this does not open the connection
self.connection = qpid.messaging.Connection(self.broker) self.connection = qpid.messaging.Connection(broker)
# Check if flags are set and if so set them for the connection # Check if flags are set and if so set them for the connection
# before we call open # before we call open
@ -320,10 +331,14 @@ class Connection(object):
except qpid.messaging.exceptions.ConnectionError: except qpid.messaging.exceptions.ConnectionError:
pass pass
attempt = 0
delay = 1 delay = 1
while True: while True:
broker = self.brokers[attempt % len(self.brokers)]
attempt += 1
try: try:
self.connection_create() self.connection_create(broker)
self.connection.open() self.connection.open()
except qpid.messaging.exceptions.ConnectionError, e: except qpid.messaging.exceptions.ConnectionError, e:
msg_dict = dict(e=e, delay=delay) msg_dict = dict(e=e, delay=delay)
@ -333,10 +348,9 @@ class Connection(object):
time.sleep(delay) time.sleep(delay)
delay = min(2 * delay, 60) delay = min(2 * delay, 60)
else: else:
LOG.info(_('Connected to AMQP server on %s'), broker)
break break
LOG.info(_('Connected to AMQP server on %s'), self.broker)
self.session = self.connection.session() self.session = self.connection.session()
if self.consumers: if self.consumers:
@ -362,12 +376,14 @@ class Connection(object):
def close(self): def close(self):
"""Close/release this connection""" """Close/release this connection"""
self.cancel_consumer_thread() self.cancel_consumer_thread()
self.wait_on_proxy_callbacks()
self.connection.close() self.connection.close()
self.connection = None self.connection = None
def reset(self): def reset(self):
"""Reset a connection so it can be used again""" """Reset a connection so it can be used again"""
self.cancel_consumer_thread() self.cancel_consumer_thread()
self.wait_on_proxy_callbacks()
self.session.close() self.session.close()
self.session = self.connection.session() self.session = self.connection.session()
self.consumers = {} self.consumers = {}
@ -422,6 +438,11 @@ class Connection(object):
pass pass
self.consumer_thread = None self.consumer_thread = None
def wait_on_proxy_callbacks(self):
"""Wait for all proxy callback threads to exit."""
for proxy_cb in self.proxy_callbacks:
proxy_cb.wait()
def publisher_send(self, cls, topic, msg): def publisher_send(self, cls, topic, msg):
"""Send to a publisher based on the publisher class""" """Send to a publisher based on the publisher class"""
@ -497,6 +518,7 @@ class Connection(object):
proxy_cb = rpc_amqp.ProxyCallback( proxy_cb = rpc_amqp.ProxyCallback(
self.conf, proxy, self.conf, proxy,
rpc_amqp.get_connection_pool(self.conf, Connection)) rpc_amqp.get_connection_pool(self.conf, Connection))
self.proxy_callbacks.append(proxy_cb)
if fanout: if fanout:
consumer = FanoutConsumer(self.conf, self.session, topic, proxy_cb) consumer = FanoutConsumer(self.conf, self.session, topic, proxy_cb)
@ -512,6 +534,7 @@ class Connection(object):
proxy_cb = rpc_amqp.ProxyCallback( proxy_cb = rpc_amqp.ProxyCallback(
self.conf, proxy, self.conf, proxy,
rpc_amqp.get_connection_pool(self.conf, Connection)) rpc_amqp.get_connection_pool(self.conf, Connection))
self.proxy_callbacks.append(proxy_cb)
consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb, consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb,
name=pool_name) name=pool_name)
@ -570,10 +593,11 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg):
rpc_amqp.get_connection_pool(conf, Connection)) rpc_amqp.get_connection_pool(conf, Connection))
def notify(conf, context, topic, msg): def notify(conf, context, topic, msg, envelope):
"""Sends a notification event on a topic.""" """Sends a notification event on a topic."""
return rpc_amqp.notify(conf, context, topic, msg, return rpc_amqp.notify(conf, context, topic, msg,
rpc_amqp.get_connection_pool(conf, Connection)) rpc_amqp.get_connection_pool(conf, Connection),
envelope)
def cleanup(): def cleanup():

34
kwapi/openstack/common/rpc/impl_zmq.py
View File

@ -49,7 +49,7 @@ zmq_opts = [
# The module.Class to use for matchmaking. # The module.Class to use for matchmaking.
cfg.StrOpt( cfg.StrOpt(
'rpc_zmq_matchmaker', 'rpc_zmq_matchmaker',
default=('kwapi.openstack.common.rpc.' default=('openstack.common.rpc.'
'matchmaker.MatchMakerLocalhost'), 'matchmaker.MatchMakerLocalhost'),
help='MatchMaker driver', help='MatchMaker driver',
), ),
@ -205,7 +205,9 @@ class ZmqClient(object):
def __init__(self, addr, socket_type=zmq.PUSH, bind=False): def __init__(self, addr, socket_type=zmq.PUSH, bind=False):
self.outq = ZmqSocket(addr, socket_type, bind=bind) self.outq = ZmqSocket(addr, socket_type, bind=bind)
def cast(self, msg_id, topic, data): def cast(self, msg_id, topic, data, serialize=True, force_envelope=False):
if serialize:
data = rpc_common.serialize_msg(data, force_envelope)
self.outq.send([str(msg_id), str(topic), str('cast'), self.outq.send([str(msg_id), str(topic), str('cast'),
_serialize(data)]) _serialize(data)])
@ -250,7 +252,7 @@ class InternalContext(object):
"""Process a curried message and cast the result to topic.""" """Process a curried message and cast the result to topic."""
LOG.debug(_("Running func with context: %s"), ctx.to_dict()) LOG.debug(_("Running func with context: %s"), ctx.to_dict())
data.setdefault('version', None) data.setdefault('version', None)
data.setdefault('args', []) data.setdefault('args', {})
try: try:
result = proxy.dispatch( result = proxy.dispatch(
@ -259,7 +261,14 @@ class InternalContext(object):
except greenlet.GreenletExit: except greenlet.GreenletExit:
# ignore these since they are just from shutdowns # ignore these since they are just from shutdowns
pass pass
except rpc_common.ClientException, e:
LOG.debug(_("Expected exception during message handling (%s)") %
e._exc_info[1])
return {'exc':
rpc_common.serialize_remote_exception(e._exc_info,
log_failure=False)}
except Exception: except Exception:
LOG.error(_("Exception during message handling"))
return {'exc': return {'exc':
rpc_common.serialize_remote_exception(sys.exc_info())} rpc_common.serialize_remote_exception(sys.exc_info())}
@ -314,7 +323,7 @@ class ConsumerBase(object):
return return
data.setdefault('version', None) data.setdefault('version', None)
data.setdefault('args', []) data.setdefault('args', {})
proxy.dispatch(ctx, data['version'], proxy.dispatch(ctx, data['version'],
data['method'], **data['args']) data['method'], **data['args'])
@ -426,7 +435,7 @@ class ZmqProxy(ZmqBaseReactor):
sock_type = zmq.PUB sock_type = zmq.PUB
elif topic.startswith('zmq_replies'): elif topic.startswith('zmq_replies'):
sock_type = zmq.PUB sock_type = zmq.PUB
inside = _deserialize(in_msg) inside = rpc_common.deserialize_msg(_deserialize(in_msg))
msg_id = inside[-1]['args']['msg_id'] msg_id = inside[-1]['args']['msg_id']
response = inside[-1]['args']['response'] response = inside[-1]['args']['response']
LOG.debug(_("->response->%s"), response) LOG.debug(_("->response->%s"), response)
@ -473,7 +482,7 @@ class ZmqReactor(ZmqBaseReactor):
msg_id, topic, style, in_msg = data msg_id, topic, style, in_msg = data
ctx, request = _deserialize(in_msg) ctx, request = rpc_common.deserialize_msg(_deserialize(in_msg))
ctx = RpcContext.unmarshal(ctx) ctx = RpcContext.unmarshal(ctx)
proxy = self.proxies[sock] proxy = self.proxies[sock]
@ -524,7 +533,8 @@ class Connection(rpc_common.Connection):
self.reactor.consume_in_thread() self.reactor.consume_in_thread()
def _cast(addr, context, msg_id, topic, msg, timeout=None): def _cast(addr, context, msg_id, topic, msg, timeout=None, serialize=True,
force_envelope=False):
timeout_cast = timeout or CONF.rpc_cast_timeout timeout_cast = timeout or CONF.rpc_cast_timeout
payload = [RpcContext.marshal(context), msg] payload = [RpcContext.marshal(context), msg]
@ -533,7 +543,7 @@ def _cast(addr, context, msg_id, topic, msg, timeout=None):
conn = ZmqClient(addr) conn = ZmqClient(addr)
# assumes cast can't return an exception # assumes cast can't return an exception
conn.cast(msg_id, topic, payload) conn.cast(msg_id, topic, payload, serialize, force_envelope)
except zmq.ZMQError: except zmq.ZMQError:
raise RPCException("Cast failed. ZMQ Socket Exception") raise RPCException("Cast failed. ZMQ Socket Exception")
finally: finally:
@ -602,7 +612,8 @@ def _call(addr, context, msg_id, topic, msg, timeout=None):
return responses[-1] return responses[-1]
def _multi_send(method, context, topic, msg, timeout=None): def _multi_send(method, context, topic, msg, timeout=None, serialize=True,
force_envelope=False):
""" """
Wraps the sending of messages, Wraps the sending of messages,
dispatches to the matchmaker and sends dispatches to the matchmaker and sends
@ -628,7 +639,8 @@ def _multi_send(method, context, topic, msg, timeout=None):
if method.__name__ == '_cast': if method.__name__ == '_cast':
eventlet.spawn_n(method, _addr, context, eventlet.spawn_n(method, _addr, context,
_topic, _topic, msg, timeout) _topic, _topic, msg, timeout, serialize,
force_envelope)
return return
return method(_addr, context, _topic, _topic, msg, timeout) return method(_addr, context, _topic, _topic, msg, timeout)
@ -669,6 +681,8 @@ def notify(conf, context, topic, msg, **kwargs):
# NOTE(ewindisch): dot-priority in rpc notifier does not # NOTE(ewindisch): dot-priority in rpc notifier does not
# work with our assumptions. # work with our assumptions.
topic.replace('.', '-') topic.replace('.', '-')
kwargs['serialize'] = kwargs.pop('envelope')
kwargs['force_envelope'] = True
cast(conf, context, topic, msg, **kwargs) cast(conf, context, topic, msg, **kwargs)

2
kwapi/openstack/common/rpc/matchmaker.py
View File

@ -21,10 +21,10 @@ return keys for direct exchanges, per (approximate) AMQP parlance.
import contextlib import contextlib
import itertools import itertools
import json import json
import logging
from kwapi.openstack.common import cfg from kwapi.openstack.common import cfg
from kwapi.openstack.common.gettextutils import _ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging
matchmaker_opts = [ matchmaker_opts = [

0
kwapi/openstack/common/scheduler/__init__.py Normal file
View File

71
kwapi/openstack/common/scheduler/filter.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Filter support
"""
import inspect
from stevedore import extension
class BaseFilter(object):
"""Base class for all filter classes."""
def _filter_one(self, obj, filter_properties):
"""Return True if it passes the filter, False otherwise.
Override this in a subclass.
"""
return True
def filter_all(self, filter_obj_list, filter_properties):
"""Yield objects that pass the filter.
Can be overriden in a subclass, if you need to base filtering
decisions on all objects. Otherwise, one can just override
_filter_one() to filter a single object.
"""
for obj in filter_obj_list:
if self._filter_one(obj, filter_properties):
yield obj
class BaseFilterHandler(object):
""" Base class to handle loading filter classes.
This class should be subclassed where one needs to use filters.
"""
def __init__(self, filter_class_type, filter_namespace):
self.namespace = filter_namespace
self.filter_class_type = filter_class_type
self.filter_manager = extension.ExtensionManager(filter_namespace)
def _is_correct_class(self, obj):
"""Return whether an object is a class of the correct type and
is not prefixed with an underscore.
"""
return (inspect.isclass(obj) and
not obj.__name__.startswith('_') and
issubclass(obj, self.filter_class_type))
def get_all_classes(self):
return [x.plugin for x in self.filter_manager
if self._is_correct_class(x.plugin)]
def get_filtered_objects(self, filter_classes, objs,
filter_properties):
for filter_cls in filter_classes:
objs = filter_cls().filter_all(objs, filter_properties)
return list(objs)

41
kwapi/openstack/common/scheduler/filters/__init__.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler host filters
"""
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.scheduler import filter
LOG = logging.getLogger(__name__)
class BaseHostFilter(filter.BaseFilter):
"""Base class for host filters."""
def _filter_one(self, obj, filter_properties):
"""Return True if the object passes the filter, otherwise False."""
return self.host_passes(obj, filter_properties)
def host_passes(self, host_state, filter_properties):
"""Return True if the HostState passes the filter, otherwise False.
Override this in a subclass.
"""
raise NotImplementedError()
class HostFilterHandler(filter.BaseFilterHandler):
def __init__(self, namespace):
super(HostFilterHandler, self).__init__(BaseHostFilter, namespace)

30
kwapi/openstack/common/scheduler/filters/availability_zone_filter.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common.scheduler import filters
class AvailabilityZoneFilter(filters.BaseHostFilter):
"""Filters Hosts by availability zone."""
def host_passes(self, host_state, filter_properties):
spec = filter_properties.get('request_spec', {})
props = spec.get('resource_properties', [])
availability_zone = props.get('availability_zone')
if availability_zone:
return availability_zone == host_state.service['availability_zone']
return True

63
kwapi/openstack/common/scheduler/filters/capabilities_filter.py Normal file
View File

@ -0,0 +1,63 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kwapi.openstack.common import log as logging
from kwapi.openstack.common.scheduler import filters
from kwapi.openstack.common.scheduler.filters import extra_specs_ops
LOG = logging.getLogger(__name__)
class CapabilitiesFilter(filters.BaseHostFilter):
"""HostFilter to work with resource (instance & volume) type records."""
def _satisfies_extra_specs(self, capabilities, resource_type):
"""Check that the capabilities provided by the services
satisfy the extra specs associated with the instance type"""
extra_specs = resource_type.get('extra_specs', [])
if not extra_specs:
return True
for key, req in extra_specs.iteritems():
# Either not scope format, or in capabilities scope
scope = key.split(':')
if len(scope) > 1 and scope[0] != "capabilities":
continue
elif scope[0] == "capabilities":
del scope[0]
cap = capabilities
for index in range(0, len(scope)):
try:
cap = cap.get(scope[index], None)
except AttributeError:
return False
if cap is None:
return False
if not extra_specs_ops.match(cap, req):
return False
return True
def host_passes(self, host_state, filter_properties):
"""Return a list of hosts that can create instance_type."""
# Note(zhiteng) Currently only Cinder and Nova are using
# this filter, so the resource type is either instance or
# volume.
resource_type = filter_properties.get('resource_type')
if not self._satisfies_extra_specs(host_state.capabilities,
resource_type):
return False
return True

68
kwapi/openstack/common/scheduler/filters/extra_specs_ops.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import operator
# 1. The following operations are supported:
# =, s==, s!=, s>=, s>, s<=, s<, <in>, <or>, ==, !=, >=, <=
# 2. Note that <or> is handled in a different way below.
# 3. If the first word in the extra_specs is not one of the operators,
# it is ignored.
_op_methods = {'=': lambda x, y: float(x) >= float(y),
'<in>': lambda x, y: y in x,
'==': lambda x, y: float(x) == float(y),
'!=': lambda x, y: float(x) != float(y),
'>=': lambda x, y: float(x) >= float(y),
'<=': lambda x, y: float(x) <= float(y),
's==': operator.eq,
's!=': operator.ne,
's<': operator.lt,
's<=': operator.le,
's>': operator.gt,
's>=': operator.ge}
def match(value, req):
words = req.split()
op = method = None
if words:
op = words.pop(0)
method = _op_methods.get(op)
if op != '<or>' and not method:
return value == req
if value is None:
return False
if op == '<or>': # Ex: <or> v1 <or> v2 <or> v3
while True:
if words.pop(0) == value:
return True
if not words:
break
op = words.pop(0) # remove a keyword <or>
if not words:
break
return False
try:
if words and method(value, words[0]):
return True
except ValueError:
pass
return False

150
kwapi/openstack/common/scheduler/filters/json_filter.py Normal file
View File

@ -0,0 +1,150 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import operator
from kwapi.openstack.common import jsonutils
from kwapi.openstack.common.scheduler import filters
class JsonFilter(filters.BaseHostFilter):
"""Host Filter to allow simple JSON-based grammar for
selecting hosts.
"""
def _op_compare(self, args, op):
"""Returns True if the specified operator can successfully
compare the first item in the args with all the rest. Will
return False if only one item is in the list.
"""
if len(args) < 2:
return False
if op is operator.contains:
bad = not args[0] in args[1:]
else:
bad = [arg for arg in args[1:]
if not op(args[0], arg)]
return not bool(bad)
def _equals(self, args):
"""First term is == all the other terms."""
return self._op_compare(args, operator.eq)
def _less_than(self, args):
"""First term is < all the other terms."""
return self._op_compare(args, operator.lt)
def _greater_than(self, args):
"""First term is > all the other terms."""
return self._op_compare(args, operator.gt)
def _in(self, args):
"""First term is in set of remaining terms"""
return self._op_compare(args, operator.contains)
def _less_than_equal(self, args):
"""First term is <= all the other terms."""
return self._op_compare(args, operator.le)
def _greater_than_equal(self, args):
"""First term is >= all the other terms."""
return self._op_compare(args, operator.ge)
def _not(self, args):
"""Flip each of the arguments."""
return [not arg for arg in args]
def _or(self, args):
"""True if any arg is True."""
return any(args)
def _and(self, args):
"""True if all args are True."""
return all(args)
commands = {
'=': _equals,
'<': _less_than,
'>': _greater_than,
'in': _in,
'<=': _less_than_equal,
'>=': _greater_than_equal,
'not': _not,
'or': _or,
'and': _and,
}
def _parse_string(self, string, host_state):
"""Strings prefixed with $ are capability lookups in the
form '$variable' where 'variable' is an attribute in the
HostState class. If $variable is a dictionary, you may
use: $variable.dictkey
"""
if not string:
return None
if not string.startswith("$"):
return string
path = string[1:].split(".")
obj = getattr(host_state, path[0], None)
if obj is None:
return None
for item in path[1:]:
obj = obj.get(item, None)
if obj is None:
return None
return obj
def _process_filter(self, query, host_state):
"""Recursively parse the query structure."""
if not query:
return True
cmd = query[0]
method = self.commands[cmd]
cooked_args = []
for arg in query[1:]:
if isinstance(arg, list):
arg = self._process_filter(arg, host_state)
elif isinstance(arg, basestring):
arg = self._parse_string(arg, host_state)
if arg is not None:
cooked_args.append(arg)
result = method(self, cooked_args)
return result
def host_passes(self, host_state, filter_properties):
"""Return a list of hosts that can fulfill the requirements
specified in the query.
"""
# TODO(zhiteng) Add description for filter_properties structure
# and scheduler_hints.
try:
query = filter_properties['scheduler_hints']['query']
except KeyError:
query = None
if not query:
return True
# NOTE(comstud): Not checking capabilities or service for
# enabled/disabled so that a provided json filter can decide
result = self._process_filter(jsonutils.loads(query), host_state)
if isinstance(result, list):
# If any succeeded, include the host
result = any(result)
if result:
# Filter it out.
return True
return False

91
kwapi/openstack/common/scheduler/weight.py Normal file
View File

@ -0,0 +1,91 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Pluggable Weighing support
"""
import inspect
from stevedore import extension
class WeighedObject(object):
"""Object with weight information."""
def __init__(self, obj, weight):
self.obj = obj
self.weight = weight
def __repr__(self):
return "<WeighedObject '%s': %s>" % (self.obj, self.weight)
class BaseWeigher(object):
"""Base class for pluggable weighers."""
def _weight_multiplier(self):
"""How weighted this weigher should be. Normally this would
be overriden in a subclass based on a config value.
"""
return 1.0
def _weigh_object(self, obj, weight_properties):
"""Override in a subclass to specify a weight for a specific
object.
"""
return 0.0
def weigh_objects(self, weighed_obj_list, weight_properties):
"""Weigh multiple objects. Override in a subclass if you need
need access to all objects in order to manipulate weights.
"""
constant = self._weight_multiplier()
for obj in weighed_obj_list:
obj.weight += (constant *
self._weigh_object(obj.obj, weight_properties))
class BaseWeightHandler(object):
object_class = WeighedObject
def __init__(self, weighed_object_type, weight_namespace):
self.namespace = weight_namespace
self.weighed_object_type = weighed_object_type
self.weight_manager = extension.ExtensionManager(weight_namespace)
def _is_correct_class(self, obj):
"""Return whether an object is a class of the correct type and
is not prefixed with an underscore.
"""
return (inspect.isclass(obj) and
not obj.__name__.startswith('_') and
issubclass(obj, self.weighed_object_type))
def get_all_classes(self):
return [x.plugin for x in self.weight_manager
if self._is_correct_class(x.plugin)]
def get_weighed_objects(self, weigher_classes, obj_list,
weighing_properties):
"""Return a sorted (highest score first) list of WeighedObjects."""
if not obj_list:
return []
weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list]
for weigher_cls in weigher_classes:
weigher = weigher_cls()
weigher.weigh_objects(weighed_objs, weighing_properties)
return sorted(weighed_objs, key=lambda x: x.weight, reverse=True)

45
kwapi/openstack/common/scheduler/weights/__init__.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler host weights
"""
from kwapi.openstack.common.scheduler import weight
class WeighedHost(weight.WeighedObject):
def to_dict(self):
return {
'weight': self.weight,
'host': self.obj.host,
}
def __repr__(self):
return ("WeighedHost [host: %s, weight: %s]" %
(self.obj.host, self.weight))
class BaseHostWeigher(weight.BaseWeigher):
"""Base class for host weights."""
pass
class HostWeightHandler(weight.BaseWeightHandler):
object_class = WeighedHost
def __init__(self, namespace):
super(HostWeightHandler, self).__init__(BaseHostWeigher, namespace)

42
kwapi/openstack/common/service.py
View File

@ -27,7 +27,7 @@ import sys
import time import time
import eventlet import eventlet
import greenlet import extras
import logging as std_logging import logging as std_logging
from kwapi.openstack.common import cfg from kwapi.openstack.common import cfg
@ -36,11 +36,8 @@ from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging from kwapi.openstack.common import log as logging
from kwapi.openstack.common import threadgroup from kwapi.openstack.common import threadgroup
try:
from kwapi.openstack.common import rpc
except ImportError:
rpc = None
rpc = extras.try_import('openstack.common.rpc')
CONF = cfg.CONF CONF = cfg.CONF
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -54,7 +51,7 @@ class Launcher(object):
:returns: None :returns: None
""" """
self._services = [] self._services = threadgroup.ThreadGroup('launcher')
eventlet_backdoor.initialize_if_enabled() eventlet_backdoor.initialize_if_enabled()
@staticmethod @staticmethod
@ -75,8 +72,7 @@ class Launcher(object):
:returns: None :returns: None
""" """
gt = eventlet.spawn(self.run_service, service) self._services.add_thread(self.run_service, service)
self._services.append(gt)
def stop(self): def stop(self):
"""Stop all services which are currently running. """Stop all services which are currently running.
@ -84,8 +80,7 @@ class Launcher(object):
:returns: None :returns: None
""" """
for service in self._services: self._services.stop()
service.kill()
def wait(self): def wait(self):
"""Waits until all services have been stopped, and then returns. """Waits until all services have been stopped, and then returns.
@ -93,11 +88,7 @@ class Launcher(object):
:returns: None :returns: None
""" """
for service in self._services: self._services.wait()
try:
service.wait()
except greenlet.GreenletExit:
pass
class SignalExit(SystemExit): class SignalExit(SystemExit):
@ -132,9 +123,9 @@ class ServiceLauncher(Launcher):
except SystemExit as exc: except SystemExit as exc:
status = exc.code status = exc.code
finally: finally:
self.stop()
if rpc: if rpc:
rpc.cleanup() rpc.cleanup()
self.stop()
return status return status
@ -252,7 +243,10 @@ class ProcessLauncher(object):
def _wait_child(self): def _wait_child(self):
try: try:
pid, status = os.wait() # Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc: except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD): if exc.errno not in (errno.EINTR, errno.ECHILD):
raise raise
@ -260,10 +254,12 @@ class ProcessLauncher(object):
if os.WIFSIGNALED(status): if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status) sig = os.WTERMSIG(status)
LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals()) LOG.info(_('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else: else:
code = os.WEXITSTATUS(status) code = os.WEXITSTATUS(status)
LOG.info(_('Child %(pid)d exited with status %(code)d'), locals()) LOG.info(_('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children: if pid not in self.children:
LOG.warning(_('pid %d not in child list'), pid) LOG.warning(_('pid %d not in child list'), pid)
@ -282,6 +278,10 @@ class ProcessLauncher(object):
while self.running: while self.running:
wrap = self._wait_child() wrap = self._wait_child()
if not wrap: if not wrap:
# Yield to other threads if no children have exited
# Sleep for a short time to avoid excessive CPU usage
# (see bug #1095346)
eventlet.greenthread.sleep(.01)
continue continue
while self.running and len(wrap.children) < wrap.workers: while self.running and len(wrap.children) < wrap.workers:
@ -309,8 +309,8 @@ class ProcessLauncher(object):
class Service(object): class Service(object):
"""Service object for binaries running on hosts.""" """Service object for binaries running on hosts."""
def __init__(self): def __init__(self, threads=1000):
self.tg = threadgroup.ThreadGroup('service') self.tg = threadgroup.ThreadGroup('service', threads)
def start(self): def start(self):
pass pass

10
kwapi/openstack/common/setup.py
View File

@ -276,6 +276,9 @@ def get_cmdclass():
from sphinx.setup_command import BuildDoc from sphinx.setup_command import BuildDoc
class LocalBuildDoc(BuildDoc): class LocalBuildDoc(BuildDoc):
builders = ['html', 'man']
def generate_autoindex(self): def generate_autoindex(self):
print "**Autodocumenting from %s" % os.path.abspath(os.curdir) print "**Autodocumenting from %s" % os.path.abspath(os.curdir)
modules = {} modules = {}
@ -311,14 +314,19 @@ def get_cmdclass():
if not os.getenv('SPHINX_DEBUG'): if not os.getenv('SPHINX_DEBUG'):
self.generate_autoindex() self.generate_autoindex()
for builder in ['html', 'man']: for builder in self.builders:
self.builder = builder self.builder = builder
self.finalize_options() self.finalize_options()
self.project = self.distribution.get_name() self.project = self.distribution.get_name()
self.version = self.distribution.get_version() self.version = self.distribution.get_version()
self.release = self.distribution.get_version() self.release = self.distribution.get_version()
BuildDoc.run(self) BuildDoc.run(self)
class LocalBuildLatex(LocalBuildDoc):
builders = ['latex']
cmdclass['build_sphinx'] = LocalBuildDoc cmdclass['build_sphinx'] = LocalBuildDoc
cmdclass['build_sphinx_latex'] = LocalBuildLatex
except ImportError: except ImportError:
pass pass

59
kwapi/openstack/common/strutils.py Normal file
View File

@ -0,0 +1,59 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import logging
LOG = logging.getLogger(__name__)
def int_from_bool_as_string(subject):
"""
Interpret a string as a boolean and return either 1 or 0.
Any string value in:
('True', 'true', 'On', 'on', '1')
is interpreted as a boolean True.
Useful for JSON-decoded stuff and config file parsing
"""
return bool_from_string(subject) and 1 or 0
def bool_from_string(subject):
"""
Interpret a string as a boolean.
Any string value in:
('True', 'true', 'On', 'on', 'Yes', 'yes', '1')
is interpreted as a boolean True.
Useful for JSON-decoded stuff and config file parsing
"""
if isinstance(subject, bool):
return subject
if isinstance(subject, basestring):
if subject.strip().lower() in ('true', 'on', 'yes', '1'):
return True
return False

68
kwapi/openstack/common/testutils.py Normal file
View File

@ -0,0 +1,68 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Utilities for unit tests."""
import functools
import nose
class skip_test(object):
"""Decorator that skips a test."""
# TODO(tr3buchet): remember forever what comstud did here
def __init__(self, msg):
self.message = msg
def __call__(self, func):
@functools.wraps(func)
def _skipper(*args, **kw):
"""Wrapped skipper function."""
raise nose.SkipTest(self.message)
return _skipper
class skip_if(object):
"""Decorator that skips a test if condition is true."""
def __init__(self, condition, msg):
self.condition = condition
self.message = msg
def __call__(self, func):
@functools.wraps(func)
def _skipper(*args, **kw):
"""Wrapped skipper function."""
if self.condition:
raise nose.SkipTest(self.message)
func(*args, **kw)
return _skipper
class skip_unless(object):
"""Decorator that skips a test if condition is not true."""
def __init__(self, condition, msg):
self.condition = condition
self.message = msg
def __call__(self, func):
@functools.wraps(func)
def _skipper(*args, **kw):
"""Wrapped skipper function."""
if not self.condition:
raise nose.SkipTest(self.message)
func(*args, **kw)
return _skipper

25
kwapi/openstack/common/threadgroup.py
View File

@ -18,7 +18,6 @@ from eventlet import greenlet
from eventlet import greenpool from eventlet import greenpool
from eventlet import greenthread from eventlet import greenthread
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import log as logging from kwapi.openstack.common import log as logging
from kwapi.openstack.common import loopingcall from kwapi.openstack.common import loopingcall
@ -27,19 +26,17 @@ LOG = logging.getLogger(__name__)
def _thread_done(gt, *args, **kwargs): def _thread_done(gt, *args, **kwargs):
''' """ Callback function to be passed to GreenThread.link() when we spawn()
Callback function to be passed to GreenThread.link() when we spawn() Calls the :class:`ThreadGroup` to notify if.
Calls the ThreadGroup to notify if.
''' """
kwargs['group'].thread_done(kwargs['thread']) kwargs['group'].thread_done(kwargs['thread'])
class Thread(object): class Thread(object):
""" """ Wrapper around a greenthread, that holds a reference to the
Wrapper around a greenthread, that holds a reference to :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
the ThreadGroup. The Thread will notify the ThreadGroup it has done so it can be removed from the threads list.
when it has done so it can be removed from the threads
list.
""" """
def __init__(self, name, thread, group): def __init__(self, name, thread, group):
self.name = name self.name = name
@ -54,11 +51,11 @@ class Thread(object):
class ThreadGroup(object): class ThreadGroup(object):
""" """ The point of the ThreadGroup classis to:
The point of this class is to:
- keep track of timers and greenthreads (making it easier to stop them * keep track of timers and greenthreads (making it easier to stop them
when need be). when need be).
- provide an easy API to add timers. * provide an easy API to add timers.
""" """
def __init__(self, name, thread_pool_size=10): def __init__(self, name, thread_pool_size=10):
self.name = name self.name = name

29
kwapi/openstack/common/timeutils.py
View File

@ -71,11 +71,15 @@ def normalize_time(timestamp):
def is_older_than(before, seconds): def is_older_than(before, seconds):
"""Return True if before is older than seconds.""" """Return True if before is older than seconds."""
if isinstance(before, basestring):
before = parse_strtime(before).replace(tzinfo=None)
return utcnow() - before > datetime.timedelta(seconds=seconds) return utcnow() - before > datetime.timedelta(seconds=seconds)
def is_newer_than(after, seconds): def is_newer_than(after, seconds):
"""Return True if after is newer than seconds.""" """Return True if after is newer than seconds."""
if isinstance(after, basestring):
after = parse_strtime(after).replace(tzinfo=None)
return after - utcnow() > datetime.timedelta(seconds=seconds) return after - utcnow() > datetime.timedelta(seconds=seconds)
@ -87,6 +91,9 @@ def utcnow_ts():
def utcnow(): def utcnow():
"""Overridable version of utils.utcnow.""" """Overridable version of utils.utcnow."""
if utcnow.override_time: if utcnow.override_time:
try:
return utcnow.override_time.pop(0)
except AttributeError:
return utcnow.override_time return utcnow.override_time
return datetime.datetime.utcnow() return datetime.datetime.utcnow()
@ -95,13 +102,20 @@ utcnow.override_time = None
def set_time_override(override_time=datetime.datetime.utcnow()): def set_time_override(override_time=datetime.datetime.utcnow()):
"""Override utils.utcnow to return a constant time.""" """
Override utils.utcnow to return a constant time or a list thereof,
one at a time.
"""
utcnow.override_time = override_time utcnow.override_time = override_time
def advance_time_delta(timedelta): def advance_time_delta(timedelta):
"""Advance overridden time using a datetime.timedelta.""" """Advance overridden time using a datetime.timedelta."""
assert(not utcnow.override_time is None) assert(not utcnow.override_time is None)
try:
for dt in utcnow.override_time:
dt += timedelta
except TypeError:
utcnow.override_time += timedelta utcnow.override_time += timedelta
@ -135,3 +149,16 @@ def unmarshall_time(tyme):
minute=tyme['minute'], minute=tyme['minute'],
second=tyme['second'], second=tyme['second'],
microsecond=tyme['microsecond']) microsecond=tyme['microsecond'])
def delta_seconds(before, after):
"""
Compute the difference in seconds between two date, time, or
datetime objects (as a float, to microsecond resolution).
"""
delta = after - before
try:
return delta.total_seconds()
except AttributeError:
return ((delta.days * 24 * 3600) + delta.seconds +
float(delta.microseconds) / (10 ** 6))

39
kwapi/openstack/common/uuidutils.py Normal file
View File

@ -0,0 +1,39 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2012 Intel Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
UUID related utilities and helper functions.
"""
import uuid
def generate_uuid():
return str(uuid.uuid4())
def is_uuid_like(val):
"""Returns validation of a value as a UUID.
For our purposes, a UUID is a canonical form string:
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
"""
try:
return str(uuid.UUID(val)) == val
except (TypeError, ValueError, AttributeError):
return False

25
kwapi/openstack/common/version.py
View File

@ -24,19 +24,6 @@ import pkg_resources
import setup import setup
class _deferred_version_string(object):
"""Internal helper class which provides delayed version calculation."""
def __init__(self, version_info, prefix):
self.version_info = version_info
self.prefix = prefix
def __str__(self):
return "%s%s" % (self.prefix, self.version_info.version_string())
def __repr__(self):
return "%s%s" % (self.prefix, self.version_info.version_string())
class VersionInfo(object): class VersionInfo(object):
def __init__(self, package, python_package=None, pre_version=None): def __init__(self, package, python_package=None, pre_version=None):
@ -57,14 +44,15 @@ class VersionInfo(object):
self.python_package = python_package self.python_package = python_package
self.pre_version = pre_version self.pre_version = pre_version
self.version = None self.version = None
self._cached_version = None
def _generate_version(self): def _generate_version(self):
"""Defer to the openstack.common.setup routines for making a """Defer to the openstack.common.setup routines for making a
version from git.""" version from git."""
if self.pre_version is None: if self.pre_version is None:
return setup.get_post_version(self.python_package) return setup.get_post_version(self.package)
else: else:
return setup.get_pre_version(self.python_package, self.pre_version) return setup.get_pre_version(self.package, self.pre_version)
def _newer_version(self, pending_version): def _newer_version(self, pending_version):
"""Check to see if we're working with a stale version or not. """Check to see if we're working with a stale version or not.
@ -138,11 +126,14 @@ class VersionInfo(object):
else: else:
return '%s-dev' % (version_parts[0],) return '%s-dev' % (version_parts[0],)
def deferred_version_string(self, prefix=""): def cached_version_string(self, prefix=""):
"""Generate an object which will expand in a string context to """Generate an object which will expand in a string context to
the results of version_string(). We do this so that don't the results of version_string(). We do this so that don't
call into pkg_resources every time we start up a program when call into pkg_resources every time we start up a program when
passing version information into the CONF constructor, but passing version information into the CONF constructor, but
rather only do the calculation when and if a version is requested rather only do the calculation when and if a version is requested
""" """
return _deferred_version_string(self, prefix) if not self._cached_version:
self._cached_version = "%s%s" % (prefix,
self.version_string())
return self._cached_version

733
kwapi/openstack/common/wsgi.py Normal file
View File

@ -0,0 +1,733 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Utility methods for working with WSGI servers."""
import datetime
import eventlet
import eventlet.wsgi
eventlet.patcher.monkey_patch(all=False, socket=True)
import routes
import routes.middleware
import sys
import webob.dec
import webob.exc
from xml.dom import minidom
from xml.parsers import expat
from kwapi.openstack.common import exception
from kwapi.openstack.common.gettextutils import _
from kwapi.openstack.common import jsonutils
from kwapi.openstack.common import log as logging
from kwapi.openstack.common import service
LOG = logging.getLogger(__name__)
def run_server(application, port):
"""Run a WSGI server with the given application."""
sock = eventlet.listen(('0.0.0.0', port))
eventlet.wsgi.server(sock, application)
class Service(service.Service):
"""
Provides a Service API for wsgi servers.
This gives us the ability to launch wsgi servers with the
Launcher classes in service.py.
"""
def __init__(self, application, port,
host='0.0.0.0', backlog=128, threads=1000):
self.application = application
self._port = port
self._host = host
self.backlog = backlog
super(Service, self).__init__(threads)
def start(self):
"""Start serving this service using the provided server instance.
:returns: None
"""
super(Service, self).start()
self._socket = eventlet.listen((self._host, self._port),
backlog=self.backlog)
self.tg.add_thread(self._run, self.application, self._socket)
@property
def host(self):
return self._socket.getsockname()[0] if self._socket else self._host
@property
def port(self):
return self._socket.getsockname()[1] if self._socket else self._port
def stop(self):
"""Stop serving this API.
:returns: None
"""
super(Service, self).stop()
def _run(self, application, socket):
"""Start a WSGI server in a new green thread."""
logger = logging.getLogger('eventlet.wsgi')
eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool,
log=logging.WritableLogger(logger))
class Middleware(object):
"""
Base WSGI middleware wrapper. These classes require an application to be
initialized that will be called next. By default the middleware will
simply call its wrapped app, or you can override __call__ to customize its
behavior.
"""
def __init__(self, application):
self.application = application
def process_request(self, req):
"""
Called on each request.
If this returns None, the next application down the stack will be
executed. If it returns a response then that response will be returned
and execution will stop here.
"""
return None
def process_response(self, response):
"""Do whatever you'd like to the response."""
return response
@webob.dec.wsgify
def __call__(self, req):
response = self.process_request(req)
if response:
return response
response = req.get_response(self.application)
return self.process_response(response)
class Debug(Middleware):
"""
Helper class that can be inserted into any WSGI application chain
to get information about the request and response.
"""
@webob.dec.wsgify
def __call__(self, req):
print ("*" * 40) + " REQUEST ENVIRON"
for key, value in req.environ.items():
print key, "=", value
print
resp = req.get_response(self.application)
print ("*" * 40) + " RESPONSE HEADERS"
for (key, value) in resp.headers.iteritems():
print key, "=", value
print
resp.app_iter = self.print_generator(resp.app_iter)
return resp
@staticmethod
def print_generator(app_iter):
"""
Iterator that prints the contents of a wrapper string iterator
when iterated.
"""
print ("*" * 40) + " BODY"
for part in app_iter:
sys.stdout.write(part)
sys.stdout.flush()
yield part
print
class Router(object):
"""
WSGI middleware that maps incoming requests to WSGI apps.
"""
def __init__(self, mapper):
"""
Create a router for the given routes.Mapper.
Each route in `mapper` must specify a 'controller', which is a
WSGI app to call. You'll probably want to specify an 'action' as
well and have your controller be a wsgi.Controller, who will route
the request to the action method.
Examples:
mapper = routes.Mapper()
sc = ServerController()
# Explicit mapping of one route to a controller+action
mapper.connect(None, "/svrlist", controller=sc, action="list")
# Actions are all implicitly defined
mapper.resource("server", "servers", controller=sc)
# Pointing to an arbitrary WSGI app. You can specify the
# {path_info:.*} parameter so the target app can be handed just that
# section of the URL.
mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
"""
self.map = mapper
self._router = routes.middleware.RoutesMiddleware(self._dispatch,
self.map)
@webob.dec.wsgify
def __call__(self, req):
"""
Route the incoming request to a controller based on self.map.
If no match, return a 404.
"""
return self._router
@staticmethod
@webob.dec.wsgify
def _dispatch(req):
"""
Called by self._router after matching the incoming request to a route
and putting the information into req.environ. Either returns 404
or the routed WSGI app's response.
"""
match = req.environ['wsgiorg.routing_args'][1]
if not match:
return webob.exc.HTTPNotFound()
app = match['controller']
return app
class Request(webob.Request):
"""Add some Openstack API-specific logic to the base webob.Request."""
default_request_content_types = ('application/json', 'application/xml')
default_accept_types = ('application/json', 'application/xml')
default_accept_type = 'application/json'
def best_match_content_type(self, supported_content_types=None):
"""Determine the requested response content-type.
Based on the query extension then the Accept header.
Defaults to default_accept_type if we don't find a preference
"""
supported_content_types = (supported_content_types or
self.default_accept_types)
parts = self.path.rsplit('.', 1)
if len(parts) > 1:
ctype = 'application/{0}'.format(parts[1])
if ctype in supported_content_types:
return ctype
bm = self.accept.best_match(supported_content_types)
return bm or self.default_accept_type
def get_content_type(self, allowed_content_types=None):
"""Determine content type of the request body.
Does not do any body introspection, only checks header
"""
if not "Content-Type" in self.headers:
return None
content_type = self.content_type
allowed_content_types = (allowed_content_types or
self.default_request_content_types)
if content_type not in allowed_content_types:
raise exception.InvalidContentType(content_type=content_type)
return content_type
class Resource(object):
"""
WSGI app that handles (de)serialization and controller dispatch.
Reads routing information supplied by RoutesMiddleware and calls
the requested action method upon its deserializer, controller,
and serializer. Those three objects may implement any of the basic
controller action methods (create, update, show, index, delete)
along with any that may be specified in the api router. A 'default'
method may also be implemented to be used in place of any
non-implemented actions. Deserializer methods must accept a request
argument and return a dictionary. Controller methods must accept a
request argument. Additionally, they must also accept keyword
arguments that represent the keys returned by the Deserializer. They
may raise a webob.exc exception or return a dict, which will be
serialized by requested content type.
"""
def __init__(self, controller, deserializer=None, serializer=None):
"""
:param controller: object that implement methods created by routes lib
:param deserializer: object that supports webob request deserialization
through controller-like actions
:param serializer: object that supports webob response serialization
through controller-like actions
"""
self.controller = controller
self.serializer = serializer or ResponseSerializer()
self.deserializer = deserializer or RequestDeserializer()
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
try:
action, action_args, accept = self.deserialize_request(request)
except exception.InvalidContentType:
msg = _("Unsupported Content-Type")
return webob.exc.HTTPUnsupportedMediaType(explanation=msg)
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return webob.exc.HTTPBadRequest(explanation=msg)
action_result = self.execute_action(action, request, **action_args)
try:
return self.serialize_response(action, action_result, accept)
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def deserialize_request(self, request):
return self.deserializer.deserialize(request)
def serialize_response(self, action, action_result, accept):
return self.serializer.serialize(action_result, accept, action)
def execute_action(self, action, request, **action_args):
return self.dispatch(self.controller, action, request, **action_args)
def dispatch(self, obj, action, *args, **kwargs):
"""Find action-specific method on self and call it."""
try:
method = getattr(obj, action)
except AttributeError:
method = getattr(obj, 'default')
return method(*args, **kwargs)
def get_action_args(self, request_environment):
"""Parse dictionary created by routes library."""
try:
args = request_environment['wsgiorg.routing_args'][1].copy()
except Exception:
return {}
try:
del args['controller']
except KeyError:
pass
try:
del args['format']
except KeyError:
pass
return args
class ActionDispatcher(object):
"""Maps method name to local methods through action name."""
def dispatch(self, *args, **kwargs):
"""Find and call local method."""
action = kwargs.pop('action', 'default')
action_method = getattr(self, str(action), self.default)
return action_method(*args, **kwargs)
def default(self, data):
raise NotImplementedError()
class DictSerializer(ActionDispatcher):
"""Default request body serialization"""
def serialize(self, data, action='default'):
return self.dispatch(data, action=action)
def default(self, data):
return ""
class JSONDictSerializer(DictSerializer):
"""Default JSON request body serialization"""
def default(self, data):
def sanitizer(obj):
if isinstance(obj, datetime.datetime):
_dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
return _dtime.isoformat()
return unicode(obj)
return jsonutils.dumps(data, default=sanitizer)
class XMLDictSerializer(DictSerializer):
def __init__(self, metadata=None, xmlns=None):
"""
:param metadata: information needed to deserialize xml into
a dictionary.
:param xmlns: XML namespace to include with serialized xml
"""
super(XMLDictSerializer, self).__init__()
self.metadata = metadata or {}
self.xmlns = xmlns
def default(self, data):
# We expect data to contain a single key which is the XML root.
root_key = data.keys()[0]
doc = minidom.Document()
node = self._to_xml_node(doc, self.metadata, root_key, data[root_key])
return self.to_xml_string(node)
def to_xml_string(self, node, has_atom=False):
self._add_xmlns(node, has_atom)
return node.toprettyxml(indent=' ', encoding='UTF-8')
#NOTE (ameade): the has_atom should be removed after all of the
# xml serializers and view builders have been updated to the current
# spec that required all responses include the xmlns:atom, the has_atom
# flag is to prevent current tests from breaking
def _add_xmlns(self, node, has_atom=False):
if self.xmlns is not None:
node.setAttribute('xmlns', self.xmlns)
if has_atom:
node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom")
def _to_xml_node(self, doc, metadata, nodename, data):
"""Recursive method to convert data members to XML nodes."""
result = doc.createElement(nodename)
# Set the xml namespace if one is specified
# TODO(justinsb): We could also use prefixes on the keys
xmlns = metadata.get('xmlns', None)
if xmlns:
result.setAttribute('xmlns', xmlns)
#TODO(bcwaldon): accomplish this without a type-check
if type(data) is list:
collections = metadata.get('list_collections', {})
if nodename in collections:
metadata = collections[nodename]
for item in data:
node = doc.createElement(metadata['item_name'])
node.setAttribute(metadata['item_key'], str(item))
result.appendChild(node)
return result
singular = metadata.get('plurals', {}).get(nodename, None)
if singular is None:
if nodename.endswith('s'):
singular = nodename[:-1]
else:
singular = 'item'
for item in data:
node = self._to_xml_node(doc, metadata, singular, item)
result.appendChild(node)
#TODO(bcwaldon): accomplish this without a type-check
elif type(data) is dict:
collections = metadata.get('dict_collections', {})
if nodename in collections:
metadata = collections[nodename]
for k, v in data.items():
node = doc.createElement(metadata['item_name'])
node.setAttribute(metadata['item_key'], str(k))
text = doc.createTextNode(str(v))
node.appendChild(text)
result.appendChild(node)
return result
attrs = metadata.get('attributes', {}).get(nodename, {})
for k, v in data.items():
if k in attrs:
result.setAttribute(k, str(v))
else:
node = self._to_xml_node(doc, metadata, k, v)
result.appendChild(node)
else:
# Type is atom
node = doc.createTextNode(str(data))
result.appendChild(node)
return result
def _create_link_nodes(self, xml_doc, links):
link_nodes = []
for link in links:
link_node = xml_doc.createElement('atom:link')
link_node.setAttribute('rel', link['rel'])
link_node.setAttribute('href', link['href'])
if 'type' in link:
link_node.setAttribute('type', link['type'])
link_nodes.append(link_node)
return link_nodes
class ResponseHeadersSerializer(ActionDispatcher):
"""Default response headers serialization"""
def serialize(self, response, data, action):
self.dispatch(response, data, action=action)
def default(self, response, data):
response.status_int = 200
class ResponseSerializer(object):
"""Encode the necessary pieces into a response object"""
def __init__(self, body_serializers=None, headers_serializer=None):
self.body_serializers = {
'application/xml': XMLDictSerializer(),
'application/json': JSONDictSerializer(),
}
self.body_serializers.update(body_serializers or {})
self.headers_serializer = (headers_serializer or
ResponseHeadersSerializer())
def serialize(self, response_data, content_type, action='default'):
"""Serialize a dict into a string and wrap in a wsgi.Request object.
:param response_data: dict produced by the Controller
:param content_type: expected mimetype of serialized response body
"""
response = webob.Response()
self.serialize_headers(response, response_data, action)
self.serialize_body(response, response_data, content_type, action)
return response
def serialize_headers(self, response, data, action):
self.headers_serializer.serialize(response, data, action)
def serialize_body(self, response, data, content_type, action):
response.headers['Content-Type'] = content_type
if data is not None:
serializer = self.get_body_serializer(content_type)
response.body = serializer.serialize(data, action)
def get_body_serializer(self, content_type):
try:
return self.body_serializers[content_type]
except (KeyError, TypeError):
raise exception.InvalidContentType(content_type=content_type)
class RequestHeadersDeserializer(ActionDispatcher):
"""Default request headers deserializer"""
def deserialize(self, request, action):
return self.dispatch(request, action=action)
def default(self, request):
return {}
class RequestDeserializer(object):
"""Break up a Request object into more useful pieces."""
def __init__(self, body_deserializers=None, headers_deserializer=None,
supported_content_types=None):
self.supported_content_types = supported_content_types
self.body_deserializers = {
'application/xml': XMLDeserializer(),
'application/json': JSONDeserializer(),
}
self.body_deserializers.update(body_deserializers or {})
self.headers_deserializer = (headers_deserializer or
RequestHeadersDeserializer())
def deserialize(self, request):
"""Extract necessary pieces of the request.
:param request: Request object
:returns: tuple of (expected controller action name, dictionary of
keyword arguments to pass to the controller, the expected
content type of the response)
"""
action_args = self.get_action_args(request.environ)
action = action_args.pop('action', None)
action_args.update(self.deserialize_headers(request, action))
action_args.update(self.deserialize_body(request, action))
accept = self.get_expected_content_type(request)
return (action, action_args, accept)
def deserialize_headers(self, request, action):
return self.headers_deserializer.deserialize(request, action)
def deserialize_body(self, request, action):
if not len(request.body) > 0:
LOG.debug(_("Empty body provided in request"))
return {}
try:
content_type = request.get_content_type()
except exception.InvalidContentType:
LOG.debug(_("Unrecognized Content-Type provided in request"))
raise
if content_type is None:
LOG.debug(_("No Content-Type provided in request"))
return {}
try:
deserializer = self.get_body_deserializer(content_type)
except exception.InvalidContentType:
LOG.debug(_("Unable to deserialize body as provided Content-Type"))
raise
return deserializer.deserialize(request.body, action)
def get_body_deserializer(self, content_type):
try:
return self.body_deserializers[content_type]
except (KeyError, TypeError):
raise exception.InvalidContentType(content_type=content_type)
def get_expected_content_type(self, request):
return request.best_match_content_type(self.supported_content_types)
def get_action_args(self, request_environment):
"""Parse dictionary created by routes library."""
try:
args = request_environment['wsgiorg.routing_args'][1].copy()
except Exception:
return {}
try:
del args['controller']
except KeyError:
pass
try:
del args['format']
except KeyError:
pass
return args
class TextDeserializer(ActionDispatcher):
"""Default request body deserialization"""
def deserialize(self, datastring, action='default'):
return self.dispatch(datastring, action=action)
def default(self, datastring):
return {}
class JSONDeserializer(TextDeserializer):
def _from_json(self, datastring):
try:
return jsonutils.loads(datastring)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def default(self, datastring):
return {'body': self._from_json(datastring)}
class XMLDeserializer(TextDeserializer):
def __init__(self, metadata=None):
"""
:param metadata: information needed to deserialize xml into
a dictionary.
"""
super(XMLDeserializer, self).__init__()
self.metadata = metadata or {}
def _from_xml(self, datastring):
plurals = set(self.metadata.get('plurals', {}))
try:
node = minidom.parseString(datastring).childNodes[0]
return {node.nodeName: self._from_xml_node(node, plurals)}
except expat.ExpatError:
msg = _("cannot understand XML")
raise exception.MalformedRequestBody(reason=msg)
def _from_xml_node(self, node, listnames):
"""Convert a minidom node to a simple Python type.
:param listnames: list of XML node names whose subnodes should
be considered list items.
"""
if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3:
return node.childNodes[0].nodeValue
elif node.nodeName in listnames:
return [self._from_xml_node(n, listnames) for n in node.childNodes]
else:
result = dict()
for attr in node.attributes.keys():
result[attr] = node.attributes[attr].nodeValue
for child in node.childNodes:
if child.nodeType != node.TEXT_NODE:
result[child.nodeName] = self._from_xml_node(child,
listnames)
return result
def find_first_child_named(self, parent, name):
"""Search a nodes children for the first child with a given name"""
for node in parent.childNodes:
if node.nodeName == name:
return node
return None
def find_children_named(self, parent, name):
"""Return all of a nodes children who have the given name"""
for node in parent.childNodes:
if node.nodeName == name:
yield node
def extract_text(self, node):
"""Get the text field contained by the given node"""
if len(node.childNodes) == 1:
child = node.childNodes[0]
if child.nodeType == child.TEXT_NODE:
return child.nodeValue
return ""
def default(self, datastring):
return {'body': self._from_xml(datastring)}

19
kwapi/plugins/api/acl.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Set up the ACL to access the API.""" """Set up the ACL to access the API."""
@ -15,16 +29,19 @@ acl_opts = [
cfg.CONF.register_opts(acl_opts) cfg.CONF.register_opts(acl_opts)
def install(app): def install(app):
"""Installs ACL check on application.""" """Installs ACL check on application."""
app.before_request(check) app.before_request(check)
return app return app
def check(): def check():
"""Checks application access.""" """Checks application access."""
headers = flask.request.headers headers = flask.request.headers
try: try:
client = Client(token=headers.get('X-Auth-Token'), auth_url=cfg.CONF.acl_auth_url) client = Client(token=headers.get('X-Auth-Token'),
auth_url=cfg.CONF.acl_auth_url)
except: except:
return "Access denied", 401 return "Access denied", 401
else: else:

15
kwapi/plugins/api/app.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Set up the API server application instance.""" """Set up the API server application instance."""
@ -19,6 +33,7 @@ app_opts = [
cfg.CONF.register_opts(app_opts) cfg.CONF.register_opts(app_opts)
def make_app(): def make_app():
"""Instantiates Flask app, attaches collector database, installs acl.""" """Instantiates Flask app, attaches collector database, installs acl."""
LOG.info('Starting API') LOG.info('Starting API')

49
kwapi/plugins/api/collector.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json import json
import threading import threading
@ -28,8 +42,12 @@ collector_opts = [
cfg.CONF.register_opts(collector_opts) cfg.CONF.register_opts(collector_opts)
class Record(dict): class Record(dict):
"""Contains fields (timestamp, kwh, w) and a method to update consumption.""" """Contains fields (timestamp, kwh, w) and a method to update
consumption.
"""
def __init__(self, timestamp, kwh, watts): def __init__(self, timestamp, kwh, watts):
"""Initializes fields with the given arguments.""" """Initializes fields with the given arguments."""
@ -42,12 +60,17 @@ class Record(dict):
def add(self, watts): def add(self, watts):
"""Updates fields with consumption data.""" """Updates fields with consumption data."""
currentTime = time.time() currentTime = time.time()
self['kwh'] += (currentTime - self['timestamp']) / 3600.0 * (watts / 1000.0) self['kwh'] += (currentTime - self['timestamp']) / 3600.0 * \
(watts / 1000.0)
self['w'] = watts self['w'] = watts
self['timestamp'] = currentTime self['timestamp'] = currentTime
class Collector: class Collector:
"""Collector gradually fills its database with received values from wattmeter drivers.""" """Collector gradually fills its database with received values from
wattmeter drivers.
"""
def __init__(self): def __init__(self):
"""Initializes an empty database and start listening the endpoint.""" """Initializes an empty database and start listening the endpoint."""
@ -74,14 +97,16 @@ class Collector:
return False return False
def clean(self): def clean(self):
"""Removes probes from database if they didn't send new values over the last period (seconds). """Removes probes from database if they didn't send new values over
If periodic, this method is executed automatically after the timeout interval. the last period (seconds). If periodic, this method is executed
automatically after the timeout interval.
""" """
LOG.info('Cleaning collector') LOG.info('Cleaning collector')
# Cleaning # Cleaning
for probe in self.database.keys(): for probe in self.database.keys():
if time.time() - self.database[probe]['timestamp'] > cfg.CONF.cleaning_interval: if time.time() - self.database[probe]['timestamp'] > \
cfg.CONF.cleaning_interval:
LOG.info('Removing data of probe %s' % probe) LOG.info('Removing data of probe %s' % probe)
self.remove(probe) self.remove(probe)
@ -92,8 +117,8 @@ class Collector:
timer.start() timer.start()
def listen(self): def listen(self):
"""Subscribes to ZeroMQ messages, and adds received measurements to the database. """Subscribes to ZeroMQ messages, and adds received measurements to the
Messages are dictionaries dumped in JSON format. database. Messages are dictionaries dumped in JSON format.
""" """
LOG.info('Collector listenig to %s' % cfg.CONF.probes_endpoint) LOG.info('Collector listenig to %s' % cfg.CONF.probes_endpoint)
@ -109,10 +134,14 @@ class Collector:
measurements = json.loads(message) measurements = json.loads(message)
if not isinstance(measurements, dict): if not isinstance(measurements, dict):
LOG.error('Bad message type (not a dict)') LOG.error('Bad message type (not a dict)')
elif cfg.CONF.signature_checking and not security.verify_signature(measurements, cfg.CONF.driver_metering_secret): elif cfg.CONF.signature_checking and \
not security.verify_signature(
measurements,
cfg.CONF.driver_metering_secret):
LOG.error('Bad message signature') LOG.error('Bad message signature')
else: else:
try: try:
self.add(measurements['probe_id'], float(measurements['w'])) self.add(measurements['probe_id'],
float(measurements['w']))
except KeyError: except KeyError:
LOG.error('Malformed message (missing required key)') LOG.error('Malformed message (missing required key)')

24
kwapi/plugins/api/v1.py
View File

@ -1,21 +1,32 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""This blueprint defines all URLs and answers.""" """This blueprint defines all URLs and answers."""
import hashlib
import hmac
import flask import flask
from kwapi.openstack.common import cfg
blueprint = flask.Blueprint('v1', __name__) blueprint = flask.Blueprint('v1', __name__)
@blueprint.route('/') @blueprint.route('/')
def welcome(): def welcome():
"""Returns detailed information about this specific version of the API.""" """Returns detailed information about this specific version of the API."""
return 'Welcome to Kwapi!' return 'Welcome to Kwapi!'
@blueprint.route('/probe-ids/') @blueprint.route('/probe-ids/')
def list_probes_ids(): def list_probes_ids():
"""Returns all known probes IDs.""" """Returns all known probes IDs."""
@ -23,6 +34,7 @@ def list_probes_ids():
message['probe_ids'] = flask.request.database.keys() message['probe_ids'] = flask.request.database.keys()
return flask.jsonify(message) return flask.jsonify(message)
@blueprint.route('/probes/') @blueprint.route('/probes/')
def list_probes(): def list_probes():
"""Returns all information about all known probes.""" """Returns all information about all known probes."""
@ -30,6 +42,7 @@ def list_probes():
message['probes'] = flask.request.database message['probes'] = flask.request.database
return flask.jsonify(message) return flask.jsonify(message)
@blueprint.route('/probes/<probe>/') @blueprint.route('/probes/<probe>/')
def probe_info(probe): def probe_info(probe):
"""Returns all information about this probe (id, timestamp, kWh, W).""" """Returns all information about this probe (id, timestamp, kWh, W)."""
@ -40,6 +53,7 @@ def probe_info(probe):
flask.abort(404) flask.abort(404)
return flask.jsonify(message) return flask.jsonify(message)
@blueprint.route('/probes/<probe>/<meter>/') @blueprint.route('/probes/<probe>/<meter>/')
def probe_value(probe, meter): def probe_value(probe, meter):
"""Returns the probe meter value.""" """Returns the probe meter value."""

15
kwapi/plugins/rrd/app.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Set up the RRD server application instance.""" """Set up the RRD server application instance."""
@ -12,6 +26,7 @@ import v1
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
def make_app(): def make_app():
"""Instantiates Flask app, attaches collector database. """ """Instantiates Flask app, attaches collector database. """
LOG.info('Starting RRD') LOG.info('Starting RRD')

98
kwapi/plugins/rrd/rrd.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Defines functions to build and update rrd database and graph.""" """Defines functions to build and update rrd database and graph."""
@ -45,27 +59,37 @@ rrd_opts = [
cfg.CONF.register_opts(rrd_opts) cfg.CONF.register_opts(rrd_opts)
scales = collections.OrderedDict() scales = collections.OrderedDict()
scales['minute'] = {'interval':300, 'resolution':1, 'label': '5 minutes'}, # Resolution = 1 second # Resolution = 1 second
scales['hour'] = {'interval':3600, 'resolution':10, 'label': 'hour'}, # Resolution = 10 seconds scales['minute'] = {'interval': 300, 'resolution': 1, 'label': '5 minutes'},
scales['day'] = {'interval':86400, 'resolution':900, 'label': 'day'}, # Resolution = 15 minutes # Resolution = 10 seconds
scales['week'] = {'interval':604800, 'resolution':7200, 'label': 'week'}, # Resolution = 2 hours scales['hour'] = {'interval': 3600, 'resolution': 10, 'label': 'hour'},
scales['month'] = {'interval':2678400, 'resolution':21600, 'label': 'month'}, # Resolution = 6 hours # Resolution = 15 minutes
scales['year'] = {'interval':31622400, 'resolution':604800, 'label': 'year'}, # Resolution = 1 week scales['day'] = {'interval': 86400, 'resolution': 900, 'label': 'day'},
# Resolution = 2 hours
scales['week'] = {'interval': 604800, 'resolution': 7200, 'label': 'week'},
# Resolution = 6 hours
scales['month'] = {'interval': 2678400, 'resolution': 21600, 'label': 'month'},
# Resolution = 1 week
scales['year'] = {'interval': 31622400, 'resolution': 604800, 'label': 'year'},
colors = ['#EA644A', '#EC9D48', '#ECD748', '#54EC48', '#48C4EC', '#7648EC', '#DE48EC', '#8A8187'] colors = ['#EA644A', '#EC9D48', '#ECD748', '#54EC48', '#48C4EC', '#7648EC',
'#DE48EC', '#8A8187']
probes = set() probes = set()
probe_colors = {} probe_colors = {}
def create_dirs(): def create_dirs():
"""Creates all required directories.""" """Creates all required directories."""
directories = [] directories = []
directories.append(cfg.CONF.png_dir) directories.append(cfg.CONF.png_dir)
directories.append(cfg.CONF.rrd_dir) directories.append(cfg.CONF.rrd_dir)
# Build a list of directory names # Build a list of directory names
# Avoid loop in try block (problem if exception occurs), and avoid multiple try blocks (too long) # Avoid loop in try block (problem if exception occurs), and avoid multiple
# try blocks (too long)
for scale in scales.keys(): for scale in scales.keys():
directories.append(cfg.CONF.png_dir + '/' + scale) directories.append(cfg.CONF.png_dir + '/' + scale)
# Create each directory in a try block, and continue if directory already exist # Create each directory in a try block, and continue if directory already
# exist
for directory in directories: for directory in directories:
try: try:
os.makedirs(directory) os.makedirs(directory)
@ -73,13 +97,18 @@ def create_dirs():
if exception.errno != errno.EEXIST: if exception.errno != errno.EEXIST:
raise raise
def get_png_filename(scale, probe): def get_png_filename(scale, probe):
"""Returns the png filename.""" """Returns the png filename."""
return cfg.CONF.png_dir + '/' + scale + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.png' return cfg.CONF.png_dir + '/' + scale + '/' + \
str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.png'
def get_rrd_filename(probe): def get_rrd_filename(probe):
"""Returns the rrd filename.""" """Returns the rrd filename."""
return cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd' return cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS,
str(probe))) + '.rrd'
def create_rrd_file(filename): def create_rrd_file(filename):
"""Creates a RRD file.""" """Creates a RRD file."""
@ -91,12 +120,17 @@ def create_rrd_file(filename):
'DS:w:GAUGE:600:0:U', 'DS:w:GAUGE:600:0:U',
] ]
for scale in scales.keys(): for scale in scales.keys():
args.append('RRA:AVERAGE:0.5:%s:%s' % (scales[scale][0]['resolution'], scales[scale][0]['interval']/scales[scale][0]['resolution'])) args.append('RRA:AVERAGE:0.5:%s:%s'
% (scales[scale][0]['resolution'],
scales[scale][0]['interval'] /
scales[scale][0]['resolution']))
rrdtool.create(args) rrdtool.create(args)
def update_rrd(probe, watts): def update_rrd(probe, watts):
"""Updates RRD file associated with this probe.""" """Updates RRD file associated with this probe."""
filename = cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd' filename = cfg.CONF.rrd_dir + '/' + \
str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd'
if not os.path.isfile(filename): if not os.path.isfile(filename):
create_rrd_file(filename) create_rrd_file(filename)
try: try:
@ -104,16 +138,19 @@ def update_rrd(probe, watts):
except rrdtool.error, e: except rrdtool.error, e:
LOG.error('Error updating RRD: %s' % e) LOG.error('Error updating RRD: %s' % e)
def build_graph(scale, probe=None): def build_graph(scale, probe=None):
"""Builds the graph for the probe, or a summary graph.""" """Builds the graph for the probe, or a summary graph."""
if scale in scales.keys() and len(probes) > 0 and (probe is None or probe in probes): if scale in scales.keys() and len(probes) > 0 \
and (probe is None or probe in probes):
# Get PNG filename # Get PNG filename
if probe is not None: if probe is not None:
png_file = get_png_filename(scale, probe) png_file = get_png_filename(scale, probe)
else: else:
png_file = cfg.CONF.png_dir + '/' + scale + '/summary.png' png_file = cfg.CONF.png_dir + '/' + scale + '/summary.png'
# Build required (PNG file not found or outdated) # Build required (PNG file not found or outdated)
if not os.path.exists(png_file) or os.path.getmtime(png_file) < time.time() - scales[scale][0]['resolution']: if not os.path.exists(png_file) or os.path.getmtime(png_file) < \
time.time() - scales[scale][0]['resolution']:
if probe is not None: if probe is not None:
# Specific arguments for probe graph # Specific arguments for probe graph
args = [png_file, args = [png_file,
@ -153,20 +190,26 @@ def build_graph(scale, probe=None):
probe_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, probe) probe_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, probe)
rrd_file = get_rrd_filename(probe) rrd_file = get_rrd_filename(probe)
# Data source # Data source
args.append('DEF:watt_with_unknown_%s=%s:w:AVERAGE' % (probe_uuid, rrd_file)) args.append('DEF:watt_with_unknown_%s=%s:w:AVERAGE'
% (probe_uuid, rrd_file))
# Data source without unknown values # Data source without unknown values
args.append('CDEF:watt_%s=watt_with_unknown_%s,UN,0,watt_with_unknown_%s,IF' % (probe_uuid, probe_uuid, probe_uuid)) args.append('CDEF:watt_%s=watt_with_unknown_%s,UN,0,watt_with_'
'unknown_%s,IF'
% (probe_uuid, probe_uuid, probe_uuid))
# Prepare CDEF expression of total watt consumption # Prepare CDEF expression of total watt consumption
cdef_watt += 'watt_%s,' % probe_uuid cdef_watt += 'watt_%s,' % probe_uuid
cdef_watt_with_unknown += 'watt_with_unknown_%s,' % probe_uuid cdef_watt_with_unknown += 'watt_with_unknown_%s,' % probe_uuid
# Draw the area for the probe # Draw the area for the probe
color = probe_colors[probe] color = probe_colors[probe]
args.append('AREA:watt_with_unknown_%s%s::STACK' % (probe_uuid, color + 'AA')) args.append('AREA:watt_with_unknown_%s%s::STACK'
% (probe_uuid, color + 'AA'))
if not stack: if not stack:
graph_lines.append('LINE:watt_with_unknown_%s%s::' % (probe_uuid, color)) graph_lines.append('LINE:watt_with_unknown_%s%s::'
% (probe_uuid, color))
stack = True stack = True
else: else:
graph_lines.append('LINE:watt_with_unknown_%s%s::STACK' % (probe_uuid, color)) graph_lines.append('LINE:watt_with_unknown_%s%s::STACK'
% (probe_uuid, color))
if len(probe_list) >= 2: if len(probe_list) >= 2:
# Prepare CDEF expression by adding the required number of '+' # Prepare CDEF expression by adding the required number of '+'
cdef_watt += '+,' * int(len(probe_list)-2) + '+' cdef_watt += '+,' * int(len(probe_list)-2) + '+'
@ -183,8 +226,10 @@ def build_graph(scale, probe=None):
# Real average (to compute kWh) # Real average (to compute kWh)
args.append('VDEF:wattavg=watt,AVERAGE') args.append('VDEF:wattavg=watt,AVERAGE')
# Compute kWh for the probe # Compute kWh for the probe
# RPN expressions must contain DEF or CDEF variables, so we pop a CDEF value # RPN expressions must contain DEF or CDEF variables, so we pop a
args.append('CDEF:kwh=watt,POP,wattavg,1000.0,/,%s,3600.0,/,*' % str(scales[scale][0]['interval'])) # CDEF value
args.append('CDEF:kwh=watt,POP,wattavg,1000.0,/,%s,3600.0,/,*'
% str(scales[scale][0]['interval']))
# Compute cost # Compute cost
args.append('CDEF:cost=watt,POP,kwh,%f,*' % cfg.CONF.kwh_price) args.append('CDEF:cost=watt,POP,kwh,%f,*' % cfg.CONF.kwh_price)
# Legend # Legend
@ -202,9 +247,10 @@ def build_graph(scale, probe=None):
LOG.info('Retrieve PNG summary graph from cache') LOG.info('Retrieve PNG summary graph from cache')
return png_file return png_file
def listen(): def listen():
"""Subscribes to ZeroMQ messages, and adds received measurements to the database. """Subscribes to ZeroMQ messages, and adds received measurements to the
Messages are dictionaries dumped in JSON format. database. Messages are dictionaries dumped in JSON format.
""" """
LOG.info('RRD listenig to %s' % cfg.CONF.probes_endpoint) LOG.info('RRD listenig to %s' % cfg.CONF.probes_endpoint)
@ -222,7 +268,9 @@ def listen():
measurements = json.loads(message) measurements = json.loads(message)
if not isinstance(measurements, dict): if not isinstance(measurements, dict):
LOG.error('Bad message type (not a dict)') LOG.error('Bad message type (not a dict)')
elif cfg.CONF.signature_checking and not security.verify_signature(measurements, cfg.CONF.driver_metering_secret): elif cfg.CONF.signature_checking and \
not security.verify_signature(measurements,
cfg.CONF.driver_metering_secret):
LOG.error('Bad message signature') LOG.error('Bad message signature')
else: else:
try: try:

30
kwapi/plugins/rrd/v1.py
View File

@ -1,4 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""This blueprint defines all URLs and answers.""" """This blueprint defines all URLs and answers."""
@ -9,29 +23,40 @@ import rrd
blueprint = flask.Blueprint('v1', __name__, static_folder='static') blueprint = flask.Blueprint('v1', __name__, static_folder='static')
@blueprint.route('/') @blueprint.route('/')
def welcome(): def welcome():
"""Shows specified page.""" """Shows specified page."""
return flask.redirect('/last/minute/') return flask.redirect('/last/minute/')
@blueprint.route('/last/<scale>/') @blueprint.route('/last/<scale>/')
def welcome_scale(scale): def welcome_scale(scale):
if scale not in flask.request.scales: if scale not in flask.request.scales:
flask.abort(404) flask.abort(404)
try: try:
return flask.render_template('index.html', probes=sorted(flask.request.probes), scales=flask.request.scales, scale=scale, view='scale') return flask.render_template('index.html',
probes=sorted(flask.request.probes),
scales=flask.request.scales,
scale=scale,
view='scale')
except TemplateNotFound: except TemplateNotFound:
flask.abort(404) flask.abort(404)
@blueprint.route('/probe/<probe>/') @blueprint.route('/probe/<probe>/')
def welcome_probe(probe): def welcome_probe(probe):
if probe not in flask.request.probes: if probe not in flask.request.probes:
flask.abort(404) flask.abort(404)
try: try:
return flask.render_template('index.html', probe=probe, scales=flask.request.scales, view='probe') return flask.render_template('index.html',
probe=probe,
scales=flask.request.scales,
view='probe')
except TemplateNotFound: except TemplateNotFound:
flask.abort(404) flask.abort(404)
@blueprint.route('/graph/<scale>/') @blueprint.route('/graph/<scale>/')
def send_summary_graph(scale): def send_summary_graph(scale):
"""Sends summary graph.""" """Sends summary graph."""
@ -42,6 +67,7 @@ def send_summary_graph(scale):
except: except:
flask.abort(404) flask.abort(404)
@blueprint.route('/graph/<scale>/<probe>/') @blueprint.route('/graph/<scale>/<probe>/')
def send_probe_graph(scale, probe): def send_probe_graph(scale, probe):
"""Sends graph.""" """Sends graph."""

23
kwapi/security.py
View File

@ -1,10 +1,25 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Computes the signature of a metering message.""" """Computes the signature of a metering message."""
import hashlib import hashlib
import hmac import hmac
def recursive_keypairs(d): def recursive_keypairs(d):
"""Generator that produces sequence of keypairs for nested dictionaries.""" """Generator that produces sequence of keypairs for nested dictionaries."""
for name, value in sorted(d.iteritems()): for name, value in sorted(d.iteritems()):
@ -14,6 +29,7 @@ def recursive_keypairs(d):
else: else:
yield name, value yield name, value
def compute_signature(message, secret): def compute_signature(message, secret):
"""Returns the signature for a message dictionary.""" """Returns the signature for a message dictionary."""
digest_maker = hmac.new(secret, '', hashlib.sha256) digest_maker = hmac.new(secret, '', hashlib.sha256)
@ -24,12 +40,17 @@ def compute_signature(message, secret):
digest_maker.update(unicode(value).encode('utf-8')) digest_maker.update(unicode(value).encode('utf-8'))
return digest_maker.hexdigest() return digest_maker.hexdigest()
def append_signature(message, secret): def append_signature(message, secret):
"""Sets the message signature key.""" """Sets the message signature key."""
message['message_signature'] = compute_signature(message, secret) message['message_signature'] = compute_signature(message, secret)
def verify_signature(message, secret): def verify_signature(message, secret):
"""Checks the signature in the message against the value computed from the rest of the contents.""" """Checks the signature in the message against the value computed from the
rest of the contents.
"""
old_sig = message.get('message_signature') old_sig = message.get('message_signature')
new_sig = compute_signature(message, secret) new_sig = compute_signature(message, secret)
return new_sig == old_sig return new_sig == old_sig

24
setup.py
View File

@ -1,5 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import setuptools import setuptools
@ -36,8 +50,14 @@ setuptools.setup(
'bin/kwapi-drivers', 'bin/kwapi-drivers',
'bin/kwapi-rrd'], 'bin/kwapi-rrd'],
data_files=[('/etc/kwapi', ['etc/kwapi/api.conf', 'etc/kwapi/drivers.conf', 'etc/kwapi/rrd.conf'])], data_files=[('/etc/kwapi', ['etc/kwapi/api.conf',
'etc/kwapi/drivers.conf',
'etc/kwapi/rrd.conf'])],
install_requires=['flask', 'pyserial', 'python-keystoneclient', 'pyzmq', 'py-rrdtool'] install_requires=['flask',
'pyserial',
'python-keystoneclient',
'pyzmq',
'py-rrdtool']
) )