PEP 8 validation (version 1.4).
Update openstack.common. License in each file.
This commit is contained in:
parent
7d294500da
commit
f925e6a8dc
|
@ -1,5 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
|
|
|
@ -1,5 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
import signal
|
||||
|
|
|
@ -1,5 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
|
|
|
@ -1,2 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from wattsup import Wattsup
|
||||
from dummy import Dummy
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
from threading import Thread, Event
|
||||
|
@ -12,21 +26,23 @@ LOG = log.getLogger(__name__)
|
|||
|
||||
driver_opts = [
|
||||
cfg.BoolOpt('enable_signing',
|
||||
required=True,
|
||||
),
|
||||
required=True,
|
||||
),
|
||||
cfg.StrOpt('metering_secret',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(driver_opts)
|
||||
|
||||
|
||||
class Driver(Thread):
|
||||
"""Generic driver class, derived from Thread."""
|
||||
|
||||
|
||||
def __init__(self, probe_ids, kwargs):
|
||||
"""Initializes driver."""
|
||||
LOG.info('Loading driver %s(probe_ids=%s, kwargs=%s)' % (self.__class__.__name__, probe_ids, kwargs))
|
||||
LOG.info('Loading driver %s(probe_ids=%s, kwargs=%s)'
|
||||
% (self.__class__.__name__, probe_ids, kwargs))
|
||||
Thread.__init__(self)
|
||||
self.probe_ids = probe_ids
|
||||
self.kwargs = kwargs
|
||||
|
@ -34,31 +50,34 @@ class Driver(Thread):
|
|||
self.stop_request = Event()
|
||||
self.publisher = zmq.Context.instance().socket(zmq.PUB)
|
||||
self.publisher.connect('inproc://drivers')
|
||||
|
||||
|
||||
def run(self):
|
||||
"""Run the driver thread. Needs to be implemented in a derived class."""
|
||||
"""Runs the driver thread. Needs to be implemented in a derived
|
||||
class.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def join(self):
|
||||
"""Asks the driver thread to terminate."""
|
||||
self.stop_request.set()
|
||||
super(Driver, self).join()
|
||||
|
||||
|
||||
def stop_request_pending(self):
|
||||
"""Returns true if a stop request is pending."""
|
||||
return self.stop_request.is_set()
|
||||
|
||||
|
||||
def send_measurements(self, probe_id, measurements):
|
||||
"""Sends a message via ZeroMQ (dictionary dumped in JSON format)."""
|
||||
measurements['probe_id'] = probe_id
|
||||
if cfg.CONF.enable_signing:
|
||||
security.append_signature(measurements, cfg.CONF.metering_secret)
|
||||
self.publisher.send(json.dumps(measurements))
|
||||
|
||||
|
||||
def subscribe(self, observer):
|
||||
"""Appends the observer (callback method) to the observers list."""
|
||||
self.probe_observers.append(observer)
|
||||
|
||||
|
||||
def stop(self):
|
||||
"""Asks the probe to terminate."""
|
||||
self.terminate = True
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Loads and checks driver threads."""
|
||||
|
||||
|
@ -21,12 +35,13 @@ driver_manager_opts = [
|
|||
cfg.StrOpt('probes_endpoint',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(driver_manager_opts)
|
||||
|
||||
threads = []
|
||||
|
||||
|
||||
def load_all_drivers():
|
||||
"""Loads all drivers from config file."""
|
||||
parser = cfg.ConfigParser(cfg.CONF.config_file[0], {})
|
||||
|
@ -42,6 +57,7 @@ def load_all_drivers():
|
|||
if thread is not None:
|
||||
threads.append(thread)
|
||||
|
||||
|
||||
def load_driver(class_name, probe_ids, kwargs):
|
||||
"""Starts a probe thread."""
|
||||
try:
|
||||
|
@ -51,30 +67,36 @@ def load_driver(class_name, probe_ids, kwargs):
|
|||
try:
|
||||
probeObject = probeClass(probe_ids, **kwargs)
|
||||
except Exception as exception:
|
||||
LOG.error('Exception occurred while initializing %s(%s, %s): %s' % (class_name, probe_ids, kwargs, exception))
|
||||
LOG.error('Exception occurred while initializing %s(%s, %s): %s'
|
||||
% (class_name, probe_ids, kwargs, exception))
|
||||
else:
|
||||
probeObject.start()
|
||||
return probeObject
|
||||
|
||||
|
||||
def check_drivers_alive():
|
||||
"""Checks all drivers and reloads those that crashed.
|
||||
This method is executed automatically at the given interval.
|
||||
|
||||
|
||||
"""
|
||||
LOG.info('Checks driver threads')
|
||||
for index, thread in enumerate(threads):
|
||||
if not thread.is_alive():
|
||||
LOG.warning('%s(probe_ids=%s, kwargs=%s) is crashed' % (thread.__class__.__name__, thread.probe_ids, thread.kwargs))
|
||||
new_thread = load_driver(thread.__class__.__name__, thread.probe_ids, thread.kwargs)
|
||||
LOG.warning('%s(probe_ids=%s, kwargs=%s) is crashed'
|
||||
% (thread.__class__.__name__,
|
||||
thread.probe_ids, thread.kwargs))
|
||||
new_thread = load_driver(thread.__class__.__name__,
|
||||
thread.probe_ids, thread.kwargs)
|
||||
if new_thread is not None:
|
||||
threads[index] = new_thread
|
||||
|
||||
|
||||
# Schedule periodic execution of this function
|
||||
if cfg.CONF.check_drivers_interval > 0:
|
||||
timer = Timer(cfg.CONF.check_drivers_interval, check_drivers_alive)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def start_zmq_server():
|
||||
"""Forwards probe values to the probes_endpoint."""
|
||||
context = zmq.Context.instance()
|
||||
|
@ -85,11 +107,13 @@ def start_zmq_server():
|
|||
backend.bind(cfg.CONF.probes_endpoint)
|
||||
thread.start_new_thread(zmq.device, (zmq.FORWARDER, frontend, backend))
|
||||
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
"""Intercepts TERM signal and properly terminates probe threads."""
|
||||
if signum is signal.SIGTERM:
|
||||
terminate()
|
||||
|
||||
|
||||
def terminate():
|
||||
"""Terminates driver threads."""
|
||||
for driver in threads:
|
||||
|
|
|
@ -1,25 +1,42 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from random import randrange
|
||||
import time
|
||||
|
||||
from driver import Driver
|
||||
|
||||
|
||||
class Dummy(Driver):
|
||||
"""Dummy driver derived from Driver class. Usefull for tests."""
|
||||
|
||||
|
||||
def __init__(self, probe_ids, **kwargs):
|
||||
"""Initializes the dummy driver.
|
||||
|
||||
|
||||
Keyword arguments:
|
||||
probe_ids -- list containing the probes IDs (a wattmeter monitor sometimes several probes
|
||||
kwargs -- keywords (min_value and max_value) defining the random value interval
|
||||
|
||||
probe_ids -- list containing the probes IDs
|
||||
(a wattmeter monitor sometimes several probes)
|
||||
kwargs -- keywords (min_value and max_value)
|
||||
defining the random value interval
|
||||
|
||||
"""
|
||||
Driver.__init__(self, probe_ids, kwargs)
|
||||
self.min_value = int(kwargs.get('min', 75))
|
||||
self.max_value = int(kwargs.get('max', 100))
|
||||
|
||||
|
||||
def run(self):
|
||||
"""Starts the driver thread."""
|
||||
while not self.stop_request_pending():
|
||||
|
|
|
@ -1,23 +1,39 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import serial
|
||||
from serial.serialutil import SerialException
|
||||
|
||||
from driver import Driver
|
||||
|
||||
|
||||
class Wattsup(Driver):
|
||||
"""Driver for Wattsup wattmeters."""
|
||||
|
||||
|
||||
def __init__(self, probe_ids, **kwargs):
|
||||
"""Initializes the Wattsup driver.
|
||||
|
||||
|
||||
Keyword arguments:
|
||||
probe_ids -- list containing the probes IDs (a wattmeter monitor sometimes several probes
|
||||
probe_ids -- list containing the probes IDs
|
||||
(a wattmeter monitor sometimes several probes)
|
||||
kwargs -- keyword (device) defining the device to read (/dev/ttyUSB0)
|
||||
|
||||
|
||||
"""
|
||||
Driver.__init__(self, probe_ids, kwargs)
|
||||
|
||||
|
||||
# Configure serial port
|
||||
self.serial = serial.Serial(
|
||||
port=kwargs.get('device', '/dev/ttyUSB0'),
|
||||
|
@ -27,15 +43,15 @@ class Wattsup(Driver):
|
|||
bytesize=serial.EIGHTBITS,
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
|
||||
# Clear memory
|
||||
self.serial.write('#R,W,0;')
|
||||
self.serial.read(256)
|
||||
|
||||
|
||||
# Start external logging with interval = 1
|
||||
self.serial.write('#L,W,3,E,1,1;')
|
||||
self.serial.read(256)
|
||||
|
||||
|
||||
def run(self):
|
||||
"""Starts the driver thread."""
|
||||
while not self.stop_request_pending():
|
||||
|
@ -47,7 +63,7 @@ class Wattsup(Driver):
|
|||
measurements = {}
|
||||
measurements['w'] = self.extract_watts(packet)
|
||||
self.send_measurements(self.probe_ids[0], measurements)
|
||||
|
||||
|
||||
def get_packet(self):
|
||||
"""Returns the next packet sent by the wattmeter."""
|
||||
packet = ''
|
||||
|
@ -58,7 +74,7 @@ class Wattsup(Driver):
|
|||
packet += char
|
||||
if char == ';':
|
||||
return packet
|
||||
|
||||
|
||||
def extract_watts(self, packet):
|
||||
"""Extracts the consumption data (watts) from the packet."""
|
||||
value = float(packet.split(',')[3])/10.0
|
||||
|
|
|
@ -0,0 +1,24 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This ensures the openstack namespace is defined
|
||||
try:
|
||||
import pkg_resources
|
||||
pkg_resources.declare_namespace(__name__)
|
||||
except ImportError:
|
||||
import pkgutil
|
||||
__path__ = pkgutil.extend_path(__path__, __name__)
|
|
@ -0,0 +1,19 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# TODO(jaypipes) Code in this module is intended to be ported to the eventual
|
||||
# openstack-common library
|
|
@ -0,0 +1,44 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Authentication related utilities and helper functions.
|
||||
"""
|
||||
|
||||
|
||||
def auth_str_equal(provided, known):
|
||||
"""Constant-time string comparison.
|
||||
|
||||
:params provided: the first string
|
||||
:params known: the second string
|
||||
|
||||
:return: True if the strings are equal.
|
||||
|
||||
This function takes two strings and compares them. It is intended to be
|
||||
used when doing a comparison for authentication purposes to help guard
|
||||
against timing attacks. When using the function for this purpose, always
|
||||
provide the user-provided password as the first argument. The time this
|
||||
function will take is always a factor of the length of this string.
|
||||
"""
|
||||
result = 0
|
||||
p_len = len(provided)
|
||||
k_len = len(known)
|
||||
for i in xrange(p_len):
|
||||
a = ord(provided[i]) if i < p_len else 0
|
||||
b = ord(known[i]) if i < k_len else 0
|
||||
result |= a ^ b
|
||||
return (p_len == k_len) & (result == 0)
|
|
@ -205,27 +205,11 @@ Option values may reference other values using PEP 292 string substitution::
|
|||
|
||||
Note that interpolation can be avoided by using '$$'.
|
||||
|
||||
For command line utilities that dispatch to other command line utilities, the
|
||||
disable_interspersed_args() method is available. If this this method is called,
|
||||
then parsing e.g.::
|
||||
|
||||
script --verbose cmd --debug /tmp/mything
|
||||
|
||||
will no longer return::
|
||||
|
||||
['cmd', '/tmp/mything']
|
||||
|
||||
as the leftover arguments, but will instead return::
|
||||
|
||||
['cmd', '--debug', '/tmp/mything']
|
||||
|
||||
i.e. argument parsing is stopped at the first non-option argument.
|
||||
|
||||
Options may be declared as required so that an error is raised if the user
|
||||
does not supply a value for the option.
|
||||
|
||||
Options may be declared as secret so that their values are not leaked into
|
||||
log files:
|
||||
log files::
|
||||
|
||||
opts = [
|
||||
cfg.StrOpt('s3_store_access_key', secret=True),
|
||||
|
@ -234,28 +218,50 @@ log files:
|
|||
]
|
||||
|
||||
This module also contains a global instance of the CommonConfigOpts class
|
||||
in order to support a common usage pattern in OpenStack:
|
||||
in order to support a common usage pattern in OpenStack::
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common import cfg
|
||||
|
||||
opts = [
|
||||
cfg.StrOpt('bind_host', default='0.0.0.0'),
|
||||
cfg.IntOpt('bind_port', default=9292),
|
||||
]
|
||||
opts = [
|
||||
cfg.StrOpt('bind_host', default='0.0.0.0'),
|
||||
cfg.IntOpt('bind_port', default=9292),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(opts)
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(opts)
|
||||
|
||||
def start(server, app):
|
||||
server.start(app, CONF.bind_port, CONF.bind_host)
|
||||
def start(server, app):
|
||||
server.start(app, CONF.bind_port, CONF.bind_host)
|
||||
|
||||
Positional command line arguments are supported via a 'positional' Opt
|
||||
constructor argument::
|
||||
|
||||
>>> CONF.register_cli_opt(MultiStrOpt('bar', positional=True))
|
||||
True
|
||||
>>> CONF(['a', 'b'])
|
||||
>>> CONF.bar
|
||||
['a', 'b']
|
||||
|
||||
It is also possible to use argparse "sub-parsers" to parse additional
|
||||
command line arguments using the SubCommandOpt class:
|
||||
|
||||
>>> def add_parsers(subparsers):
|
||||
... list_action = subparsers.add_parser('list')
|
||||
... list_action.add_argument('id')
|
||||
...
|
||||
>>> CONF.register_cli_opt(SubCommandOpt('action', handler=add_parsers))
|
||||
True
|
||||
>>> CONF(['list', '10'])
|
||||
>>> CONF.action.name, CONF.action.id
|
||||
('list', '10')
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import glob
|
||||
import optparse
|
||||
import os
|
||||
import string
|
||||
import sys
|
||||
|
@ -474,6 +480,13 @@ def _is_opt_registered(opts, opt):
|
|||
return False
|
||||
|
||||
|
||||
def set_defaults(opts, **kwargs):
|
||||
for opt in opts:
|
||||
if opt.dest in kwargs:
|
||||
opt.default = kwargs[opt.dest]
|
||||
break
|
||||
|
||||
|
||||
class Opt(object):
|
||||
|
||||
"""Base class for all configuration options.
|
||||
|
@ -489,6 +502,8 @@ class Opt(object):
|
|||
a single character CLI option name
|
||||
default:
|
||||
the default value of the option
|
||||
positional:
|
||||
True if the option is a positional CLI argument
|
||||
metavar:
|
||||
the name shown as the argument to a CLI option in --help output
|
||||
help:
|
||||
|
@ -497,8 +512,8 @@ class Opt(object):
|
|||
multi = False
|
||||
|
||||
def __init__(self, name, dest=None, short=None, default=None,
|
||||
metavar=None, help=None, secret=False, required=False,
|
||||
deprecated_name=None):
|
||||
positional=False, metavar=None, help=None,
|
||||
secret=False, required=False, deprecated_name=None):
|
||||
"""Construct an Opt object.
|
||||
|
||||
The only required parameter is the option's name. However, it is
|
||||
|
@ -508,6 +523,7 @@ class Opt(object):
|
|||
:param dest: the name of the corresponding ConfigOpts property
|
||||
:param short: a single character CLI option name
|
||||
:param default: the default value of the option
|
||||
:param positional: True if the option is a positional CLI argument
|
||||
:param metavar: the option argument to show in --help
|
||||
:param help: an explanation of how the option is used
|
||||
:param secret: true iff the value should be obfuscated in log output
|
||||
|
@ -521,6 +537,7 @@ class Opt(object):
|
|||
self.dest = dest
|
||||
self.short = short
|
||||
self.default = default
|
||||
self.positional = positional
|
||||
self.metavar = metavar
|
||||
self.help = help
|
||||
self.secret = secret
|
||||
|
@ -561,64 +578,73 @@ class Opt(object):
|
|||
:param parser: the CLI option parser
|
||||
:param group: an optional OptGroup object
|
||||
"""
|
||||
container = self._get_optparse_container(parser, group)
|
||||
kwargs = self._get_optparse_kwargs(group)
|
||||
prefix = self._get_optparse_prefix('', group)
|
||||
self._add_to_optparse(container, self.name, self.short, kwargs, prefix,
|
||||
self.deprecated_name)
|
||||
container = self._get_argparse_container(parser, group)
|
||||
kwargs = self._get_argparse_kwargs(group)
|
||||
prefix = self._get_argparse_prefix('', group)
|
||||
self._add_to_argparse(container, self.name, self.short, kwargs, prefix,
|
||||
self.positional, self.deprecated_name)
|
||||
|
||||
def _add_to_optparse(self, container, name, short, kwargs, prefix='',
|
||||
deprecated_name=None):
|
||||
"""Add an option to an optparse parser or group.
|
||||
def _add_to_argparse(self, container, name, short, kwargs, prefix='',
|
||||
positional=False, deprecated_name=None):
|
||||
"""Add an option to an argparse parser or group.
|
||||
|
||||
:param container: an optparse.OptionContainer object
|
||||
:param container: an argparse._ArgumentGroup object
|
||||
:param name: the opt name
|
||||
:param short: the short opt name
|
||||
:param kwargs: the keyword arguments for add_option()
|
||||
:param kwargs: the keyword arguments for add_argument()
|
||||
:param prefix: an optional prefix to prepend to the opt name
|
||||
:param position: whether the optional is a positional CLI argument
|
||||
:raises: DuplicateOptError if a naming confict is detected
|
||||
"""
|
||||
args = ['--' + prefix + name]
|
||||
def hyphen(arg):
|
||||
return arg if not positional else ''
|
||||
|
||||
args = [hyphen('--') + prefix + name]
|
||||
if short:
|
||||
args += ['-' + short]
|
||||
args.append(hyphen('-') + short)
|
||||
if deprecated_name:
|
||||
args += ['--' + prefix + deprecated_name]
|
||||
for a in args:
|
||||
if container.has_option(a):
|
||||
raise DuplicateOptError(a)
|
||||
container.add_option(*args, **kwargs)
|
||||
args.append(hyphen('--') + prefix + deprecated_name)
|
||||
|
||||
def _get_optparse_container(self, parser, group):
|
||||
"""Returns an optparse.OptionContainer.
|
||||
try:
|
||||
container.add_argument(*args, **kwargs)
|
||||
except argparse.ArgumentError as e:
|
||||
raise DuplicateOptError(e)
|
||||
|
||||
:param parser: an optparse.OptionParser
|
||||
def _get_argparse_container(self, parser, group):
|
||||
"""Returns an argparse._ArgumentGroup.
|
||||
|
||||
:param parser: an argparse.ArgumentParser
|
||||
:param group: an (optional) OptGroup object
|
||||
:returns: an optparse.OptionGroup if a group is given, else the parser
|
||||
:returns: an argparse._ArgumentGroup if group is given, else parser
|
||||
"""
|
||||
if group is not None:
|
||||
return group._get_optparse_group(parser)
|
||||
return group._get_argparse_group(parser)
|
||||
else:
|
||||
return parser
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Build a dict of keyword arguments for optparse's add_option().
|
||||
def _get_argparse_kwargs(self, group, **kwargs):
|
||||
"""Build a dict of keyword arguments for argparse's add_argument().
|
||||
|
||||
Most opt types extend this method to customize the behaviour of the
|
||||
options added to optparse.
|
||||
options added to argparse.
|
||||
|
||||
:param group: an optional group
|
||||
:param kwargs: optional keyword arguments to add to
|
||||
:returns: a dict of keyword arguments
|
||||
"""
|
||||
dest = self.dest
|
||||
if group is not None:
|
||||
dest = group.name + '_' + dest
|
||||
kwargs.update({'dest': dest,
|
||||
if not self.positional:
|
||||
dest = self.dest
|
||||
if group is not None:
|
||||
dest = group.name + '_' + dest
|
||||
kwargs['dest'] = dest
|
||||
else:
|
||||
kwargs['nargs'] = '?'
|
||||
kwargs.update({'default': None,
|
||||
'metavar': self.metavar,
|
||||
'help': self.help, })
|
||||
return kwargs
|
||||
|
||||
def _get_optparse_prefix(self, prefix, group):
|
||||
def _get_argparse_prefix(self, prefix, group):
|
||||
"""Build a prefix for the CLI option name, if required.
|
||||
|
||||
CLI options in a group are prefixed with the group's name in order
|
||||
|
@ -656,6 +682,11 @@ class BoolOpt(Opt):
|
|||
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
|
||||
'0': False, 'no': False, 'false': False, 'off': False}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
if 'positional' in kwargs:
|
||||
raise ValueError('positional boolean args not supported')
|
||||
super(BoolOpt, self).__init__(*args, **kwargs)
|
||||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a boolean from ConfigParser."""
|
||||
def convert_bool(v):
|
||||
|
@ -671,21 +702,32 @@ class BoolOpt(Opt):
|
|||
def _add_to_cli(self, parser, group=None):
|
||||
"""Extends the base class method to add the --nooptname option."""
|
||||
super(BoolOpt, self)._add_to_cli(parser, group)
|
||||
self._add_inverse_to_optparse(parser, group)
|
||||
self._add_inverse_to_argparse(parser, group)
|
||||
|
||||
def _add_inverse_to_optparse(self, parser, group):
|
||||
def _add_inverse_to_argparse(self, parser, group):
|
||||
"""Add the --nooptname option to the option parser."""
|
||||
container = self._get_optparse_container(parser, group)
|
||||
kwargs = self._get_optparse_kwargs(group, action='store_false')
|
||||
prefix = self._get_optparse_prefix('no', group)
|
||||
container = self._get_argparse_container(parser, group)
|
||||
kwargs = self._get_argparse_kwargs(group, action='store_false')
|
||||
prefix = self._get_argparse_prefix('no', group)
|
||||
kwargs["help"] = "The inverse of --" + self.name
|
||||
self._add_to_optparse(container, self.name, None, kwargs, prefix,
|
||||
self.deprecated_name)
|
||||
self._add_to_argparse(container, self.name, None, kwargs, prefix,
|
||||
self.positional, self.deprecated_name)
|
||||
|
||||
def _get_optparse_kwargs(self, group, action='store_true', **kwargs):
|
||||
"""Extends the base optparse keyword dict for boolean options."""
|
||||
return super(BoolOpt,
|
||||
self)._get_optparse_kwargs(group, action=action, **kwargs)
|
||||
def _get_argparse_kwargs(self, group, action='store_true', **kwargs):
|
||||
"""Extends the base argparse keyword dict for boolean options."""
|
||||
|
||||
kwargs = super(BoolOpt, self)._get_argparse_kwargs(group, **kwargs)
|
||||
|
||||
# metavar has no effect for BoolOpt
|
||||
if 'metavar' in kwargs:
|
||||
del kwargs['metavar']
|
||||
|
||||
if action != 'store_true':
|
||||
action = 'store_false'
|
||||
|
||||
kwargs['action'] = action
|
||||
|
||||
return kwargs
|
||||
|
||||
|
||||
class IntOpt(Opt):
|
||||
|
@ -697,10 +739,10 @@ class IntOpt(Opt):
|
|||
return [int(v) for v in self._cparser_get_with_deprecated(cparser,
|
||||
section)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for integer options."""
|
||||
def _get_argparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base argparse keyword dict for integer options."""
|
||||
return super(IntOpt,
|
||||
self)._get_optparse_kwargs(group, type='int', **kwargs)
|
||||
self)._get_argparse_kwargs(group, type=int, **kwargs)
|
||||
|
||||
|
||||
class FloatOpt(Opt):
|
||||
|
@ -712,10 +754,10 @@ class FloatOpt(Opt):
|
|||
return [float(v) for v in
|
||||
self._cparser_get_with_deprecated(cparser, section)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for float options."""
|
||||
return super(FloatOpt,
|
||||
self)._get_optparse_kwargs(group, type='float', **kwargs)
|
||||
def _get_argparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base argparse keyword dict for float options."""
|
||||
return super(FloatOpt, self)._get_argparse_kwargs(group,
|
||||
type=float, **kwargs)
|
||||
|
||||
|
||||
class ListOpt(Opt):
|
||||
|
@ -725,23 +767,26 @@ class ListOpt(Opt):
|
|||
is a list containing these strings.
|
||||
"""
|
||||
|
||||
class _StoreListAction(argparse.Action):
|
||||
"""
|
||||
An argparse action for parsing an option value into a list.
|
||||
"""
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
if values is not None:
|
||||
values = [a.strip() for a in values.split(',')]
|
||||
setattr(namespace, self.dest, values)
|
||||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a list from ConfigParser."""
|
||||
return [v.split(',') for v in
|
||||
return [[a.strip() for a in v.split(',')] for v in
|
||||
self._cparser_get_with_deprecated(cparser, section)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for list options."""
|
||||
return super(ListOpt,
|
||||
self)._get_optparse_kwargs(group,
|
||||
type='string',
|
||||
action='callback',
|
||||
callback=self._parse_list,
|
||||
**kwargs)
|
||||
|
||||
def _parse_list(self, option, opt, value, parser):
|
||||
"""An optparse callback for parsing an option value into a list."""
|
||||
setattr(parser.values, self.dest, value.split(','))
|
||||
def _get_argparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base argparse keyword dict for list options."""
|
||||
return Opt._get_argparse_kwargs(self,
|
||||
group,
|
||||
action=ListOpt._StoreListAction,
|
||||
**kwargs)
|
||||
|
||||
|
||||
class MultiStrOpt(Opt):
|
||||
|
@ -752,10 +797,14 @@ class MultiStrOpt(Opt):
|
|||
"""
|
||||
multi = True
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for multi str options."""
|
||||
return super(MultiStrOpt,
|
||||
self)._get_optparse_kwargs(group, action='append')
|
||||
def _get_argparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base argparse keyword dict for multi str options."""
|
||||
kwargs = super(MultiStrOpt, self)._get_argparse_kwargs(group)
|
||||
if not self.positional:
|
||||
kwargs['action'] = 'append'
|
||||
else:
|
||||
kwargs['nargs'] = '*'
|
||||
return kwargs
|
||||
|
||||
def _cparser_get_with_deprecated(self, cparser, section):
|
||||
"""If cannot find option as dest try deprecated_name alias."""
|
||||
|
@ -765,6 +814,57 @@ class MultiStrOpt(Opt):
|
|||
return cparser.get(section, [self.dest], multi=True)
|
||||
|
||||
|
||||
class SubCommandOpt(Opt):
|
||||
|
||||
"""
|
||||
Sub-command options allow argparse sub-parsers to be used to parse
|
||||
additional command line arguments.
|
||||
|
||||
The handler argument to the SubCommandOpt contructor is a callable
|
||||
which is supplied an argparse subparsers object. Use this handler
|
||||
callable to add sub-parsers.
|
||||
|
||||
The opt value is SubCommandAttr object with the name of the chosen
|
||||
sub-parser stored in the 'name' attribute and the values of other
|
||||
sub-parser arguments available as additional attributes.
|
||||
"""
|
||||
|
||||
def __init__(self, name, dest=None, handler=None,
|
||||
title=None, description=None, help=None):
|
||||
"""Construct an sub-command parsing option.
|
||||
|
||||
This behaves similarly to other Opt sub-classes but adds a
|
||||
'handler' argument. The handler is a callable which is supplied
|
||||
an subparsers object when invoked. The add_parser() method on
|
||||
this subparsers object can be used to register parsers for
|
||||
sub-commands.
|
||||
|
||||
:param name: the option's name
|
||||
:param dest: the name of the corresponding ConfigOpts property
|
||||
:param title: title of the sub-commands group in help output
|
||||
:param description: description of the group in help output
|
||||
:param help: a help string giving an overview of available sub-commands
|
||||
"""
|
||||
super(SubCommandOpt, self).__init__(name, dest=dest, help=help)
|
||||
self.handler = handler
|
||||
self.title = title
|
||||
self.description = description
|
||||
|
||||
def _add_to_cli(self, parser, group=None):
|
||||
"""Add argparse sub-parsers and invoke the handler method."""
|
||||
dest = self.dest
|
||||
if group is not None:
|
||||
dest = group.name + '_' + dest
|
||||
|
||||
subparsers = parser.add_subparsers(dest=dest,
|
||||
title=self.title,
|
||||
description=self.description,
|
||||
help=self.help)
|
||||
|
||||
if not self.handler is None:
|
||||
self.handler(subparsers)
|
||||
|
||||
|
||||
class OptGroup(object):
|
||||
|
||||
"""
|
||||
|
@ -800,19 +900,20 @@ class OptGroup(object):
|
|||
self.help = help
|
||||
|
||||
self._opts = {} # dict of dicts of (opt:, override:, default:)
|
||||
self._optparse_group = None
|
||||
self._argparse_group = None
|
||||
|
||||
def _register_opt(self, opt):
|
||||
def _register_opt(self, opt, cli=False):
|
||||
"""Add an opt to this group.
|
||||
|
||||
:param opt: an Opt object
|
||||
:param cli: whether this is a CLI option
|
||||
:returns: False if previously registered, True otherwise
|
||||
:raises: DuplicateOptError if a naming conflict is detected
|
||||
"""
|
||||
if _is_opt_registered(self._opts, opt):
|
||||
return False
|
||||
|
||||
self._opts[opt.dest] = {'opt': opt}
|
||||
self._opts[opt.dest] = {'opt': opt, 'cli': cli}
|
||||
|
||||
return True
|
||||
|
||||
|
@ -824,16 +925,16 @@ class OptGroup(object):
|
|||
if opt.dest in self._opts:
|
||||
del self._opts[opt.dest]
|
||||
|
||||
def _get_optparse_group(self, parser):
|
||||
"""Build an optparse.OptionGroup for this group."""
|
||||
if self._optparse_group is None:
|
||||
self._optparse_group = optparse.OptionGroup(parser, self.title,
|
||||
self.help)
|
||||
return self._optparse_group
|
||||
def _get_argparse_group(self, parser):
|
||||
if self._argparse_group is None:
|
||||
"""Build an argparse._ArgumentGroup for this group."""
|
||||
self._argparse_group = parser.add_argument_group(self.title,
|
||||
self.help)
|
||||
return self._argparse_group
|
||||
|
||||
def _clear(self):
|
||||
"""Clear this group's option parsing state."""
|
||||
self._optparse_group = None
|
||||
self._argparse_group = None
|
||||
|
||||
|
||||
class ParseError(iniparser.ParseError):
|
||||
|
@ -928,26 +1029,31 @@ class ConfigOpts(collections.Mapping):
|
|||
self._groups = {}
|
||||
|
||||
self._args = None
|
||||
|
||||
self._oparser = None
|
||||
self._cparser = None
|
||||
self._cli_values = {}
|
||||
self.__cache = {}
|
||||
self._config_opts = []
|
||||
self._disable_interspersed_args = False
|
||||
|
||||
def _setup(self, project, prog, version, usage, default_config_files):
|
||||
"""Initialize a ConfigOpts object for option parsing."""
|
||||
def _pre_setup(self, project, prog, version, usage, default_config_files):
|
||||
"""Initialize a ConfigCliParser object for option parsing."""
|
||||
|
||||
if prog is None:
|
||||
prog = os.path.basename(sys.argv[0])
|
||||
|
||||
if default_config_files is None:
|
||||
default_config_files = find_config_files(project, prog)
|
||||
|
||||
self._oparser = optparse.OptionParser(prog=prog,
|
||||
version=version,
|
||||
usage=usage)
|
||||
if self._disable_interspersed_args:
|
||||
self._oparser.disable_interspersed_args()
|
||||
self._oparser = argparse.ArgumentParser(prog=prog, usage=usage)
|
||||
self._oparser.add_argument('--version',
|
||||
action='version',
|
||||
version=version)
|
||||
|
||||
return prog, default_config_files
|
||||
|
||||
def _setup(self, project, prog, version, usage, default_config_files):
|
||||
"""Initialize a ConfigOpts object for option parsing."""
|
||||
|
||||
self._config_opts = [
|
||||
MultiStrOpt('config-file',
|
||||
|
@ -1017,18 +1123,23 @@ class ConfigOpts(collections.Mapping):
|
|||
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
|
||||
RequiredOptError, DuplicateOptError
|
||||
"""
|
||||
|
||||
self.clear()
|
||||
|
||||
prog, default_config_files = self._pre_setup(project,
|
||||
prog,
|
||||
version,
|
||||
usage,
|
||||
default_config_files)
|
||||
|
||||
self._setup(project, prog, version, usage, default_config_files)
|
||||
|
||||
self._cli_values, leftovers = self._parse_cli_opts(args)
|
||||
self._cli_values = self._parse_cli_opts(args)
|
||||
|
||||
self._parse_config_files()
|
||||
|
||||
self._check_required_opts()
|
||||
|
||||
return leftovers
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Look up an option value and perform string substitution.
|
||||
|
||||
|
@ -1062,17 +1173,21 @@ class ConfigOpts(collections.Mapping):
|
|||
|
||||
@__clear_cache
|
||||
def clear(self):
|
||||
"""Clear the state of the object to before it was called."""
|
||||
"""Clear the state of the object to before it was called.
|
||||
|
||||
Any subparsers added using the add_cli_subparsers() will also be
|
||||
removed as a side-effect of this method.
|
||||
"""
|
||||
self._args = None
|
||||
self._cli_values.clear()
|
||||
self._oparser = None
|
||||
self._oparser = argparse.ArgumentParser()
|
||||
self._cparser = None
|
||||
self.unregister_opts(self._config_opts)
|
||||
for group in self._groups.values():
|
||||
group._clear()
|
||||
|
||||
@__clear_cache
|
||||
def register_opt(self, opt, group=None):
|
||||
def register_opt(self, opt, group=None, cli=False):
|
||||
"""Register an option schema.
|
||||
|
||||
Registering an option schema makes any option value which is previously
|
||||
|
@ -1080,17 +1195,19 @@ class ConfigOpts(collections.Mapping):
|
|||
as an attribute of this object.
|
||||
|
||||
:param opt: an instance of an Opt sub-class
|
||||
:param cli: whether this is a CLI option
|
||||
:param group: an optional OptGroup object or group name
|
||||
:return: False if the opt was already register, True otherwise
|
||||
:raises: DuplicateOptError
|
||||
"""
|
||||
if group is not None:
|
||||
return self._get_group(group, autocreate=True)._register_opt(opt)
|
||||
group = self._get_group(group, autocreate=True)
|
||||
return group._register_opt(opt, cli)
|
||||
|
||||
if _is_opt_registered(self._opts, opt):
|
||||
return False
|
||||
|
||||
self._opts[opt.dest] = {'opt': opt}
|
||||
self._opts[opt.dest] = {'opt': opt, 'cli': cli}
|
||||
|
||||
return True
|
||||
|
||||
|
@ -1116,7 +1233,7 @@ class ConfigOpts(collections.Mapping):
|
|||
if self._args is not None:
|
||||
raise ArgsAlreadyParsedError("cannot register CLI option")
|
||||
|
||||
return self.register_opt(opt, group, clear_cache=False)
|
||||
return self.register_opt(opt, group, cli=True, clear_cache=False)
|
||||
|
||||
@__clear_cache
|
||||
def register_cli_opts(self, opts, group=None):
|
||||
|
@ -1243,10 +1360,11 @@ class ConfigOpts(collections.Mapping):
|
|||
for info in group._opts.values():
|
||||
yield info, group
|
||||
|
||||
def _all_opts(self):
|
||||
"""A generator function for iteration opts."""
|
||||
def _all_cli_opts(self):
|
||||
"""A generator function for iterating CLI opts."""
|
||||
for info, group in self._all_opt_infos():
|
||||
yield info['opt'], group
|
||||
if info['cli']:
|
||||
yield info['opt'], group
|
||||
|
||||
def _unset_defaults_and_overrides(self):
|
||||
"""Unset any default or override on all options."""
|
||||
|
@ -1254,31 +1372,6 @@ class ConfigOpts(collections.Mapping):
|
|||
info.pop('default', None)
|
||||
info.pop('override', None)
|
||||
|
||||
def disable_interspersed_args(self):
|
||||
"""Set parsing to stop on the first non-option.
|
||||
|
||||
If this this method is called, then parsing e.g.
|
||||
|
||||
script --verbose cmd --debug /tmp/mything
|
||||
|
||||
will no longer return:
|
||||
|
||||
['cmd', '/tmp/mything']
|
||||
|
||||
as the leftover arguments, but will instead return:
|
||||
|
||||
['cmd', '--debug', '/tmp/mything']
|
||||
|
||||
i.e. argument parsing is stopped at the first non-option argument.
|
||||
"""
|
||||
self._disable_interspersed_args = True
|
||||
|
||||
def enable_interspersed_args(self):
|
||||
"""Set parsing to not stop on the first non-option.
|
||||
|
||||
This it the default behaviour."""
|
||||
self._disable_interspersed_args = False
|
||||
|
||||
def find_file(self, name):
|
||||
"""Locate a file located alongside the config files.
|
||||
|
||||
|
@ -1377,6 +1470,9 @@ class ConfigOpts(collections.Mapping):
|
|||
info = self._get_opt_info(name, group)
|
||||
opt = info['opt']
|
||||
|
||||
if isinstance(opt, SubCommandOpt):
|
||||
return self.SubCommandAttr(self, group, opt.dest)
|
||||
|
||||
if 'override' in info:
|
||||
return info['override']
|
||||
|
||||
|
@ -1401,6 +1497,10 @@ class ConfigOpts(collections.Mapping):
|
|||
if not opt.multi:
|
||||
return value
|
||||
|
||||
# argparse ignores default=None for nargs='*'
|
||||
if opt.positional and not value:
|
||||
value = opt.default
|
||||
|
||||
return value + values
|
||||
|
||||
if values:
|
||||
|
@ -1523,12 +1623,10 @@ class ConfigOpts(collections.Mapping):
|
|||
"""
|
||||
self._args = args
|
||||
|
||||
for opt, group in self._all_opts():
|
||||
for opt, group in self._all_cli_opts():
|
||||
opt._add_to_cli(self._oparser, group)
|
||||
|
||||
values, leftovers = self._oparser.parse_args(args)
|
||||
|
||||
return vars(values), leftovers
|
||||
return vars(self._oparser.parse_args(args))
|
||||
|
||||
class GroupAttr(collections.Mapping):
|
||||
|
||||
|
@ -1543,12 +1641,12 @@ class ConfigOpts(collections.Mapping):
|
|||
:param conf: a ConfigOpts object
|
||||
:param group: an OptGroup object
|
||||
"""
|
||||
self.conf = conf
|
||||
self.group = group
|
||||
self._conf = conf
|
||||
self._group = group
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Look up an option value and perform template substitution."""
|
||||
return self.conf._get(name, self.group)
|
||||
return self._conf._get(name, self._group)
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""Look up an option value and perform string substitution."""
|
||||
|
@ -1556,16 +1654,50 @@ class ConfigOpts(collections.Mapping):
|
|||
|
||||
def __contains__(self, key):
|
||||
"""Return True if key is the name of a registered opt or group."""
|
||||
return key in self.group._opts
|
||||
return key in self._group._opts
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over all registered opt and group names."""
|
||||
for key in self.group._opts.keys():
|
||||
for key in self._group._opts.keys():
|
||||
yield key
|
||||
|
||||
def __len__(self):
|
||||
"""Return the number of options and option groups."""
|
||||
return len(self.group._opts)
|
||||
return len(self._group._opts)
|
||||
|
||||
class SubCommandAttr(object):
|
||||
|
||||
"""
|
||||
A helper class representing the name and arguments of an argparse
|
||||
sub-parser.
|
||||
"""
|
||||
|
||||
def __init__(self, conf, group, dest):
|
||||
"""Construct a SubCommandAttr object.
|
||||
|
||||
:param conf: a ConfigOpts object
|
||||
:param group: an OptGroup object
|
||||
:param dest: the name of the sub-parser
|
||||
"""
|
||||
self._conf = conf
|
||||
self._group = group
|
||||
self._dest = dest
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Look up a sub-parser name or argument value."""
|
||||
if name == 'name':
|
||||
name = self._dest
|
||||
if self._group is not None:
|
||||
name = self._group.name + '_' + name
|
||||
return self._conf._cli_values[name]
|
||||
|
||||
if name in self._conf:
|
||||
raise DuplicateOptError(name)
|
||||
|
||||
try:
|
||||
return self._conf._cli_values[name]
|
||||
except KeyError:
|
||||
raise NoSuchOptError(name)
|
||||
|
||||
class StrSubWrapper(object):
|
||||
|
||||
|
@ -1623,19 +1755,21 @@ class CommonConfigOpts(ConfigOpts):
|
|||
metavar='FORMAT',
|
||||
help='A logging.Formatter log message format string which may '
|
||||
'use any of the available logging.LogRecord attributes. '
|
||||
'Default: %default'),
|
||||
'Default: %(default)s'),
|
||||
StrOpt('log-date-format',
|
||||
default=DEFAULT_LOG_DATE_FORMAT,
|
||||
metavar='DATE_FORMAT',
|
||||
help='Format string for %(asctime)s in log records. '
|
||||
'Default: %default'),
|
||||
help='Format string for %%(asctime)s in log records. '
|
||||
'Default: %(default)s'),
|
||||
StrOpt('log-file',
|
||||
metavar='PATH',
|
||||
deprecated_name='logfile',
|
||||
help='(Optional) Name of log file to output to. '
|
||||
'If not set, logging will go to stdout.'),
|
||||
StrOpt('log-dir',
|
||||
deprecated_name='logdir',
|
||||
help='(Optional) The directory to keep log files in '
|
||||
'(will be prepended to --logfile)'),
|
||||
'(will be prepended to --log-file)'),
|
||||
BoolOpt('use-syslog',
|
||||
default=False,
|
||||
help='Use syslog for logging.'),
|
||||
|
|
|
@ -0,0 +1,63 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import inspect
|
||||
|
||||
|
||||
class MissingArgs(Exception):
|
||||
|
||||
def __init__(self, missing):
|
||||
self.missing = missing
|
||||
|
||||
def __str__(self):
|
||||
if len(self.missing) == 1:
|
||||
return "An argument is missing"
|
||||
else:
|
||||
return ("%(num)d arguments are missing" %
|
||||
dict(num=len(self.missing)))
|
||||
|
||||
|
||||
def validate_args(fn, *args, **kwargs):
|
||||
"""Check that the supplied args are sufficient for calling a function.
|
||||
|
||||
>>> validate_args(lambda a: None)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
MissingArgs: An argument is missing: a
|
||||
>>> validate_args(lambda a, b, c, d: None, 0, c=1)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
MissingArgs: 2 arguments are missing: b, d
|
||||
|
||||
:param fn: the function to check
|
||||
:param arg: the positional arguments supplied
|
||||
:param kwargs: the keyword arguments supplied
|
||||
"""
|
||||
argspec = inspect.getargspec(fn)
|
||||
|
||||
num_defaults = len(argspec.defaults or [])
|
||||
required_args = argspec.args[:len(argspec.args) - num_defaults]
|
||||
|
||||
def isbound(method):
|
||||
return getattr(method, 'im_self', None) is not None
|
||||
|
||||
if isbound(fn):
|
||||
required_args.pop(0)
|
||||
|
||||
missing = [arg for arg in required_args if arg not in kwargs]
|
||||
missing = missing[len(args):]
|
||||
if missing:
|
||||
raise MissingArgs(missing)
|
|
@ -46,7 +46,7 @@ def _find_objects(t):
|
|||
|
||||
|
||||
def _print_greenthreads():
|
||||
for i, gt in enumerate(find_objects(greenlet.greenlet)):
|
||||
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
|
||||
print i, gt
|
||||
traceback.print_stack(gt.gr_frame)
|
||||
print
|
||||
|
|
|
@ -0,0 +1,137 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Exceptions common to OpenStack projects
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
def __init__(self, message=None):
|
||||
super(Error, self).__init__(message)
|
||||
|
||||
|
||||
class ApiError(Error):
|
||||
def __init__(self, message='Unknown', code='Unknown'):
|
||||
self.message = message
|
||||
self.code = code
|
||||
super(ApiError, self).__init__('%s: %s' % (code, message))
|
||||
|
||||
|
||||
class NotFound(Error):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownScheme(Error):
|
||||
|
||||
msg = "Unknown scheme '%s' found in URI"
|
||||
|
||||
def __init__(self, scheme):
|
||||
msg = self.__class__.msg % scheme
|
||||
super(UnknownScheme, self).__init__(msg)
|
||||
|
||||
|
||||
class BadStoreUri(Error):
|
||||
|
||||
msg = "The Store URI %s was malformed. Reason: %s"
|
||||
|
||||
def __init__(self, uri, reason):
|
||||
msg = self.__class__.msg % (uri, reason)
|
||||
super(BadStoreUri, self).__init__(msg)
|
||||
|
||||
|
||||
class Duplicate(Error):
|
||||
pass
|
||||
|
||||
|
||||
class NotAuthorized(Error):
|
||||
pass
|
||||
|
||||
|
||||
class NotEmpty(Error):
|
||||
pass
|
||||
|
||||
|
||||
class Invalid(Error):
|
||||
pass
|
||||
|
||||
|
||||
class BadInputError(Exception):
|
||||
"""Error resulting from a client sending bad input to a server"""
|
||||
pass
|
||||
|
||||
|
||||
class MissingArgumentError(Error):
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseMigrationError(Error):
|
||||
pass
|
||||
|
||||
|
||||
class ClientConnectionError(Exception):
|
||||
"""Error resulting from a client connecting to a server"""
|
||||
pass
|
||||
|
||||
|
||||
def wrap_exception(f):
|
||||
def _wrap(*args, **kw):
|
||||
try:
|
||||
return f(*args, **kw)
|
||||
except Exception, e:
|
||||
if not isinstance(e, Error):
|
||||
#exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
logging.exception(_('Uncaught exception'))
|
||||
#logging.error(traceback.extract_stack(exc_traceback))
|
||||
raise Error(str(e))
|
||||
raise
|
||||
_wrap.func_name = f.func_name
|
||||
return _wrap
|
||||
|
||||
|
||||
class OpenstackException(Exception):
|
||||
"""
|
||||
Base Exception
|
||||
|
||||
To correctly use this class, inherit from it and define
|
||||
a 'message' property. That message will get printf'd
|
||||
with the keyword arguments provided to the constructor.
|
||||
"""
|
||||
message = "An unknown exception occurred"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
try:
|
||||
self._error_string = self.message % kwargs
|
||||
|
||||
except Exception:
|
||||
# at least get the core message out if something happened
|
||||
self._error_string = self.message
|
||||
|
||||
def __str__(self):
|
||||
return self._error_string
|
||||
|
||||
|
||||
class MalformedRequestBody(OpenstackException):
|
||||
message = "Malformed message body: %(reason)s"
|
||||
|
||||
|
||||
class InvalidContentType(OpenstackException):
|
||||
message = "Invalid content type %(content_type)s"
|
|
@ -24,6 +24,8 @@ import logging
|
|||
import sys
|
||||
import traceback
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def save_and_reraise_exception():
|
||||
|
@ -43,7 +45,7 @@ def save_and_reraise_exception():
|
|||
try:
|
||||
yield
|
||||
except Exception:
|
||||
logging.error('Original exception being dropped: %s' %
|
||||
(traceback.format_exception(type_, value, tb)))
|
||||
logging.error(_('Original exception being dropped: %s'),
|
||||
traceback.format_exception(type_, value, tb))
|
||||
raise
|
||||
raise type_, value, tb
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import errno
|
||||
import os
|
||||
|
||||
|
||||
def ensure_tree(path):
|
||||
"""Create a directory (and any ancestor directories required)
|
||||
|
||||
:param path: Directory to create
|
||||
"""
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
if not os.path.isdir(path):
|
||||
raise
|
||||
else:
|
||||
raise
|
|
@ -29,7 +29,7 @@ def import_class(import_str):
|
|||
try:
|
||||
__import__(mod_str)
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except (ValueError, AttributeError), exc:
|
||||
except (ValueError, AttributeError):
|
||||
raise ImportError('Class %s cannot be found (%s)' %
|
||||
(class_str,
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
|
|
|
@ -120,7 +120,7 @@ def to_primitive(value, convert_instances=False, level=0):
|
|||
level=level + 1)
|
||||
else:
|
||||
return value
|
||||
except TypeError, e:
|
||||
except TypeError:
|
||||
# Class objects are tricky since they may define something like
|
||||
# __iter__ defined but it isn't callable as list().
|
||||
return unicode(value)
|
||||
|
|
|
@ -0,0 +1,233 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import errno
|
||||
import functools
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import weakref
|
||||
|
||||
from eventlet import semaphore
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common import fileutils
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
util_opts = [
|
||||
cfg.BoolOpt('disable_process_locking', default=False,
|
||||
help='Whether to disable inter-process locks'),
|
||||
cfg.StrOpt('lock_path',
|
||||
default=os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
'../')),
|
||||
help='Directory to use for lock files')
|
||||
]
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(util_opts)
|
||||
|
||||
|
||||
class _InterProcessLock(object):
|
||||
"""Lock implementation which allows multiple locks, working around
|
||||
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
|
||||
not require any cleanup. Since the lock is always held on a file
|
||||
descriptor rather than outside of the process, the lock gets dropped
|
||||
automatically if the process crashes, even if __exit__ is not executed.
|
||||
|
||||
There are no guarantees regarding usage by multiple green threads in a
|
||||
single process here. This lock works only between processes. Exclusive
|
||||
access between local threads should be achieved using the semaphores
|
||||
in the @synchronized decorator.
|
||||
|
||||
Note these locks are released when the descriptor is closed, so it's not
|
||||
safe to close the file descriptor while another green thread holds the
|
||||
lock. Just opening and closing the lock file can break synchronisation,
|
||||
so lock files must be accessed only using this abstraction.
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.lockfile = None
|
||||
self.fname = name
|
||||
|
||||
def __enter__(self):
|
||||
self.lockfile = open(self.fname, 'w')
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Using non-blocking locks since green threads are not
|
||||
# patched to deal with blocking locking calls.
|
||||
# Also upon reading the MSDN docs for locking(), it seems
|
||||
# to have a laughable 10 attempts "blocking" mechanism.
|
||||
self.trylock()
|
||||
return self
|
||||
except IOError, e:
|
||||
if e.errno in (errno.EACCES, errno.EAGAIN):
|
||||
# external locks synchronise things like iptables
|
||||
# updates - give it some time to prevent busy spinning
|
||||
time.sleep(0.01)
|
||||
else:
|
||||
raise
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
try:
|
||||
self.unlock()
|
||||
self.lockfile.close()
|
||||
except IOError:
|
||||
LOG.exception(_("Could not release the acquired lock `%s`"),
|
||||
self.fname)
|
||||
|
||||
def trylock(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def unlock(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class _WindowsLock(_InterProcessLock):
|
||||
def trylock(self):
|
||||
msvcrt.locking(self.lockfile, msvcrt.LK_NBLCK, 1)
|
||||
|
||||
def unlock(self):
|
||||
msvcrt.locking(self.lockfile, msvcrt.LK_UNLCK, 1)
|
||||
|
||||
|
||||
class _PosixLock(_InterProcessLock):
|
||||
def trylock(self):
|
||||
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
|
||||
def unlock(self):
|
||||
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
InterProcessLock = _WindowsLock
|
||||
else:
|
||||
import fcntl
|
||||
InterProcessLock = _PosixLock
|
||||
|
||||
_semaphores = weakref.WeakValueDictionary()
|
||||
|
||||
|
||||
def synchronized(name, lock_file_prefix, external=False, lock_path=None):
|
||||
"""Synchronization decorator.
|
||||
|
||||
Decorating a method like so::
|
||||
|
||||
@synchronized('mylock')
|
||||
def foo(self, *args):
|
||||
...
|
||||
|
||||
ensures that only one thread will execute the bar method at a time.
|
||||
|
||||
Different methods can share the same lock::
|
||||
|
||||
@synchronized('mylock')
|
||||
def foo(self, *args):
|
||||
...
|
||||
|
||||
@synchronized('mylock')
|
||||
def bar(self, *args):
|
||||
...
|
||||
|
||||
This way only one of either foo or bar can be executing at a time.
|
||||
|
||||
The lock_file_prefix argument is used to provide lock files on disk with a
|
||||
meaningful prefix. The prefix should end with a hyphen ('-') if specified.
|
||||
|
||||
The external keyword argument denotes whether this lock should work across
|
||||
multiple processes. This means that if two different workers both run a
|
||||
a method decorated with @synchronized('mylock', external=True), only one
|
||||
of them will execute at a time.
|
||||
|
||||
The lock_path keyword argument is used to specify a special location for
|
||||
external lock files to live. If nothing is set, then CONF.lock_path is
|
||||
used as a default.
|
||||
"""
|
||||
|
||||
def wrap(f):
|
||||
@functools.wraps(f)
|
||||
def inner(*args, **kwargs):
|
||||
# NOTE(soren): If we ever go natively threaded, this will be racy.
|
||||
# See http://stackoverflow.com/questions/5390569/dyn
|
||||
# amically-allocating-and-destroying-mutexes
|
||||
sem = _semaphores.get(name, semaphore.Semaphore())
|
||||
if name not in _semaphores:
|
||||
# this check is not racy - we're already holding ref locally
|
||||
# so GC won't remove the item and there was no IO switch
|
||||
# (only valid in greenthreads)
|
||||
_semaphores[name] = sem
|
||||
|
||||
with sem:
|
||||
LOG.debug(_('Got semaphore "%(lock)s" for method '
|
||||
'"%(method)s"...'), {'lock': name,
|
||||
'method': f.__name__})
|
||||
if external and not CONF.disable_process_locking:
|
||||
LOG.debug(_('Attempting to grab file lock "%(lock)s" for '
|
||||
'method "%(method)s"...'),
|
||||
{'lock': name, 'method': f.__name__})
|
||||
cleanup_dir = False
|
||||
|
||||
# We need a copy of lock_path because it is non-local
|
||||
local_lock_path = lock_path
|
||||
if not local_lock_path:
|
||||
local_lock_path = CONF.lock_path
|
||||
|
||||
if not local_lock_path:
|
||||
cleanup_dir = True
|
||||
local_lock_path = tempfile.mkdtemp()
|
||||
|
||||
if not os.path.exists(local_lock_path):
|
||||
cleanup_dir = True
|
||||
fileutils.ensure_tree(local_lock_path)
|
||||
|
||||
# NOTE(mikal): the lock name cannot contain directory
|
||||
# separators
|
||||
safe_name = name.replace(os.sep, '_')
|
||||
lock_file_name = '%s%s' % (lock_file_prefix, safe_name)
|
||||
lock_file_path = os.path.join(local_lock_path,
|
||||
lock_file_name)
|
||||
|
||||
try:
|
||||
lock = InterProcessLock(lock_file_path)
|
||||
with lock:
|
||||
LOG.debug(_('Got file lock "%(lock)s" at %(path)s '
|
||||
'for method "%(method)s"...'),
|
||||
{'lock': name,
|
||||
'path': lock_file_path,
|
||||
'method': f.__name__})
|
||||
retval = f(*args, **kwargs)
|
||||
finally:
|
||||
# NOTE(vish): This removes the tempdir if we needed
|
||||
# to create one. This is used to cleanup
|
||||
# the locks left behind by unit tests.
|
||||
if cleanup_dir:
|
||||
shutil.rmtree(local_lock_path)
|
||||
else:
|
||||
retval = f(*args, **kwargs)
|
||||
|
||||
return retval
|
||||
return inner
|
||||
return wrap
|
|
@ -49,19 +49,20 @@ from kwapi.openstack.common import notifier
|
|||
|
||||
log_opts = [
|
||||
cfg.StrOpt('logging_context_format_string',
|
||||
default='%(asctime)s %(levelname)s %(name)s [%(request_id)s '
|
||||
'%(user)s %(tenant)s] %(instance)s'
|
||||
default='%(asctime)s.%(msecs)d %(levelname)s %(name)s '
|
||||
'[%(request_id)s %(user)s %(tenant)s] %(instance)s'
|
||||
'%(message)s',
|
||||
help='format string to use for log messages with context'),
|
||||
cfg.StrOpt('logging_default_format_string',
|
||||
default='%(asctime)s %(process)d %(levelname)s %(name)s [-]'
|
||||
' %(instance)s%(message)s',
|
||||
default='%(asctime)s.%(msecs)d %(process)d %(levelname)s '
|
||||
'%(name)s [-] %(instance)s%(message)s',
|
||||
help='format string to use for log messages without context'),
|
||||
cfg.StrOpt('logging_debug_format_suffix',
|
||||
default='%(funcName)s %(pathname)s:%(lineno)d',
|
||||
help='data to append to log format when level is DEBUG'),
|
||||
cfg.StrOpt('logging_exception_prefix',
|
||||
default='%(asctime)s %(process)d TRACE %(name)s %(instance)s',
|
||||
default='%(asctime)s.%(msecs)d %(process)d TRACE %(name)s '
|
||||
'%(instance)s',
|
||||
help='prefix each line of exception output with this format'),
|
||||
cfg.ListOpt('default_log_levels',
|
||||
default=[
|
||||
|
@ -174,7 +175,7 @@ class ContextAdapter(logging.LoggerAdapter):
|
|||
self.log(logging.AUDIT, msg, *args, **kwargs)
|
||||
|
||||
def deprecated(self, msg, *args, **kwargs):
|
||||
stdmsg = _("Deprecated Config: %s") % msg
|
||||
stdmsg = _("Deprecated: %s") % msg
|
||||
if CONF.fatal_deprecations:
|
||||
self.critical(stdmsg, *args, **kwargs)
|
||||
raise DeprecatedConfig(msg=stdmsg)
|
||||
|
@ -257,7 +258,7 @@ class JSONFormatter(logging.Formatter):
|
|||
|
||||
class PublishErrorsHandler(logging.Handler):
|
||||
def emit(self, record):
|
||||
if ('kwapi.openstack.common.notifier.log_notifier' in
|
||||
if ('openstack.common.notifier.log_notifier' in
|
||||
CONF.notification_driver):
|
||||
return
|
||||
notifier.api.notify(None, 'error.publisher',
|
||||
|
@ -289,6 +290,12 @@ def setup(product_name):
|
|||
_setup_logging_from_conf(product_name)
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
cfg.set_defaults(log_opts,
|
||||
logging_context_format_string=
|
||||
logging_context_format_string)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||
facility = getattr(logging.handlers.SysLogHandler,
|
||||
|
|
|
@ -24,6 +24,7 @@ from eventlet import greenthread
|
|||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common import timeutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -62,10 +63,16 @@ class LoopingCall(object):
|
|||
|
||||
try:
|
||||
while self._running:
|
||||
start = timeutils.utcnow()
|
||||
self.f(*self.args, **self.kw)
|
||||
end = timeutils.utcnow()
|
||||
if not self._running:
|
||||
break
|
||||
greenthread.sleep(interval)
|
||||
delay = interval - timeutils.delta_seconds(start, end)
|
||||
if delay <= 0:
|
||||
LOG.warn(_('task run outlasted interval by %s sec') %
|
||||
-delay)
|
||||
greenthread.sleep(delay if delay > 0 else 0)
|
||||
except LoopingCallDone, e:
|
||||
self.stop()
|
||||
done.send(e.retvalue)
|
||||
|
|
|
@ -0,0 +1,64 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Middleware that attaches a context to the WSGI request
|
||||
"""
|
||||
|
||||
from kwapi.openstack.common import context
|
||||
from kwapi.openstack.common import importutils
|
||||
from kwapi.openstack.common import wsgi
|
||||
|
||||
|
||||
class ContextMiddleware(wsgi.Middleware):
|
||||
def __init__(self, app, options):
|
||||
self.options = options
|
||||
super(ContextMiddleware, self).__init__(app)
|
||||
|
||||
def make_context(self, *args, **kwargs):
|
||||
"""
|
||||
Create a context with the given arguments.
|
||||
"""
|
||||
|
||||
# Determine the context class to use
|
||||
ctxcls = context.RequestContext
|
||||
if 'context_class' in self.options:
|
||||
ctxcls = importutils.import_class(self.options['context_class'])
|
||||
|
||||
return ctxcls(*args, **kwargs)
|
||||
|
||||
def process_request(self, req):
|
||||
"""
|
||||
Extract any authentication information in the request and
|
||||
construct an appropriate context from it.
|
||||
"""
|
||||
# Use the default empty context, with admin turned on for
|
||||
# backwards compatibility
|
||||
req.context = self.make_context(is_admin=True)
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
"""
|
||||
Factory method for paste.deploy
|
||||
"""
|
||||
conf = global_conf.copy()
|
||||
conf.update(local_conf)
|
||||
|
||||
def filter(app):
|
||||
return ContextMiddleware(app, conf)
|
||||
|
||||
return filter
|
|
@ -137,10 +137,11 @@ def notify(context, publisher_id, event_type, priority, payload):
|
|||
for driver in _get_drivers():
|
||||
try:
|
||||
driver.notify(context, msg)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
LOG.exception(_("Problem '%(e)s' attempting to "
|
||||
"send to notification system. "
|
||||
"Payload=%(payload)s") % locals())
|
||||
"Payload=%(payload)s")
|
||||
% dict(e=e, payload=payload))
|
||||
|
||||
|
||||
_drivers = None
|
||||
|
@ -166,7 +167,7 @@ def add_driver(notification_driver):
|
|||
try:
|
||||
driver = importutils.import_module(notification_driver)
|
||||
_drivers[notification_driver] = driver
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
LOG.exception(_("Failed to load notifier %s. "
|
||||
"These notifications will not be sent.") %
|
||||
notification_driver)
|
||||
|
|
|
@ -1,118 +0,0 @@
|
|||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import importutils
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
|
||||
list_notifier_drivers_opt = cfg.MultiStrOpt(
|
||||
'list_notifier_drivers',
|
||||
default=['kwapi.openstack.common.notifier.no_op_notifier'],
|
||||
help='List of drivers to send notifications')
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opt(list_notifier_drivers_opt)
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
drivers = None
|
||||
|
||||
|
||||
class ImportFailureNotifier(object):
|
||||
"""Noisily re-raises some exception over-and-over when notify is called."""
|
||||
|
||||
def __init__(self, exception):
|
||||
self.exception = exception
|
||||
|
||||
def notify(self, context, message):
|
||||
raise self.exception
|
||||
|
||||
|
||||
def _get_drivers():
|
||||
"""Instantiates and returns drivers based on the flag values."""
|
||||
global drivers
|
||||
if drivers is None:
|
||||
drivers = []
|
||||
for notification_driver in CONF.list_notifier_drivers:
|
||||
try:
|
||||
drivers.append(importutils.import_module(notification_driver))
|
||||
except ImportError as e:
|
||||
drivers.append(ImportFailureNotifier(e))
|
||||
return drivers
|
||||
|
||||
|
||||
def add_driver(notification_driver):
|
||||
"""Add a notification driver at runtime."""
|
||||
# Make sure the driver list is initialized.
|
||||
_get_drivers()
|
||||
if isinstance(notification_driver, basestring):
|
||||
# Load and add
|
||||
try:
|
||||
drivers.append(importutils.import_module(notification_driver))
|
||||
except ImportError as e:
|
||||
drivers.append(ImportFailureNotifier(e))
|
||||
else:
|
||||
# Driver is already loaded; just add the object.
|
||||
drivers.append(notification_driver)
|
||||
|
||||
|
||||
def _object_name(obj):
|
||||
name = []
|
||||
if hasattr(obj, '__module__'):
|
||||
name.append(obj.__module__)
|
||||
if hasattr(obj, '__name__'):
|
||||
name.append(obj.__name__)
|
||||
else:
|
||||
name.append(obj.__class__.__name__)
|
||||
return '.'.join(name)
|
||||
|
||||
|
||||
def remove_driver(notification_driver):
|
||||
"""Remove a notification driver at runtime."""
|
||||
# Make sure the driver list is initialized.
|
||||
_get_drivers()
|
||||
removed = False
|
||||
if notification_driver in drivers:
|
||||
# We're removing an object. Easy.
|
||||
drivers.remove(notification_driver)
|
||||
removed = True
|
||||
else:
|
||||
# We're removing a driver by name. Search for it.
|
||||
for driver in drivers:
|
||||
if _object_name(driver) == notification_driver:
|
||||
drivers.remove(driver)
|
||||
removed = True
|
||||
|
||||
if not removed:
|
||||
raise ValueError("Cannot remove; %s is not in list" %
|
||||
notification_driver)
|
||||
|
||||
|
||||
def notify(context, message):
|
||||
"""Passes notification to multiple notifiers in a list."""
|
||||
for driver in _get_drivers():
|
||||
try:
|
||||
driver.notify(context, message)
|
||||
except Exception as e:
|
||||
LOG.exception(_("Problem '%(e)s' attempting to send to "
|
||||
"notification driver %(driver)s."), locals())
|
||||
|
||||
|
||||
def _reset_drivers():
|
||||
"""Used by unit tests to reset the drivers."""
|
||||
global drivers
|
||||
drivers = None
|
|
@ -30,6 +30,6 @@ def notify(_context, message):
|
|||
CONF.default_notification_level)
|
||||
priority = priority.lower()
|
||||
logger = logging.getLogger(
|
||||
'kwapi.openstack.common.notification.%s' %
|
||||
'openstack.common.notification.%s' %
|
||||
message['event_type'])
|
||||
getattr(logger, priority)(jsonutils.dumps(message))
|
||||
|
|
|
@ -41,6 +41,6 @@ def notify(context, message):
|
|||
topic = '%s.%s' % (topic, priority)
|
||||
try:
|
||||
rpc.notify(context, topic, message)
|
||||
except Exception, e:
|
||||
except Exception:
|
||||
LOG.exception(_("Could not send notification to %(topic)s. "
|
||||
"Payload=%(message)s"), locals())
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
'''messaging based notification driver, with message envelopes'''
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common import context as req_context
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common import rpc
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
notification_topic_opt = cfg.ListOpt(
|
||||
'topics', default=['notifications', ],
|
||||
help='AMQP topic(s) used for openstack notifications')
|
||||
|
||||
opt_group = cfg.OptGroup(name='rpc_notifier2',
|
||||
title='Options for rpc_notifier2')
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_group(opt_group)
|
||||
CONF.register_opt(notification_topic_opt, opt_group)
|
||||
|
||||
|
||||
def notify(context, message):
|
||||
"""Sends a notification via RPC"""
|
||||
if not context:
|
||||
context = req_context.get_admin_context()
|
||||
priority = message.get('priority',
|
||||
CONF.default_notification_level)
|
||||
priority = priority.lower()
|
||||
for topic in CONF.rpc_notifier2.topics:
|
||||
topic = '%s.%s' % (topic, priority)
|
||||
try:
|
||||
rpc.notify(context, topic, message, envelope=True)
|
||||
except Exception:
|
||||
LOG.exception(_("Could not send notification to %(topic)s. "
|
||||
"Payload=%(message)s"), locals())
|
|
@ -0,0 +1,164 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
from paste import deploy
|
||||
|
||||
from kwapi.openstack.common import local
|
||||
|
||||
|
||||
class BasePasteFactory(object):
|
||||
|
||||
"""A base class for paste app and filter factories.
|
||||
|
||||
Sub-classes must override the KEY class attribute and provide
|
||||
a __call__ method.
|
||||
"""
|
||||
|
||||
KEY = None
|
||||
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
def _import_factory(self, local_conf):
|
||||
"""Import an app/filter class.
|
||||
|
||||
Lookup the KEY from the PasteDeploy local conf and import the
|
||||
class named there. This class can then be used as an app or
|
||||
filter factory.
|
||||
|
||||
Note we support the <module>:<class> format.
|
||||
|
||||
Note also that if you do e.g.
|
||||
|
||||
key =
|
||||
value
|
||||
|
||||
then ConfigParser returns a value with a leading newline, so
|
||||
we strip() the value before using it.
|
||||
"""
|
||||
mod_str, _sep, class_str = local_conf[self.KEY].strip().rpartition(':')
|
||||
del local_conf[self.KEY]
|
||||
|
||||
__import__(mod_str)
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
|
||||
|
||||
class AppFactory(BasePasteFactory):
|
||||
|
||||
"""A Generic paste.deploy app factory.
|
||||
|
||||
This requires openstack.app_factory to be set to a callable which returns a
|
||||
WSGI app when invoked. The format of the name is <module>:<callable> e.g.
|
||||
|
||||
[app:myfooapp]
|
||||
paste.app_factory = openstack.common.pastedeploy:app_factory
|
||||
openstack.app_factory = myapp:Foo
|
||||
|
||||
The WSGI app constructor must accept a data object and a local config
|
||||
dict as its two arguments.
|
||||
"""
|
||||
|
||||
KEY = 'openstack.app_factory'
|
||||
|
||||
def __call__(self, global_conf, **local_conf):
|
||||
"""The actual paste.app_factory protocol method."""
|
||||
factory = self._import_factory(local_conf)
|
||||
return factory(self.data, **local_conf)
|
||||
|
||||
|
||||
class FilterFactory(AppFactory):
|
||||
|
||||
"""A Generic paste.deploy filter factory.
|
||||
|
||||
This requires openstack.filter_factory to be set to a callable which
|
||||
returns a WSGI filter when invoked. The format is <module>:<callable> e.g.
|
||||
|
||||
[filter:myfoofilter]
|
||||
paste.filter_factory = openstack.common.pastedeploy:filter_factory
|
||||
openstack.filter_factory = myfilter:Foo
|
||||
|
||||
The WSGI filter constructor must accept a WSGI app, a data object and
|
||||
a local config dict as its three arguments.
|
||||
"""
|
||||
|
||||
KEY = 'openstack.filter_factory'
|
||||
|
||||
def __call__(self, global_conf, **local_conf):
|
||||
"""The actual paste.filter_factory protocol method."""
|
||||
factory = self._import_factory(local_conf)
|
||||
|
||||
def filter(app):
|
||||
return factory(app, self.data, **local_conf)
|
||||
|
||||
return filter
|
||||
|
||||
|
||||
def app_factory(global_conf, **local_conf):
|
||||
"""A paste app factory used with paste_deploy_app()."""
|
||||
return local.store.app_factory(global_conf, **local_conf)
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
"""A paste filter factory used with paste_deploy_app()."""
|
||||
return local.store.filter_factory(global_conf, **local_conf)
|
||||
|
||||
|
||||
def paste_deploy_app(paste_config_file, app_name, data):
|
||||
"""Load a WSGI app from a PasteDeploy configuration.
|
||||
|
||||
Use deploy.loadapp() to load the app from the PasteDeploy configuration,
|
||||
ensuring that the supplied data object is passed to the app and filter
|
||||
factories defined in this module.
|
||||
|
||||
To use these factories and the data object, the configuration should look
|
||||
like this:
|
||||
|
||||
[app:myapp]
|
||||
paste.app_factory = openstack.common.pastedeploy:app_factory
|
||||
openstack.app_factory = myapp:App
|
||||
...
|
||||
[filter:myfilter]
|
||||
paste.filter_factory = openstack.common.pastedeploy:filter_factory
|
||||
openstack.filter_factory = myapp:Filter
|
||||
|
||||
and then:
|
||||
|
||||
myapp.py:
|
||||
|
||||
class App(object):
|
||||
def __init__(self, data):
|
||||
...
|
||||
|
||||
class Filter(object):
|
||||
def __init__(self, app, data):
|
||||
...
|
||||
|
||||
:param paste_config_file: a PasteDeploy config file
|
||||
:param app_name: the name of the app/pipeline to load from the file
|
||||
:param data: a data object to supply to the app and its filters
|
||||
:returns: the WSGI app
|
||||
"""
|
||||
(af, ff) = (AppFactory(data), FilterFactory(data))
|
||||
|
||||
local.store.app_factory = af
|
||||
local.store.filter_factory = ff
|
||||
try:
|
||||
return deploy.loadapp("config:%s" % paste_config_file, name=app_name)
|
||||
finally:
|
||||
del local.store.app_factory
|
||||
del local.store.filter_factory
|
|
@ -0,0 +1,115 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def periodic_task(*args, **kwargs):
|
||||
"""Decorator to indicate that a method is a periodic task.
|
||||
|
||||
This decorator can be used in two ways:
|
||||
|
||||
1. Without arguments '@periodic_task', this will be run on every tick
|
||||
of the periodic scheduler.
|
||||
|
||||
2. With arguments, @periodic_task(ticks_between_runs=N), this will be
|
||||
run on every N ticks of the periodic scheduler.
|
||||
"""
|
||||
def decorator(f):
|
||||
f._periodic_task = True
|
||||
f._ticks_between_runs = kwargs.pop('ticks_between_runs', 0)
|
||||
return f
|
||||
|
||||
# NOTE(sirp): The `if` is necessary to allow the decorator to be used with
|
||||
# and without parens.
|
||||
#
|
||||
# In the 'with-parens' case (with kwargs present), this function needs to
|
||||
# return a decorator function since the interpreter will invoke it like:
|
||||
#
|
||||
# periodic_task(*args, **kwargs)(f)
|
||||
#
|
||||
# In the 'without-parens' case, the original function will be passed
|
||||
# in as the first argument, like:
|
||||
#
|
||||
# periodic_task(f)
|
||||
if kwargs:
|
||||
return decorator
|
||||
else:
|
||||
return decorator(args[0])
|
||||
|
||||
|
||||
class _PeriodicTasksMeta(type):
|
||||
def __init__(cls, names, bases, dict_):
|
||||
"""Metaclass that allows us to collect decorated periodic tasks."""
|
||||
super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_)
|
||||
|
||||
# NOTE(sirp): if the attribute is not present then we must be the base
|
||||
# class, so, go ahead and initialize it. If the attribute is present,
|
||||
# then we're a subclass so make a copy of it so we don't step on our
|
||||
# parent's toes.
|
||||
try:
|
||||
cls._periodic_tasks = cls._periodic_tasks[:]
|
||||
except AttributeError:
|
||||
cls._periodic_tasks = []
|
||||
|
||||
try:
|
||||
cls._ticks_to_skip = cls._ticks_to_skip.copy()
|
||||
except AttributeError:
|
||||
cls._ticks_to_skip = {}
|
||||
|
||||
# This uses __dict__ instead of
|
||||
# inspect.getmembers(cls, inspect.ismethod) so only the methods of the
|
||||
# current class are added when this class is scanned, and base classes
|
||||
# are not added redundantly.
|
||||
for value in cls.__dict__.values():
|
||||
if getattr(value, '_periodic_task', False):
|
||||
task = value
|
||||
name = task.__name__
|
||||
cls._periodic_tasks.append((name, task))
|
||||
cls._ticks_to_skip[name] = task._ticks_between_runs
|
||||
|
||||
|
||||
class PeriodicTasks(object):
|
||||
__metaclass__ = _PeriodicTasksMeta
|
||||
|
||||
def run_periodic_tasks(self, context, raise_on_error=False):
|
||||
"""Tasks to be run at a periodic interval."""
|
||||
for task_name, task in self._periodic_tasks:
|
||||
full_task_name = '.'.join([self.__class__.__name__, task_name])
|
||||
|
||||
ticks_to_skip = self._ticks_to_skip[task_name]
|
||||
if ticks_to_skip > 0:
|
||||
LOG.debug(_("Skipping %(full_task_name)s, %(ticks_to_skip)s"
|
||||
" ticks left until next run"),
|
||||
dict(full_task_name=full_task_name,
|
||||
ticks_to_skip=ticks_to_skip))
|
||||
self._ticks_to_skip[task_name] -= 1
|
||||
continue
|
||||
|
||||
self._ticks_to_skip[task_name] = task._ticks_between_runs
|
||||
LOG.debug(_("Running periodic task %(full_task_name)s"),
|
||||
dict(full_task_name=full_task_name))
|
||||
|
||||
try:
|
||||
task(self, context)
|
||||
except Exception as e:
|
||||
if raise_on_error:
|
||||
raise
|
||||
LOG.exception(_("Error during %(full_task_name)s:"
|
||||
" %(e)s"),
|
||||
dict(e=e, full_task_name=full_task_name))
|
|
@ -0,0 +1,14 @@
|
|||
# Copyright 2012 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,93 @@
|
|||
# Copyright 2012 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.plugin import plugin
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _CallbackNotifier(object):
|
||||
"""Manages plugin-defined notification callbacks.
|
||||
|
||||
For each Plugin, a CallbackNotifier will be added to the
|
||||
notification driver list. Calls to notify() with appropriate
|
||||
messages will be hooked and prompt callbacks.
|
||||
|
||||
A callback should look like this:
|
||||
def callback(context, message, user_data)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._callback_dict = {}
|
||||
|
||||
def _add_callback(self, event_type, callback, user_data):
|
||||
callback_list = self._callback_dict.get(event_type, [])
|
||||
callback_list.append({'function': callback,
|
||||
'user_data': user_data})
|
||||
self._callback_dict[event_type] = callback_list
|
||||
|
||||
def _remove_callback(self, callback):
|
||||
for callback_list in self._callback_dict.values():
|
||||
for entry in callback_list:
|
||||
if entry['function'] == callback:
|
||||
callback_list.remove(entry)
|
||||
|
||||
def notify(self, context, message):
|
||||
if message.get('event_type') not in self._callback_dict:
|
||||
return
|
||||
|
||||
for entry in self._callback_dict[message.get('event_type')]:
|
||||
entry['function'](context, message, entry.get('user_data'))
|
||||
|
||||
def callbacks(self):
|
||||
return self._callback_dict
|
||||
|
||||
|
||||
class CallbackPlugin(plugin.Plugin):
|
||||
""" Plugin with a simple callback interface.
|
||||
|
||||
This class is provided as a convenience for producing a simple
|
||||
plugin that only watches a couple of events. For example, here's
|
||||
a subclass which prints a line the first time an instance is created.
|
||||
|
||||
class HookInstanceCreation(CallbackPlugin):
|
||||
|
||||
def __init__(self, _service_name):
|
||||
super(HookInstanceCreation, self).__init__()
|
||||
self._add_callback(self.magic, 'compute.instance.create.start')
|
||||
|
||||
def magic(self):
|
||||
print "An instance was created!"
|
||||
self._remove_callback(self, self.magic)
|
||||
"""
|
||||
|
||||
def __init__(self, service_name):
|
||||
super(CallbackPlugin, self).__init__(service_name)
|
||||
self._callback_notifier = _CallbackNotifier()
|
||||
self._add_notifier(self._callback_notifier)
|
||||
|
||||
def _add_callback(self, callback, event_type, user_data=None):
|
||||
"""Add callback for a given event notification.
|
||||
|
||||
Subclasses can call this as an alternative to implementing
|
||||
a fullblown notify notifier.
|
||||
"""
|
||||
self._callback_notifier._add_callback(event_type, callback, user_data)
|
||||
|
||||
def _remove_callback(self, callback):
|
||||
"""Remove all notification callbacks to specified function."""
|
||||
self._callback_notifier._remove_callback(callback)
|
|
@ -0,0 +1,86 @@
|
|||
# Copyright 2012 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Plugin(object):
|
||||
"""Defines an interface for adding functionality to an OpenStack service.
|
||||
|
||||
A plugin interacts with a service via the following pathways:
|
||||
|
||||
- An optional set of notifiers, managed by calling add_notifier()
|
||||
or by overriding _notifiers()
|
||||
|
||||
- A set of api extensions, managed via add_api_extension_descriptor()
|
||||
|
||||
- Direct calls to service functions.
|
||||
|
||||
- Whatever else the plugin wants to do on its own.
|
||||
|
||||
This is the reference implementation.
|
||||
"""
|
||||
|
||||
# The following functions are provided as convenience methods
|
||||
# for subclasses. Subclasses should call them but probably not
|
||||
# override them.
|
||||
def _add_api_extension_descriptor(self, descriptor):
|
||||
"""Subclass convenience method which adds an extension descriptor.
|
||||
|
||||
Subclass constructors should call this method when
|
||||
extending a project's REST interface.
|
||||
|
||||
Note that once the api service has loaded, the
|
||||
API extension set is more-or-less fixed, so
|
||||
this should mainly be called by subclass constructors.
|
||||
"""
|
||||
self._api_extension_descriptors.append(descriptor)
|
||||
|
||||
def _add_notifier(self, notifier):
|
||||
"""Subclass convenience method which adds a notifier.
|
||||
|
||||
Notifier objects should implement the function notify(message).
|
||||
Each notifier receives a notify() call whenever an openstack
|
||||
service broadcasts a notification.
|
||||
|
||||
Best to call this during construction. Notifiers are enumerated
|
||||
and registered by the pluginmanager at plugin load time.
|
||||
"""
|
||||
self._notifiers.append(notifier)
|
||||
|
||||
# The following methods are called by OpenStack services to query
|
||||
# plugin features. Subclasses should probably not override these.
|
||||
def _notifiers(self):
|
||||
"""Returns list of notifiers for this plugin."""
|
||||
return self._notifiers
|
||||
|
||||
notifiers = property(_notifiers)
|
||||
|
||||
def _api_extension_descriptors(self):
|
||||
"""Return a list of API extension descriptors.
|
||||
|
||||
Called by a project API during its load sequence.
|
||||
"""
|
||||
return self._api_extension_descriptors
|
||||
|
||||
api_extension_descriptors = property(_api_extension_descriptors)
|
||||
|
||||
# Most plugins will override this:
|
||||
def __init__(self, service_name):
|
||||
self._notifiers = []
|
||||
self._api_extension_descriptors = []
|
|
@ -0,0 +1,77 @@
|
|||
# Copyright 2012 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.notifier import api as notifier_api
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PluginManager(object):
|
||||
"""Manages plugin entrypoints and loading.
|
||||
|
||||
For a service to implement this plugin interface for callback purposes:
|
||||
|
||||
- Make use of the openstack-common notifier system
|
||||
- Instantiate this manager in each process (passing in
|
||||
project and service name)
|
||||
|
||||
For an API service to extend itself using this plugin interface,
|
||||
it needs to query the plugin_extension_factory provided by
|
||||
the already-instantiated PluginManager.
|
||||
"""
|
||||
|
||||
def __init__(self, project_name, service_name):
|
||||
""" Construct Plugin Manager; load and initialize plugins.
|
||||
|
||||
project_name (e.g. 'nova' or 'glance') is used
|
||||
to construct the entry point that identifies plugins.
|
||||
|
||||
The service_name (e.g. 'compute') is passed on to
|
||||
each plugin as a raw string for it to do what it will.
|
||||
"""
|
||||
self._project_name = project_name
|
||||
self._service_name = service_name
|
||||
self.plugins = []
|
||||
|
||||
def load_plugins(self):
|
||||
self.plugins = []
|
||||
|
||||
for entrypoint in pkg_resources.iter_entry_points('%s.plugin' %
|
||||
self._project_name):
|
||||
try:
|
||||
pluginclass = entrypoint.load()
|
||||
plugin = pluginclass(self._service_name)
|
||||
self.plugins.append(plugin)
|
||||
except Exception, exc:
|
||||
LOG.error(_("Failed to load plugin %(plug)s: %(exc)s") %
|
||||
{'plug': entrypoint, 'exc': exc})
|
||||
|
||||
# Register individual notifiers.
|
||||
for plugin in self.plugins:
|
||||
for notifier in plugin.notifiers:
|
||||
notifier_api.add_driver(notifier)
|
||||
|
||||
def plugin_extension_factory(self, ext_mgr):
|
||||
for plugin in self.plugins:
|
||||
descriptors = plugin.api_extension_descriptors
|
||||
for descriptor in descriptors:
|
||||
ext_mgr.load_extension(descriptor)
|
|
@ -1,6 +1,6 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# Copyright (c) 2012 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -15,10 +15,52 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Common Policy Engine Implementation"""
|
||||
"""
|
||||
Common Policy Engine Implementation
|
||||
|
||||
Policies can be expressed in one of two forms: A list of lists, or a
|
||||
string written in the new policy language.
|
||||
|
||||
In the list-of-lists representation, each check inside the innermost
|
||||
list is combined as with an "and" conjunction--for that check to pass,
|
||||
all the specified checks must pass. These innermost lists are then
|
||||
combined as with an "or" conjunction. This is the original way of
|
||||
expressing policies, but there now exists a new way: the policy
|
||||
language.
|
||||
|
||||
In the policy language, each check is specified the same way as in the
|
||||
list-of-lists representation: a simple "a:b" pair that is matched to
|
||||
the correct code to perform that check. However, conjunction
|
||||
operators are available, allowing for more expressiveness in crafting
|
||||
policies.
|
||||
|
||||
As an example, take the following rule, expressed in the list-of-lists
|
||||
representation::
|
||||
|
||||
[["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]]
|
||||
|
||||
In the policy language, this becomes::
|
||||
|
||||
role:admin or (project_id:%(project_id)s and role:projectadmin)
|
||||
|
||||
The policy language also has the "not" operator, allowing a richer
|
||||
policy rule::
|
||||
|
||||
project_id:%(project_id)s and not role:dunce
|
||||
|
||||
Finally, two special policy checks should be mentioned; the policy
|
||||
check "@" will always accept an access, and the policy check "!" will
|
||||
always reject an access. (Note that if a rule is either the empty
|
||||
list ("[]") or the empty string, this is equivalent to the "@" policy
|
||||
check.) Of these, the "!" policy check is probably the most useful,
|
||||
as it allows particular rules to be explicitly disabled.
|
||||
"""
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import re
|
||||
import urllib
|
||||
|
||||
import urllib2
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
|
@ -28,218 +70,650 @@ from kwapi.openstack.common import jsonutils
|
|||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_BRAIN = None
|
||||
_rules = None
|
||||
_checks = {}
|
||||
|
||||
|
||||
def set_brain(brain):
|
||||
"""Set the brain used by enforce().
|
||||
|
||||
Defaults use Brain() if not set.
|
||||
|
||||
class Rules(dict):
|
||||
"""
|
||||
global _BRAIN
|
||||
_BRAIN = brain
|
||||
|
||||
|
||||
def reset():
|
||||
"""Clear the brain used by enforce()."""
|
||||
global _BRAIN
|
||||
_BRAIN = None
|
||||
|
||||
|
||||
def enforce(match_list, target_dict, credentials_dict, exc=None,
|
||||
*args, **kwargs):
|
||||
"""Enforces authorization of some rules against credentials.
|
||||
|
||||
:param match_list: nested tuples of data to match against
|
||||
|
||||
The basic brain supports three types of match lists:
|
||||
|
||||
1) rules
|
||||
|
||||
looks like: ``('rule:compute:get_instance',)``
|
||||
|
||||
Retrieves the named rule from the rules dict and recursively
|
||||
checks against the contents of the rule.
|
||||
|
||||
2) roles
|
||||
|
||||
looks like: ``('role:compute:admin',)``
|
||||
|
||||
Matches if the specified role is in credentials_dict['roles'].
|
||||
|
||||
3) generic
|
||||
|
||||
looks like: ``('tenant_id:%(tenant_id)s',)``
|
||||
|
||||
Substitutes values from the target dict into the match using
|
||||
the % operator and matches them against the creds dict.
|
||||
|
||||
Combining rules:
|
||||
|
||||
The brain returns True if any of the outer tuple of rules
|
||||
match and also True if all of the inner tuples match. You
|
||||
can use this to perform simple boolean logic. For
|
||||
example, the following rule would return True if the creds
|
||||
contain the role 'admin' OR the if the tenant_id matches
|
||||
the target dict AND the the creds contains the role
|
||||
'compute_sysadmin':
|
||||
|
||||
::
|
||||
|
||||
{
|
||||
"rule:combined": (
|
||||
'role:admin',
|
||||
('tenant_id:%(tenant_id)s', 'role:compute_sysadmin')
|
||||
)
|
||||
}
|
||||
|
||||
Note that rule and role are reserved words in the credentials match, so
|
||||
you can't match against properties with those names. Custom brains may
|
||||
also add new reserved words. For example, the HttpBrain adds http as a
|
||||
reserved word.
|
||||
|
||||
:param target_dict: dict of object properties
|
||||
|
||||
Target dicts contain as much information as we can about the object being
|
||||
operated on.
|
||||
|
||||
:param credentials_dict: dict of actor properties
|
||||
|
||||
Credentials dicts contain as much information as we can about the user
|
||||
performing the action.
|
||||
|
||||
:param exc: exception to raise
|
||||
|
||||
Class of the exception to raise if the check fails. Any remaining
|
||||
arguments passed to enforce() (both positional and keyword arguments)
|
||||
will be passed to the exception class. If exc is not provided, returns
|
||||
False.
|
||||
|
||||
:return: True if the policy allows the action
|
||||
:return: False if the policy does not allow the action and exc is not set
|
||||
A store for rules. Handles the default_rule setting directly.
|
||||
"""
|
||||
global _BRAIN
|
||||
if not _BRAIN:
|
||||
_BRAIN = Brain()
|
||||
if not _BRAIN.check(match_list, target_dict, credentials_dict):
|
||||
if exc:
|
||||
raise exc(*args, **kwargs)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class Brain(object):
|
||||
"""Implements policy checking."""
|
||||
|
||||
_checks = {}
|
||||
|
||||
@classmethod
|
||||
def _register(cls, name, func):
|
||||
cls._checks[name] = func
|
||||
|
||||
@classmethod
|
||||
def load_json(cls, data, default_rule=None):
|
||||
"""Init a brain using json instead of a rules dictionary."""
|
||||
rules_dict = jsonutils.loads(data)
|
||||
return cls(rules=rules_dict, default_rule=default_rule)
|
||||
"""
|
||||
Allow loading of JSON rule data.
|
||||
"""
|
||||
|
||||
# Suck in the JSON data and parse the rules
|
||||
rules = dict((k, parse_rule(v)) for k, v in
|
||||
jsonutils.loads(data).items())
|
||||
|
||||
return cls(rules, default_rule)
|
||||
|
||||
def __init__(self, rules=None, default_rule=None):
|
||||
if self.__class__ != Brain:
|
||||
LOG.warning(_("Inheritance-based rules are deprecated; use "
|
||||
"the default brain instead of %s.") %
|
||||
self.__class__.__name__)
|
||||
"""Initialize the Rules store."""
|
||||
|
||||
self.rules = rules or {}
|
||||
super(Rules, self).__init__(rules or {})
|
||||
self.default_rule = default_rule
|
||||
|
||||
def add_rule(self, key, match):
|
||||
self.rules[key] = match
|
||||
def __missing__(self, key):
|
||||
"""Implements the default rule handling."""
|
||||
|
||||
def _check(self, match, target_dict, cred_dict):
|
||||
# If the default rule isn't actually defined, do something
|
||||
# reasonably intelligent
|
||||
if not self.default_rule or self.default_rule not in self:
|
||||
raise KeyError(key)
|
||||
|
||||
return self[self.default_rule]
|
||||
|
||||
def __str__(self):
|
||||
"""Dumps a string representation of the rules."""
|
||||
|
||||
# Start by building the canonical strings for the rules
|
||||
out_rules = {}
|
||||
for key, value in self.items():
|
||||
# Use empty string for singleton TrueCheck instances
|
||||
if isinstance(value, TrueCheck):
|
||||
out_rules[key] = ''
|
||||
else:
|
||||
out_rules[key] = str(value)
|
||||
|
||||
# Dump a pretty-printed JSON representation
|
||||
return jsonutils.dumps(out_rules, indent=4)
|
||||
|
||||
|
||||
# Really have to figure out a way to deprecate this
|
||||
def set_rules(rules):
|
||||
"""Set the rules in use for policy checks."""
|
||||
|
||||
global _rules
|
||||
|
||||
_rules = rules
|
||||
|
||||
|
||||
# Ditto
|
||||
def reset():
|
||||
"""Clear the rules used for policy checks."""
|
||||
|
||||
global _rules
|
||||
|
||||
_rules = None
|
||||
|
||||
|
||||
def check(rule, target, creds, exc=None, *args, **kwargs):
|
||||
"""
|
||||
Checks authorization of a rule against the target and credentials.
|
||||
|
||||
:param rule: The rule to evaluate.
|
||||
:param target: As much information about the object being operated
|
||||
on as possible, as a dictionary.
|
||||
:param creds: As much information about the user performing the
|
||||
action as possible, as a dictionary.
|
||||
:param exc: Class of the exception to raise if the check fails.
|
||||
Any remaining arguments passed to check() (both
|
||||
positional and keyword arguments) will be passed to
|
||||
the exception class. If exc is not provided, returns
|
||||
False.
|
||||
|
||||
:return: Returns False if the policy does not allow the action and
|
||||
exc is not provided; otherwise, returns a value that
|
||||
evaluates to True. Note: for rules using the "case"
|
||||
expression, this True value will be the specified string
|
||||
from the expression.
|
||||
"""
|
||||
|
||||
# Allow the rule to be a Check tree
|
||||
if isinstance(rule, BaseCheck):
|
||||
result = rule(target, creds)
|
||||
elif not _rules:
|
||||
# No rules to reference means we're going to fail closed
|
||||
result = False
|
||||
else:
|
||||
try:
|
||||
match_kind, match_value = match.split(':', 1)
|
||||
except Exception:
|
||||
LOG.exception(_("Failed to understand rule %(match)r") % locals())
|
||||
# If the rule is invalid, fail closed
|
||||
return False
|
||||
# Evaluate the rule
|
||||
result = _rules[rule](target, creds)
|
||||
except KeyError:
|
||||
# If the rule doesn't exist, fail closed
|
||||
result = False
|
||||
|
||||
func = None
|
||||
try:
|
||||
old_func = getattr(self, '_check_%s' % match_kind)
|
||||
except AttributeError:
|
||||
func = self._checks.get(match_kind, self._checks.get(None, None))
|
||||
else:
|
||||
LOG.warning(_("Inheritance-based rules are deprecated; update "
|
||||
"_check_%s") % match_kind)
|
||||
func = lambda brain, kind, value, target, cred: old_func(value,
|
||||
target,
|
||||
cred)
|
||||
# If it is False, raise the exception if requested
|
||||
if exc and result is False:
|
||||
raise exc(*args, **kwargs)
|
||||
|
||||
if not func:
|
||||
LOG.error(_("No handler for matches of kind %s") % match_kind)
|
||||
# Fail closed
|
||||
return False
|
||||
return result
|
||||
|
||||
return func(self, match_kind, match_value, target_dict, cred_dict)
|
||||
|
||||
def check(self, match_list, target_dict, cred_dict):
|
||||
"""Checks authorization of some rules against credentials.
|
||||
class BaseCheck(object):
|
||||
"""
|
||||
Abstract base class for Check classes.
|
||||
"""
|
||||
|
||||
Detailed description of the check with examples in policy.enforce().
|
||||
|
||||
:param match_list: nested tuples of data to match against
|
||||
:param target_dict: dict of object properties
|
||||
:param credentials_dict: dict of actor properties
|
||||
|
||||
:returns: True if the check passes
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
@abc.abstractmethod
|
||||
def __str__(self):
|
||||
"""
|
||||
if not match_list:
|
||||
return True
|
||||
for and_list in match_list:
|
||||
if isinstance(and_list, basestring):
|
||||
and_list = (and_list,)
|
||||
if all([self._check(item, target_dict, cred_dict)
|
||||
for item in and_list]):
|
||||
return True
|
||||
Retrieve a string representation of the Check tree rooted at
|
||||
this node.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def __call__(self, target, cred):
|
||||
"""
|
||||
Perform the check. Returns False to reject the access or a
|
||||
true value (not necessary True) to accept the access.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class FalseCheck(BaseCheck):
|
||||
"""
|
||||
A policy check that always returns False (disallow).
|
||||
"""
|
||||
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "!"
|
||||
|
||||
def __call__(self, target, cred):
|
||||
"""Check the policy."""
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class HttpBrain(Brain):
|
||||
"""A brain that can check external urls for policy.
|
||||
|
||||
Posts json blobs for target and credentials.
|
||||
|
||||
Note that this brain is deprecated; the http check is registered
|
||||
by default.
|
||||
class TrueCheck(BaseCheck):
|
||||
"""
|
||||
A policy check that always returns True (allow).
|
||||
"""
|
||||
|
||||
pass
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "@"
|
||||
|
||||
def __call__(self, target, cred):
|
||||
"""Check the policy."""
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class Check(BaseCheck):
|
||||
"""
|
||||
A base class to allow for user-defined policy checks.
|
||||
"""
|
||||
|
||||
def __init__(self, kind, match):
|
||||
"""
|
||||
:param kind: The kind of the check, i.e., the field before the
|
||||
':'.
|
||||
:param match: The match of the check, i.e., the field after
|
||||
the ':'.
|
||||
"""
|
||||
|
||||
self.kind = kind
|
||||
self.match = match
|
||||
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "%s:%s" % (self.kind, self.match)
|
||||
|
||||
|
||||
class NotCheck(BaseCheck):
|
||||
"""
|
||||
A policy check that inverts the result of another policy check.
|
||||
Implements the "not" operator.
|
||||
"""
|
||||
|
||||
def __init__(self, rule):
|
||||
"""
|
||||
Initialize the 'not' check.
|
||||
|
||||
:param rule: The rule to negate. Must be a Check.
|
||||
"""
|
||||
|
||||
self.rule = rule
|
||||
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "not %s" % self.rule
|
||||
|
||||
def __call__(self, target, cred):
|
||||
"""
|
||||
Check the policy. Returns the logical inverse of the wrapped
|
||||
check.
|
||||
"""
|
||||
|
||||
return not self.rule(target, cred)
|
||||
|
||||
|
||||
class AndCheck(BaseCheck):
|
||||
"""
|
||||
A policy check that requires that a list of other checks all
|
||||
return True. Implements the "and" operator.
|
||||
"""
|
||||
|
||||
def __init__(self, rules):
|
||||
"""
|
||||
Initialize the 'and' check.
|
||||
|
||||
:param rules: A list of rules that will be tested.
|
||||
"""
|
||||
|
||||
self.rules = rules
|
||||
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "(%s)" % ' and '.join(str(r) for r in self.rules)
|
||||
|
||||
def __call__(self, target, cred):
|
||||
"""
|
||||
Check the policy. Requires that all rules accept in order to
|
||||
return True.
|
||||
"""
|
||||
|
||||
for rule in self.rules:
|
||||
if not rule(target, cred):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def add_check(self, rule):
|
||||
"""
|
||||
Allows addition of another rule to the list of rules that will
|
||||
be tested. Returns the AndCheck object for convenience.
|
||||
"""
|
||||
|
||||
self.rules.append(rule)
|
||||
return self
|
||||
|
||||
|
||||
class OrCheck(BaseCheck):
|
||||
"""
|
||||
A policy check that requires that at least one of a list of other
|
||||
checks returns True. Implements the "or" operator.
|
||||
"""
|
||||
|
||||
def __init__(self, rules):
|
||||
"""
|
||||
Initialize the 'or' check.
|
||||
|
||||
:param rules: A list of rules that will be tested.
|
||||
"""
|
||||
|
||||
self.rules = rules
|
||||
|
||||
def __str__(self):
|
||||
"""Return a string representation of this check."""
|
||||
|
||||
return "(%s)" % ' or '.join(str(r) for r in self.rules)
|
||||
|
||||
def __call__(self, target, cred):
|
||||
"""
|
||||
Check the policy. Requires that at least one rule accept in
|
||||
order to return True.
|
||||
"""
|
||||
|
||||
for rule in self.rules:
|
||||
if rule(target, cred):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def add_check(self, rule):
|
||||
"""
|
||||
Allows addition of another rule to the list of rules that will
|
||||
be tested. Returns the OrCheck object for convenience.
|
||||
"""
|
||||
|
||||
self.rules.append(rule)
|
||||
return self
|
||||
|
||||
|
||||
def _parse_check(rule):
|
||||
"""
|
||||
Parse a single base check rule into an appropriate Check object.
|
||||
"""
|
||||
|
||||
# Handle the special checks
|
||||
if rule == '!':
|
||||
return FalseCheck()
|
||||
elif rule == '@':
|
||||
return TrueCheck()
|
||||
|
||||
try:
|
||||
kind, match = rule.split(':', 1)
|
||||
except Exception:
|
||||
LOG.exception(_("Failed to understand rule %(rule)s") % locals())
|
||||
# If the rule is invalid, we'll fail closed
|
||||
return FalseCheck()
|
||||
|
||||
# Find what implements the check
|
||||
if kind in _checks:
|
||||
return _checks[kind](kind, match)
|
||||
elif None in _checks:
|
||||
return _checks[None](kind, match)
|
||||
else:
|
||||
LOG.error(_("No handler for matches of kind %s") % kind)
|
||||
return FalseCheck()
|
||||
|
||||
|
||||
def _parse_list_rule(rule):
|
||||
"""
|
||||
Provided for backwards compatibility. Translates the old
|
||||
list-of-lists syntax into a tree of Check objects.
|
||||
"""
|
||||
|
||||
# Empty rule defaults to True
|
||||
if not rule:
|
||||
return TrueCheck()
|
||||
|
||||
# Outer list is joined by "or"; inner list by "and"
|
||||
or_list = []
|
||||
for inner_rule in rule:
|
||||
# Elide empty inner lists
|
||||
if not inner_rule:
|
||||
continue
|
||||
|
||||
# Handle bare strings
|
||||
if isinstance(inner_rule, basestring):
|
||||
inner_rule = [inner_rule]
|
||||
|
||||
# Parse the inner rules into Check objects
|
||||
and_list = [_parse_check(r) for r in inner_rule]
|
||||
|
||||
# Append the appropriate check to the or_list
|
||||
if len(and_list) == 1:
|
||||
or_list.append(and_list[0])
|
||||
else:
|
||||
or_list.append(AndCheck(and_list))
|
||||
|
||||
# If we have only one check, omit the "or"
|
||||
if len(or_list) == 0:
|
||||
return FalseCheck()
|
||||
elif len(or_list) == 1:
|
||||
return or_list[0]
|
||||
|
||||
return OrCheck(or_list)
|
||||
|
||||
|
||||
# Used for tokenizing the policy language
|
||||
_tokenize_re = re.compile(r'\s+')
|
||||
|
||||
|
||||
def _parse_tokenize(rule):
|
||||
"""
|
||||
Tokenizer for the policy language.
|
||||
|
||||
Most of the single-character tokens are specified in the
|
||||
_tokenize_re; however, parentheses need to be handled specially,
|
||||
because they can appear inside a check string. Thankfully, those
|
||||
parentheses that appear inside a check string can never occur at
|
||||
the very beginning or end ("%(variable)s" is the correct syntax).
|
||||
"""
|
||||
|
||||
for tok in _tokenize_re.split(rule):
|
||||
# Skip empty tokens
|
||||
if not tok or tok.isspace():
|
||||
continue
|
||||
|
||||
# Handle leading parens on the token
|
||||
clean = tok.lstrip('(')
|
||||
for i in range(len(tok) - len(clean)):
|
||||
yield '(', '('
|
||||
|
||||
# If it was only parentheses, continue
|
||||
if not clean:
|
||||
continue
|
||||
else:
|
||||
tok = clean
|
||||
|
||||
# Handle trailing parens on the token
|
||||
clean = tok.rstrip(')')
|
||||
trail = len(tok) - len(clean)
|
||||
|
||||
# Yield the cleaned token
|
||||
lowered = clean.lower()
|
||||
if lowered in ('and', 'or', 'not'):
|
||||
# Special tokens
|
||||
yield lowered, clean
|
||||
elif clean:
|
||||
# Not a special token, but not composed solely of ')'
|
||||
if len(tok) >= 2 and ((tok[0], tok[-1]) in
|
||||
[('"', '"'), ("'", "'")]):
|
||||
# It's a quoted string
|
||||
yield 'string', tok[1:-1]
|
||||
else:
|
||||
yield 'check', _parse_check(clean)
|
||||
|
||||
# Yield the trailing parens
|
||||
for i in range(trail):
|
||||
yield ')', ')'
|
||||
|
||||
|
||||
class ParseStateMeta(type):
|
||||
"""
|
||||
Metaclass for the ParseState class. Facilitates identifying
|
||||
reduction methods.
|
||||
"""
|
||||
|
||||
def __new__(mcs, name, bases, cls_dict):
|
||||
"""
|
||||
Create the class. Injects the 'reducers' list, a list of
|
||||
tuples matching token sequences to the names of the
|
||||
corresponding reduction methods.
|
||||
"""
|
||||
|
||||
reducers = []
|
||||
|
||||
for key, value in cls_dict.items():
|
||||
if not hasattr(value, 'reducers'):
|
||||
continue
|
||||
for reduction in value.reducers:
|
||||
reducers.append((reduction, key))
|
||||
|
||||
cls_dict['reducers'] = reducers
|
||||
|
||||
return super(ParseStateMeta, mcs).__new__(mcs, name, bases, cls_dict)
|
||||
|
||||
|
||||
def reducer(*tokens):
|
||||
"""
|
||||
Decorator for reduction methods. Arguments are a sequence of
|
||||
tokens, in order, which should trigger running this reduction
|
||||
method.
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
# Make sure we have a list of reducer sequences
|
||||
if not hasattr(func, 'reducers'):
|
||||
func.reducers = []
|
||||
|
||||
# Add the tokens to the list of reducer sequences
|
||||
func.reducers.append(list(tokens))
|
||||
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
class ParseState(object):
|
||||
"""
|
||||
Implement the core of parsing the policy language. Uses a greedy
|
||||
reduction algorithm to reduce a sequence of tokens into a single
|
||||
terminal, the value of which will be the root of the Check tree.
|
||||
|
||||
Note: error reporting is rather lacking. The best we can get with
|
||||
this parser formulation is an overall "parse failed" error.
|
||||
Fortunately, the policy language is simple enough that this
|
||||
shouldn't be that big a problem.
|
||||
"""
|
||||
|
||||
__metaclass__ = ParseStateMeta
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the ParseState."""
|
||||
|
||||
self.tokens = []
|
||||
self.values = []
|
||||
|
||||
def reduce(self):
|
||||
"""
|
||||
Perform a greedy reduction of the token stream. If a reducer
|
||||
method matches, it will be executed, then the reduce() method
|
||||
will be called recursively to search for any more possible
|
||||
reductions.
|
||||
"""
|
||||
|
||||
for reduction, methname in self.reducers:
|
||||
if (len(self.tokens) >= len(reduction) and
|
||||
self.tokens[-len(reduction):] == reduction):
|
||||
# Get the reduction method
|
||||
meth = getattr(self, methname)
|
||||
|
||||
# Reduce the token stream
|
||||
results = meth(*self.values[-len(reduction):])
|
||||
|
||||
# Update the tokens and values
|
||||
self.tokens[-len(reduction):] = [r[0] for r in results]
|
||||
self.values[-len(reduction):] = [r[1] for r in results]
|
||||
|
||||
# Check for any more reductions
|
||||
return self.reduce()
|
||||
|
||||
def shift(self, tok, value):
|
||||
"""Adds one more token to the state. Calls reduce()."""
|
||||
|
||||
self.tokens.append(tok)
|
||||
self.values.append(value)
|
||||
|
||||
# Do a greedy reduce...
|
||||
self.reduce()
|
||||
|
||||
@property
|
||||
def result(self):
|
||||
"""
|
||||
Obtain the final result of the parse. Raises ValueError if
|
||||
the parse failed to reduce to a single result.
|
||||
"""
|
||||
|
||||
if len(self.values) != 1:
|
||||
raise ValueError("Could not parse rule")
|
||||
return self.values[0]
|
||||
|
||||
@reducer('(', 'check', ')')
|
||||
@reducer('(', 'and_expr', ')')
|
||||
@reducer('(', 'or_expr', ')')
|
||||
def _wrap_check(self, _p1, check, _p2):
|
||||
"""Turn parenthesized expressions into a 'check' token."""
|
||||
|
||||
return [('check', check)]
|
||||
|
||||
@reducer('check', 'and', 'check')
|
||||
def _make_and_expr(self, check1, _and, check2):
|
||||
"""
|
||||
Create an 'and_expr' from two checks joined by the 'and'
|
||||
operator.
|
||||
"""
|
||||
|
||||
return [('and_expr', AndCheck([check1, check2]))]
|
||||
|
||||
@reducer('and_expr', 'and', 'check')
|
||||
def _extend_and_expr(self, and_expr, _and, check):
|
||||
"""
|
||||
Extend an 'and_expr' by adding one more check.
|
||||
"""
|
||||
|
||||
return [('and_expr', and_expr.add_check(check))]
|
||||
|
||||
@reducer('check', 'or', 'check')
|
||||
def _make_or_expr(self, check1, _or, check2):
|
||||
"""
|
||||
Create an 'or_expr' from two checks joined by the 'or'
|
||||
operator.
|
||||
"""
|
||||
|
||||
return [('or_expr', OrCheck([check1, check2]))]
|
||||
|
||||
@reducer('or_expr', 'or', 'check')
|
||||
def _extend_or_expr(self, or_expr, _or, check):
|
||||
"""
|
||||
Extend an 'or_expr' by adding one more check.
|
||||
"""
|
||||
|
||||
return [('or_expr', or_expr.add_check(check))]
|
||||
|
||||
@reducer('not', 'check')
|
||||
def _make_not_expr(self, _not, check):
|
||||
"""Invert the result of another check."""
|
||||
|
||||
return [('check', NotCheck(check))]
|
||||
|
||||
|
||||
def _parse_text_rule(rule):
|
||||
"""
|
||||
Translates a policy written in the policy language into a tree of
|
||||
Check objects.
|
||||
"""
|
||||
|
||||
# Empty rule means always accept
|
||||
if not rule:
|
||||
return TrueCheck()
|
||||
|
||||
# Parse the token stream
|
||||
state = ParseState()
|
||||
for tok, value in _parse_tokenize(rule):
|
||||
state.shift(tok, value)
|
||||
|
||||
try:
|
||||
return state.result
|
||||
except ValueError:
|
||||
# Couldn't parse the rule
|
||||
LOG.exception(_("Failed to understand rule %(rule)r") % locals())
|
||||
|
||||
# Fail closed
|
||||
return FalseCheck()
|
||||
|
||||
|
||||
def parse_rule(rule):
|
||||
"""
|
||||
Parses a policy rule into a tree of Check objects.
|
||||
"""
|
||||
|
||||
# If the rule is a string, it's in the policy language
|
||||
if isinstance(rule, basestring):
|
||||
return _parse_text_rule(rule)
|
||||
return _parse_list_rule(rule)
|
||||
|
||||
|
||||
def register(name, func=None):
|
||||
"""
|
||||
Register a function as a policy check.
|
||||
Register a function or Check class as a policy check.
|
||||
|
||||
:param name: Gives the name of the check type, e.g., 'rule',
|
||||
'role', etc. If name is None, a default function
|
||||
'role', etc. If name is None, a default check type
|
||||
will be registered.
|
||||
:param func: If given, provides the function to register. If not
|
||||
given, returns a function taking one argument to
|
||||
specify the function to register, allowing use as a
|
||||
decorator.
|
||||
:param func: If given, provides the function or class to register.
|
||||
If not given, returns a function taking one argument
|
||||
to specify the function or class to register,
|
||||
allowing use as a decorator.
|
||||
"""
|
||||
|
||||
# Perform the actual decoration by registering the function.
|
||||
# Returns the function for compliance with the decorator
|
||||
# interface.
|
||||
# Perform the actual decoration by registering the function or
|
||||
# class. Returns the function or class for compliance with the
|
||||
# decorator interface.
|
||||
def decorator(func):
|
||||
# Register the function
|
||||
Brain._register(name, func)
|
||||
_checks[name] = func
|
||||
return func
|
||||
|
||||
# If the function is given, do the registration
|
||||
# If the function or class is given, do the registration
|
||||
if func:
|
||||
return decorator(func)
|
||||
|
||||
|
@ -247,55 +721,59 @@ def register(name, func=None):
|
|||
|
||||
|
||||
@register("rule")
|
||||
def _check_rule(brain, match_kind, match, target_dict, cred_dict):
|
||||
"""Recursively checks credentials based on the brains rules."""
|
||||
try:
|
||||
new_match_list = brain.rules[match]
|
||||
except KeyError:
|
||||
if brain.default_rule and match != brain.default_rule:
|
||||
new_match_list = ('rule:%s' % brain.default_rule,)
|
||||
else:
|
||||
return False
|
||||
class RuleCheck(Check):
|
||||
def __call__(self, target, creds):
|
||||
"""
|
||||
Recursively checks credentials based on the defined rules.
|
||||
"""
|
||||
|
||||
return brain.check(new_match_list, target_dict, cred_dict)
|
||||
try:
|
||||
return _rules[self.match](target, creds)
|
||||
except KeyError:
|
||||
# We don't have any matching rule; fail closed
|
||||
return False
|
||||
|
||||
|
||||
@register("role")
|
||||
def _check_role(brain, match_kind, match, target_dict, cred_dict):
|
||||
"""Check that there is a matching role in the cred dict."""
|
||||
return match.lower() in [x.lower() for x in cred_dict['roles']]
|
||||
class RoleCheck(Check):
|
||||
def __call__(self, target, creds):
|
||||
"""Check that there is a matching role in the cred dict."""
|
||||
|
||||
return self.match.lower() in [x.lower() for x in creds['roles']]
|
||||
|
||||
|
||||
@register('http')
|
||||
def _check_http(brain, match_kind, match, target_dict, cred_dict):
|
||||
"""Check http: rules by calling to a remote server.
|
||||
class HttpCheck(Check):
|
||||
def __call__(self, target, creds):
|
||||
"""
|
||||
Check http: rules by calling to a remote server.
|
||||
|
||||
This example implementation simply verifies that the response is
|
||||
exactly 'True'. A custom brain using response codes could easily
|
||||
be implemented.
|
||||
This example implementation simply verifies that the response
|
||||
is exactly 'True'.
|
||||
"""
|
||||
|
||||
"""
|
||||
url = 'http:' + (match % target_dict)
|
||||
data = {'target': jsonutils.dumps(target_dict),
|
||||
'credentials': jsonutils.dumps(cred_dict)}
|
||||
post_data = urllib.urlencode(data)
|
||||
f = urllib2.urlopen(url, post_data)
|
||||
return f.read() == "True"
|
||||
url = ('http:' + self.match) % target
|
||||
data = {'target': jsonutils.dumps(target),
|
||||
'credentials': jsonutils.dumps(creds)}
|
||||
post_data = urllib.urlencode(data)
|
||||
f = urllib2.urlopen(url, post_data)
|
||||
return f.read() == "True"
|
||||
|
||||
|
||||
@register(None)
|
||||
def _check_generic(brain, match_kind, match, target_dict, cred_dict):
|
||||
"""Check an individual match.
|
||||
class GenericCheck(Check):
|
||||
def __call__(self, target, creds):
|
||||
"""
|
||||
Check an individual match.
|
||||
|
||||
Matches look like:
|
||||
Matches look like:
|
||||
|
||||
tenant:%(tenant_id)s
|
||||
role:compute:admin
|
||||
tenant:%(tenant_id)s
|
||||
role:compute:admin
|
||||
"""
|
||||
|
||||
"""
|
||||
|
||||
# TODO(termie): do dict inspection via dot syntax
|
||||
match = match % target_dict
|
||||
if match_kind in cred_dict:
|
||||
return match == unicode(cred_dict[match_kind])
|
||||
return False
|
||||
# TODO(termie): do dict inspection via dot syntax
|
||||
match = self.match % target
|
||||
if self.kind in creds:
|
||||
return match == unicode(creds[self.kind])
|
||||
return False
|
||||
|
|
|
@ -0,0 +1,135 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import random
|
||||
import shlex
|
||||
|
||||
from eventlet.green import subprocess
|
||||
from eventlet import greenthread
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnknownArgumentError(Exception):
|
||||
def __init__(self, message=None):
|
||||
super(UnknownArgumentError, self).__init__(message)
|
||||
|
||||
|
||||
class ProcessExecutionError(Exception):
|
||||
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
|
||||
description=None):
|
||||
if description is None:
|
||||
description = "Unexpected error while running command."
|
||||
if exit_code is None:
|
||||
exit_code = '-'
|
||||
message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
|
||||
% (description, cmd, exit_code, stdout, stderr))
|
||||
super(ProcessExecutionError, self).__init__(message)
|
||||
|
||||
|
||||
def execute(*cmd, **kwargs):
|
||||
"""
|
||||
Helper method to shell out and execute a command through subprocess with
|
||||
optional retry.
|
||||
|
||||
:param cmd: Passed to subprocess.Popen.
|
||||
:type cmd: string
|
||||
:param process_input: Send to opened process.
|
||||
:type proces_input: string
|
||||
:param check_exit_code: Defaults to 0. Will raise
|
||||
:class:`ProcessExecutionError`
|
||||
if the command exits without returning this value
|
||||
as a returncode
|
||||
:type check_exit_code: int
|
||||
:param delay_on_retry: True | False. Defaults to True. If set to True,
|
||||
wait a short amount of time before retrying.
|
||||
:type delay_on_retry: boolean
|
||||
:param attempts: How many times to retry cmd.
|
||||
:type attempts: int
|
||||
:param run_as_root: True | False. Defaults to False. If set to True,
|
||||
the command is prefixed by the command specified
|
||||
in the root_helper kwarg.
|
||||
:type run_as_root: boolean
|
||||
:param root_helper: command to prefix all cmd's with
|
||||
:type root_helper: string
|
||||
:returns: (stdout, stderr) from process execution
|
||||
:raises: :class:`UnknownArgumentError` on
|
||||
receiving unknown arguments
|
||||
:raises: :class:`ProcessExecutionError`
|
||||
"""
|
||||
|
||||
process_input = kwargs.pop('process_input', None)
|
||||
check_exit_code = kwargs.pop('check_exit_code', 0)
|
||||
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
||||
attempts = kwargs.pop('attempts', 1)
|
||||
run_as_root = kwargs.pop('run_as_root', False)
|
||||
root_helper = kwargs.pop('root_helper', '')
|
||||
if len(kwargs):
|
||||
raise UnknownArgumentError(_('Got unknown keyword args '
|
||||
'to utils.execute: %r') % kwargs)
|
||||
if run_as_root:
|
||||
cmd = shlex.split(root_helper) + list(cmd)
|
||||
cmd = map(str, cmd)
|
||||
|
||||
while attempts > 0:
|
||||
attempts -= 1
|
||||
try:
|
||||
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
|
||||
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
||||
obj = subprocess.Popen(cmd,
|
||||
stdin=_PIPE,
|
||||
stdout=_PIPE,
|
||||
stderr=_PIPE,
|
||||
close_fds=True)
|
||||
result = None
|
||||
if process_input is not None:
|
||||
result = obj.communicate(process_input)
|
||||
else:
|
||||
result = obj.communicate()
|
||||
obj.stdin.close() # pylint: disable=E1101
|
||||
_returncode = obj.returncode # pylint: disable=E1101
|
||||
if _returncode:
|
||||
LOG.debug(_('Result was %s') % _returncode)
|
||||
if (isinstance(check_exit_code, int) and
|
||||
not isinstance(check_exit_code, bool) and
|
||||
_returncode != check_exit_code):
|
||||
(stdout, stderr) = result
|
||||
raise ProcessExecutionError(exit_code=_returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cmd=' '.join(cmd))
|
||||
return result
|
||||
except ProcessExecutionError:
|
||||
if not attempts:
|
||||
raise
|
||||
else:
|
||||
LOG.debug(_('%r failed. Retrying.'), cmd)
|
||||
if delay_on_retry:
|
||||
greenthread.sleep(random.randint(20, 200) / 100.0)
|
||||
finally:
|
||||
# NOTE(termie): this appears to be necessary to let the subprocess
|
||||
# call clean something up in between calls, without
|
||||
# it two execute calls in a row hangs the second one
|
||||
greenthread.sleep(0)
|
|
@ -0,0 +1,16 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,180 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
|
||||
class CommandFilter(object):
|
||||
"""Command filter only checking that the 1st argument matches exec_path"""
|
||||
|
||||
def __init__(self, exec_path, run_as, *args):
|
||||
self.name = ''
|
||||
self.exec_path = exec_path
|
||||
self.run_as = run_as
|
||||
self.args = args
|
||||
self.real_exec = None
|
||||
|
||||
def get_exec(self, exec_dirs=[]):
|
||||
"""Returns existing executable, or empty string if none found"""
|
||||
if self.real_exec is not None:
|
||||
return self.real_exec
|
||||
self.real_exec = ""
|
||||
if self.exec_path.startswith('/'):
|
||||
if os.access(self.exec_path, os.X_OK):
|
||||
self.real_exec = self.exec_path
|
||||
else:
|
||||
for binary_path in exec_dirs:
|
||||
expanded_path = os.path.join(binary_path, self.exec_path)
|
||||
if os.access(expanded_path, os.X_OK):
|
||||
self.real_exec = expanded_path
|
||||
break
|
||||
return self.real_exec
|
||||
|
||||
def match(self, userargs):
|
||||
"""Only check that the first argument (command) matches exec_path"""
|
||||
if (os.path.basename(self.exec_path) == userargs[0]):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_command(self, userargs, exec_dirs=[]):
|
||||
"""Returns command to execute (with sudo -u if run_as != root)."""
|
||||
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
|
||||
if (self.run_as != 'root'):
|
||||
# Used to run commands at lesser privileges
|
||||
return ['sudo', '-u', self.run_as, to_exec] + userargs[1:]
|
||||
return [to_exec] + userargs[1:]
|
||||
|
||||
def get_environment(self, userargs):
|
||||
"""Returns specific environment to set, None if none"""
|
||||
return None
|
||||
|
||||
|
||||
class RegExpFilter(CommandFilter):
|
||||
"""Command filter doing regexp matching for every argument"""
|
||||
|
||||
def match(self, userargs):
|
||||
# Early skip if command or number of args don't match
|
||||
if (len(self.args) != len(userargs)):
|
||||
# DENY: argument numbers don't match
|
||||
return False
|
||||
# Compare each arg (anchoring pattern explicitly at end of string)
|
||||
for (pattern, arg) in zip(self.args, userargs):
|
||||
try:
|
||||
if not re.match(pattern + '$', arg):
|
||||
break
|
||||
except re.error:
|
||||
# DENY: Badly-formed filter
|
||||
return False
|
||||
else:
|
||||
# ALLOW: All arguments matched
|
||||
return True
|
||||
|
||||
# DENY: Some arguments did not match
|
||||
return False
|
||||
|
||||
|
||||
class DnsmasqFilter(CommandFilter):
|
||||
"""Specific filter for the dnsmasq call (which includes env)"""
|
||||
|
||||
CONFIG_FILE_ARG = 'CONFIG_FILE'
|
||||
|
||||
def match(self, userargs):
|
||||
if (userargs[0] == 'env' and
|
||||
userargs[1].startswith(self.CONFIG_FILE_ARG) and
|
||||
userargs[2].startswith('NETWORK_ID=') and
|
||||
userargs[3] == 'dnsmasq'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_command(self, userargs, exec_dirs=[]):
|
||||
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
|
||||
dnsmasq_pos = userargs.index('dnsmasq')
|
||||
return [to_exec] + userargs[dnsmasq_pos + 1:]
|
||||
|
||||
def get_environment(self, userargs):
|
||||
env = os.environ.copy()
|
||||
env[self.CONFIG_FILE_ARG] = userargs[1].split('=')[-1]
|
||||
env['NETWORK_ID'] = userargs[2].split('=')[-1]
|
||||
return env
|
||||
|
||||
|
||||
class DeprecatedDnsmasqFilter(DnsmasqFilter):
|
||||
"""Variant of dnsmasq filter to support old-style FLAGFILE"""
|
||||
CONFIG_FILE_ARG = 'FLAGFILE'
|
||||
|
||||
|
||||
class KillFilter(CommandFilter):
|
||||
"""Specific filter for the kill calls.
|
||||
1st argument is the user to run /bin/kill under
|
||||
2nd argument is the location of the affected executable
|
||||
Subsequent arguments list the accepted signals (if any)
|
||||
|
||||
This filter relies on /proc to accurately determine affected
|
||||
executable, so it will only work on procfs-capable systems (not OSX).
|
||||
"""
|
||||
|
||||
def __init__(self, *args):
|
||||
super(KillFilter, self).__init__("/bin/kill", *args)
|
||||
|
||||
def match(self, userargs):
|
||||
if userargs[0] != "kill":
|
||||
return False
|
||||
args = list(userargs)
|
||||
if len(args) == 3:
|
||||
# A specific signal is requested
|
||||
signal = args.pop(1)
|
||||
if signal not in self.args[1:]:
|
||||
# Requested signal not in accepted list
|
||||
return False
|
||||
else:
|
||||
if len(args) != 2:
|
||||
# Incorrect number of arguments
|
||||
return False
|
||||
if len(self.args) > 1:
|
||||
# No signal requested, but filter requires specific signal
|
||||
return False
|
||||
try:
|
||||
command = os.readlink("/proc/%d/exe" % int(args[1]))
|
||||
# NOTE(dprince): /proc/PID/exe may have ' (deleted)' on
|
||||
# the end if an executable is updated or deleted
|
||||
if command.endswith(" (deleted)"):
|
||||
command = command[:command.rindex(" ")]
|
||||
if command != self.args[0]:
|
||||
# Affected executable does not match
|
||||
return False
|
||||
except (ValueError, OSError):
|
||||
# Incorrect PID
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class ReadFileFilter(CommandFilter):
|
||||
"""Specific filter for the utils.read_file_as_root call"""
|
||||
|
||||
def __init__(self, file_path, *args):
|
||||
self.file_path = file_path
|
||||
super(ReadFileFilter, self).__init__("/bin/cat", "root", *args)
|
||||
|
||||
def match(self, userargs):
|
||||
if userargs[0] != 'cat':
|
||||
return False
|
||||
if userargs[1] != self.file_path:
|
||||
return False
|
||||
if len(userargs) != 2:
|
||||
return False
|
||||
return True
|
|
@ -0,0 +1,149 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import ConfigParser
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import string
|
||||
|
||||
from kwapi.openstack.common.rootwrap import filters
|
||||
|
||||
|
||||
class NoFilterMatched(Exception):
|
||||
"""This exception is raised when no filter matched."""
|
||||
pass
|
||||
|
||||
|
||||
class FilterMatchNotExecutable(Exception):
|
||||
"""
|
||||
This exception is raised when a filter matched but no executable was
|
||||
found.
|
||||
"""
|
||||
def __init__(self, match=None, **kwargs):
|
||||
self.match = match
|
||||
|
||||
|
||||
class RootwrapConfig(object):
|
||||
|
||||
def __init__(self, config):
|
||||
# filters_path
|
||||
self.filters_path = config.get("DEFAULT", "filters_path").split(",")
|
||||
|
||||
# exec_dirs
|
||||
if config.has_option("DEFAULT", "exec_dirs"):
|
||||
self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",")
|
||||
else:
|
||||
# Use system PATH if exec_dirs is not specified
|
||||
self.exec_dirs = os.environ["PATH"].split(':')
|
||||
|
||||
# syslog_log_facility
|
||||
if config.has_option("DEFAULT", "syslog_log_facility"):
|
||||
v = config.get("DEFAULT", "syslog_log_facility")
|
||||
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||
self.syslog_log_facility = getattr(logging.handlers.SysLogHandler,
|
||||
v, None)
|
||||
if self.syslog_log_facility is None and v in facility_names:
|
||||
self.syslog_log_facility = facility_names.get(v)
|
||||
if self.syslog_log_facility is None:
|
||||
raise ValueError('Unexpected syslog_log_facility: %s' % v)
|
||||
else:
|
||||
default_facility = logging.handlers.SysLogHandler.LOG_SYSLOG
|
||||
self.syslog_log_facility = default_facility
|
||||
|
||||
# syslog_log_level
|
||||
if config.has_option("DEFAULT", "syslog_log_level"):
|
||||
v = config.get("DEFAULT", "syslog_log_level")
|
||||
self.syslog_log_level = logging.getLevelName(v.upper())
|
||||
if (self.syslog_log_level == "Level %s" % v.upper()):
|
||||
raise ValueError('Unexepected syslog_log_level: %s' % v)
|
||||
else:
|
||||
self.syslog_log_level = logging.ERROR
|
||||
|
||||
# use_syslog
|
||||
if config.has_option("DEFAULT", "use_syslog"):
|
||||
self.use_syslog = config.getboolean("DEFAULT", "use_syslog")
|
||||
else:
|
||||
self.use_syslog = False
|
||||
|
||||
|
||||
def setup_syslog(execname, facility, level):
|
||||
rootwrap_logger = logging.getLogger()
|
||||
rootwrap_logger.setLevel(level)
|
||||
handler = logging.handlers.SysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
handler.setFormatter(logging.Formatter(
|
||||
os.path.basename(execname) + ': %(message)s'))
|
||||
rootwrap_logger.addHandler(handler)
|
||||
|
||||
|
||||
def build_filter(class_name, *args):
|
||||
"""Returns a filter object of class class_name"""
|
||||
if not hasattr(filters, class_name):
|
||||
logging.warning("Skipping unknown filter class (%s) specified "
|
||||
"in filter definitions" % class_name)
|
||||
return None
|
||||
filterclass = getattr(filters, class_name)
|
||||
return filterclass(*args)
|
||||
|
||||
|
||||
def load_filters(filters_path):
|
||||
"""Load filters from a list of directories"""
|
||||
filterlist = []
|
||||
for filterdir in filters_path:
|
||||
if not os.path.isdir(filterdir):
|
||||
continue
|
||||
for filterfile in os.listdir(filterdir):
|
||||
filterconfig = ConfigParser.RawConfigParser()
|
||||
filterconfig.read(os.path.join(filterdir, filterfile))
|
||||
for (name, value) in filterconfig.items("Filters"):
|
||||
filterdefinition = [string.strip(s) for s in value.split(',')]
|
||||
newfilter = build_filter(*filterdefinition)
|
||||
if newfilter is None:
|
||||
continue
|
||||
newfilter.name = name
|
||||
filterlist.append(newfilter)
|
||||
return filterlist
|
||||
|
||||
|
||||
def match_filter(filters, userargs, exec_dirs=[]):
|
||||
"""
|
||||
Checks user command and arguments through command filters and
|
||||
returns the first matching filter.
|
||||
Raises NoFilterMatched if no filter matched.
|
||||
Raises FilterMatchNotExecutable if no executable was found for the
|
||||
best filter match.
|
||||
"""
|
||||
first_not_executable_filter = None
|
||||
|
||||
for f in filters:
|
||||
if f.match(userargs):
|
||||
# Try other filters if executable is absent
|
||||
if not f.get_exec(exec_dirs=exec_dirs):
|
||||
if not first_not_executable_filter:
|
||||
first_not_executable_filter = f
|
||||
continue
|
||||
# Otherwise return matching filter for execution
|
||||
return f
|
||||
|
||||
if first_not_executable_filter:
|
||||
# A filter matched, but no executable was found for it
|
||||
raise FilterMatchNotExecutable(match=first_not_executable_filter)
|
||||
|
||||
# No filter matched
|
||||
raise NoFilterMatched()
|
|
@ -47,28 +47,29 @@ rpc_opts = [
|
|||
help='Seconds to wait before a cast expires (TTL). '
|
||||
'Only supported by impl_zmq.'),
|
||||
cfg.ListOpt('allowed_rpc_exception_modules',
|
||||
default=['kwapi.openstack.common.exception',
|
||||
default=['openstack.common.exception',
|
||||
'nova.exception',
|
||||
'cinder.exception',
|
||||
'exceptions',
|
||||
],
|
||||
help='Modules of exceptions that are permitted to be recreated'
|
||||
'upon receiving exception data from an rpc call.'),
|
||||
cfg.BoolOpt('fake_rabbit',
|
||||
default=False,
|
||||
help='If passed, use a fake RabbitMQ provider'),
|
||||
#
|
||||
# The following options are not registered here, but are expected to be
|
||||
# present. The project using this library must register these options with
|
||||
# the configuration so that project-specific defaults may be defined.
|
||||
#
|
||||
#cfg.StrOpt('control_exchange',
|
||||
# default='nova',
|
||||
# help='AMQP exchange to connect to if using RabbitMQ or Qpid'),
|
||||
cfg.StrOpt('control_exchange',
|
||||
default='openstack',
|
||||
help='AMQP exchange to connect to if using RabbitMQ or Qpid'),
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(rpc_opts)
|
||||
|
||||
|
||||
def set_defaults(control_exchange):
|
||||
cfg.set_defaults(rpc_opts,
|
||||
control_exchange=control_exchange)
|
||||
|
||||
|
||||
def create_connection(new=True):
|
||||
"""Create a connection to the message bus used for rpc.
|
||||
|
||||
|
@ -177,17 +178,18 @@ def multicall(context, topic, msg, timeout=None):
|
|||
return _get_impl().multicall(cfg.CONF, context, topic, msg, timeout)
|
||||
|
||||
|
||||
def notify(context, topic, msg):
|
||||
def notify(context, topic, msg, envelope=False):
|
||||
"""Send notification event.
|
||||
|
||||
:param context: Information that identifies the user that has made this
|
||||
request.
|
||||
:param topic: The topic to send the notification to.
|
||||
:param msg: This is a dict of content of event.
|
||||
:param envelope: Set to True to enable message envelope for notifications.
|
||||
|
||||
:returns: None
|
||||
"""
|
||||
return _get_impl().notify(cfg.CONF, context, topic, msg)
|
||||
return _get_impl().notify(cfg.CONF, context, topic, msg, envelope)
|
||||
|
||||
|
||||
def cleanup():
|
||||
|
|
|
@ -26,7 +26,6 @@ AMQP, but is deprecated and predates this code.
|
|||
"""
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
|
@ -34,10 +33,10 @@ from eventlet import greenpool
|
|||
from eventlet import pools
|
||||
from eventlet import semaphore
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common import excutils
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import local
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.rpc import common as rpc_common
|
||||
|
||||
|
||||
|
@ -55,7 +54,7 @@ class Pool(pools.Pool):
|
|||
|
||||
# TODO(comstud): Timeout connections not used in a while
|
||||
def create(self):
|
||||
LOG.debug('Pool creating new connection')
|
||||
LOG.debug(_('Pool creating new connection'))
|
||||
return self.connection_cls(self.conf)
|
||||
|
||||
def empty(self):
|
||||
|
@ -150,7 +149,7 @@ class ConnectionContext(rpc_common.Connection):
|
|||
|
||||
|
||||
def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
|
||||
ending=False):
|
||||
ending=False, log_failure=True):
|
||||
"""Sends a reply or an error on the channel signified by msg_id.
|
||||
|
||||
Failure should be a sys.exc_info() tuple.
|
||||
|
@ -158,7 +157,8 @@ def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
|
|||
"""
|
||||
with ConnectionContext(conf, connection_pool) as conn:
|
||||
if failure:
|
||||
failure = rpc_common.serialize_remote_exception(failure)
|
||||
failure = rpc_common.serialize_remote_exception(failure,
|
||||
log_failure)
|
||||
|
||||
try:
|
||||
msg = {'result': reply, 'failure': failure}
|
||||
|
@ -168,7 +168,7 @@ def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None,
|
|||
'failure': failure}
|
||||
if ending:
|
||||
msg['ending'] = True
|
||||
conn.direct_send(msg_id, msg)
|
||||
conn.direct_send(msg_id, rpc_common.serialize_msg(msg))
|
||||
|
||||
|
||||
class RpcContext(rpc_common.CommonRpcContext):
|
||||
|
@ -185,10 +185,10 @@ class RpcContext(rpc_common.CommonRpcContext):
|
|||
return self.__class__(**values)
|
||||
|
||||
def reply(self, reply=None, failure=None, ending=False,
|
||||
connection_pool=None):
|
||||
connection_pool=None, log_failure=True):
|
||||
if self.msg_id:
|
||||
msg_reply(self.conf, self.msg_id, connection_pool, reply, failure,
|
||||
ending)
|
||||
ending, log_failure)
|
||||
if ending:
|
||||
self.msg_id = None
|
||||
|
||||
|
@ -282,11 +282,21 @@ class ProxyCallback(object):
|
|||
ctxt.reply(rval, None, connection_pool=self.connection_pool)
|
||||
# This final None tells multicall that it is done.
|
||||
ctxt.reply(ending=True, connection_pool=self.connection_pool)
|
||||
except Exception as e:
|
||||
LOG.exception('Exception during message handling')
|
||||
except rpc_common.ClientException as e:
|
||||
LOG.debug(_('Expected exception during message handling (%s)') %
|
||||
e._exc_info[1])
|
||||
ctxt.reply(None, e._exc_info,
|
||||
connection_pool=self.connection_pool,
|
||||
log_failure=False)
|
||||
except Exception:
|
||||
LOG.exception(_('Exception during message handling'))
|
||||
ctxt.reply(None, sys.exc_info(),
|
||||
connection_pool=self.connection_pool)
|
||||
|
||||
def wait(self):
|
||||
"""Wait for all callback threads to exit."""
|
||||
self.pool.waitall()
|
||||
|
||||
|
||||
class MulticallWaiter(object):
|
||||
def __init__(self, conf, connection, timeout):
|
||||
|
@ -349,7 +359,7 @@ def multicall(conf, context, topic, msg, timeout, connection_pool):
|
|||
# that will continue to use the connection. When it's done,
|
||||
# connection.close() will get called which will put it back into
|
||||
# the pool
|
||||
LOG.debug(_('Making asynchronous call on %s ...'), topic)
|
||||
LOG.debug(_('Making synchronous call on %s ...'), topic)
|
||||
msg_id = uuid.uuid4().hex
|
||||
msg.update({'_msg_id': msg_id})
|
||||
LOG.debug(_('MSG_ID is %s') % (msg_id))
|
||||
|
@ -358,7 +368,7 @@ def multicall(conf, context, topic, msg, timeout, connection_pool):
|
|||
conn = ConnectionContext(conf, connection_pool)
|
||||
wait_msg = MulticallWaiter(conf, conn, timeout)
|
||||
conn.declare_direct_consumer(msg_id, wait_msg)
|
||||
conn.topic_send(topic, msg)
|
||||
conn.topic_send(topic, rpc_common.serialize_msg(msg))
|
||||
return wait_msg
|
||||
|
||||
|
||||
|
@ -377,7 +387,7 @@ def cast(conf, context, topic, msg, connection_pool):
|
|||
LOG.debug(_('Making asynchronous cast on %s...'), topic)
|
||||
pack_context(msg, context)
|
||||
with ConnectionContext(conf, connection_pool) as conn:
|
||||
conn.topic_send(topic, msg)
|
||||
conn.topic_send(topic, rpc_common.serialize_msg(msg))
|
||||
|
||||
|
||||
def fanout_cast(conf, context, topic, msg, connection_pool):
|
||||
|
@ -385,7 +395,7 @@ def fanout_cast(conf, context, topic, msg, connection_pool):
|
|||
LOG.debug(_('Making asynchronous fanout cast...'))
|
||||
pack_context(msg, context)
|
||||
with ConnectionContext(conf, connection_pool) as conn:
|
||||
conn.fanout_send(topic, msg)
|
||||
conn.fanout_send(topic, rpc_common.serialize_msg(msg))
|
||||
|
||||
|
||||
def cast_to_server(conf, context, server_params, topic, msg, connection_pool):
|
||||
|
@ -393,7 +403,7 @@ def cast_to_server(conf, context, server_params, topic, msg, connection_pool):
|
|||
pack_context(msg, context)
|
||||
with ConnectionContext(conf, connection_pool, pooled=False,
|
||||
server_params=server_params) as conn:
|
||||
conn.topic_send(topic, msg)
|
||||
conn.topic_send(topic, rpc_common.serialize_msg(msg))
|
||||
|
||||
|
||||
def fanout_cast_to_server(conf, context, server_params, topic, msg,
|
||||
|
@ -402,15 +412,18 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg,
|
|||
pack_context(msg, context)
|
||||
with ConnectionContext(conf, connection_pool, pooled=False,
|
||||
server_params=server_params) as conn:
|
||||
conn.fanout_send(topic, msg)
|
||||
conn.fanout_send(topic, rpc_common.serialize_msg(msg))
|
||||
|
||||
|
||||
def notify(conf, context, topic, msg, connection_pool):
|
||||
def notify(conf, context, topic, msg, connection_pool, envelope):
|
||||
"""Sends a notification event on a topic."""
|
||||
event_type = msg.get('event_type')
|
||||
LOG.debug(_('Sending %(event_type)s on %(topic)s'), locals())
|
||||
LOG.debug(_('Sending %(event_type)s on %(topic)s'),
|
||||
dict(event_type=msg.get('event_type'),
|
||||
topic=topic))
|
||||
pack_context(msg, context)
|
||||
with ConnectionContext(conf, connection_pool) as conn:
|
||||
if envelope:
|
||||
msg = rpc_common.serialize_msg(msg, force_envelope=True)
|
||||
conn.notify_send(topic, msg)
|
||||
|
||||
|
||||
|
@ -420,7 +433,4 @@ def cleanup(connection_pool):
|
|||
|
||||
|
||||
def get_control_exchange(conf):
|
||||
try:
|
||||
return conf.control_exchange
|
||||
except cfg.NoSuchOptError:
|
||||
return 'openstack'
|
||||
return conf.control_exchange
|
||||
|
|
|
@ -18,18 +18,61 @@
|
|||
# under the License.
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import importutils
|
||||
from kwapi.openstack.common import jsonutils
|
||||
from kwapi.openstack.common import local
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
'''RPC Envelope Version.
|
||||
|
||||
This version number applies to the top level structure of messages sent out.
|
||||
It does *not* apply to the message payload, which must be versioned
|
||||
independently. For example, when using rpc APIs, a version number is applied
|
||||
for changes to the API being exposed over rpc. This version number is handled
|
||||
in the rpc proxy and dispatcher modules.
|
||||
|
||||
This version number applies to the message envelope that is used in the
|
||||
serialization done inside the rpc layer. See serialize_msg() and
|
||||
deserialize_msg().
|
||||
|
||||
The current message format (version 2.0) is very simple. It is:
|
||||
|
||||
{
|
||||
'oslo.version': <RPC Envelope Version as a String>,
|
||||
'oslo.message': <Application Message Payload, JSON encoded>
|
||||
}
|
||||
|
||||
Message format version '1.0' is just considered to be the messages we sent
|
||||
without a message envelope.
|
||||
|
||||
So, the current message envelope just includes the envelope version. It may
|
||||
eventually contain additional information, such as a signature for the message
|
||||
payload.
|
||||
|
||||
We will JSON encode the application message payload. The message envelope,
|
||||
which includes the JSON encoded application message body, will be passed down
|
||||
to the messaging libraries as a dict.
|
||||
'''
|
||||
_RPC_ENVELOPE_VERSION = '2.0'
|
||||
|
||||
_VERSION_KEY = 'oslo.version'
|
||||
_MESSAGE_KEY = 'oslo.message'
|
||||
|
||||
|
||||
# TODO(russellb) Turn this on after Grizzly.
|
||||
_SEND_RPC_ENVELOPE = False
|
||||
|
||||
|
||||
class RPCException(Exception):
|
||||
message = _("An unknown RPC related exception occurred.")
|
||||
|
||||
|
@ -40,7 +83,7 @@ class RPCException(Exception):
|
|||
try:
|
||||
message = self.message % kwargs
|
||||
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# kwargs doesn't match a variable in the message
|
||||
# log the issue and the kwargs
|
||||
LOG.exception(_('Exception in string format operation'))
|
||||
|
@ -90,6 +133,11 @@ class UnsupportedRpcVersion(RPCException):
|
|||
"this endpoint.")
|
||||
|
||||
|
||||
class UnsupportedRpcEnvelopeVersion(RPCException):
|
||||
message = _("Specified RPC envelope version, %(version)s, "
|
||||
"not supported by this endpoint.")
|
||||
|
||||
|
||||
class Connection(object):
|
||||
"""A connection, returned by rpc.create_connection().
|
||||
|
||||
|
@ -164,8 +212,12 @@ class Connection(object):
|
|||
|
||||
def _safe_log(log_func, msg, msg_data):
|
||||
"""Sanitizes the msg_data field before logging."""
|
||||
SANITIZE = {'set_admin_password': ('new_pass',),
|
||||
'run_instance': ('admin_password',), }
|
||||
SANITIZE = {'set_admin_password': [('args', 'new_pass')],
|
||||
'run_instance': [('args', 'admin_password')],
|
||||
'route_message': [('args', 'message', 'args', 'method_info',
|
||||
'method_kwargs', 'password'),
|
||||
('args', 'message', 'args', 'method_info',
|
||||
'method_kwargs', 'admin_password')]}
|
||||
|
||||
has_method = 'method' in msg_data and msg_data['method'] in SANITIZE
|
||||
has_context_token = '_context_auth_token' in msg_data
|
||||
|
@ -177,14 +229,16 @@ def _safe_log(log_func, msg, msg_data):
|
|||
msg_data = copy.deepcopy(msg_data)
|
||||
|
||||
if has_method:
|
||||
method = msg_data['method']
|
||||
if method in SANITIZE:
|
||||
args_to_sanitize = SANITIZE[method]
|
||||
for arg in args_to_sanitize:
|
||||
try:
|
||||
msg_data['args'][arg] = "<SANITIZED>"
|
||||
except KeyError:
|
||||
pass
|
||||
for arg in SANITIZE.get(msg_data['method'], []):
|
||||
try:
|
||||
d = msg_data
|
||||
for elem in arg[:-1]:
|
||||
d = d[elem]
|
||||
d[arg[-1]] = '<SANITIZED>'
|
||||
except KeyError, e:
|
||||
LOG.info(_('Failed to sanitize %(item)s. Key error %(err)s'),
|
||||
{'item': arg,
|
||||
'err': e})
|
||||
|
||||
if has_context_token:
|
||||
msg_data['_context_auth_token'] = '<SANITIZED>'
|
||||
|
@ -195,7 +249,7 @@ def _safe_log(log_func, msg, msg_data):
|
|||
return log_func(msg, msg_data)
|
||||
|
||||
|
||||
def serialize_remote_exception(failure_info):
|
||||
def serialize_remote_exception(failure_info, log_failure=True):
|
||||
"""Prepares exception data to be sent over rpc.
|
||||
|
||||
Failure_info should be a sys.exc_info() tuple.
|
||||
|
@ -203,8 +257,9 @@ def serialize_remote_exception(failure_info):
|
|||
"""
|
||||
tb = traceback.format_exception(*failure_info)
|
||||
failure = failure_info[1]
|
||||
LOG.error(_("Returning exception %s to caller"), unicode(failure))
|
||||
LOG.error(tb)
|
||||
if log_failure:
|
||||
LOG.error(_("Returning exception %s to caller"), unicode(failure))
|
||||
LOG.error(tb)
|
||||
|
||||
kwargs = {}
|
||||
if hasattr(failure, 'kwargs'):
|
||||
|
@ -258,7 +313,7 @@ def deserialize_remote_exception(conf, data):
|
|||
# we cannot necessarily change an exception message so we must override
|
||||
# the __str__ method.
|
||||
failure.__class__ = new_ex_type
|
||||
except TypeError as e:
|
||||
except TypeError:
|
||||
# NOTE(ameade): If a core exception then just add the traceback to the
|
||||
# first exception argument.
|
||||
failure.args = (message,) + failure.args[1:]
|
||||
|
@ -309,3 +364,107 @@ class CommonRpcContext(object):
|
|||
context.values['read_deleted'] = read_deleted
|
||||
|
||||
return context
|
||||
|
||||
|
||||
class ClientException(Exception):
|
||||
"""This encapsulates some actual exception that is expected to be
|
||||
hit by an RPC proxy object. Merely instantiating it records the
|
||||
current exception information, which will be passed back to the
|
||||
RPC client without exceptional logging."""
|
||||
def __init__(self):
|
||||
self._exc_info = sys.exc_info()
|
||||
|
||||
|
||||
def catch_client_exception(exceptions, func, *args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception, e:
|
||||
if type(e) in exceptions:
|
||||
raise ClientException()
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def client_exceptions(*exceptions):
|
||||
"""Decorator for manager methods that raise expected exceptions.
|
||||
Marking a Manager method with this decorator allows the declaration
|
||||
of expected exceptions that the RPC layer should not consider fatal,
|
||||
and not log as if they were generated in a real error scenario. Note
|
||||
that this will cause listed exceptions to be wrapped in a
|
||||
ClientException, which is used internally by the RPC layer."""
|
||||
def outer(func):
|
||||
def inner(*args, **kwargs):
|
||||
return catch_client_exception(exceptions, func, *args, **kwargs)
|
||||
return inner
|
||||
return outer
|
||||
|
||||
|
||||
def version_is_compatible(imp_version, version):
|
||||
"""Determine whether versions are compatible.
|
||||
|
||||
:param imp_version: The version implemented
|
||||
:param version: The version requested by an incoming message.
|
||||
"""
|
||||
version_parts = version.split('.')
|
||||
imp_version_parts = imp_version.split('.')
|
||||
if int(version_parts[0]) != int(imp_version_parts[0]): # Major
|
||||
return False
|
||||
if int(version_parts[1]) > int(imp_version_parts[1]): # Minor
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def serialize_msg(raw_msg, force_envelope=False):
|
||||
if not _SEND_RPC_ENVELOPE and not force_envelope:
|
||||
return raw_msg
|
||||
|
||||
# NOTE(russellb) See the docstring for _RPC_ENVELOPE_VERSION for more
|
||||
# information about this format.
|
||||
msg = {_VERSION_KEY: _RPC_ENVELOPE_VERSION,
|
||||
_MESSAGE_KEY: jsonutils.dumps(raw_msg)}
|
||||
|
||||
return msg
|
||||
|
||||
|
||||
def deserialize_msg(msg):
|
||||
# NOTE(russellb): Hang on to your hats, this road is about to
|
||||
# get a little bumpy.
|
||||
#
|
||||
# Robustness Principle:
|
||||
# "Be strict in what you send, liberal in what you accept."
|
||||
#
|
||||
# At this point we have to do a bit of guessing about what it
|
||||
# is we just received. Here is the set of possibilities:
|
||||
#
|
||||
# 1) We received a dict. This could be 2 things:
|
||||
#
|
||||
# a) Inspect it to see if it looks like a standard message envelope.
|
||||
# If so, great!
|
||||
#
|
||||
# b) If it doesn't look like a standard message envelope, it could either
|
||||
# be a notification, or a message from before we added a message
|
||||
# envelope (referred to as version 1.0).
|
||||
# Just return the message as-is.
|
||||
#
|
||||
# 2) It's any other non-dict type. Just return it and hope for the best.
|
||||
# This case covers return values from rpc.call() from before message
|
||||
# envelopes were used. (messages to call a method were always a dict)
|
||||
|
||||
if not isinstance(msg, dict):
|
||||
# See #2 above.
|
||||
return msg
|
||||
|
||||
base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY)
|
||||
if not all(map(lambda key: key in msg, base_envelope_keys)):
|
||||
# See #1.b above.
|
||||
return msg
|
||||
|
||||
# At this point we think we have the message envelope
|
||||
# format we were expecting. (#1.a above)
|
||||
|
||||
if not version_is_compatible(_RPC_ENVELOPE_VERSION, msg[_VERSION_KEY]):
|
||||
raise UnsupportedRpcEnvelopeVersion(version=msg[_VERSION_KEY])
|
||||
|
||||
raw_msg = jsonutils.loads(msg[_MESSAGE_KEY])
|
||||
|
||||
return raw_msg
|
||||
|
|
|
@ -41,8 +41,8 @@ server side of the API at the same time. However, as the code stands today,
|
|||
there can be both versioned and unversioned APIs implemented in the same code
|
||||
base.
|
||||
|
||||
|
||||
EXAMPLES:
|
||||
EXAMPLES
|
||||
========
|
||||
|
||||
Nova was the first project to use versioned rpc APIs. Consider the compute rpc
|
||||
API as an example. The client side is in nova/compute/rpcapi.py and the server
|
||||
|
@ -50,12 +50,13 @@ side is in nova/compute/manager.py.
|
|||
|
||||
|
||||
Example 1) Adding a new method.
|
||||
-------------------------------
|
||||
|
||||
Adding a new method is a backwards compatible change. It should be added to
|
||||
nova/compute/manager.py, and RPC_API_VERSION should be bumped from X.Y to
|
||||
X.Y+1. On the client side, the new method in nova/compute/rpcapi.py should
|
||||
have a specific version specified to indicate the minimum API version that must
|
||||
be implemented for the method to be supported. For example:
|
||||
be implemented for the method to be supported. For example::
|
||||
|
||||
def get_host_uptime(self, ctxt, host):
|
||||
topic = _compute_topic(self.topic, ctxt, host, None)
|
||||
|
@ -67,10 +68,11 @@ get_host_uptime() method.
|
|||
|
||||
|
||||
Example 2) Adding a new parameter.
|
||||
----------------------------------
|
||||
|
||||
Adding a new parameter to an rpc method can be made backwards compatible. The
|
||||
RPC_API_VERSION on the server side (nova/compute/manager.py) should be bumped.
|
||||
The implementation of the method must not expect the parameter to be present.
|
||||
The implementation of the method must not expect the parameter to be present.::
|
||||
|
||||
def some_remote_method(self, arg1, arg2, newarg=None):
|
||||
# The code needs to deal with newarg=None for cases
|
||||
|
@ -101,21 +103,6 @@ class RpcDispatcher(object):
|
|||
self.callbacks = callbacks
|
||||
super(RpcDispatcher, self).__init__()
|
||||
|
||||
@staticmethod
|
||||
def _is_compatible(mversion, version):
|
||||
"""Determine whether versions are compatible.
|
||||
|
||||
:param mversion: The API version implemented by a callback.
|
||||
:param version: The API version requested by an incoming message.
|
||||
"""
|
||||
version_parts = version.split('.')
|
||||
mversion_parts = mversion.split('.')
|
||||
if int(version_parts[0]) != int(mversion_parts[0]): # Major
|
||||
return False
|
||||
if int(version_parts[1]) > int(mversion_parts[1]): # Minor
|
||||
return False
|
||||
return True
|
||||
|
||||
def dispatch(self, ctxt, version, method, **kwargs):
|
||||
"""Dispatch a message based on a requested version.
|
||||
|
||||
|
@ -137,7 +124,8 @@ class RpcDispatcher(object):
|
|||
rpc_api_version = proxyobj.RPC_API_VERSION
|
||||
else:
|
||||
rpc_api_version = '1.0'
|
||||
is_compatible = self._is_compatible(rpc_api_version, version)
|
||||
is_compatible = rpc_common.version_is_compatible(rpc_api_version,
|
||||
version)
|
||||
had_compatible = had_compatible or is_compatible
|
||||
if not hasattr(proxyobj, method):
|
||||
continue
|
||||
|
|
|
@ -18,11 +18,15 @@ queues. Casts will block, but this is very useful for tests.
|
|||
"""
|
||||
|
||||
import inspect
|
||||
# NOTE(russellb): We specifically want to use json, not our own jsonutils.
|
||||
# jsonutils has some extra logic to automatically convert objects to primitive
|
||||
# types so that they can be serialized. We want to catch all cases where
|
||||
# non-primitive types make it into this code and treat it as an error.
|
||||
import json
|
||||
import time
|
||||
|
||||
import eventlet
|
||||
|
||||
from kwapi.openstack.common import jsonutils
|
||||
from kwapi.openstack.common.rpc import common as rpc_common
|
||||
|
||||
CONSUMERS = {}
|
||||
|
@ -75,6 +79,8 @@ class Consumer(object):
|
|||
else:
|
||||
res.append(rval)
|
||||
done.send(res)
|
||||
except rpc_common.ClientException as e:
|
||||
done.send_exception(e._exc_info[1])
|
||||
except Exception as e:
|
||||
done.send_exception(e)
|
||||
|
||||
|
@ -121,7 +127,7 @@ def create_connection(conf, new=True):
|
|||
|
||||
def check_serialize(msg):
|
||||
"""Make sure a message intended for rpc can be serialized."""
|
||||
jsonutils.dumps(msg)
|
||||
json.dumps(msg)
|
||||
|
||||
|
||||
def multicall(conf, context, topic, msg, timeout=None):
|
||||
|
@ -154,6 +160,7 @@ def call(conf, context, topic, msg, timeout=None):
|
|||
|
||||
|
||||
def cast(conf, context, topic, msg):
|
||||
check_serialize(msg)
|
||||
try:
|
||||
call(conf, context, topic, msg)
|
||||
except Exception:
|
||||
|
|
|
@ -162,7 +162,8 @@ class ConsumerBase(object):
|
|||
def _callback(raw_message):
|
||||
message = self.channel.message_to_python(raw_message)
|
||||
try:
|
||||
callback(message.payload)
|
||||
msg = rpc_common.deserialize_msg(message.payload)
|
||||
callback(msg)
|
||||
message.ack()
|
||||
except Exception:
|
||||
LOG.exception(_("Failed to process message... skipping it."))
|
||||
|
@ -196,7 +197,7 @@ class DirectConsumer(ConsumerBase):
|
|||
# Default options
|
||||
options = {'durable': False,
|
||||
'auto_delete': True,
|
||||
'exclusive': True}
|
||||
'exclusive': False}
|
||||
options.update(kwargs)
|
||||
exchange = kombu.entity.Exchange(name=msg_id,
|
||||
type='direct',
|
||||
|
@ -269,7 +270,7 @@ class FanoutConsumer(ConsumerBase):
|
|||
options = {'durable': False,
|
||||
'queue_arguments': _get_queue_arguments(conf),
|
||||
'auto_delete': True,
|
||||
'exclusive': True}
|
||||
'exclusive': False}
|
||||
options.update(kwargs)
|
||||
exchange = kombu.entity.Exchange(name=exchange_name, type='fanout',
|
||||
durable=options['durable'],
|
||||
|
@ -316,7 +317,7 @@ class DirectPublisher(Publisher):
|
|||
|
||||
options = {'durable': False,
|
||||
'auto_delete': True,
|
||||
'exclusive': True}
|
||||
'exclusive': False}
|
||||
options.update(kwargs)
|
||||
super(DirectPublisher, self).__init__(channel, msg_id, msg_id,
|
||||
type='direct', **options)
|
||||
|
@ -350,7 +351,7 @@ class FanoutPublisher(Publisher):
|
|||
"""
|
||||
options = {'durable': False,
|
||||
'auto_delete': True,
|
||||
'exclusive': True}
|
||||
'exclusive': False}
|
||||
options.update(kwargs)
|
||||
super(FanoutPublisher, self).__init__(channel, '%s_fanout' % topic,
|
||||
None, type='fanout', **options)
|
||||
|
@ -387,6 +388,7 @@ class Connection(object):
|
|||
def __init__(self, conf, server_params=None):
|
||||
self.consumers = []
|
||||
self.consumer_thread = None
|
||||
self.proxy_callbacks = []
|
||||
self.conf = conf
|
||||
self.max_retries = self.conf.rabbit_max_retries
|
||||
# Try forever?
|
||||
|
@ -469,7 +471,7 @@ class Connection(object):
|
|||
LOG.info(_("Reconnecting to AMQP server on "
|
||||
"%(hostname)s:%(port)d") % params)
|
||||
try:
|
||||
self.connection.close()
|
||||
self.connection.release()
|
||||
except self.connection_errors:
|
||||
pass
|
||||
# Setting this in case the next statement fails, though
|
||||
|
@ -573,12 +575,14 @@ class Connection(object):
|
|||
def close(self):
|
||||
"""Close/release this connection"""
|
||||
self.cancel_consumer_thread()
|
||||
self.wait_on_proxy_callbacks()
|
||||
self.connection.release()
|
||||
self.connection = None
|
||||
|
||||
def reset(self):
|
||||
"""Reset a connection so it can be used again"""
|
||||
self.cancel_consumer_thread()
|
||||
self.wait_on_proxy_callbacks()
|
||||
self.channel.close()
|
||||
self.channel = self.connection.channel()
|
||||
# work around 'memory' transport bug in 1.1.3
|
||||
|
@ -644,6 +648,11 @@ class Connection(object):
|
|||
pass
|
||||
self.consumer_thread = None
|
||||
|
||||
def wait_on_proxy_callbacks(self):
|
||||
"""Wait for all proxy callback threads to exit."""
|
||||
for proxy_cb in self.proxy_callbacks:
|
||||
proxy_cb.wait()
|
||||
|
||||
def publisher_send(self, cls, topic, msg, **kwargs):
|
||||
"""Send to a publisher based on the publisher class"""
|
||||
|
||||
|
@ -719,6 +728,7 @@ class Connection(object):
|
|||
proxy_cb = rpc_amqp.ProxyCallback(
|
||||
self.conf, proxy,
|
||||
rpc_amqp.get_connection_pool(self.conf, Connection))
|
||||
self.proxy_callbacks.append(proxy_cb)
|
||||
|
||||
if fanout:
|
||||
self.declare_fanout_consumer(topic, proxy_cb)
|
||||
|
@ -730,6 +740,7 @@ class Connection(object):
|
|||
proxy_cb = rpc_amqp.ProxyCallback(
|
||||
self.conf, proxy,
|
||||
rpc_amqp.get_connection_pool(self.conf, Connection))
|
||||
self.proxy_callbacks.append(proxy_cb)
|
||||
self.declare_topic_consumer(topic, proxy_cb, pool_name)
|
||||
|
||||
|
||||
|
@ -782,11 +793,12 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg):
|
|||
rpc_amqp.get_connection_pool(conf, Connection))
|
||||
|
||||
|
||||
def notify(conf, context, topic, msg):
|
||||
def notify(conf, context, topic, msg, envelope):
|
||||
"""Sends a notification event on a topic."""
|
||||
return rpc_amqp.notify(
|
||||
conf, context, topic, msg,
|
||||
rpc_amqp.get_connection_pool(conf, Connection))
|
||||
rpc_amqp.get_connection_pool(conf, Connection),
|
||||
envelope)
|
||||
|
||||
|
||||
def cleanup():
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
import functools
|
||||
import itertools
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
@ -29,6 +28,7 @@ import qpid.messaging.exceptions
|
|||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import jsonutils
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.rpc import amqp as rpc_amqp
|
||||
from kwapi.openstack.common.rpc import common as rpc_common
|
||||
|
||||
|
@ -41,6 +41,9 @@ qpid_opts = [
|
|||
cfg.StrOpt('qpid_port',
|
||||
default='5672',
|
||||
help='Qpid broker port'),
|
||||
cfg.ListOpt('qpid_hosts',
|
||||
default=['$qpid_hostname:$qpid_port'],
|
||||
help='Qpid HA cluster host:port pairs'),
|
||||
cfg.StrOpt('qpid_username',
|
||||
default='',
|
||||
help='Username for qpid connection'),
|
||||
|
@ -121,7 +124,8 @@ class ConsumerBase(object):
|
|||
"""Fetch the message and pass it to the callback object"""
|
||||
message = self.receiver.fetch()
|
||||
try:
|
||||
self.callback(message.content)
|
||||
msg = rpc_common.deserialize_msg(message.content)
|
||||
self.callback(msg)
|
||||
except Exception:
|
||||
LOG.exception(_("Failed to process message... skipping it."))
|
||||
finally:
|
||||
|
@ -274,25 +278,32 @@ class Connection(object):
|
|||
self.session = None
|
||||
self.consumers = {}
|
||||
self.consumer_thread = None
|
||||
self.proxy_callbacks = []
|
||||
self.conf = conf
|
||||
|
||||
if server_params and 'hostname' in server_params:
|
||||
# NOTE(russellb) This enables support for cast_to_server.
|
||||
server_params['qpid_hosts'] = [
|
||||
'%s:%d' % (server_params['hostname'],
|
||||
server_params.get('port', 5672))
|
||||
]
|
||||
|
||||
params = {
|
||||
'hostname': self.conf.qpid_hostname,
|
||||
'port': self.conf.qpid_port,
|
||||
'qpid_hosts': self.conf.qpid_hosts,
|
||||
'username': self.conf.qpid_username,
|
||||
'password': self.conf.qpid_password,
|
||||
}
|
||||
params.update(server_params or {})
|
||||
|
||||
self.broker = params['hostname'] + ":" + str(params['port'])
|
||||
self.brokers = params['qpid_hosts']
|
||||
self.username = params['username']
|
||||
self.password = params['password']
|
||||
self.connection_create()
|
||||
self.connection_create(self.brokers[0])
|
||||
self.reconnect()
|
||||
|
||||
def connection_create(self):
|
||||
def connection_create(self, broker):
|
||||
# Create the connection - this does not open the connection
|
||||
self.connection = qpid.messaging.Connection(self.broker)
|
||||
self.connection = qpid.messaging.Connection(broker)
|
||||
|
||||
# Check if flags are set and if so set them for the connection
|
||||
# before we call open
|
||||
|
@ -320,10 +331,14 @@ class Connection(object):
|
|||
except qpid.messaging.exceptions.ConnectionError:
|
||||
pass
|
||||
|
||||
attempt = 0
|
||||
delay = 1
|
||||
while True:
|
||||
broker = self.brokers[attempt % len(self.brokers)]
|
||||
attempt += 1
|
||||
|
||||
try:
|
||||
self.connection_create()
|
||||
self.connection_create(broker)
|
||||
self.connection.open()
|
||||
except qpid.messaging.exceptions.ConnectionError, e:
|
||||
msg_dict = dict(e=e, delay=delay)
|
||||
|
@ -333,10 +348,9 @@ class Connection(object):
|
|||
time.sleep(delay)
|
||||
delay = min(2 * delay, 60)
|
||||
else:
|
||||
LOG.info(_('Connected to AMQP server on %s'), broker)
|
||||
break
|
||||
|
||||
LOG.info(_('Connected to AMQP server on %s'), self.broker)
|
||||
|
||||
self.session = self.connection.session()
|
||||
|
||||
if self.consumers:
|
||||
|
@ -362,12 +376,14 @@ class Connection(object):
|
|||
def close(self):
|
||||
"""Close/release this connection"""
|
||||
self.cancel_consumer_thread()
|
||||
self.wait_on_proxy_callbacks()
|
||||
self.connection.close()
|
||||
self.connection = None
|
||||
|
||||
def reset(self):
|
||||
"""Reset a connection so it can be used again"""
|
||||
self.cancel_consumer_thread()
|
||||
self.wait_on_proxy_callbacks()
|
||||
self.session.close()
|
||||
self.session = self.connection.session()
|
||||
self.consumers = {}
|
||||
|
@ -422,6 +438,11 @@ class Connection(object):
|
|||
pass
|
||||
self.consumer_thread = None
|
||||
|
||||
def wait_on_proxy_callbacks(self):
|
||||
"""Wait for all proxy callback threads to exit."""
|
||||
for proxy_cb in self.proxy_callbacks:
|
||||
proxy_cb.wait()
|
||||
|
||||
def publisher_send(self, cls, topic, msg):
|
||||
"""Send to a publisher based on the publisher class"""
|
||||
|
||||
|
@ -497,6 +518,7 @@ class Connection(object):
|
|||
proxy_cb = rpc_amqp.ProxyCallback(
|
||||
self.conf, proxy,
|
||||
rpc_amqp.get_connection_pool(self.conf, Connection))
|
||||
self.proxy_callbacks.append(proxy_cb)
|
||||
|
||||
if fanout:
|
||||
consumer = FanoutConsumer(self.conf, self.session, topic, proxy_cb)
|
||||
|
@ -512,6 +534,7 @@ class Connection(object):
|
|||
proxy_cb = rpc_amqp.ProxyCallback(
|
||||
self.conf, proxy,
|
||||
rpc_amqp.get_connection_pool(self.conf, Connection))
|
||||
self.proxy_callbacks.append(proxy_cb)
|
||||
|
||||
consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb,
|
||||
name=pool_name)
|
||||
|
@ -570,10 +593,11 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg):
|
|||
rpc_amqp.get_connection_pool(conf, Connection))
|
||||
|
||||
|
||||
def notify(conf, context, topic, msg):
|
||||
def notify(conf, context, topic, msg, envelope):
|
||||
"""Sends a notification event on a topic."""
|
||||
return rpc_amqp.notify(conf, context, topic, msg,
|
||||
rpc_amqp.get_connection_pool(conf, Connection))
|
||||
rpc_amqp.get_connection_pool(conf, Connection),
|
||||
envelope)
|
||||
|
||||
|
||||
def cleanup():
|
||||
|
|
|
@ -49,7 +49,7 @@ zmq_opts = [
|
|||
# The module.Class to use for matchmaking.
|
||||
cfg.StrOpt(
|
||||
'rpc_zmq_matchmaker',
|
||||
default=('kwapi.openstack.common.rpc.'
|
||||
default=('openstack.common.rpc.'
|
||||
'matchmaker.MatchMakerLocalhost'),
|
||||
help='MatchMaker driver',
|
||||
),
|
||||
|
@ -205,7 +205,9 @@ class ZmqClient(object):
|
|||
def __init__(self, addr, socket_type=zmq.PUSH, bind=False):
|
||||
self.outq = ZmqSocket(addr, socket_type, bind=bind)
|
||||
|
||||
def cast(self, msg_id, topic, data):
|
||||
def cast(self, msg_id, topic, data, serialize=True, force_envelope=False):
|
||||
if serialize:
|
||||
data = rpc_common.serialize_msg(data, force_envelope)
|
||||
self.outq.send([str(msg_id), str(topic), str('cast'),
|
||||
_serialize(data)])
|
||||
|
||||
|
@ -250,7 +252,7 @@ class InternalContext(object):
|
|||
"""Process a curried message and cast the result to topic."""
|
||||
LOG.debug(_("Running func with context: %s"), ctx.to_dict())
|
||||
data.setdefault('version', None)
|
||||
data.setdefault('args', [])
|
||||
data.setdefault('args', {})
|
||||
|
||||
try:
|
||||
result = proxy.dispatch(
|
||||
|
@ -259,7 +261,14 @@ class InternalContext(object):
|
|||
except greenlet.GreenletExit:
|
||||
# ignore these since they are just from shutdowns
|
||||
pass
|
||||
except rpc_common.ClientException, e:
|
||||
LOG.debug(_("Expected exception during message handling (%s)") %
|
||||
e._exc_info[1])
|
||||
return {'exc':
|
||||
rpc_common.serialize_remote_exception(e._exc_info,
|
||||
log_failure=False)}
|
||||
except Exception:
|
||||
LOG.error(_("Exception during message handling"))
|
||||
return {'exc':
|
||||
rpc_common.serialize_remote_exception(sys.exc_info())}
|
||||
|
||||
|
@ -314,7 +323,7 @@ class ConsumerBase(object):
|
|||
return
|
||||
|
||||
data.setdefault('version', None)
|
||||
data.setdefault('args', [])
|
||||
data.setdefault('args', {})
|
||||
proxy.dispatch(ctx, data['version'],
|
||||
data['method'], **data['args'])
|
||||
|
||||
|
@ -426,7 +435,7 @@ class ZmqProxy(ZmqBaseReactor):
|
|||
sock_type = zmq.PUB
|
||||
elif topic.startswith('zmq_replies'):
|
||||
sock_type = zmq.PUB
|
||||
inside = _deserialize(in_msg)
|
||||
inside = rpc_common.deserialize_msg(_deserialize(in_msg))
|
||||
msg_id = inside[-1]['args']['msg_id']
|
||||
response = inside[-1]['args']['response']
|
||||
LOG.debug(_("->response->%s"), response)
|
||||
|
@ -473,7 +482,7 @@ class ZmqReactor(ZmqBaseReactor):
|
|||
|
||||
msg_id, topic, style, in_msg = data
|
||||
|
||||
ctx, request = _deserialize(in_msg)
|
||||
ctx, request = rpc_common.deserialize_msg(_deserialize(in_msg))
|
||||
ctx = RpcContext.unmarshal(ctx)
|
||||
|
||||
proxy = self.proxies[sock]
|
||||
|
@ -524,7 +533,8 @@ class Connection(rpc_common.Connection):
|
|||
self.reactor.consume_in_thread()
|
||||
|
||||
|
||||
def _cast(addr, context, msg_id, topic, msg, timeout=None):
|
||||
def _cast(addr, context, msg_id, topic, msg, timeout=None, serialize=True,
|
||||
force_envelope=False):
|
||||
timeout_cast = timeout or CONF.rpc_cast_timeout
|
||||
payload = [RpcContext.marshal(context), msg]
|
||||
|
||||
|
@ -533,7 +543,7 @@ def _cast(addr, context, msg_id, topic, msg, timeout=None):
|
|||
conn = ZmqClient(addr)
|
||||
|
||||
# assumes cast can't return an exception
|
||||
conn.cast(msg_id, topic, payload)
|
||||
conn.cast(msg_id, topic, payload, serialize, force_envelope)
|
||||
except zmq.ZMQError:
|
||||
raise RPCException("Cast failed. ZMQ Socket Exception")
|
||||
finally:
|
||||
|
@ -602,7 +612,8 @@ def _call(addr, context, msg_id, topic, msg, timeout=None):
|
|||
return responses[-1]
|
||||
|
||||
|
||||
def _multi_send(method, context, topic, msg, timeout=None):
|
||||
def _multi_send(method, context, topic, msg, timeout=None, serialize=True,
|
||||
force_envelope=False):
|
||||
"""
|
||||
Wraps the sending of messages,
|
||||
dispatches to the matchmaker and sends
|
||||
|
@ -628,7 +639,8 @@ def _multi_send(method, context, topic, msg, timeout=None):
|
|||
|
||||
if method.__name__ == '_cast':
|
||||
eventlet.spawn_n(method, _addr, context,
|
||||
_topic, _topic, msg, timeout)
|
||||
_topic, _topic, msg, timeout, serialize,
|
||||
force_envelope)
|
||||
return
|
||||
return method(_addr, context, _topic, _topic, msg, timeout)
|
||||
|
||||
|
@ -669,6 +681,8 @@ def notify(conf, context, topic, msg, **kwargs):
|
|||
# NOTE(ewindisch): dot-priority in rpc notifier does not
|
||||
# work with our assumptions.
|
||||
topic.replace('.', '-')
|
||||
kwargs['serialize'] = kwargs.pop('envelope')
|
||||
kwargs['force_envelope'] = True
|
||||
cast(conf, context, topic, msg, **kwargs)
|
||||
|
||||
|
||||
|
|
|
@ -21,10 +21,10 @@ return keys for direct exchanges, per (approximate) AMQP parlance.
|
|||
import contextlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
|
||||
|
||||
matchmaker_opts = [
|
||||
|
|
|
@ -0,0 +1,71 @@
|
|||
# Copyright (c) 2011-2012 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Filter support
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
from stevedore import extension
|
||||
|
||||
|
||||
class BaseFilter(object):
|
||||
"""Base class for all filter classes."""
|
||||
def _filter_one(self, obj, filter_properties):
|
||||
"""Return True if it passes the filter, False otherwise.
|
||||
Override this in a subclass.
|
||||
"""
|
||||
return True
|
||||
|
||||
def filter_all(self, filter_obj_list, filter_properties):
|
||||
"""Yield objects that pass the filter.
|
||||
|
||||
Can be overriden in a subclass, if you need to base filtering
|
||||
decisions on all objects. Otherwise, one can just override
|
||||
_filter_one() to filter a single object.
|
||||
"""
|
||||
for obj in filter_obj_list:
|
||||
if self._filter_one(obj, filter_properties):
|
||||
yield obj
|
||||
|
||||
|
||||
class BaseFilterHandler(object):
|
||||
""" Base class to handle loading filter classes.
|
||||
|
||||
This class should be subclassed where one needs to use filters.
|
||||
"""
|
||||
def __init__(self, filter_class_type, filter_namespace):
|
||||
self.namespace = filter_namespace
|
||||
self.filter_class_type = filter_class_type
|
||||
self.filter_manager = extension.ExtensionManager(filter_namespace)
|
||||
|
||||
def _is_correct_class(self, obj):
|
||||
"""Return whether an object is a class of the correct type and
|
||||
is not prefixed with an underscore.
|
||||
"""
|
||||
return (inspect.isclass(obj) and
|
||||
not obj.__name__.startswith('_') and
|
||||
issubclass(obj, self.filter_class_type))
|
||||
|
||||
def get_all_classes(self):
|
||||
return [x.plugin for x in self.filter_manager
|
||||
if self._is_correct_class(x.plugin)]
|
||||
|
||||
def get_filtered_objects(self, filter_classes, objs,
|
||||
filter_properties):
|
||||
for filter_cls in filter_classes:
|
||||
objs = filter_cls().filter_all(objs, filter_properties)
|
||||
return list(objs)
|
|
@ -0,0 +1,41 @@
|
|||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Scheduler host filters
|
||||
"""
|
||||
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.scheduler import filter
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseHostFilter(filter.BaseFilter):
|
||||
"""Base class for host filters."""
|
||||
def _filter_one(self, obj, filter_properties):
|
||||
"""Return True if the object passes the filter, otherwise False."""
|
||||
return self.host_passes(obj, filter_properties)
|
||||
|
||||
def host_passes(self, host_state, filter_properties):
|
||||
"""Return True if the HostState passes the filter, otherwise False.
|
||||
Override this in a subclass.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class HostFilterHandler(filter.BaseFilterHandler):
|
||||
def __init__(self, namespace):
|
||||
super(HostFilterHandler, self).__init__(BaseHostFilter, namespace)
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright (c) 2011-2012 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from kwapi.openstack.common.scheduler import filters
|
||||
|
||||
|
||||
class AvailabilityZoneFilter(filters.BaseHostFilter):
|
||||
"""Filters Hosts by availability zone."""
|
||||
|
||||
def host_passes(self, host_state, filter_properties):
|
||||
spec = filter_properties.get('request_spec', {})
|
||||
props = spec.get('resource_properties', [])
|
||||
availability_zone = props.get('availability_zone')
|
||||
|
||||
if availability_zone:
|
||||
return availability_zone == host_state.service['availability_zone']
|
||||
return True
|
|
@ -0,0 +1,63 @@
|
|||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common.scheduler import filters
|
||||
from kwapi.openstack.common.scheduler.filters import extra_specs_ops
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CapabilitiesFilter(filters.BaseHostFilter):
|
||||
"""HostFilter to work with resource (instance & volume) type records."""
|
||||
|
||||
def _satisfies_extra_specs(self, capabilities, resource_type):
|
||||
"""Check that the capabilities provided by the services
|
||||
satisfy the extra specs associated with the instance type"""
|
||||
extra_specs = resource_type.get('extra_specs', [])
|
||||
if not extra_specs:
|
||||
return True
|
||||
|
||||
for key, req in extra_specs.iteritems():
|
||||
# Either not scope format, or in capabilities scope
|
||||
scope = key.split(':')
|
||||
if len(scope) > 1 and scope[0] != "capabilities":
|
||||
continue
|
||||
elif scope[0] == "capabilities":
|
||||
del scope[0]
|
||||
|
||||
cap = capabilities
|
||||
for index in range(0, len(scope)):
|
||||
try:
|
||||
cap = cap.get(scope[index], None)
|
||||
except AttributeError:
|
||||
return False
|
||||
if cap is None:
|
||||
return False
|
||||
if not extra_specs_ops.match(cap, req):
|
||||
return False
|
||||
return True
|
||||
|
||||
def host_passes(self, host_state, filter_properties):
|
||||
"""Return a list of hosts that can create instance_type."""
|
||||
# Note(zhiteng) Currently only Cinder and Nova are using
|
||||
# this filter, so the resource type is either instance or
|
||||
# volume.
|
||||
resource_type = filter_properties.get('resource_type')
|
||||
if not self._satisfies_extra_specs(host_state.capabilities,
|
||||
resource_type):
|
||||
return False
|
||||
return True
|
|
@ -0,0 +1,68 @@
|
|||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import operator
|
||||
|
||||
# 1. The following operations are supported:
|
||||
# =, s==, s!=, s>=, s>, s<=, s<, <in>, <or>, ==, !=, >=, <=
|
||||
# 2. Note that <or> is handled in a different way below.
|
||||
# 3. If the first word in the extra_specs is not one of the operators,
|
||||
# it is ignored.
|
||||
_op_methods = {'=': lambda x, y: float(x) >= float(y),
|
||||
'<in>': lambda x, y: y in x,
|
||||
'==': lambda x, y: float(x) == float(y),
|
||||
'!=': lambda x, y: float(x) != float(y),
|
||||
'>=': lambda x, y: float(x) >= float(y),
|
||||
'<=': lambda x, y: float(x) <= float(y),
|
||||
's==': operator.eq,
|
||||
's!=': operator.ne,
|
||||
's<': operator.lt,
|
||||
's<=': operator.le,
|
||||
's>': operator.gt,
|
||||
's>=': operator.ge}
|
||||
|
||||
|
||||
def match(value, req):
|
||||
words = req.split()
|
||||
|
||||
op = method = None
|
||||
if words:
|
||||
op = words.pop(0)
|
||||
method = _op_methods.get(op)
|
||||
|
||||
if op != '<or>' and not method:
|
||||
return value == req
|
||||
|
||||
if value is None:
|
||||
return False
|
||||
|
||||
if op == '<or>': # Ex: <or> v1 <or> v2 <or> v3
|
||||
while True:
|
||||
if words.pop(0) == value:
|
||||
return True
|
||||
if not words:
|
||||
break
|
||||
op = words.pop(0) # remove a keyword <or>
|
||||
if not words:
|
||||
break
|
||||
return False
|
||||
|
||||
try:
|
||||
if words and method(value, words[0]):
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return False
|
|
@ -0,0 +1,150 @@
|
|||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import operator
|
||||
|
||||
from kwapi.openstack.common import jsonutils
|
||||
from kwapi.openstack.common.scheduler import filters
|
||||
|
||||
|
||||
class JsonFilter(filters.BaseHostFilter):
|
||||
"""Host Filter to allow simple JSON-based grammar for
|
||||
selecting hosts.
|
||||
"""
|
||||
def _op_compare(self, args, op):
|
||||
"""Returns True if the specified operator can successfully
|
||||
compare the first item in the args with all the rest. Will
|
||||
return False if only one item is in the list.
|
||||
"""
|
||||
if len(args) < 2:
|
||||
return False
|
||||
if op is operator.contains:
|
||||
bad = not args[0] in args[1:]
|
||||
else:
|
||||
bad = [arg for arg in args[1:]
|
||||
if not op(args[0], arg)]
|
||||
return not bool(bad)
|
||||
|
||||
def _equals(self, args):
|
||||
"""First term is == all the other terms."""
|
||||
return self._op_compare(args, operator.eq)
|
||||
|
||||
def _less_than(self, args):
|
||||
"""First term is < all the other terms."""
|
||||
return self._op_compare(args, operator.lt)
|
||||
|
||||
def _greater_than(self, args):
|
||||
"""First term is > all the other terms."""
|
||||
return self._op_compare(args, operator.gt)
|
||||
|
||||
def _in(self, args):
|
||||
"""First term is in set of remaining terms"""
|
||||
return self._op_compare(args, operator.contains)
|
||||
|
||||
def _less_than_equal(self, args):
|
||||
"""First term is <= all the other terms."""
|
||||
return self._op_compare(args, operator.le)
|
||||
|
||||
def _greater_than_equal(self, args):
|
||||
"""First term is >= all the other terms."""
|
||||
return self._op_compare(args, operator.ge)
|
||||
|
||||
def _not(self, args):
|
||||
"""Flip each of the arguments."""
|
||||
return [not arg for arg in args]
|
||||
|
||||
def _or(self, args):
|
||||
"""True if any arg is True."""
|
||||
return any(args)
|
||||
|
||||
def _and(self, args):
|
||||
"""True if all args are True."""
|
||||
return all(args)
|
||||
|
||||
commands = {
|
||||
'=': _equals,
|
||||
'<': _less_than,
|
||||
'>': _greater_than,
|
||||
'in': _in,
|
||||
'<=': _less_than_equal,
|
||||
'>=': _greater_than_equal,
|
||||
'not': _not,
|
||||
'or': _or,
|
||||
'and': _and,
|
||||
}
|
||||
|
||||
def _parse_string(self, string, host_state):
|
||||
"""Strings prefixed with $ are capability lookups in the
|
||||
form '$variable' where 'variable' is an attribute in the
|
||||
HostState class. If $variable is a dictionary, you may
|
||||
use: $variable.dictkey
|
||||
"""
|
||||
if not string:
|
||||
return None
|
||||
if not string.startswith("$"):
|
||||
return string
|
||||
|
||||
path = string[1:].split(".")
|
||||
obj = getattr(host_state, path[0], None)
|
||||
if obj is None:
|
||||
return None
|
||||
for item in path[1:]:
|
||||
obj = obj.get(item, None)
|
||||
if obj is None:
|
||||
return None
|
||||
return obj
|
||||
|
||||
def _process_filter(self, query, host_state):
|
||||
"""Recursively parse the query structure."""
|
||||
if not query:
|
||||
return True
|
||||
cmd = query[0]
|
||||
method = self.commands[cmd]
|
||||
cooked_args = []
|
||||
for arg in query[1:]:
|
||||
if isinstance(arg, list):
|
||||
arg = self._process_filter(arg, host_state)
|
||||
elif isinstance(arg, basestring):
|
||||
arg = self._parse_string(arg, host_state)
|
||||
if arg is not None:
|
||||
cooked_args.append(arg)
|
||||
result = method(self, cooked_args)
|
||||
return result
|
||||
|
||||
def host_passes(self, host_state, filter_properties):
|
||||
"""Return a list of hosts that can fulfill the requirements
|
||||
specified in the query.
|
||||
"""
|
||||
# TODO(zhiteng) Add description for filter_properties structure
|
||||
# and scheduler_hints.
|
||||
try:
|
||||
query = filter_properties['scheduler_hints']['query']
|
||||
except KeyError:
|
||||
query = None
|
||||
if not query:
|
||||
return True
|
||||
|
||||
# NOTE(comstud): Not checking capabilities or service for
|
||||
# enabled/disabled so that a provided json filter can decide
|
||||
|
||||
result = self._process_filter(jsonutils.loads(query), host_state)
|
||||
if isinstance(result, list):
|
||||
# If any succeeded, include the host
|
||||
result = any(result)
|
||||
if result:
|
||||
# Filter it out.
|
||||
return True
|
||||
return False
|
|
@ -0,0 +1,91 @@
|
|||
# Copyright (c) 2011-2012 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Pluggable Weighing support
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
from stevedore import extension
|
||||
|
||||
|
||||
class WeighedObject(object):
|
||||
"""Object with weight information."""
|
||||
def __init__(self, obj, weight):
|
||||
self.obj = obj
|
||||
self.weight = weight
|
||||
|
||||
def __repr__(self):
|
||||
return "<WeighedObject '%s': %s>" % (self.obj, self.weight)
|
||||
|
||||
|
||||
class BaseWeigher(object):
|
||||
"""Base class for pluggable weighers."""
|
||||
def _weight_multiplier(self):
|
||||
"""How weighted this weigher should be. Normally this would
|
||||
be overriden in a subclass based on a config value.
|
||||
"""
|
||||
return 1.0
|
||||
|
||||
def _weigh_object(self, obj, weight_properties):
|
||||
"""Override in a subclass to specify a weight for a specific
|
||||
object.
|
||||
"""
|
||||
return 0.0
|
||||
|
||||
def weigh_objects(self, weighed_obj_list, weight_properties):
|
||||
"""Weigh multiple objects. Override in a subclass if you need
|
||||
need access to all objects in order to manipulate weights.
|
||||
"""
|
||||
constant = self._weight_multiplier()
|
||||
for obj in weighed_obj_list:
|
||||
obj.weight += (constant *
|
||||
self._weigh_object(obj.obj, weight_properties))
|
||||
|
||||
|
||||
class BaseWeightHandler(object):
|
||||
object_class = WeighedObject
|
||||
|
||||
def __init__(self, weighed_object_type, weight_namespace):
|
||||
self.namespace = weight_namespace
|
||||
self.weighed_object_type = weighed_object_type
|
||||
self.weight_manager = extension.ExtensionManager(weight_namespace)
|
||||
|
||||
def _is_correct_class(self, obj):
|
||||
"""Return whether an object is a class of the correct type and
|
||||
is not prefixed with an underscore.
|
||||
"""
|
||||
return (inspect.isclass(obj) and
|
||||
not obj.__name__.startswith('_') and
|
||||
issubclass(obj, self.weighed_object_type))
|
||||
|
||||
def get_all_classes(self):
|
||||
return [x.plugin for x in self.weight_manager
|
||||
if self._is_correct_class(x.plugin)]
|
||||
|
||||
def get_weighed_objects(self, weigher_classes, obj_list,
|
||||
weighing_properties):
|
||||
"""Return a sorted (highest score first) list of WeighedObjects."""
|
||||
|
||||
if not obj_list:
|
||||
return []
|
||||
|
||||
weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list]
|
||||
for weigher_cls in weigher_classes:
|
||||
weigher = weigher_cls()
|
||||
weigher.weigh_objects(weighed_objs, weighing_properties)
|
||||
|
||||
return sorted(weighed_objs, key=lambda x: x.weight, reverse=True)
|
|
@ -0,0 +1,45 @@
|
|||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Scheduler host weights
|
||||
"""
|
||||
|
||||
|
||||
from kwapi.openstack.common.scheduler import weight
|
||||
|
||||
|
||||
class WeighedHost(weight.WeighedObject):
|
||||
def to_dict(self):
|
||||
return {
|
||||
'weight': self.weight,
|
||||
'host': self.obj.host,
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return ("WeighedHost [host: %s, weight: %s]" %
|
||||
(self.obj.host, self.weight))
|
||||
|
||||
|
||||
class BaseHostWeigher(weight.BaseWeigher):
|
||||
"""Base class for host weights."""
|
||||
pass
|
||||
|
||||
|
||||
class HostWeightHandler(weight.BaseWeightHandler):
|
||||
object_class = WeighedHost
|
||||
|
||||
def __init__(self, namespace):
|
||||
super(HostWeightHandler, self).__init__(BaseHostWeigher, namespace)
|
|
@ -27,7 +27,7 @@ import sys
|
|||
import time
|
||||
|
||||
import eventlet
|
||||
import greenlet
|
||||
import extras
|
||||
import logging as std_logging
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
|
@ -36,11 +36,8 @@ from kwapi.openstack.common.gettextutils import _
|
|||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common import threadgroup
|
||||
|
||||
try:
|
||||
from kwapi.openstack.common import rpc
|
||||
except ImportError:
|
||||
rpc = None
|
||||
|
||||
rpc = extras.try_import('openstack.common.rpc')
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -54,7 +51,7 @@ class Launcher(object):
|
|||
:returns: None
|
||||
|
||||
"""
|
||||
self._services = []
|
||||
self._services = threadgroup.ThreadGroup('launcher')
|
||||
eventlet_backdoor.initialize_if_enabled()
|
||||
|
||||
@staticmethod
|
||||
|
@ -75,8 +72,7 @@ class Launcher(object):
|
|||
:returns: None
|
||||
|
||||
"""
|
||||
gt = eventlet.spawn(self.run_service, service)
|
||||
self._services.append(gt)
|
||||
self._services.add_thread(self.run_service, service)
|
||||
|
||||
def stop(self):
|
||||
"""Stop all services which are currently running.
|
||||
|
@ -84,8 +80,7 @@ class Launcher(object):
|
|||
:returns: None
|
||||
|
||||
"""
|
||||
for service in self._services:
|
||||
service.kill()
|
||||
self._services.stop()
|
||||
|
||||
def wait(self):
|
||||
"""Waits until all services have been stopped, and then returns.
|
||||
|
@ -93,11 +88,7 @@ class Launcher(object):
|
|||
:returns: None
|
||||
|
||||
"""
|
||||
for service in self._services:
|
||||
try:
|
||||
service.wait()
|
||||
except greenlet.GreenletExit:
|
||||
pass
|
||||
self._services.wait()
|
||||
|
||||
|
||||
class SignalExit(SystemExit):
|
||||
|
@ -132,9 +123,9 @@ class ServiceLauncher(Launcher):
|
|||
except SystemExit as exc:
|
||||
status = exc.code
|
||||
finally:
|
||||
self.stop()
|
||||
if rpc:
|
||||
rpc.cleanup()
|
||||
self.stop()
|
||||
return status
|
||||
|
||||
|
||||
|
@ -252,7 +243,10 @@ class ProcessLauncher(object):
|
|||
|
||||
def _wait_child(self):
|
||||
try:
|
||||
pid, status = os.wait()
|
||||
# Don't block if no child processes have exited
|
||||
pid, status = os.waitpid(0, os.WNOHANG)
|
||||
if not pid:
|
||||
return None
|
||||
except OSError as exc:
|
||||
if exc.errno not in (errno.EINTR, errno.ECHILD):
|
||||
raise
|
||||
|
@ -260,10 +254,12 @@ class ProcessLauncher(object):
|
|||
|
||||
if os.WIFSIGNALED(status):
|
||||
sig = os.WTERMSIG(status)
|
||||
LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals())
|
||||
LOG.info(_('Child %(pid)d killed by signal %(sig)d'),
|
||||
dict(pid=pid, sig=sig))
|
||||
else:
|
||||
code = os.WEXITSTATUS(status)
|
||||
LOG.info(_('Child %(pid)d exited with status %(code)d'), locals())
|
||||
LOG.info(_('Child %(pid)s exited with status %(code)d'),
|
||||
dict(pid=pid, code=code))
|
||||
|
||||
if pid not in self.children:
|
||||
LOG.warning(_('pid %d not in child list'), pid)
|
||||
|
@ -282,6 +278,10 @@ class ProcessLauncher(object):
|
|||
while self.running:
|
||||
wrap = self._wait_child()
|
||||
if not wrap:
|
||||
# Yield to other threads if no children have exited
|
||||
# Sleep for a short time to avoid excessive CPU usage
|
||||
# (see bug #1095346)
|
||||
eventlet.greenthread.sleep(.01)
|
||||
continue
|
||||
|
||||
while self.running and len(wrap.children) < wrap.workers:
|
||||
|
@ -309,8 +309,8 @@ class ProcessLauncher(object):
|
|||
class Service(object):
|
||||
"""Service object for binaries running on hosts."""
|
||||
|
||||
def __init__(self):
|
||||
self.tg = threadgroup.ThreadGroup('service')
|
||||
def __init__(self, threads=1000):
|
||||
self.tg = threadgroup.ThreadGroup('service', threads)
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
|
|
@ -276,6 +276,9 @@ def get_cmdclass():
|
|||
from sphinx.setup_command import BuildDoc
|
||||
|
||||
class LocalBuildDoc(BuildDoc):
|
||||
|
||||
builders = ['html', 'man']
|
||||
|
||||
def generate_autoindex(self):
|
||||
print "**Autodocumenting from %s" % os.path.abspath(os.curdir)
|
||||
modules = {}
|
||||
|
@ -311,14 +314,19 @@ def get_cmdclass():
|
|||
if not os.getenv('SPHINX_DEBUG'):
|
||||
self.generate_autoindex()
|
||||
|
||||
for builder in ['html', 'man']:
|
||||
for builder in self.builders:
|
||||
self.builder = builder
|
||||
self.finalize_options()
|
||||
self.project = self.distribution.get_name()
|
||||
self.version = self.distribution.get_version()
|
||||
self.release = self.distribution.get_version()
|
||||
BuildDoc.run(self)
|
||||
|
||||
class LocalBuildLatex(LocalBuildDoc):
|
||||
builders = ['latex']
|
||||
|
||||
cmdclass['build_sphinx'] = LocalBuildDoc
|
||||
cmdclass['build_sphinx_latex'] = LocalBuildLatex
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def int_from_bool_as_string(subject):
|
||||
"""
|
||||
Interpret a string as a boolean and return either 1 or 0.
|
||||
|
||||
Any string value in:
|
||||
|
||||
('True', 'true', 'On', 'on', '1')
|
||||
|
||||
is interpreted as a boolean True.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing
|
||||
"""
|
||||
return bool_from_string(subject) and 1 or 0
|
||||
|
||||
|
||||
def bool_from_string(subject):
|
||||
"""
|
||||
Interpret a string as a boolean.
|
||||
|
||||
Any string value in:
|
||||
|
||||
('True', 'true', 'On', 'on', 'Yes', 'yes', '1')
|
||||
|
||||
is interpreted as a boolean True.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing
|
||||
"""
|
||||
if isinstance(subject, bool):
|
||||
return subject
|
||||
if isinstance(subject, basestring):
|
||||
if subject.strip().lower() in ('true', 'on', 'yes', '1'):
|
||||
return True
|
||||
return False
|
|
@ -0,0 +1,68 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Utilities for unit tests."""
|
||||
|
||||
import functools
|
||||
import nose
|
||||
|
||||
|
||||
class skip_test(object):
|
||||
"""Decorator that skips a test."""
|
||||
# TODO(tr3buchet): remember forever what comstud did here
|
||||
def __init__(self, msg):
|
||||
self.message = msg
|
||||
|
||||
def __call__(self, func):
|
||||
@functools.wraps(func)
|
||||
def _skipper(*args, **kw):
|
||||
"""Wrapped skipper function."""
|
||||
raise nose.SkipTest(self.message)
|
||||
return _skipper
|
||||
|
||||
|
||||
class skip_if(object):
|
||||
"""Decorator that skips a test if condition is true."""
|
||||
def __init__(self, condition, msg):
|
||||
self.condition = condition
|
||||
self.message = msg
|
||||
|
||||
def __call__(self, func):
|
||||
@functools.wraps(func)
|
||||
def _skipper(*args, **kw):
|
||||
"""Wrapped skipper function."""
|
||||
if self.condition:
|
||||
raise nose.SkipTest(self.message)
|
||||
func(*args, **kw)
|
||||
return _skipper
|
||||
|
||||
|
||||
class skip_unless(object):
|
||||
"""Decorator that skips a test if condition is not true."""
|
||||
def __init__(self, condition, msg):
|
||||
self.condition = condition
|
||||
self.message = msg
|
||||
|
||||
def __call__(self, func):
|
||||
@functools.wraps(func)
|
||||
def _skipper(*args, **kw):
|
||||
"""Wrapped skipper function."""
|
||||
if not self.condition:
|
||||
raise nose.SkipTest(self.message)
|
||||
func(*args, **kw)
|
||||
return _skipper
|
|
@ -18,7 +18,6 @@ from eventlet import greenlet
|
|||
from eventlet import greenpool
|
||||
from eventlet import greenthread
|
||||
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common import loopingcall
|
||||
|
||||
|
@ -27,19 +26,17 @@ LOG = logging.getLogger(__name__)
|
|||
|
||||
|
||||
def _thread_done(gt, *args, **kwargs):
|
||||
'''
|
||||
Callback function to be passed to GreenThread.link() when we spawn()
|
||||
Calls the ThreadGroup to notify if.
|
||||
'''
|
||||
""" Callback function to be passed to GreenThread.link() when we spawn()
|
||||
Calls the :class:`ThreadGroup` to notify if.
|
||||
|
||||
"""
|
||||
kwargs['group'].thread_done(kwargs['thread'])
|
||||
|
||||
|
||||
class Thread(object):
|
||||
"""
|
||||
Wrapper around a greenthread, that holds a reference to
|
||||
the ThreadGroup. The Thread will notify the ThreadGroup
|
||||
when it has done so it can be removed from the threads
|
||||
list.
|
||||
""" Wrapper around a greenthread, that holds a reference to the
|
||||
:class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
|
||||
it has done so it can be removed from the threads list.
|
||||
"""
|
||||
def __init__(self, name, thread, group):
|
||||
self.name = name
|
||||
|
@ -54,11 +51,11 @@ class Thread(object):
|
|||
|
||||
|
||||
class ThreadGroup(object):
|
||||
"""
|
||||
The point of this class is to:
|
||||
- keep track of timers and greenthreads (making it easier to stop them
|
||||
""" The point of the ThreadGroup classis to:
|
||||
|
||||
* keep track of timers and greenthreads (making it easier to stop them
|
||||
when need be).
|
||||
- provide an easy API to add timers.
|
||||
* provide an easy API to add timers.
|
||||
"""
|
||||
def __init__(self, name, thread_pool_size=10):
|
||||
self.name = name
|
||||
|
|
|
@ -71,11 +71,15 @@ def normalize_time(timestamp):
|
|||
|
||||
def is_older_than(before, seconds):
|
||||
"""Return True if before is older than seconds."""
|
||||
if isinstance(before, basestring):
|
||||
before = parse_strtime(before).replace(tzinfo=None)
|
||||
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def is_newer_than(after, seconds):
|
||||
"""Return True if after is newer than seconds."""
|
||||
if isinstance(after, basestring):
|
||||
after = parse_strtime(after).replace(tzinfo=None)
|
||||
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
|
@ -87,7 +91,10 @@ def utcnow_ts():
|
|||
def utcnow():
|
||||
"""Overridable version of utils.utcnow."""
|
||||
if utcnow.override_time:
|
||||
return utcnow.override_time
|
||||
try:
|
||||
return utcnow.override_time.pop(0)
|
||||
except AttributeError:
|
||||
return utcnow.override_time
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
|
@ -95,14 +102,21 @@ utcnow.override_time = None
|
|||
|
||||
|
||||
def set_time_override(override_time=datetime.datetime.utcnow()):
|
||||
"""Override utils.utcnow to return a constant time."""
|
||||
"""
|
||||
Override utils.utcnow to return a constant time or a list thereof,
|
||||
one at a time.
|
||||
"""
|
||||
utcnow.override_time = override_time
|
||||
|
||||
|
||||
def advance_time_delta(timedelta):
|
||||
"""Advance overridden time using a datetime.timedelta."""
|
||||
assert(not utcnow.override_time is None)
|
||||
utcnow.override_time += timedelta
|
||||
try:
|
||||
for dt in utcnow.override_time:
|
||||
dt += timedelta
|
||||
except TypeError:
|
||||
utcnow.override_time += timedelta
|
||||
|
||||
|
||||
def advance_time_seconds(seconds):
|
||||
|
@ -135,3 +149,16 @@ def unmarshall_time(tyme):
|
|||
minute=tyme['minute'],
|
||||
second=tyme['second'],
|
||||
microsecond=tyme['microsecond'])
|
||||
|
||||
|
||||
def delta_seconds(before, after):
|
||||
"""
|
||||
Compute the difference in seconds between two date, time, or
|
||||
datetime objects (as a float, to microsecond resolution).
|
||||
"""
|
||||
delta = after - before
|
||||
try:
|
||||
return delta.total_seconds()
|
||||
except AttributeError:
|
||||
return ((delta.days * 24 * 3600) + delta.seconds +
|
||||
float(delta.microseconds) / (10 ** 6))
|
||||
|
|
|
@ -0,0 +1,39 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2012 Intel Corporation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
UUID related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
|
||||
def generate_uuid():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def is_uuid_like(val):
|
||||
"""Returns validation of a value as a UUID.
|
||||
|
||||
For our purposes, a UUID is a canonical form string:
|
||||
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
|
||||
|
||||
"""
|
||||
try:
|
||||
return str(uuid.UUID(val)) == val
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
return False
|
|
@ -24,19 +24,6 @@ import pkg_resources
|
|||
import setup
|
||||
|
||||
|
||||
class _deferred_version_string(object):
|
||||
"""Internal helper class which provides delayed version calculation."""
|
||||
def __init__(self, version_info, prefix):
|
||||
self.version_info = version_info
|
||||
self.prefix = prefix
|
||||
|
||||
def __str__(self):
|
||||
return "%s%s" % (self.prefix, self.version_info.version_string())
|
||||
|
||||
def __repr__(self):
|
||||
return "%s%s" % (self.prefix, self.version_info.version_string())
|
||||
|
||||
|
||||
class VersionInfo(object):
|
||||
|
||||
def __init__(self, package, python_package=None, pre_version=None):
|
||||
|
@ -57,14 +44,15 @@ class VersionInfo(object):
|
|||
self.python_package = python_package
|
||||
self.pre_version = pre_version
|
||||
self.version = None
|
||||
self._cached_version = None
|
||||
|
||||
def _generate_version(self):
|
||||
"""Defer to the openstack.common.setup routines for making a
|
||||
version from git."""
|
||||
if self.pre_version is None:
|
||||
return setup.get_post_version(self.python_package)
|
||||
return setup.get_post_version(self.package)
|
||||
else:
|
||||
return setup.get_pre_version(self.python_package, self.pre_version)
|
||||
return setup.get_pre_version(self.package, self.pre_version)
|
||||
|
||||
def _newer_version(self, pending_version):
|
||||
"""Check to see if we're working with a stale version or not.
|
||||
|
@ -138,11 +126,14 @@ class VersionInfo(object):
|
|||
else:
|
||||
return '%s-dev' % (version_parts[0],)
|
||||
|
||||
def deferred_version_string(self, prefix=""):
|
||||
def cached_version_string(self, prefix=""):
|
||||
"""Generate an object which will expand in a string context to
|
||||
the results of version_string(). We do this so that don't
|
||||
call into pkg_resources every time we start up a program when
|
||||
passing version information into the CONF constructor, but
|
||||
rather only do the calculation when and if a version is requested
|
||||
"""
|
||||
return _deferred_version_string(self, prefix)
|
||||
if not self._cached_version:
|
||||
self._cached_version = "%s%s" % (prefix,
|
||||
self.version_string())
|
||||
return self._cached_version
|
||||
|
|
|
@ -0,0 +1,733 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Utility methods for working with WSGI servers."""
|
||||
|
||||
import datetime
|
||||
import eventlet
|
||||
import eventlet.wsgi
|
||||
|
||||
eventlet.patcher.monkey_patch(all=False, socket=True)
|
||||
|
||||
import routes
|
||||
import routes.middleware
|
||||
import sys
|
||||
import webob.dec
|
||||
import webob.exc
|
||||
from xml.dom import minidom
|
||||
from xml.parsers import expat
|
||||
|
||||
from kwapi.openstack.common import exception
|
||||
from kwapi.openstack.common.gettextutils import _
|
||||
from kwapi.openstack.common import jsonutils
|
||||
from kwapi.openstack.common import log as logging
|
||||
from kwapi.openstack.common import service
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_server(application, port):
|
||||
"""Run a WSGI server with the given application."""
|
||||
sock = eventlet.listen(('0.0.0.0', port))
|
||||
eventlet.wsgi.server(sock, application)
|
||||
|
||||
|
||||
class Service(service.Service):
|
||||
"""
|
||||
Provides a Service API for wsgi servers.
|
||||
|
||||
This gives us the ability to launch wsgi servers with the
|
||||
Launcher classes in service.py.
|
||||
"""
|
||||
|
||||
def __init__(self, application, port,
|
||||
host='0.0.0.0', backlog=128, threads=1000):
|
||||
self.application = application
|
||||
self._port = port
|
||||
self._host = host
|
||||
self.backlog = backlog
|
||||
super(Service, self).__init__(threads)
|
||||
|
||||
def start(self):
|
||||
"""Start serving this service using the provided server instance.
|
||||
|
||||
:returns: None
|
||||
|
||||
"""
|
||||
super(Service, self).start()
|
||||
self._socket = eventlet.listen((self._host, self._port),
|
||||
backlog=self.backlog)
|
||||
self.tg.add_thread(self._run, self.application, self._socket)
|
||||
|
||||
@property
|
||||
def host(self):
|
||||
return self._socket.getsockname()[0] if self._socket else self._host
|
||||
|
||||
@property
|
||||
def port(self):
|
||||
return self._socket.getsockname()[1] if self._socket else self._port
|
||||
|
||||
def stop(self):
|
||||
"""Stop serving this API.
|
||||
|
||||
:returns: None
|
||||
|
||||
"""
|
||||
super(Service, self).stop()
|
||||
|
||||
def _run(self, application, socket):
|
||||
"""Start a WSGI server in a new green thread."""
|
||||
logger = logging.getLogger('eventlet.wsgi')
|
||||
eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool,
|
||||
log=logging.WritableLogger(logger))
|
||||
|
||||
|
||||
class Middleware(object):
|
||||
"""
|
||||
Base WSGI middleware wrapper. These classes require an application to be
|
||||
initialized that will be called next. By default the middleware will
|
||||
simply call its wrapped app, or you can override __call__ to customize its
|
||||
behavior.
|
||||
"""
|
||||
|
||||
def __init__(self, application):
|
||||
self.application = application
|
||||
|
||||
def process_request(self, req):
|
||||
"""
|
||||
Called on each request.
|
||||
|
||||
If this returns None, the next application down the stack will be
|
||||
executed. If it returns a response then that response will be returned
|
||||
and execution will stop here.
|
||||
"""
|
||||
return None
|
||||
|
||||
def process_response(self, response):
|
||||
"""Do whatever you'd like to the response."""
|
||||
return response
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, req):
|
||||
response = self.process_request(req)
|
||||
if response:
|
||||
return response
|
||||
response = req.get_response(self.application)
|
||||
return self.process_response(response)
|
||||
|
||||
|
||||
class Debug(Middleware):
|
||||
"""
|
||||
Helper class that can be inserted into any WSGI application chain
|
||||
to get information about the request and response.
|
||||
"""
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, req):
|
||||
print ("*" * 40) + " REQUEST ENVIRON"
|
||||
for key, value in req.environ.items():
|
||||
print key, "=", value
|
||||
print
|
||||
resp = req.get_response(self.application)
|
||||
|
||||
print ("*" * 40) + " RESPONSE HEADERS"
|
||||
for (key, value) in resp.headers.iteritems():
|
||||
print key, "=", value
|
||||
print
|
||||
|
||||
resp.app_iter = self.print_generator(resp.app_iter)
|
||||
|
||||
return resp
|
||||
|
||||
@staticmethod
|
||||
def print_generator(app_iter):
|
||||
"""
|
||||
Iterator that prints the contents of a wrapper string iterator
|
||||
when iterated.
|
||||
"""
|
||||
print ("*" * 40) + " BODY"
|
||||
for part in app_iter:
|
||||
sys.stdout.write(part)
|
||||
sys.stdout.flush()
|
||||
yield part
|
||||
print
|
||||
|
||||
|
||||
class Router(object):
|
||||
|
||||
"""
|
||||
WSGI middleware that maps incoming requests to WSGI apps.
|
||||
"""
|
||||
|
||||
def __init__(self, mapper):
|
||||
"""
|
||||
Create a router for the given routes.Mapper.
|
||||
|
||||
Each route in `mapper` must specify a 'controller', which is a
|
||||
WSGI app to call. You'll probably want to specify an 'action' as
|
||||
well and have your controller be a wsgi.Controller, who will route
|
||||
the request to the action method.
|
||||
|
||||
Examples:
|
||||
mapper = routes.Mapper()
|
||||
sc = ServerController()
|
||||
|
||||
# Explicit mapping of one route to a controller+action
|
||||
mapper.connect(None, "/svrlist", controller=sc, action="list")
|
||||
|
||||
# Actions are all implicitly defined
|
||||
mapper.resource("server", "servers", controller=sc)
|
||||
|
||||
# Pointing to an arbitrary WSGI app. You can specify the
|
||||
# {path_info:.*} parameter so the target app can be handed just that
|
||||
# section of the URL.
|
||||
mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
|
||||
"""
|
||||
self.map = mapper
|
||||
self._router = routes.middleware.RoutesMiddleware(self._dispatch,
|
||||
self.map)
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, req):
|
||||
"""
|
||||
Route the incoming request to a controller based on self.map.
|
||||
If no match, return a 404.
|
||||
"""
|
||||
return self._router
|
||||
|
||||
@staticmethod
|
||||
@webob.dec.wsgify
|
||||
def _dispatch(req):
|
||||
"""
|
||||
Called by self._router after matching the incoming request to a route
|
||||
and putting the information into req.environ. Either returns 404
|
||||
or the routed WSGI app's response.
|
||||
"""
|
||||
match = req.environ['wsgiorg.routing_args'][1]
|
||||
if not match:
|
||||
return webob.exc.HTTPNotFound()
|
||||
app = match['controller']
|
||||
return app
|
||||
|
||||
|
||||
class Request(webob.Request):
|
||||
"""Add some Openstack API-specific logic to the base webob.Request."""
|
||||
|
||||
default_request_content_types = ('application/json', 'application/xml')
|
||||
default_accept_types = ('application/json', 'application/xml')
|
||||
default_accept_type = 'application/json'
|
||||
|
||||
def best_match_content_type(self, supported_content_types=None):
|
||||
"""Determine the requested response content-type.
|
||||
|
||||
Based on the query extension then the Accept header.
|
||||
Defaults to default_accept_type if we don't find a preference
|
||||
|
||||
"""
|
||||
supported_content_types = (supported_content_types or
|
||||
self.default_accept_types)
|
||||
|
||||
parts = self.path.rsplit('.', 1)
|
||||
if len(parts) > 1:
|
||||
ctype = 'application/{0}'.format(parts[1])
|
||||
if ctype in supported_content_types:
|
||||
return ctype
|
||||
|
||||
bm = self.accept.best_match(supported_content_types)
|
||||
return bm or self.default_accept_type
|
||||
|
||||
def get_content_type(self, allowed_content_types=None):
|
||||
"""Determine content type of the request body.
|
||||
|
||||
Does not do any body introspection, only checks header
|
||||
|
||||
"""
|
||||
if not "Content-Type" in self.headers:
|
||||
return None
|
||||
|
||||
content_type = self.content_type
|
||||
allowed_content_types = (allowed_content_types or
|
||||
self.default_request_content_types)
|
||||
|
||||
if content_type not in allowed_content_types:
|
||||
raise exception.InvalidContentType(content_type=content_type)
|
||||
return content_type
|
||||
|
||||
|
||||
class Resource(object):
|
||||
"""
|
||||
WSGI app that handles (de)serialization and controller dispatch.
|
||||
|
||||
Reads routing information supplied by RoutesMiddleware and calls
|
||||
the requested action method upon its deserializer, controller,
|
||||
and serializer. Those three objects may implement any of the basic
|
||||
controller action methods (create, update, show, index, delete)
|
||||
along with any that may be specified in the api router. A 'default'
|
||||
method may also be implemented to be used in place of any
|
||||
non-implemented actions. Deserializer methods must accept a request
|
||||
argument and return a dictionary. Controller methods must accept a
|
||||
request argument. Additionally, they must also accept keyword
|
||||
arguments that represent the keys returned by the Deserializer. They
|
||||
may raise a webob.exc exception or return a dict, which will be
|
||||
serialized by requested content type.
|
||||
"""
|
||||
def __init__(self, controller, deserializer=None, serializer=None):
|
||||
"""
|
||||
:param controller: object that implement methods created by routes lib
|
||||
:param deserializer: object that supports webob request deserialization
|
||||
through controller-like actions
|
||||
:param serializer: object that supports webob response serialization
|
||||
through controller-like actions
|
||||
"""
|
||||
self.controller = controller
|
||||
self.serializer = serializer or ResponseSerializer()
|
||||
self.deserializer = deserializer or RequestDeserializer()
|
||||
|
||||
@webob.dec.wsgify(RequestClass=Request)
|
||||
def __call__(self, request):
|
||||
"""WSGI method that controls (de)serialization and method dispatch."""
|
||||
|
||||
try:
|
||||
action, action_args, accept = self.deserialize_request(request)
|
||||
except exception.InvalidContentType:
|
||||
msg = _("Unsupported Content-Type")
|
||||
return webob.exc.HTTPUnsupportedMediaType(explanation=msg)
|
||||
except exception.MalformedRequestBody:
|
||||
msg = _("Malformed request body")
|
||||
return webob.exc.HTTPBadRequest(explanation=msg)
|
||||
|
||||
action_result = self.execute_action(action, request, **action_args)
|
||||
try:
|
||||
return self.serialize_response(action, action_result, accept)
|
||||
# return unserializable result (typically a webob exc)
|
||||
except Exception:
|
||||
return action_result
|
||||
|
||||
def deserialize_request(self, request):
|
||||
return self.deserializer.deserialize(request)
|
||||
|
||||
def serialize_response(self, action, action_result, accept):
|
||||
return self.serializer.serialize(action_result, accept, action)
|
||||
|
||||
def execute_action(self, action, request, **action_args):
|
||||
return self.dispatch(self.controller, action, request, **action_args)
|
||||
|
||||
def dispatch(self, obj, action, *args, **kwargs):
|
||||
"""Find action-specific method on self and call it."""
|
||||
try:
|
||||
method = getattr(obj, action)
|
||||
except AttributeError:
|
||||
method = getattr(obj, 'default')
|
||||
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def get_action_args(self, request_environment):
|
||||
"""Parse dictionary created by routes library."""
|
||||
try:
|
||||
args = request_environment['wsgiorg.routing_args'][1].copy()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
try:
|
||||
del args['controller']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
try:
|
||||
del args['format']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
return args
|
||||
|
||||
|
||||
class ActionDispatcher(object):
|
||||
"""Maps method name to local methods through action name."""
|
||||
|
||||
def dispatch(self, *args, **kwargs):
|
||||
"""Find and call local method."""
|
||||
action = kwargs.pop('action', 'default')
|
||||
action_method = getattr(self, str(action), self.default)
|
||||
return action_method(*args, **kwargs)
|
||||
|
||||
def default(self, data):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class DictSerializer(ActionDispatcher):
|
||||
"""Default request body serialization"""
|
||||
|
||||
def serialize(self, data, action='default'):
|
||||
return self.dispatch(data, action=action)
|
||||
|
||||
def default(self, data):
|
||||
return ""
|
||||
|
||||
|
||||
class JSONDictSerializer(DictSerializer):
|
||||
"""Default JSON request body serialization"""
|
||||
|
||||
def default(self, data):
|
||||
def sanitizer(obj):
|
||||
if isinstance(obj, datetime.datetime):
|
||||
_dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
|
||||
return _dtime.isoformat()
|
||||
return unicode(obj)
|
||||
return jsonutils.dumps(data, default=sanitizer)
|
||||
|
||||
|
||||
class XMLDictSerializer(DictSerializer):
|
||||
|
||||
def __init__(self, metadata=None, xmlns=None):
|
||||
"""
|
||||
:param metadata: information needed to deserialize xml into
|
||||
a dictionary.
|
||||
:param xmlns: XML namespace to include with serialized xml
|
||||
"""
|
||||
super(XMLDictSerializer, self).__init__()
|
||||
self.metadata = metadata or {}
|
||||
self.xmlns = xmlns
|
||||
|
||||
def default(self, data):
|
||||
# We expect data to contain a single key which is the XML root.
|
||||
root_key = data.keys()[0]
|
||||
doc = minidom.Document()
|
||||
node = self._to_xml_node(doc, self.metadata, root_key, data[root_key])
|
||||
|
||||
return self.to_xml_string(node)
|
||||
|
||||
def to_xml_string(self, node, has_atom=False):
|
||||
self._add_xmlns(node, has_atom)
|
||||
return node.toprettyxml(indent=' ', encoding='UTF-8')
|
||||
|
||||
#NOTE (ameade): the has_atom should be removed after all of the
|
||||
# xml serializers and view builders have been updated to the current
|
||||
# spec that required all responses include the xmlns:atom, the has_atom
|
||||
# flag is to prevent current tests from breaking
|
||||
def _add_xmlns(self, node, has_atom=False):
|
||||
if self.xmlns is not None:
|
||||
node.setAttribute('xmlns', self.xmlns)
|
||||
if has_atom:
|
||||
node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom")
|
||||
|
||||
def _to_xml_node(self, doc, metadata, nodename, data):
|
||||
"""Recursive method to convert data members to XML nodes."""
|
||||
result = doc.createElement(nodename)
|
||||
|
||||
# Set the xml namespace if one is specified
|
||||
# TODO(justinsb): We could also use prefixes on the keys
|
||||
xmlns = metadata.get('xmlns', None)
|
||||
if xmlns:
|
||||
result.setAttribute('xmlns', xmlns)
|
||||
|
||||
#TODO(bcwaldon): accomplish this without a type-check
|
||||
if type(data) is list:
|
||||
collections = metadata.get('list_collections', {})
|
||||
if nodename in collections:
|
||||
metadata = collections[nodename]
|
||||
for item in data:
|
||||
node = doc.createElement(metadata['item_name'])
|
||||
node.setAttribute(metadata['item_key'], str(item))
|
||||
result.appendChild(node)
|
||||
return result
|
||||
singular = metadata.get('plurals', {}).get(nodename, None)
|
||||
if singular is None:
|
||||
if nodename.endswith('s'):
|
||||
singular = nodename[:-1]
|
||||
else:
|
||||
singular = 'item'
|
||||
for item in data:
|
||||
node = self._to_xml_node(doc, metadata, singular, item)
|
||||
result.appendChild(node)
|
||||
#TODO(bcwaldon): accomplish this without a type-check
|
||||
elif type(data) is dict:
|
||||
collections = metadata.get('dict_collections', {})
|
||||
if nodename in collections:
|
||||
metadata = collections[nodename]
|
||||
for k, v in data.items():
|
||||
node = doc.createElement(metadata['item_name'])
|
||||
node.setAttribute(metadata['item_key'], str(k))
|
||||
text = doc.createTextNode(str(v))
|
||||
node.appendChild(text)
|
||||
result.appendChild(node)
|
||||
return result
|
||||
attrs = metadata.get('attributes', {}).get(nodename, {})
|
||||
for k, v in data.items():
|
||||
if k in attrs:
|
||||
result.setAttribute(k, str(v))
|
||||
else:
|
||||
node = self._to_xml_node(doc, metadata, k, v)
|
||||
result.appendChild(node)
|
||||
else:
|
||||
# Type is atom
|
||||
node = doc.createTextNode(str(data))
|
||||
result.appendChild(node)
|
||||
return result
|
||||
|
||||
def _create_link_nodes(self, xml_doc, links):
|
||||
link_nodes = []
|
||||
for link in links:
|
||||
link_node = xml_doc.createElement('atom:link')
|
||||
link_node.setAttribute('rel', link['rel'])
|
||||
link_node.setAttribute('href', link['href'])
|
||||
if 'type' in link:
|
||||
link_node.setAttribute('type', link['type'])
|
||||
link_nodes.append(link_node)
|
||||
return link_nodes
|
||||
|
||||
|
||||
class ResponseHeadersSerializer(ActionDispatcher):
|
||||
"""Default response headers serialization"""
|
||||
|
||||
def serialize(self, response, data, action):
|
||||
self.dispatch(response, data, action=action)
|
||||
|
||||
def default(self, response, data):
|
||||
response.status_int = 200
|
||||
|
||||
|
||||
class ResponseSerializer(object):
|
||||
"""Encode the necessary pieces into a response object"""
|
||||
|
||||
def __init__(self, body_serializers=None, headers_serializer=None):
|
||||
self.body_serializers = {
|
||||
'application/xml': XMLDictSerializer(),
|
||||
'application/json': JSONDictSerializer(),
|
||||
}
|
||||
self.body_serializers.update(body_serializers or {})
|
||||
|
||||
self.headers_serializer = (headers_serializer or
|
||||
ResponseHeadersSerializer())
|
||||
|
||||
def serialize(self, response_data, content_type, action='default'):
|
||||
"""Serialize a dict into a string and wrap in a wsgi.Request object.
|
||||
|
||||
:param response_data: dict produced by the Controller
|
||||
:param content_type: expected mimetype of serialized response body
|
||||
|
||||
"""
|
||||
response = webob.Response()
|
||||
self.serialize_headers(response, response_data, action)
|
||||
self.serialize_body(response, response_data, content_type, action)
|
||||
return response
|
||||
|
||||
def serialize_headers(self, response, data, action):
|
||||
self.headers_serializer.serialize(response, data, action)
|
||||
|
||||
def serialize_body(self, response, data, content_type, action):
|
||||
response.headers['Content-Type'] = content_type
|
||||
if data is not None:
|
||||
serializer = self.get_body_serializer(content_type)
|
||||
response.body = serializer.serialize(data, action)
|
||||
|
||||
def get_body_serializer(self, content_type):
|
||||
try:
|
||||
return self.body_serializers[content_type]
|
||||
except (KeyError, TypeError):
|
||||
raise exception.InvalidContentType(content_type=content_type)
|
||||
|
||||
|
||||
class RequestHeadersDeserializer(ActionDispatcher):
|
||||
"""Default request headers deserializer"""
|
||||
|
||||
def deserialize(self, request, action):
|
||||
return self.dispatch(request, action=action)
|
||||
|
||||
def default(self, request):
|
||||
return {}
|
||||
|
||||
|
||||
class RequestDeserializer(object):
|
||||
"""Break up a Request object into more useful pieces."""
|
||||
|
||||
def __init__(self, body_deserializers=None, headers_deserializer=None,
|
||||
supported_content_types=None):
|
||||
|
||||
self.supported_content_types = supported_content_types
|
||||
|
||||
self.body_deserializers = {
|
||||
'application/xml': XMLDeserializer(),
|
||||
'application/json': JSONDeserializer(),
|
||||
}
|
||||
self.body_deserializers.update(body_deserializers or {})
|
||||
|
||||
self.headers_deserializer = (headers_deserializer or
|
||||
RequestHeadersDeserializer())
|
||||
|
||||
def deserialize(self, request):
|
||||
"""Extract necessary pieces of the request.
|
||||
|
||||
:param request: Request object
|
||||
:returns: tuple of (expected controller action name, dictionary of
|
||||
keyword arguments to pass to the controller, the expected
|
||||
content type of the response)
|
||||
|
||||
"""
|
||||
action_args = self.get_action_args(request.environ)
|
||||
action = action_args.pop('action', None)
|
||||
|
||||
action_args.update(self.deserialize_headers(request, action))
|
||||
action_args.update(self.deserialize_body(request, action))
|
||||
|
||||
accept = self.get_expected_content_type(request)
|
||||
|
||||
return (action, action_args, accept)
|
||||
|
||||
def deserialize_headers(self, request, action):
|
||||
return self.headers_deserializer.deserialize(request, action)
|
||||
|
||||
def deserialize_body(self, request, action):
|
||||
if not len(request.body) > 0:
|
||||
LOG.debug(_("Empty body provided in request"))
|
||||
return {}
|
||||
|
||||
try:
|
||||
content_type = request.get_content_type()
|
||||
except exception.InvalidContentType:
|
||||
LOG.debug(_("Unrecognized Content-Type provided in request"))
|
||||
raise
|
||||
|
||||
if content_type is None:
|
||||
LOG.debug(_("No Content-Type provided in request"))
|
||||
return {}
|
||||
|
||||
try:
|
||||
deserializer = self.get_body_deserializer(content_type)
|
||||
except exception.InvalidContentType:
|
||||
LOG.debug(_("Unable to deserialize body as provided Content-Type"))
|
||||
raise
|
||||
|
||||
return deserializer.deserialize(request.body, action)
|
||||
|
||||
def get_body_deserializer(self, content_type):
|
||||
try:
|
||||
return self.body_deserializers[content_type]
|
||||
except (KeyError, TypeError):
|
||||
raise exception.InvalidContentType(content_type=content_type)
|
||||
|
||||
def get_expected_content_type(self, request):
|
||||
return request.best_match_content_type(self.supported_content_types)
|
||||
|
||||
def get_action_args(self, request_environment):
|
||||
"""Parse dictionary created by routes library."""
|
||||
try:
|
||||
args = request_environment['wsgiorg.routing_args'][1].copy()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
try:
|
||||
del args['controller']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
try:
|
||||
del args['format']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
return args
|
||||
|
||||
|
||||
class TextDeserializer(ActionDispatcher):
|
||||
"""Default request body deserialization"""
|
||||
|
||||
def deserialize(self, datastring, action='default'):
|
||||
return self.dispatch(datastring, action=action)
|
||||
|
||||
def default(self, datastring):
|
||||
return {}
|
||||
|
||||
|
||||
class JSONDeserializer(TextDeserializer):
|
||||
|
||||
def _from_json(self, datastring):
|
||||
try:
|
||||
return jsonutils.loads(datastring)
|
||||
except ValueError:
|
||||
msg = _("cannot understand JSON")
|
||||
raise exception.MalformedRequestBody(reason=msg)
|
||||
|
||||
def default(self, datastring):
|
||||
return {'body': self._from_json(datastring)}
|
||||
|
||||
|
||||
class XMLDeserializer(TextDeserializer):
|
||||
|
||||
def __init__(self, metadata=None):
|
||||
"""
|
||||
:param metadata: information needed to deserialize xml into
|
||||
a dictionary.
|
||||
"""
|
||||
super(XMLDeserializer, self).__init__()
|
||||
self.metadata = metadata or {}
|
||||
|
||||
def _from_xml(self, datastring):
|
||||
plurals = set(self.metadata.get('plurals', {}))
|
||||
|
||||
try:
|
||||
node = minidom.parseString(datastring).childNodes[0]
|
||||
return {node.nodeName: self._from_xml_node(node, plurals)}
|
||||
except expat.ExpatError:
|
||||
msg = _("cannot understand XML")
|
||||
raise exception.MalformedRequestBody(reason=msg)
|
||||
|
||||
def _from_xml_node(self, node, listnames):
|
||||
"""Convert a minidom node to a simple Python type.
|
||||
|
||||
:param listnames: list of XML node names whose subnodes should
|
||||
be considered list items.
|
||||
|
||||
"""
|
||||
|
||||
if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3:
|
||||
return node.childNodes[0].nodeValue
|
||||
elif node.nodeName in listnames:
|
||||
return [self._from_xml_node(n, listnames) for n in node.childNodes]
|
||||
else:
|
||||
result = dict()
|
||||
for attr in node.attributes.keys():
|
||||
result[attr] = node.attributes[attr].nodeValue
|
||||
for child in node.childNodes:
|
||||
if child.nodeType != node.TEXT_NODE:
|
||||
result[child.nodeName] = self._from_xml_node(child,
|
||||
listnames)
|
||||
return result
|
||||
|
||||
def find_first_child_named(self, parent, name):
|
||||
"""Search a nodes children for the first child with a given name"""
|
||||
for node in parent.childNodes:
|
||||
if node.nodeName == name:
|
||||
return node
|
||||
return None
|
||||
|
||||
def find_children_named(self, parent, name):
|
||||
"""Return all of a nodes children who have the given name"""
|
||||
for node in parent.childNodes:
|
||||
if node.nodeName == name:
|
||||
yield node
|
||||
|
||||
def extract_text(self, node):
|
||||
"""Get the text field contained by the given node"""
|
||||
if len(node.childNodes) == 1:
|
||||
child = node.childNodes[0]
|
||||
if child.nodeType == child.TEXT_NODE:
|
||||
return child.nodeValue
|
||||
return ""
|
||||
|
||||
def default(self, datastring):
|
||||
return {'body': self._from_xml(datastring)}
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Set up the ACL to access the API."""
|
||||
|
||||
|
@ -11,20 +25,23 @@ acl_opts = [
|
|||
cfg.StrOpt('acl_auth_url',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(acl_opts)
|
||||
|
||||
|
||||
def install(app):
|
||||
"""Installs ACL check on application."""
|
||||
app.before_request(check)
|
||||
return app
|
||||
|
||||
|
||||
def check():
|
||||
"""Checks application access."""
|
||||
headers = flask.request.headers
|
||||
try:
|
||||
client = Client(token=headers.get('X-Auth-Token'), auth_url=cfg.CONF.acl_auth_url)
|
||||
client = Client(token=headers.get('X-Auth-Token'),
|
||||
auth_url=cfg.CONF.acl_auth_url)
|
||||
except:
|
||||
return "Access denied", 401
|
||||
else:
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Set up the API server application instance."""
|
||||
|
||||
|
@ -15,25 +29,26 @@ app_opts = [
|
|||
cfg.BoolOpt('acl_enabled',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(app_opts)
|
||||
|
||||
|
||||
def make_app():
|
||||
"""Instantiates Flask app, attaches collector database, installs acl."""
|
||||
LOG.info('Starting API')
|
||||
app = flask.Flask(__name__)
|
||||
app.register_blueprint(v1.blueprint, url_prefix='/v1')
|
||||
|
||||
|
||||
collector = Collector()
|
||||
collector.clean()
|
||||
|
||||
|
||||
@app.before_request
|
||||
def attach_config():
|
||||
flask.request.database = collector.database
|
||||
|
||||
|
||||
# Install the middleware wrapper
|
||||
if cfg.CONF.acl_enabled:
|
||||
return acl.install(app)
|
||||
|
||||
|
||||
return app
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import threading
|
||||
|
@ -13,8 +27,8 @@ LOG = log.getLogger(__name__)
|
|||
|
||||
collector_opts = [
|
||||
cfg.BoolOpt('signature_checking',
|
||||
required=True,
|
||||
),
|
||||
required=True,
|
||||
),
|
||||
cfg.IntOpt('cleaning_interval',
|
||||
required=True,
|
||||
),
|
||||
|
@ -24,13 +38,17 @@ collector_opts = [
|
|||
cfg.StrOpt('driver_metering_secret',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(collector_opts)
|
||||
|
||||
|
||||
class Record(dict):
|
||||
"""Contains fields (timestamp, kwh, w) and a method to update consumption."""
|
||||
|
||||
"""Contains fields (timestamp, kwh, w) and a method to update
|
||||
consumption.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, timestamp, kwh, watts):
|
||||
"""Initializes fields with the given arguments."""
|
||||
dict.__init__(self)
|
||||
|
@ -38,17 +56,22 @@ class Record(dict):
|
|||
self['timestamp'] = timestamp
|
||||
self['kwh'] = kwh
|
||||
self['w'] = watts
|
||||
|
||||
|
||||
def add(self, watts):
|
||||
"""Updates fields with consumption data."""
|
||||
currentTime = time.time()
|
||||
self['kwh'] += (currentTime - self['timestamp']) / 3600.0 * (watts / 1000.0)
|
||||
self['kwh'] += (currentTime - self['timestamp']) / 3600.0 * \
|
||||
(watts / 1000.0)
|
||||
self['w'] = watts
|
||||
self['timestamp'] = currentTime
|
||||
|
||||
|
||||
class Collector:
|
||||
"""Collector gradually fills its database with received values from wattmeter drivers."""
|
||||
|
||||
"""Collector gradually fills its database with received values from
|
||||
wattmeter drivers.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initializes an empty database and start listening the endpoint."""
|
||||
LOG.info('Starting Collector')
|
||||
|
@ -56,7 +79,7 @@ class Collector:
|
|||
thread = threading.Thread(target=self.listen)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
|
||||
def add(self, probe, watts):
|
||||
"""Creates (or updates) consumption data for this probe."""
|
||||
if probe in self.database.keys():
|
||||
|
@ -64,7 +87,7 @@ class Collector:
|
|||
else:
|
||||
record = Record(timestamp=time.time(), kwh=0.0, watts=watts)
|
||||
self.database[probe] = record
|
||||
|
||||
|
||||
def remove(self, probe):
|
||||
"""Removes this probe from database."""
|
||||
if probe in self.database.keys():
|
||||
|
@ -72,47 +95,53 @@ class Collector:
|
|||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def clean(self):
|
||||
"""Removes probes from database if they didn't send new values over the last period (seconds).
|
||||
If periodic, this method is executed automatically after the timeout interval.
|
||||
|
||||
"""Removes probes from database if they didn't send new values over
|
||||
the last period (seconds). If periodic, this method is executed
|
||||
automatically after the timeout interval.
|
||||
|
||||
"""
|
||||
LOG.info('Cleaning collector')
|
||||
# Cleaning
|
||||
for probe in self.database.keys():
|
||||
if time.time() - self.database[probe]['timestamp'] > cfg.CONF.cleaning_interval:
|
||||
if time.time() - self.database[probe]['timestamp'] > \
|
||||
cfg.CONF.cleaning_interval:
|
||||
LOG.info('Removing data of probe %s' % probe)
|
||||
self.remove(probe)
|
||||
|
||||
|
||||
# Schedule periodic execution of this function
|
||||
if cfg.CONF.cleaning_interval > 0:
|
||||
timer = threading.Timer(cfg.CONF.cleaning_interval, self.clean)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def listen(self):
|
||||
"""Subscribes to ZeroMQ messages, and adds received measurements to the database.
|
||||
Messages are dictionaries dumped in JSON format.
|
||||
|
||||
"""Subscribes to ZeroMQ messages, and adds received measurements to the
|
||||
database. Messages are dictionaries dumped in JSON format.
|
||||
|
||||
"""
|
||||
LOG.info('Collector listenig to %s' % cfg.CONF.probes_endpoint)
|
||||
|
||||
|
||||
context = zmq.Context.instance()
|
||||
subscriber = context.socket(zmq.SUB)
|
||||
subscriber.setsockopt(zmq.SUBSCRIBE, '')
|
||||
for endpoint in cfg.CONF.probes_endpoint:
|
||||
subscriber.connect(endpoint)
|
||||
|
||||
|
||||
while True:
|
||||
message = subscriber.recv()
|
||||
measurements = json.loads(message)
|
||||
if not isinstance(measurements, dict):
|
||||
LOG.error('Bad message type (not a dict)')
|
||||
elif cfg.CONF.signature_checking and not security.verify_signature(measurements, cfg.CONF.driver_metering_secret):
|
||||
elif cfg.CONF.signature_checking and \
|
||||
not security.verify_signature(
|
||||
measurements,
|
||||
cfg.CONF.driver_metering_secret):
|
||||
LOG.error('Bad message signature')
|
||||
else:
|
||||
try:
|
||||
self.add(measurements['probe_id'], float(measurements['w']))
|
||||
self.add(measurements['probe_id'],
|
||||
float(measurements['w']))
|
||||
except KeyError:
|
||||
LOG.error('Malformed message (missing required key)')
|
||||
|
|
|
@ -1,21 +1,32 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""This blueprint defines all URLs and answers."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
import flask
|
||||
|
||||
from kwapi.openstack.common import cfg
|
||||
|
||||
blueprint = flask.Blueprint('v1', __name__)
|
||||
|
||||
|
||||
@blueprint.route('/')
|
||||
def welcome():
|
||||
"""Returns detailed information about this specific version of the API."""
|
||||
return 'Welcome to Kwapi!'
|
||||
|
||||
|
||||
@blueprint.route('/probe-ids/')
|
||||
def list_probes_ids():
|
||||
"""Returns all known probes IDs."""
|
||||
|
@ -23,6 +34,7 @@ def list_probes_ids():
|
|||
message['probe_ids'] = flask.request.database.keys()
|
||||
return flask.jsonify(message)
|
||||
|
||||
|
||||
@blueprint.route('/probes/')
|
||||
def list_probes():
|
||||
"""Returns all information about all known probes."""
|
||||
|
@ -30,6 +42,7 @@ def list_probes():
|
|||
message['probes'] = flask.request.database
|
||||
return flask.jsonify(message)
|
||||
|
||||
|
||||
@blueprint.route('/probes/<probe>/')
|
||||
def probe_info(probe):
|
||||
"""Returns all information about this probe (id, timestamp, kWh, W)."""
|
||||
|
@ -40,6 +53,7 @@ def probe_info(probe):
|
|||
flask.abort(404)
|
||||
return flask.jsonify(message)
|
||||
|
||||
|
||||
@blueprint.route('/probes/<probe>/<meter>/')
|
||||
def probe_value(probe, meter):
|
||||
"""Returns the probe meter value."""
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Set up the RRD server application instance."""
|
||||
|
||||
|
@ -12,14 +26,15 @@ import v1
|
|||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
def make_app():
|
||||
"""Instantiates Flask app, attaches collector database. """
|
||||
LOG.info('Starting RRD')
|
||||
app = flask.Flask(__name__)
|
||||
app.register_blueprint(v1.blueprint)
|
||||
|
||||
|
||||
thread.start_new_thread(rrd.listen, ())
|
||||
|
||||
|
||||
@app.before_request
|
||||
def attach_config():
|
||||
flask.request.probes = rrd.probes
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Defines functions to build and update rrd database and graph."""
|
||||
|
||||
|
@ -20,8 +34,8 @@ LOG = log.getLogger(__name__)
|
|||
|
||||
rrd_opts = [
|
||||
cfg.BoolOpt('signature_checking',
|
||||
required=True,
|
||||
),
|
||||
required=True,
|
||||
),
|
||||
cfg.FloatOpt('kwh_price',
|
||||
required=True,
|
||||
),
|
||||
|
@ -40,32 +54,42 @@ rrd_opts = [
|
|||
cfg.StrOpt('rrd_dir',
|
||||
required=True,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(rrd_opts)
|
||||
|
||||
scales = collections.OrderedDict()
|
||||
scales['minute'] = {'interval':300, 'resolution':1, 'label': '5 minutes'}, # Resolution = 1 second
|
||||
scales['hour'] = {'interval':3600, 'resolution':10, 'label': 'hour'}, # Resolution = 10 seconds
|
||||
scales['day'] = {'interval':86400, 'resolution':900, 'label': 'day'}, # Resolution = 15 minutes
|
||||
scales['week'] = {'interval':604800, 'resolution':7200, 'label': 'week'}, # Resolution = 2 hours
|
||||
scales['month'] = {'interval':2678400, 'resolution':21600, 'label': 'month'}, # Resolution = 6 hours
|
||||
scales['year'] = {'interval':31622400, 'resolution':604800, 'label': 'year'}, # Resolution = 1 week
|
||||
# Resolution = 1 second
|
||||
scales['minute'] = {'interval': 300, 'resolution': 1, 'label': '5 minutes'},
|
||||
# Resolution = 10 seconds
|
||||
scales['hour'] = {'interval': 3600, 'resolution': 10, 'label': 'hour'},
|
||||
# Resolution = 15 minutes
|
||||
scales['day'] = {'interval': 86400, 'resolution': 900, 'label': 'day'},
|
||||
# Resolution = 2 hours
|
||||
scales['week'] = {'interval': 604800, 'resolution': 7200, 'label': 'week'},
|
||||
# Resolution = 6 hours
|
||||
scales['month'] = {'interval': 2678400, 'resolution': 21600, 'label': 'month'},
|
||||
# Resolution = 1 week
|
||||
scales['year'] = {'interval': 31622400, 'resolution': 604800, 'label': 'year'},
|
||||
|
||||
colors = ['#EA644A', '#EC9D48', '#ECD748', '#54EC48', '#48C4EC', '#7648EC', '#DE48EC', '#8A8187']
|
||||
colors = ['#EA644A', '#EC9D48', '#ECD748', '#54EC48', '#48C4EC', '#7648EC',
|
||||
'#DE48EC', '#8A8187']
|
||||
probes = set()
|
||||
probe_colors = {}
|
||||
|
||||
|
||||
def create_dirs():
|
||||
"""Creates all required directories."""
|
||||
directories = []
|
||||
directories.append(cfg.CONF.png_dir)
|
||||
directories.append(cfg.CONF.rrd_dir)
|
||||
# Build a list of directory names
|
||||
# Avoid loop in try block (problem if exception occurs), and avoid multiple try blocks (too long)
|
||||
# Avoid loop in try block (problem if exception occurs), and avoid multiple
|
||||
# try blocks (too long)
|
||||
for scale in scales.keys():
|
||||
directories.append(cfg.CONF.png_dir + '/' + scale)
|
||||
# Create each directory in a try block, and continue if directory already exist
|
||||
# Create each directory in a try block, and continue if directory already
|
||||
# exist
|
||||
for directory in directories:
|
||||
try:
|
||||
os.makedirs(directory)
|
||||
|
@ -73,13 +97,18 @@ def create_dirs():
|
|||
if exception.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
|
||||
def get_png_filename(scale, probe):
|
||||
"""Returns the png filename."""
|
||||
return cfg.CONF.png_dir + '/' + scale + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.png'
|
||||
return cfg.CONF.png_dir + '/' + scale + '/' + \
|
||||
str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.png'
|
||||
|
||||
|
||||
def get_rrd_filename(probe):
|
||||
"""Returns the rrd filename."""
|
||||
return cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd'
|
||||
return cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS,
|
||||
str(probe))) + '.rrd'
|
||||
|
||||
|
||||
def create_rrd_file(filename):
|
||||
"""Creates a RRD file."""
|
||||
|
@ -91,12 +120,17 @@ def create_rrd_file(filename):
|
|||
'DS:w:GAUGE:600:0:U',
|
||||
]
|
||||
for scale in scales.keys():
|
||||
args.append('RRA:AVERAGE:0.5:%s:%s' % (scales[scale][0]['resolution'], scales[scale][0]['interval']/scales[scale][0]['resolution']))
|
||||
args.append('RRA:AVERAGE:0.5:%s:%s'
|
||||
% (scales[scale][0]['resolution'],
|
||||
scales[scale][0]['interval'] /
|
||||
scales[scale][0]['resolution']))
|
||||
rrdtool.create(args)
|
||||
|
||||
|
||||
def update_rrd(probe, watts):
|
||||
"""Updates RRD file associated with this probe."""
|
||||
filename = cfg.CONF.rrd_dir + '/' + str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd'
|
||||
filename = cfg.CONF.rrd_dir + '/' + \
|
||||
str(uuid.uuid5(uuid.NAMESPACE_DNS, str(probe))) + '.rrd'
|
||||
if not os.path.isfile(filename):
|
||||
create_rrd_file(filename)
|
||||
try:
|
||||
|
@ -104,16 +138,19 @@ def update_rrd(probe, watts):
|
|||
except rrdtool.error, e:
|
||||
LOG.error('Error updating RRD: %s' % e)
|
||||
|
||||
|
||||
def build_graph(scale, probe=None):
|
||||
"""Builds the graph for the probe, or a summary graph."""
|
||||
if scale in scales.keys() and len(probes) > 0 and (probe is None or probe in probes):
|
||||
if scale in scales.keys() and len(probes) > 0 \
|
||||
and (probe is None or probe in probes):
|
||||
# Get PNG filename
|
||||
if probe is not None:
|
||||
png_file = get_png_filename(scale, probe)
|
||||
else:
|
||||
png_file = cfg.CONF.png_dir + '/' + scale + '/summary.png'
|
||||
# Build required (PNG file not found or outdated)
|
||||
if not os.path.exists(png_file) or os.path.getmtime(png_file) < time.time() - scales[scale][0]['resolution']:
|
||||
if not os.path.exists(png_file) or os.path.getmtime(png_file) < \
|
||||
time.time() - scales[scale][0]['resolution']:
|
||||
if probe is not None:
|
||||
# Specific arguments for probe graph
|
||||
args = [png_file,
|
||||
|
@ -131,14 +168,14 @@ def build_graph(scale, probe=None):
|
|||
]
|
||||
# Common arguments
|
||||
args += ['--start', '-' + str(scales[scale][0]['interval']),
|
||||
'--end', 'now',
|
||||
'--full-size-mode',
|
||||
'--imgformat', 'PNG',
|
||||
'--alt-y-grid',
|
||||
'--vertical-label', 'Watts',
|
||||
'--lower-limit', '0',
|
||||
'--rigid',
|
||||
]
|
||||
'--end', 'now',
|
||||
'--full-size-mode',
|
||||
'--imgformat', 'PNG',
|
||||
'--alt-y-grid',
|
||||
'--vertical-label', 'Watts',
|
||||
'--lower-limit', '0',
|
||||
'--rigid',
|
||||
]
|
||||
if scale == 'minute':
|
||||
args += ['--x-grid', 'SECOND:30:MINUTE:1:MINUTE:1:0:%H:%M']
|
||||
cdef_watt = 'CDEF:watt='
|
||||
|
@ -153,20 +190,26 @@ def build_graph(scale, probe=None):
|
|||
probe_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, probe)
|
||||
rrd_file = get_rrd_filename(probe)
|
||||
# Data source
|
||||
args.append('DEF:watt_with_unknown_%s=%s:w:AVERAGE' % (probe_uuid, rrd_file))
|
||||
args.append('DEF:watt_with_unknown_%s=%s:w:AVERAGE'
|
||||
% (probe_uuid, rrd_file))
|
||||
# Data source without unknown values
|
||||
args.append('CDEF:watt_%s=watt_with_unknown_%s,UN,0,watt_with_unknown_%s,IF' % (probe_uuid, probe_uuid, probe_uuid))
|
||||
args.append('CDEF:watt_%s=watt_with_unknown_%s,UN,0,watt_with_'
|
||||
'unknown_%s,IF'
|
||||
% (probe_uuid, probe_uuid, probe_uuid))
|
||||
# Prepare CDEF expression of total watt consumption
|
||||
cdef_watt += 'watt_%s,' % probe_uuid
|
||||
cdef_watt_with_unknown += 'watt_with_unknown_%s,' % probe_uuid
|
||||
# Draw the area for the probe
|
||||
color = probe_colors[probe]
|
||||
args.append('AREA:watt_with_unknown_%s%s::STACK' % (probe_uuid, color + 'AA'))
|
||||
args.append('AREA:watt_with_unknown_%s%s::STACK'
|
||||
% (probe_uuid, color + 'AA'))
|
||||
if not stack:
|
||||
graph_lines.append('LINE:watt_with_unknown_%s%s::' % (probe_uuid, color))
|
||||
graph_lines.append('LINE:watt_with_unknown_%s%s::'
|
||||
% (probe_uuid, color))
|
||||
stack = True
|
||||
else:
|
||||
graph_lines.append('LINE:watt_with_unknown_%s%s::STACK' % (probe_uuid, color))
|
||||
graph_lines.append('LINE:watt_with_unknown_%s%s::STACK'
|
||||
% (probe_uuid, color))
|
||||
if len(probe_list) >= 2:
|
||||
# Prepare CDEF expression by adding the required number of '+'
|
||||
cdef_watt += '+,' * int(len(probe_list)-2) + '+'
|
||||
|
@ -183,8 +226,10 @@ def build_graph(scale, probe=None):
|
|||
# Real average (to compute kWh)
|
||||
args.append('VDEF:wattavg=watt,AVERAGE')
|
||||
# Compute kWh for the probe
|
||||
# RPN expressions must contain DEF or CDEF variables, so we pop a CDEF value
|
||||
args.append('CDEF:kwh=watt,POP,wattavg,1000.0,/,%s,3600.0,/,*' % str(scales[scale][0]['interval']))
|
||||
# RPN expressions must contain DEF or CDEF variables, so we pop a
|
||||
# CDEF value
|
||||
args.append('CDEF:kwh=watt,POP,wattavg,1000.0,/,%s,3600.0,/,*'
|
||||
% str(scales[scale][0]['interval']))
|
||||
# Compute cost
|
||||
args.append('CDEF:cost=watt,POP,kwh,%f,*' % cfg.CONF.kwh_price)
|
||||
# Legend
|
||||
|
@ -202,27 +247,30 @@ def build_graph(scale, probe=None):
|
|||
LOG.info('Retrieve PNG summary graph from cache')
|
||||
return png_file
|
||||
|
||||
|
||||
def listen():
|
||||
"""Subscribes to ZeroMQ messages, and adds received measurements to the database.
|
||||
Messages are dictionaries dumped in JSON format.
|
||||
|
||||
"""Subscribes to ZeroMQ messages, and adds received measurements to the
|
||||
database. Messages are dictionaries dumped in JSON format.
|
||||
|
||||
"""
|
||||
LOG.info('RRD listenig to %s' % cfg.CONF.probes_endpoint)
|
||||
|
||||
|
||||
create_dirs()
|
||||
|
||||
|
||||
context = zmq.Context.instance()
|
||||
subscriber = context.socket(zmq.SUB)
|
||||
subscriber.setsockopt(zmq.SUBSCRIBE, '')
|
||||
for endpoint in cfg.CONF.probes_endpoint:
|
||||
subscriber.connect(endpoint)
|
||||
|
||||
|
||||
while True:
|
||||
message = subscriber.recv()
|
||||
measurements = json.loads(message)
|
||||
if not isinstance(measurements, dict):
|
||||
LOG.error('Bad message type (not a dict)')
|
||||
elif cfg.CONF.signature_checking and not security.verify_signature(measurements, cfg.CONF.driver_metering_secret):
|
||||
elif cfg.CONF.signature_checking and \
|
||||
not security.verify_signature(measurements,
|
||||
cfg.CONF.driver_metering_secret):
|
||||
LOG.error('Bad message signature')
|
||||
else:
|
||||
try:
|
||||
|
|
|
@ -1,4 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""This blueprint defines all URLs and answers."""
|
||||
|
||||
|
@ -9,29 +23,40 @@ import rrd
|
|||
|
||||
blueprint = flask.Blueprint('v1', __name__, static_folder='static')
|
||||
|
||||
|
||||
@blueprint.route('/')
|
||||
def welcome():
|
||||
"""Shows specified page."""
|
||||
return flask.redirect('/last/minute/')
|
||||
|
||||
|
||||
@blueprint.route('/last/<scale>/')
|
||||
def welcome_scale(scale):
|
||||
if scale not in flask.request.scales:
|
||||
flask.abort(404)
|
||||
try:
|
||||
return flask.render_template('index.html', probes=sorted(flask.request.probes), scales=flask.request.scales, scale=scale, view='scale')
|
||||
return flask.render_template('index.html',
|
||||
probes=sorted(flask.request.probes),
|
||||
scales=flask.request.scales,
|
||||
scale=scale,
|
||||
view='scale')
|
||||
except TemplateNotFound:
|
||||
flask.abort(404)
|
||||
|
||||
|
||||
@blueprint.route('/probe/<probe>/')
|
||||
def welcome_probe(probe):
|
||||
if probe not in flask.request.probes:
|
||||
flask.abort(404)
|
||||
try:
|
||||
return flask.render_template('index.html', probe=probe, scales=flask.request.scales, view='probe')
|
||||
return flask.render_template('index.html',
|
||||
probe=probe,
|
||||
scales=flask.request.scales,
|
||||
view='probe')
|
||||
except TemplateNotFound:
|
||||
flask.abort(404)
|
||||
|
||||
|
||||
@blueprint.route('/graph/<scale>/')
|
||||
def send_summary_graph(scale):
|
||||
"""Sends summary graph."""
|
||||
|
@ -42,6 +67,7 @@ def send_summary_graph(scale):
|
|||
except:
|
||||
flask.abort(404)
|
||||
|
||||
|
||||
@blueprint.route('/graph/<scale>/<probe>/')
|
||||
def send_probe_graph(scale, probe):
|
||||
"""Sends graph."""
|
||||
|
|
|
@ -1,10 +1,25 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Computes the signature of a metering message."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
|
||||
def recursive_keypairs(d):
|
||||
"""Generator that produces sequence of keypairs for nested dictionaries."""
|
||||
for name, value in sorted(d.iteritems()):
|
||||
|
@ -14,6 +29,7 @@ def recursive_keypairs(d):
|
|||
else:
|
||||
yield name, value
|
||||
|
||||
|
||||
def compute_signature(message, secret):
|
||||
"""Returns the signature for a message dictionary."""
|
||||
digest_maker = hmac.new(secret, '', hashlib.sha256)
|
||||
|
@ -24,12 +40,17 @@ def compute_signature(message, secret):
|
|||
digest_maker.update(unicode(value).encode('utf-8'))
|
||||
return digest_maker.hexdigest()
|
||||
|
||||
|
||||
def append_signature(message, secret):
|
||||
"""Sets the message signature key."""
|
||||
message['message_signature'] = compute_signature(message, secret)
|
||||
|
||||
|
||||
def verify_signature(message, secret):
|
||||
"""Checks the signature in the message against the value computed from the rest of the contents."""
|
||||
"""Checks the signature in the message against the value computed from the
|
||||
rest of the contents.
|
||||
|
||||
"""
|
||||
old_sig = message.get('message_signature')
|
||||
new_sig = compute_signature(message, secret)
|
||||
return new_sig == old_sig
|
||||
|
|
46
setup.py
46
setup.py
|
@ -1,20 +1,34 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Author: François Rossigneux <francois.rossigneux@inria.fr>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import setuptools
|
||||
|
||||
setuptools.setup(
|
||||
|
||||
|
||||
name='kwapi',
|
||||
version='1.0',
|
||||
|
||||
|
||||
description='Energy Efficiency Architecture',
|
||||
|
||||
|
||||
author='François Rossigneux',
|
||||
author_email='francois.rossigneux@inria.fr',
|
||||
|
||||
|
||||
url='http://gitorious.ow2.org/xlcloud/kwapi',
|
||||
|
||||
|
||||
classifiers=[
|
||||
'Development Status :: 3 - Alpha',
|
||||
'Framework :: Setuptools Plugin',
|
||||
|
@ -28,16 +42,22 @@ setuptools.setup(
|
|||
'Programming Language :: Python :: 2.7',
|
||||
'Topic :: System :: Monitoring',
|
||||
],
|
||||
|
||||
|
||||
packages=setuptools.find_packages(),
|
||||
package_data={'kwapi.plugins.rrd': ['templates/*', 'static/*']},
|
||||
|
||||
|
||||
scripts=['bin/kwapi-api',
|
||||
'bin/kwapi-drivers',
|
||||
'bin/kwapi-rrd'],
|
||||
|
||||
data_files=[('/etc/kwapi', ['etc/kwapi/api.conf', 'etc/kwapi/drivers.conf', 'etc/kwapi/rrd.conf'])],
|
||||
|
||||
install_requires=['flask', 'pyserial', 'python-keystoneclient', 'pyzmq', 'py-rrdtool']
|
||||
|
||||
)
|
||||
|
||||
data_files=[('/etc/kwapi', ['etc/kwapi/api.conf',
|
||||
'etc/kwapi/drivers.conf',
|
||||
'etc/kwapi/rrd.conf'])],
|
||||
|
||||
install_requires=['flask',
|
||||
'pyserial',
|
||||
'python-keystoneclient',
|
||||
'pyzmq',
|
||||
'py-rrdtool']
|
||||
|
||||
)
|
||||
|
|
Loading…
Reference in New Issue