Refresh Oslo code - add support for ssl

We want to add ssl support for the reddwarf api server.
For this, updating the openstack/common/wsgi module, and
all it's dependencies as a result of using:
$> python update.py --base reddwarf --dest-dir ../reddwarf --modules wsgi
(details:https://wiki.openstack.org/wiki/Oslo)

I'm commenting out the "six" update, since it seems to break
our fake mode testing with regard to reddwarf-cli.

I'll open a separate bug for it.

Change-Id: I763d06658488bd1ca1eac5a2ba1ffde5629f842f
Fixes: Bug #1178421
This commit is contained in:
Dror Kagan 2013-05-09 13:57:55 -07:00 committed by daniel-a-nguyen
parent 9037e6f3df
commit 9b4279d2eb
20 changed files with 405 additions and 83 deletions

View File

@ -103,3 +103,16 @@ notifier_queue_transport = memory
#log_dir = /integration/report #log_dir = /integration/report
#log_file = reddwarf-api.log #log_file = reddwarf-api.log
# ============ SSL configuration (and enablement) =============================
# In order to enable SSL for the reddwarf api server, uncomment
# the cert_file and key_file - and of course have those files
# accessible. The existance of those setting and files will
# enable SSL.
[ssl]
#cert_file = /path/to/server.crt
#key_file = /path/to/server.key
#optional:
#ca_file = /path/to/ca_file

View File

@ -184,7 +184,7 @@ def get_engine():
def load_mysqld_options(): def load_mysqld_options():
try: try:
out, err = utils.execute("/usr/sbin/mysqld", "--print-defaults", out, err = utils.execute("/usr/sbin/mysqld", "--print-defaults",
run_as_root=True) run_as_root=True, root_helper="sudo")
arglist = re.split("\n", out)[1].split() arglist = re.split("\n", out)[1].split()
args = {} args = {}
for item in arglist: for item in arglist:
@ -256,7 +256,7 @@ class MySqlAppStatus(object):
try: try:
out, err = utils.execute_with_timeout( out, err = utils.execute_with_timeout(
"/usr/bin/mysqladmin", "/usr/bin/mysqladmin",
"ping", run_as_root=True) "ping", run_as_root=True, root_helper="sudo")
LOG.info("Service Status is RUNNING.") LOG.info("Service Status is RUNNING.")
return rd_models.ServiceStatuses.RUNNING return rd_models.ServiceStatuses.RUNNING
except ProcessExecutionError as e: except ProcessExecutionError as e:
@ -791,7 +791,7 @@ class MySqlApp(object):
command = command % locals() command = command % locals()
else: else:
command = "sudo update-rc.d mysql enable" command = "sudo update-rc.d mysql enable"
utils.execute_with_timeout(command, with_shell=True) utils.execute_with_timeout(command, shell=True)
def _disable_mysql_on_boot(self): def _disable_mysql_on_boot(self):
''' '''
@ -809,7 +809,7 @@ class MySqlApp(object):
command = command % locals() command = command % locals()
else: else:
command = "sudo update-rc.d mysql disable" command = "sudo update-rc.d mysql disable"
utils.execute_with_timeout(command, with_shell=True) utils.execute_with_timeout(command, shell=True)
def stop_db(self, update_db=False, do_not_start_on_reboot=False): def stop_db(self, update_db=False, do_not_start_on_reboot=False):
LOG.info(_("Stopping mysql...")) LOG.info(_("Stopping mysql..."))

View File

@ -16,6 +16,8 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from __future__ import print_function
import gc import gc
import pprint import pprint
import sys import sys
@ -37,7 +39,7 @@ CONF.register_opts(eventlet_backdoor_opts)
def _dont_use_this(): def _dont_use_this():
print "Don't use this, just disconnect instead" print("Don't use this, just disconnect instead")
def _find_objects(t): def _find_objects(t):
@ -46,16 +48,16 @@ def _find_objects(t):
def _print_greenthreads(): def _print_greenthreads():
for i, gt in enumerate(_find_objects(greenlet.greenlet)): for i, gt in enumerate(_find_objects(greenlet.greenlet)):
print i, gt print(i, gt)
traceback.print_stack(gt.gr_frame) traceback.print_stack(gt.gr_frame)
print print()
def _print_nativethreads(): def _print_nativethreads():
for threadId, stack in sys._current_frames().items(): for threadId, stack in sys._current_frames().items():
print threadId print(threadId)
traceback.print_stack(stack) traceback.print_stack(stack)
print print()
def initialize_if_enabled(): def initialize_if_enabled():

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -23,6 +23,8 @@ import logging
from reddwarf.openstack.common.gettextutils import _ from reddwarf.openstack.common.gettextutils import _
_FATAL_EXCEPTION_FORMAT_ERRORS = False
class Error(Exception): class Error(Exception):
def __init__(self, message=None): def __init__(self, message=None):
@ -96,7 +98,7 @@ def wrap_exception(f):
def _wrap(*args, **kw): def _wrap(*args, **kw):
try: try:
return f(*args, **kw) return f(*args, **kw)
except Exception, e: except Exception as e:
if not isinstance(e, Error): if not isinstance(e, Error):
#exc_type, exc_value, exc_traceback = sys.exc_info() #exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception(_('Uncaught exception')) logging.exception(_('Uncaught exception'))
@ -121,7 +123,10 @@ class OpenstackException(Exception):
try: try:
self._error_string = self.message % kwargs self._error_string = self.message % kwargs
except Exception: except Exception as e:
if _FATAL_EXCEPTION_FORMAT_ERRORS:
raise e
else:
# at least get the core message out if something happened # at least get the core message out if something happened
self._error_string = self.message self._error_string = self.message

View File

@ -41,6 +41,8 @@ import json
import types import types
import xmlrpclib import xmlrpclib
#import six
from reddwarf.openstack.common import timeutils from reddwarf.openstack.common import timeutils
@ -94,6 +96,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
# and results in infinite loop when list(value) is called. # and results in infinite loop when list(value) is called.
if type(value) == itertools.count: if type(value) == itertools.count:
return unicode(value) return unicode(value)
# return six.text_type(value)
# FIXME(vish): Workaround for LP bug 852095. Without this workaround, # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
# tests that raise an exception in a mocked method that # tests that raise an exception in a mocked method that
@ -138,11 +141,13 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
else: else:
if any(test(value) for test in _nasty_type_tests): if any(test(value) for test in _nasty_type_tests):
return unicode(value) return unicode(value)
# return six.text_type(value)
return value return value
except TypeError: except TypeError:
# Class objects are tricky since they may define something like # Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list(). # __iter__ defined but it isn't callable as list().
return unicode(value) return unicode(value)
# return six.text_type(value)
def dumps(value, default=to_primitive, **kwargs): def dumps(value, default=to_primitive, **kwargs):

View File

@ -37,7 +37,6 @@ import logging
import logging.config import logging.config
import logging.handlers import logging.handlers
import os import os
import stat
import sys import sys
import traceback import traceback
@ -49,7 +48,6 @@ from reddwarf.openstack.common import local
from reddwarf.openstack.common import notifier from reddwarf.openstack.common import notifier
_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
common_cli_opts = [ common_cli_opts = [
@ -74,11 +72,13 @@ logging_cli_opts = [
'documentation for details on logging configuration ' 'documentation for details on logging configuration '
'files.'), 'files.'),
cfg.StrOpt('log-format', cfg.StrOpt('log-format',
default=_DEFAULT_LOG_FORMAT, default=None,
metavar='FORMAT', metavar='FORMAT',
help='A logging.Formatter log message format string which may ' help='A logging.Formatter log message format string which may '
'use any of the available logging.LogRecord attributes. ' 'use any of the available logging.LogRecord attributes. '
'Default: %(default)s'), 'This option is deprecated. Please use '
'logging_context_format_string and '
'logging_default_format_string instead.'),
cfg.StrOpt('log-date-format', cfg.StrOpt('log-date-format',
default=_DEFAULT_LOG_DATE_FORMAT, default=_DEFAULT_LOG_DATE_FORMAT,
metavar='DATE_FORMAT', metavar='DATE_FORMAT',
@ -104,10 +104,7 @@ logging_cli_opts = [
generic_log_opts = [ generic_log_opts = [
cfg.BoolOpt('use_stderr', cfg.BoolOpt('use_stderr',
default=True, default=True,
help='Log output to standard error'), help='Log output to standard error')
cfg.StrOpt('logfile_mode',
default='0644',
help='Default file mode used when creating log files'),
] ]
log_opts = [ log_opts = [
@ -211,7 +208,27 @@ def _get_log_file_path(binary=None):
return '%s.log' % (os.path.join(logdir, binary),) return '%s.log' % (os.path.join(logdir, binary),)
class ContextAdapter(logging.LoggerAdapter): class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs):
self.log(logging.AUDIT, msg, *args, **kwargs)
class LazyAdapter(BaseLoggerAdapter):
def __init__(self, name='unknown', version='unknown'):
self._logger = None
self.extra = {}
self.name = name
self.version = version
@property
def logger(self):
if not self._logger:
self._logger = getLogger(self.name, self.version)
return self._logger
class ContextAdapter(BaseLoggerAdapter):
warn = logging.LoggerAdapter.warning warn = logging.LoggerAdapter.warning
def __init__(self, logger, project_name, version_string): def __init__(self, logger, project_name, version_string):
@ -219,8 +236,9 @@ class ContextAdapter(logging.LoggerAdapter):
self.project = project_name self.project = project_name
self.version = version_string self.version = version_string
def audit(self, msg, *args, **kwargs): @property
self.log(logging.AUDIT, msg, *args, **kwargs) def handlers(self):
return self.logger.handlers
def deprecated(self, msg, *args, **kwargs): def deprecated(self, msg, *args, **kwargs):
stdmsg = _("Deprecated: %s") % msg stdmsg = _("Deprecated: %s") % msg
@ -399,11 +417,6 @@ def _setup_logging_from_conf():
filelog = logging.handlers.WatchedFileHandler(logpath) filelog = logging.handlers.WatchedFileHandler(logpath)
log_root.addHandler(filelog) log_root.addHandler(filelog)
mode = int(CONF.logfile_mode, 8)
st = os.stat(logpath)
if st.st_mode != (stat.S_IFREG | mode):
os.chmod(logpath, mode)
if CONF.use_stderr: if CONF.use_stderr:
streamlog = ColorHandler() streamlog = ColorHandler()
log_root.addHandler(streamlog) log_root.addHandler(streamlog)
@ -417,13 +430,17 @@ def _setup_logging_from_conf():
if CONF.publish_errors: if CONF.publish_errors:
log_root.addHandler(PublishErrorsHandler(logging.ERROR)) log_root.addHandler(PublishErrorsHandler(logging.ERROR))
for handler in log_root.handlers:
datefmt = CONF.log_date_format datefmt = CONF.log_date_format
for handler in log_root.handlers:
# NOTE(alaski): CONF.log_format overrides everything currently. This
# should be deprecated in favor of context aware formatting.
if CONF.log_format: if CONF.log_format:
handler.setFormatter(logging.Formatter(fmt=CONF.log_format, handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
datefmt=datefmt)) datefmt=datefmt))
log_root.info('Deprecated: log_format is now deprecated and will '
'be removed in the next release')
else: else:
handler.setFormatter(LegacyFormatter(datefmt=datefmt)) handler.setFormatter(ContextFormatter(datefmt=datefmt))
if CONF.debug: if CONF.debug:
log_root.setLevel(logging.DEBUG) log_root.setLevel(logging.DEBUG)
@ -449,6 +466,15 @@ def getLogger(name='unknown', version='unknown'):
return _loggers[name] return _loggers[name]
def getLazyLogger(name='unknown', version='unknown'):
"""
create a pass-through logger that does not create the real logger
until it is really needed and delegates all calls to the real logger
once it is created
"""
return LazyAdapter(name, version)
class WritableLogger(object): class WritableLogger(object):
"""A thin wrapper that responds to `write` and logs.""" """A thin wrapper that responds to `write` and logs."""
@ -460,7 +486,7 @@ class WritableLogger(object):
self.logger.log(self.level, msg) self.logger.log(self.level, msg)
class LegacyFormatter(logging.Formatter): class ContextFormatter(logging.Formatter):
"""A context.RequestContext aware formatter configured through flags. """A context.RequestContext aware formatter configured through flags.
The flags used to set format strings are: logging_context_format_string The flags used to set format strings are: logging_context_format_string

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -19,8 +19,10 @@
System-level utilities and helper functions. System-level utilities and helper functions.
""" """
import os
import random import random
import shlex import shlex
import signal
from eventlet.green import subprocess from eventlet.green import subprocess
from eventlet import greenthread from eventlet import greenthread
@ -40,6 +42,12 @@ class UnknownArgumentError(Exception):
class ProcessExecutionError(Exception): class ProcessExecutionError(Exception):
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
description=None): description=None):
self.exit_code = exit_code
self.stderr = stderr
self.stdout = stdout
self.cmd = cmd
self.description = description
if description is None: if description is None:
description = "Unexpected error while running command." description = "Unexpected error while running command."
if exit_code is None: if exit_code is None:
@ -49,6 +57,17 @@ class ProcessExecutionError(Exception):
super(ProcessExecutionError, self).__init__(message) super(ProcessExecutionError, self).__init__(message)
class NoRootWrapSpecified(Exception):
def __init__(self, message=None):
super(NoRootWrapSpecified, self).__init__(message)
def _subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
# non-Python subprocesses expect.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def execute(*cmd, **kwargs): def execute(*cmd, **kwargs):
""" """
Helper method to shell out and execute a command through subprocess with Helper method to shell out and execute a command through subprocess with
@ -58,11 +77,11 @@ def execute(*cmd, **kwargs):
:type cmd: string :type cmd: string
:param process_input: Send to opened process. :param process_input: Send to opened process.
:type proces_input: string :type proces_input: string
:param check_exit_code: Defaults to 0. Will raise :param check_exit_code: Single bool, int, or list of allowed exit
:class:`ProcessExecutionError` codes. Defaults to [0]. Raise
if the command exits without returning this value :class:`ProcessExecutionError` unless
as a returncode program exits with one of these code.
:type check_exit_code: int :type check_exit_code: boolean, int, or [int]
:param delay_on_retry: True | False. Defaults to True. If set to True, :param delay_on_retry: True | False. Defaults to True. If set to True,
wait a short amount of time before retrying. wait a short amount of time before retrying.
:type delay_on_retry: boolean :type delay_on_retry: boolean
@ -72,8 +91,12 @@ def execute(*cmd, **kwargs):
the command is prefixed by the command specified the command is prefixed by the command specified
in the root_helper kwarg. in the root_helper kwarg.
:type run_as_root: boolean :type run_as_root: boolean
:param root_helper: command to prefix all cmd's with :param root_helper: command to prefix to commands called with
run_as_root=True
:type root_helper: string :type root_helper: string
:param shell: whether or not there should be a shell used to
execute this command. Defaults to false.
:type shell: boolean
:returns: (stdout, stderr) from process execution :returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on :raises: :class:`UnknownArgumentError` on
receiving unknown arguments receiving unknown arguments
@ -81,17 +104,31 @@ def execute(*cmd, **kwargs):
""" """
process_input = kwargs.pop('process_input', None) process_input = kwargs.pop('process_input', None)
check_exit_code = kwargs.pop('check_exit_code', 0) check_exit_code = kwargs.pop('check_exit_code', [0])
ignore_exit_code = False
delay_on_retry = kwargs.pop('delay_on_retry', True) delay_on_retry = kwargs.pop('delay_on_retry', True)
attempts = kwargs.pop('attempts', 1) attempts = kwargs.pop('attempts', 1)
run_as_root = kwargs.pop('run_as_root', False) run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '') root_helper = kwargs.pop('root_helper', '')
with_shell = kwargs.pop('with_shell', False) shell = kwargs.pop('shell', False)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
check_exit_code = [0]
elif isinstance(check_exit_code, int):
check_exit_code = [check_exit_code]
if len(kwargs): if len(kwargs):
raise UnknownArgumentError(_('Got unknown keyword args ' raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs) 'to utils.execute: %r') % kwargs)
if run_as_root:
if run_as_root and os.geteuid() != 0:
if not root_helper:
raise NoRootWrapSpecified(
message=('Command requested root, but did not specify a root '
'helper.'))
cmd = shlex.split(root_helper) + list(cmd) cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd) cmd = map(str, cmd)
while attempts > 0: while attempts > 0:
@ -99,12 +136,21 @@ def execute(*cmd, **kwargs):
try: try:
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd)) LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
_PIPE = subprocess.PIPE # pylint: disable=E1101 _PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
preexec_fn = None
close_fds = False
else:
preexec_fn = _subprocess_setup
close_fds = True
obj = subprocess.Popen(cmd, obj = subprocess.Popen(cmd,
stdin=_PIPE, stdin=_PIPE,
stdout=_PIPE, stdout=_PIPE,
stderr=_PIPE, stderr=_PIPE,
close_fds=True, close_fds=close_fds,
shell=with_shell) preexec_fn=preexec_fn,
shell=shell)
result = None result = None
if process_input is not None: if process_input is not None:
result = obj.communicate(process_input) result = obj.communicate(process_input)
@ -114,9 +160,7 @@ def execute(*cmd, **kwargs):
_returncode = obj.returncode # pylint: disable=E1101 _returncode = obj.returncode # pylint: disable=E1101
if _returncode: if _returncode:
LOG.debug(_('Result was %s') % _returncode) LOG.debug(_('Result was %s') % _returncode)
if (isinstance(check_exit_code, int) and if not ignore_exit_code and _returncode not in check_exit_code:
not isinstance(check_exit_code, bool) and
_returncode != check_exit_code):
(stdout, stderr) = result (stdout, stderr) = result
raise ProcessExecutionError(exit_code=_returncode, raise ProcessExecutionError(exit_code=_returncode,
stdout=stdout, stdout=stdout,

View File

@ -22,6 +22,7 @@ import sys
import traceback import traceback
from oslo.config import cfg from oslo.config import cfg
#import six
from reddwarf.openstack.common.gettextutils import _ from reddwarf.openstack.common.gettextutils import _
from reddwarf.openstack.common import importutils from reddwarf.openstack.common import importutils
@ -300,6 +301,8 @@ def serialize_remote_exception(failure_info, log_failure=True):
failure = failure_info[1] failure = failure_info[1]
if log_failure: if log_failure:
LOG.error(_("Returning exception %s to caller"), unicode(failure)) LOG.error(_("Returning exception %s to caller"), unicode(failure))
# LOG.error(_("Returning exception %s to caller"),
# six.text_type(failure))
LOG.error(tb) LOG.error(tb)
kwargs = {} kwargs = {}
@ -310,6 +313,7 @@ def serialize_remote_exception(failure_info, log_failure=True):
'class': str(failure.__class__.__name__), 'class': str(failure.__class__.__name__),
'module': str(failure.__class__.__module__), 'module': str(failure.__class__.__module__),
'message': unicode(failure), 'message': unicode(failure),
# 'message': six.text_type(failure),
'tb': tb, 'tb': tb,
'args': failure.args, 'args': failure.args,
'kwargs': kwargs 'kwargs': kwargs

View File

@ -331,15 +331,16 @@ class Connection(object):
def reconnect(self): def reconnect(self):
"""Handles reconnecting and re-establishing sessions and queues""" """Handles reconnecting and re-establishing sessions and queues"""
attempt = 0
delay = 1
while True:
# Close the session if necessary
if self.connection.opened(): if self.connection.opened():
try: try:
self.connection.close() self.connection.close()
except qpid_exceptions.ConnectionError: except qpid_exceptions.ConnectionError:
pass pass
attempt = 0
delay = 1
while True:
broker = self.brokers[attempt % len(self.brokers)] broker = self.brokers[attempt % len(self.brokers)]
attempt += 1 attempt += 1

View File

@ -0,0 +1,80 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import ssl
from oslo.config import cfg
from reddwarf.openstack.common.gettextutils import _
ssl_opts = [
cfg.StrOpt('ca_file',
default=None,
help="CA certificate file to use to verify "
"connecting clients"),
cfg.StrOpt('cert_file',
default=None,
help="Certificate file to use when starting "
"the server securely"),
cfg.StrOpt('key_file',
default=None,
help="Private key file to use when starting "
"the server securely"),
]
CONF = cfg.CONF
CONF.register_opts(ssl_opts, "ssl")
def is_enabled():
cert_file = CONF.ssl.cert_file
key_file = CONF.ssl.key_file
ca_file = CONF.ssl.ca_file
use_ssl = cert_file or key_file
if cert_file and not os.path.exists(cert_file):
raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
if ca_file and not os.path.exists(ca_file):
raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)
if key_file and not os.path.exists(key_file):
raise RuntimeError(_("Unable to find key_file : %s") % key_file)
if use_ssl and (not cert_file or not key_file):
raise RuntimeError(_("When running server in SSL mode, you must "
"specify both a cert_file and key_file "
"option value in your configuration file"))
return use_ssl
def wrap(sock):
ssl_kwargs = {
'server_side': True,
'certfile': CONF.ssl.cert_file,
'keyfile': CONF.ssl.key_file,
'cert_reqs': ssl.CERT_NONE,
}
if CONF.ssl.ca_file:
ssl_kwargs['ca_certs'] = CONF.ssl.ca_file
ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
return ssl.wrap_socket(sock, **ssl_kwargs)

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -17,15 +17,22 @@
"""Utility methods for working with WSGI servers.""" """Utility methods for working with WSGI servers."""
import datetime from __future__ import print_function
import eventlet
import eventlet.wsgi
import eventlet
eventlet.patcher.monkey_patch(all=False, socket=True) eventlet.patcher.monkey_patch(all=False, socket=True)
import datetime
import errno
import socket
import sys
import time
import eventlet.wsgi
from oslo.config import cfg
import routes import routes
import routes.middleware import routes.middleware
import sys #import six
import webob.dec import webob.dec
import webob.exc import webob.exc
from xml.dom import minidom from xml.dom import minidom
@ -36,15 +43,29 @@ from reddwarf.openstack.common.gettextutils import _
from reddwarf.openstack.common import jsonutils from reddwarf.openstack.common import jsonutils
from reddwarf.openstack.common import log as logging from reddwarf.openstack.common import log as logging
from reddwarf.openstack.common import service from reddwarf.openstack.common import service
from reddwarf.openstack.common import sslutils
from reddwarf.openstack.common import xmlutils
socket_opts = [
cfg.IntOpt('backlog',
default=4096,
help="Number of backlog requests to configure the socket with"),
cfg.IntOpt('tcp_keepidle',
default=600,
help="Sets the value of TCP_KEEPIDLE in seconds for each "
"server socket. Not supported on OS X."),
]
CONF = cfg.CONF
CONF.register_opts(socket_opts)
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
def run_server(application, port): def run_server(application, port, **kwargs):
"""Run a WSGI server with the given application.""" """Run a WSGI server with the given application."""
sock = eventlet.listen(('0.0.0.0', port)) sock = eventlet.listen(('0.0.0.0', port))
eventlet.wsgi.server(sock, application) eventlet.wsgi.server(sock, application, **kwargs)
class Service(service.Service): class Service(service.Service):
@ -56,13 +77,55 @@ class Service(service.Service):
""" """
def __init__(self, application, port, def __init__(self, application, port,
host='0.0.0.0', backlog=128, threads=1000): host='0.0.0.0', backlog=4096, threads=1000):
self.application = application self.application = application
self._port = port self._port = port
self._host = host self._host = host
self.backlog = backlog self._backlog = backlog if backlog else CONF.backlog
self._socket = self._get_socket(host, port, self._backlog)
super(Service, self).__init__(threads) super(Service, self).__init__(threads)
def _get_socket(self, host, port, backlog):
# TODO(dims): eventlet's green dns/socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
info = socket.getaddrinfo(host,
port,
socket.AF_UNSPEC,
socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
sock = None
retry_until = time.time() + 30
while not sock and time.time() < retry_until:
try:
sock = eventlet.listen(bind_addr,
backlog=backlog,
family=family)
if sslutils.is_enabled():
sock = sslutils.wrap(sock)
except socket.error as err:
if err.args[0] != errno.EADDRINUSE:
raise
eventlet.sleep(0.1)
if not sock:
raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
"after trying for 30 seconds") %
{'host': host, 'port': port})
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# sockets can hang around forever without keepalive
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# This option isn't available in the OS X version of eventlet
if hasattr(socket, 'TCP_KEEPIDLE'):
sock.setsockopt(socket.IPPROTO_TCP,
socket.TCP_KEEPIDLE,
CONF.tcp_keepidle)
return sock
def start(self): def start(self):
"""Start serving this service using the provided server instance. """Start serving this service using the provided server instance.
@ -70,10 +133,12 @@ class Service(service.Service):
""" """
super(Service, self).start() super(Service, self).start()
self._socket = eventlet.listen((self._host, self._port),
backlog=self.backlog)
self.tg.add_thread(self._run, self.application, self._socket) self.tg.add_thread(self._run, self.application, self._socket)
@property
def backlog(self):
return self._backlog
@property @property
def host(self): def host(self):
return self._socket.getsockname()[0] if self._socket else self._host return self._socket.getsockname()[0] if self._socket else self._host
@ -93,7 +158,9 @@ class Service(service.Service):
def _run(self, application, socket): def _run(self, application, socket):
"""Start a WSGI server in a new green thread.""" """Start a WSGI server in a new green thread."""
logger = logging.getLogger('eventlet.wsgi') logger = logging.getLogger('eventlet.wsgi')
eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool, eventlet.wsgi.server(socket,
application,
custom_pool=self.tg.pool,
log=logging.WritableLogger(logger)) log=logging.WritableLogger(logger))
@ -139,16 +206,16 @@ class Debug(Middleware):
@webob.dec.wsgify @webob.dec.wsgify
def __call__(self, req): def __call__(self, req):
print ("*" * 40) + " REQUEST ENVIRON" print(("*" * 40) + " REQUEST ENVIRON")
for key, value in req.environ.items(): for key, value in req.environ.items():
print key, "=", value print(key, "=", value)
print print()
resp = req.get_response(self.application) resp = req.get_response(self.application)
print ("*" * 40) + " RESPONSE HEADERS" print(("*" * 40) + " RESPONSE HEADERS")
for (key, value) in resp.headers.iteritems(): for (key, value) in resp.headers.iteritems():
print key, "=", value print(key, "=", value)
print print()
resp.app_iter = self.print_generator(resp.app_iter) resp.app_iter = self.print_generator(resp.app_iter)
@ -160,12 +227,12 @@ class Debug(Middleware):
Iterator that prints the contents of a wrapper string iterator Iterator that prints the contents of a wrapper string iterator
when iterated. when iterated.
""" """
print ("*" * 40) + " BODY" print(("*" * 40) + " BODY")
for part in app_iter: for part in app_iter:
sys.stdout.write(part) sys.stdout.write(part)
sys.stdout.flush() sys.stdout.flush()
yield part yield part
print print()
class Router(object): class Router(object):
@ -257,7 +324,7 @@ class Request(webob.Request):
Does not do any body introspection, only checks header Does not do any body introspection, only checks header
""" """
if not "Content-Type" in self.headers: if "Content-Type" not in self.headers:
return None return None
content_type = self.content_type content_type = self.content_type
@ -388,6 +455,7 @@ class JSONDictSerializer(DictSerializer):
_dtime = obj - datetime.timedelta(microseconds=obj.microsecond) _dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
return _dtime.isoformat() return _dtime.isoformat()
return obj return obj
# return six.text_type(obj)
return jsonutils.dumps(data, default=sanitizer) return jsonutils.dumps(data, default=sanitizer)
@ -680,7 +748,7 @@ class XMLDeserializer(TextDeserializer):
plurals = set(self.metadata.get('plurals', {})) plurals = set(self.metadata.get('plurals', {}))
try: try:
node = minidom.parseString(datastring).childNodes[0] node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0]
return {node.nodeName: self._from_xml_node(node, plurals)} return {node.nodeName: self._from_xml_node(node, plurals)}
except expat.ExpatError: except expat.ExpatError:
msg = _("cannot understand XML") msg = _("cannot understand XML")

View File

@ -0,0 +1,74 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from xml.dom import minidom
from xml.parsers import expat
from xml import sax
from xml.sax import expatreader
class ProtectedExpatParser(expatreader.ExpatParser):
"""An expat parser which disables DTD's and entities by default."""
def __init__(self, forbid_dtd=True, forbid_entities=True,
*args, **kwargs):
# Python 2.x old style class
expatreader.ExpatParser.__init__(self, *args, **kwargs)
self.forbid_dtd = forbid_dtd
self.forbid_entities = forbid_entities
def start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
raise ValueError("Inline DTD forbidden")
def entity_decl(self, entityName, is_parameter_entity, value, base,
systemId, publicId, notationName):
raise ValueError("<!ENTITY> entity declaration forbidden")
def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
# expat 1.2
raise ValueError("<!ENTITY> unparsed entity forbidden")
def external_entity_ref(self, context, base, systemId, publicId):
raise ValueError("<!ENTITY> external entity forbidden")
def notation_decl(self, name, base, sysid, pubid):
raise ValueError("<!ENTITY> notation forbidden")
def reset(self):
expatreader.ExpatParser.reset(self)
if self.forbid_dtd:
self._parser.StartDoctypeDeclHandler = self.start_doctype_decl
self._parser.EndDoctypeDeclHandler = None
if self.forbid_entities:
self._parser.EntityDeclHandler = self.entity_decl
self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl
self._parser.ExternalEntityRefHandler = self.external_entity_ref
self._parser.NotationDeclHandler = self.notation_decl
try:
self._parser.SkippedEntityHandler = None
except AttributeError:
# some pyexpat versions do not support SkippedEntity
pass
def safe_minidom_parse_string(xml_string):
"""Parse an XML string using minidom safely.
"""
try:
return minidom.parseString(xml_string, parser=ProtectedExpatParser())
except sax.SAXParseException:
raise expat.ExpatError()

View File

@ -597,7 +597,7 @@ class UpdateGuest(object):
@test(enabled=UPDATE_GUEST_CONF is not None) @test(enabled=UPDATE_GUEST_CONF is not None)
def upload_update_to_repo(self): def upload_update_to_repo(self):
cmds = UPDATE_GUEST_CONF["install-repo-cmd"] cmds = UPDATE_GUEST_CONF["install-repo-cmd"]
utils.execute(*cmds, run_as_root=True) utils.execute(*cmds, run_as_root=True, root_helper="sudo")
@test(enabled=UPDATE_GUEST_CONF is not None, @test(enabled=UPDATE_GUEST_CONF is not None,
depends_on=[upload_update_to_repo]) depends_on=[upload_update_to_repo])

View File

@ -130,7 +130,7 @@ def call_xmllint(name, body):
if CONFIG.get('xml_xsd', None): if CONFIG.get('xml_xsd', None):
args += ["--schema", CONFIG.xml_xsd] args += ["--schema", CONFIG.xml_xsd]
output = processutils.execute(CONFIG.xmllint_bin, *args, output = processutils.execute(CONFIG.xmllint_bin, *args,
check_exit_code=0, with_shell=False) check_exit_code=0, shell=False)
except processutils.ProcessExecutionError as pe: except processutils.ProcessExecutionError as pe:
fail("Error validating XML! %s" % pe) fail("Error validating XML! %s" % pe)