From 9b4279d2ebbec1a49a54ebf767062a97f0a85c2c Mon Sep 17 00:00:00 2001 From: Dror Kagan Date: Thu, 9 May 2013 13:57:55 -0700 Subject: [PATCH] Refresh Oslo code - add support for ssl We want to add ssl support for the reddwarf api server. For this, updating the openstack/common/wsgi module, and all it's dependencies as a result of using: $> python update.py --base reddwarf --dest-dir ../reddwarf --modules wsgi (details:https://wiki.openstack.org/wiki/Oslo) I'm commenting out the "six" update, since it seems to break our fake mode testing with regard to reddwarf-cli. I'll open a separate bug for it. Change-Id: I763d06658488bd1ca1eac5a2ba1ffde5629f842f Fixes: Bug #1178421 --- etc/reddwarf/reddwarf.conf.sample | 13 ++ reddwarf/guestagent/manager/mysql.py | 8 +- .../openstack/common/eventlet_backdoor.py | 12 +- reddwarf/openstack/common/exception.py | 15 ++- reddwarf/openstack/common/jsonutils.py | 5 + reddwarf/openstack/common/log.py | 64 +++++++--- .../openstack/common/notifier/__init__.py | 2 +- reddwarf/openstack/common/notifier/api.py | 2 +- .../openstack/common/notifier/log_notifier.py | 2 +- .../common/notifier/no_op_notifier.py | 2 +- .../openstack/common/notifier/rpc_notifier.py | 2 +- .../common/notifier/test_notifier.py | 2 +- reddwarf/openstack/common/processutils.py | 72 ++++++++--- reddwarf/openstack/common/rpc/common.py | 4 + reddwarf/openstack/common/rpc/impl_qpid.py | 13 +- reddwarf/openstack/common/sslutils.py | 80 +++++++++++++ reddwarf/openstack/common/wsgi.py | 112 ++++++++++++++---- reddwarf/openstack/common/xmlutils.py | 74 ++++++++++++ reddwarf/tests/api/instances_actions.py | 2 +- reddwarf/tests/util/client.py | 2 +- 20 files changed, 405 insertions(+), 83 deletions(-) create mode 100644 reddwarf/openstack/common/sslutils.py create mode 100644 reddwarf/openstack/common/xmlutils.py diff --git a/etc/reddwarf/reddwarf.conf.sample b/etc/reddwarf/reddwarf.conf.sample index bf4061fc46..43b1b7708a 100644 --- a/etc/reddwarf/reddwarf.conf.sample +++ b/etc/reddwarf/reddwarf.conf.sample @@ -103,3 +103,16 @@ notifier_queue_transport = memory #log_dir = /integration/report #log_file = reddwarf-api.log + +# ============ SSL configuration (and enablement) ============================= +# In order to enable SSL for the reddwarf api server, uncomment +# the cert_file and key_file - and of course have those files +# accessible. The existance of those setting and files will +# enable SSL. + +[ssl] + +#cert_file = /path/to/server.crt +#key_file = /path/to/server.key +#optional: +#ca_file = /path/to/ca_file diff --git a/reddwarf/guestagent/manager/mysql.py b/reddwarf/guestagent/manager/mysql.py index 2e733072e8..0dd10387fe 100644 --- a/reddwarf/guestagent/manager/mysql.py +++ b/reddwarf/guestagent/manager/mysql.py @@ -184,7 +184,7 @@ def get_engine(): def load_mysqld_options(): try: out, err = utils.execute("/usr/sbin/mysqld", "--print-defaults", - run_as_root=True) + run_as_root=True, root_helper="sudo") arglist = re.split("\n", out)[1].split() args = {} for item in arglist: @@ -256,7 +256,7 @@ class MySqlAppStatus(object): try: out, err = utils.execute_with_timeout( "/usr/bin/mysqladmin", - "ping", run_as_root=True) + "ping", run_as_root=True, root_helper="sudo") LOG.info("Service Status is RUNNING.") return rd_models.ServiceStatuses.RUNNING except ProcessExecutionError as e: @@ -791,7 +791,7 @@ class MySqlApp(object): command = command % locals() else: command = "sudo update-rc.d mysql enable" - utils.execute_with_timeout(command, with_shell=True) + utils.execute_with_timeout(command, shell=True) def _disable_mysql_on_boot(self): ''' @@ -809,7 +809,7 @@ class MySqlApp(object): command = command % locals() else: command = "sudo update-rc.d mysql disable" - utils.execute_with_timeout(command, with_shell=True) + utils.execute_with_timeout(command, shell=True) def stop_db(self, update_db=False, do_not_start_on_reboot=False): LOG.info(_("Stopping mysql...")) diff --git a/reddwarf/openstack/common/eventlet_backdoor.py b/reddwarf/openstack/common/eventlet_backdoor.py index c0ad460fe6..57b89ae914 100644 --- a/reddwarf/openstack/common/eventlet_backdoor.py +++ b/reddwarf/openstack/common/eventlet_backdoor.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import print_function + import gc import pprint import sys @@ -37,7 +39,7 @@ CONF.register_opts(eventlet_backdoor_opts) def _dont_use_this(): - print "Don't use this, just disconnect instead" + print("Don't use this, just disconnect instead") def _find_objects(t): @@ -46,16 +48,16 @@ def _find_objects(t): def _print_greenthreads(): for i, gt in enumerate(_find_objects(greenlet.greenlet)): - print i, gt + print(i, gt) traceback.print_stack(gt.gr_frame) - print + print() def _print_nativethreads(): for threadId, stack in sys._current_frames().items(): - print threadId + print(threadId) traceback.print_stack(stack) - print + print() def initialize_if_enabled(): diff --git a/reddwarf/openstack/common/exception.py b/reddwarf/openstack/common/exception.py index 5e42a91c9f..813d4b8ae5 100644 --- a/reddwarf/openstack/common/exception.py +++ b/reddwarf/openstack/common/exception.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -23,6 +23,8 @@ import logging from reddwarf.openstack.common.gettextutils import _ +_FATAL_EXCEPTION_FORMAT_ERRORS = False + class Error(Exception): def __init__(self, message=None): @@ -96,7 +98,7 @@ def wrap_exception(f): def _wrap(*args, **kw): try: return f(*args, **kw) - except Exception, e: + except Exception as e: if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() logging.exception(_('Uncaught exception')) @@ -121,9 +123,12 @@ class OpenstackException(Exception): try: self._error_string = self.message % kwargs - except Exception: - # at least get the core message out if something happened - self._error_string = self.message + except Exception as e: + if _FATAL_EXCEPTION_FORMAT_ERRORS: + raise e + else: + # at least get the core message out if something happened + self._error_string = self.message def __str__(self): return self._error_string diff --git a/reddwarf/openstack/common/jsonutils.py b/reddwarf/openstack/common/jsonutils.py index 9d591392e7..58bc0dbf24 100644 --- a/reddwarf/openstack/common/jsonutils.py +++ b/reddwarf/openstack/common/jsonutils.py @@ -41,6 +41,8 @@ import json import types import xmlrpclib +#import six + from reddwarf.openstack.common import timeutils @@ -94,6 +96,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, # and results in infinite loop when list(value) is called. if type(value) == itertools.count: return unicode(value) +# return six.text_type(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that @@ -138,11 +141,13 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, else: if any(test(value) for test in _nasty_type_tests): return unicode(value) +# return six.text_type(value) return value except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return unicode(value) +# return six.text_type(value) def dumps(value, default=to_primitive, **kwargs): diff --git a/reddwarf/openstack/common/log.py b/reddwarf/openstack/common/log.py index a835c52047..76821b14e2 100644 --- a/reddwarf/openstack/common/log.py +++ b/reddwarf/openstack/common/log.py @@ -37,7 +37,6 @@ import logging import logging.config import logging.handlers import os -import stat import sys import traceback @@ -49,7 +48,6 @@ from reddwarf.openstack.common import local from reddwarf.openstack.common import notifier -_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" common_cli_opts = [ @@ -74,11 +72,13 @@ logging_cli_opts = [ 'documentation for details on logging configuration ' 'files.'), cfg.StrOpt('log-format', - default=_DEFAULT_LOG_FORMAT, + default=None, metavar='FORMAT', help='A logging.Formatter log message format string which may ' 'use any of the available logging.LogRecord attributes. ' - 'Default: %(default)s'), + 'This option is deprecated. Please use ' + 'logging_context_format_string and ' + 'logging_default_format_string instead.'), cfg.StrOpt('log-date-format', default=_DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', @@ -104,10 +104,7 @@ logging_cli_opts = [ generic_log_opts = [ cfg.BoolOpt('use_stderr', default=True, - help='Log output to standard error'), - cfg.StrOpt('logfile_mode', - default='0644', - help='Default file mode used when creating log files'), + help='Log output to standard error') ] log_opts = [ @@ -211,7 +208,27 @@ def _get_log_file_path(binary=None): return '%s.log' % (os.path.join(logdir, binary),) -class ContextAdapter(logging.LoggerAdapter): +class BaseLoggerAdapter(logging.LoggerAdapter): + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + +class LazyAdapter(BaseLoggerAdapter): + def __init__(self, name='unknown', version='unknown'): + self._logger = None + self.extra = {} + self.name = name + self.version = version + + @property + def logger(self): + if not self._logger: + self._logger = getLogger(self.name, self.version) + return self._logger + + +class ContextAdapter(BaseLoggerAdapter): warn = logging.LoggerAdapter.warning def __init__(self, logger, project_name, version_string): @@ -219,8 +236,9 @@ class ContextAdapter(logging.LoggerAdapter): self.project = project_name self.version = version_string - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) + @property + def handlers(self): + return self.logger.handlers def deprecated(self, msg, *args, **kwargs): stdmsg = _("Deprecated: %s") % msg @@ -399,11 +417,6 @@ def _setup_logging_from_conf(): filelog = logging.handlers.WatchedFileHandler(logpath) log_root.addHandler(filelog) - mode = int(CONF.logfile_mode, 8) - st = os.stat(logpath) - if st.st_mode != (stat.S_IFREG | mode): - os.chmod(logpath, mode) - if CONF.use_stderr: streamlog = ColorHandler() log_root.addHandler(streamlog) @@ -417,13 +430,17 @@ def _setup_logging_from_conf(): if CONF.publish_errors: log_root.addHandler(PublishErrorsHandler(logging.ERROR)) + datefmt = CONF.log_date_format for handler in log_root.handlers: - datefmt = CONF.log_date_format + # NOTE(alaski): CONF.log_format overrides everything currently. This + # should be deprecated in favor of context aware formatting. if CONF.log_format: handler.setFormatter(logging.Formatter(fmt=CONF.log_format, datefmt=datefmt)) + log_root.info('Deprecated: log_format is now deprecated and will ' + 'be removed in the next release') else: - handler.setFormatter(LegacyFormatter(datefmt=datefmt)) + handler.setFormatter(ContextFormatter(datefmt=datefmt)) if CONF.debug: log_root.setLevel(logging.DEBUG) @@ -449,6 +466,15 @@ def getLogger(name='unknown', version='unknown'): return _loggers[name] +def getLazyLogger(name='unknown', version='unknown'): + """ + create a pass-through logger that does not create the real logger + until it is really needed and delegates all calls to the real logger + once it is created + """ + return LazyAdapter(name, version) + + class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -460,7 +486,7 @@ class WritableLogger(object): self.logger.log(self.level, msg) -class LegacyFormatter(logging.Formatter): +class ContextFormatter(logging.Formatter): """A context.RequestContext aware formatter configured through flags. The flags used to set format strings are: logging_context_format_string diff --git a/reddwarf/openstack/common/notifier/__init__.py b/reddwarf/openstack/common/notifier/__init__.py index 31285c4fc2..45c3b46ae9 100644 --- a/reddwarf/openstack/common/notifier/__init__.py +++ b/reddwarf/openstack/common/notifier/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/notifier/api.py b/reddwarf/openstack/common/notifier/api.py index 7b68262bf6..e231a628e9 100644 --- a/reddwarf/openstack/common/notifier/api.py +++ b/reddwarf/openstack/common/notifier/api.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/notifier/log_notifier.py b/reddwarf/openstack/common/notifier/log_notifier.py index 61389bfd79..9c04d43576 100644 --- a/reddwarf/openstack/common/notifier/log_notifier.py +++ b/reddwarf/openstack/common/notifier/log_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/notifier/no_op_notifier.py b/reddwarf/openstack/common/notifier/no_op_notifier.py index d22d02072e..bc7a56ca7a 100644 --- a/reddwarf/openstack/common/notifier/no_op_notifier.py +++ b/reddwarf/openstack/common/notifier/no_op_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/notifier/rpc_notifier.py b/reddwarf/openstack/common/notifier/rpc_notifier.py index 93d9744b50..667d5223f2 100644 --- a/reddwarf/openstack/common/notifier/rpc_notifier.py +++ b/reddwarf/openstack/common/notifier/rpc_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/notifier/test_notifier.py b/reddwarf/openstack/common/notifier/test_notifier.py index 2e392f8b90..96c1746bf4 100644 --- a/reddwarf/openstack/common/notifier/test_notifier.py +++ b/reddwarf/openstack/common/notifier/test_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/reddwarf/openstack/common/processutils.py b/reddwarf/openstack/common/processutils.py index 53d110e553..4636b66a51 100644 --- a/reddwarf/openstack/common/processutils.py +++ b/reddwarf/openstack/common/processutils.py @@ -19,8 +19,10 @@ System-level utilities and helper functions. """ +import os import random import shlex +import signal from eventlet.green import subprocess from eventlet import greenthread @@ -40,6 +42,12 @@ class UnknownArgumentError(Exception): class ProcessExecutionError(Exception): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + if description is None: description = "Unexpected error while running command." if exit_code is None: @@ -49,6 +57,17 @@ class ProcessExecutionError(Exception): super(ProcessExecutionError, self).__init__(message) +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + def execute(*cmd, **kwargs): """ Helper method to shell out and execute a command through subprocess with @@ -58,11 +77,11 @@ def execute(*cmd, **kwargs): :type cmd: string :param process_input: Send to opened process. :type proces_input: string - :param check_exit_code: Defaults to 0. Will raise - :class:`ProcessExecutionError` - if the command exits without returning this value - as a returncode - :type check_exit_code: int + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] :param delay_on_retry: True | False. Defaults to True. If set to True, wait a short amount of time before retrying. :type delay_on_retry: boolean @@ -72,8 +91,12 @@ def execute(*cmd, **kwargs): the command is prefixed by the command specified in the root_helper kwarg. :type run_as_root: boolean - :param root_helper: command to prefix all cmd's with + :param root_helper: command to prefix to commands called with + run_as_root=True :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean :returns: (stdout, stderr) from process execution :raises: :class:`UnknownArgumentError` on receiving unknown arguments @@ -81,17 +104,31 @@ def execute(*cmd, **kwargs): """ process_input = kwargs.pop('process_input', None) - check_exit_code = kwargs.pop('check_exit_code', 0) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False delay_on_retry = kwargs.pop('delay_on_retry', True) attempts = kwargs.pop('attempts', 1) run_as_root = kwargs.pop('run_as_root', False) root_helper = kwargs.pop('root_helper', '') - with_shell = kwargs.pop('with_shell', False) + shell = kwargs.pop('shell', False) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + if len(kwargs): raise UnknownArgumentError(_('Got unknown keyword args ' 'to utils.execute: %r') % kwargs) - if run_as_root: + + if run_as_root and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=('Command requested root, but did not specify a root ' + 'helper.')) cmd = shlex.split(root_helper) + list(cmd) + cmd = map(str, cmd) while attempts > 0: @@ -99,12 +136,21 @@ def execute(*cmd, **kwargs): try: LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd)) _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, stderr=_PIPE, - close_fds=True, - shell=with_shell) + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) result = None if process_input is not None: result = obj.communicate(process_input) @@ -114,9 +160,7 @@ def execute(*cmd, **kwargs): _returncode = obj.returncode # pylint: disable=E1101 if _returncode: LOG.debug(_('Result was %s') % _returncode) - if (isinstance(check_exit_code, int) and - not isinstance(check_exit_code, bool) and - _returncode != check_exit_code): + if not ignore_exit_code and _returncode not in check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=_returncode, stdout=stdout, diff --git a/reddwarf/openstack/common/rpc/common.py b/reddwarf/openstack/common/rpc/common.py index 2fbd0482b8..15149a710d 100644 --- a/reddwarf/openstack/common/rpc/common.py +++ b/reddwarf/openstack/common/rpc/common.py @@ -22,6 +22,7 @@ import sys import traceback from oslo.config import cfg +#import six from reddwarf.openstack.common.gettextutils import _ from reddwarf.openstack.common import importutils @@ -300,6 +301,8 @@ def serialize_remote_exception(failure_info, log_failure=True): failure = failure_info[1] if log_failure: LOG.error(_("Returning exception %s to caller"), unicode(failure)) +# LOG.error(_("Returning exception %s to caller"), +# six.text_type(failure)) LOG.error(tb) kwargs = {} @@ -310,6 +313,7 @@ def serialize_remote_exception(failure_info, log_failure=True): 'class': str(failure.__class__.__name__), 'module': str(failure.__class__.__module__), 'message': unicode(failure), +# 'message': six.text_type(failure), 'tb': tb, 'args': failure.args, 'kwargs': kwargs diff --git a/reddwarf/openstack/common/rpc/impl_qpid.py b/reddwarf/openstack/common/rpc/impl_qpid.py index c7d57acc64..af902d3f35 100644 --- a/reddwarf/openstack/common/rpc/impl_qpid.py +++ b/reddwarf/openstack/common/rpc/impl_qpid.py @@ -331,15 +331,16 @@ class Connection(object): def reconnect(self): """Handles reconnecting and re-establishing sessions and queues""" - if self.connection.opened(): - try: - self.connection.close() - except qpid_exceptions.ConnectionError: - pass - attempt = 0 delay = 1 while True: + # Close the session if necessary + if self.connection.opened(): + try: + self.connection.close() + except qpid_exceptions.ConnectionError: + pass + broker = self.brokers[attempt % len(self.brokers)] attempt += 1 diff --git a/reddwarf/openstack/common/sslutils.py b/reddwarf/openstack/common/sslutils.py new file mode 100644 index 0000000000..dbabf0d094 --- /dev/null +++ b/reddwarf/openstack/common/sslutils.py @@ -0,0 +1,80 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import ssl + +from oslo.config import cfg + +from reddwarf.openstack.common.gettextutils import _ + + +ssl_opts = [ + cfg.StrOpt('ca_file', + default=None, + help="CA certificate file to use to verify " + "connecting clients"), + cfg.StrOpt('cert_file', + default=None, + help="Certificate file to use when starting " + "the server securely"), + cfg.StrOpt('key_file', + default=None, + help="Private key file to use when starting " + "the server securely"), +] + + +CONF = cfg.CONF +CONF.register_opts(ssl_opts, "ssl") + + +def is_enabled(): + cert_file = CONF.ssl.cert_file + key_file = CONF.ssl.key_file + ca_file = CONF.ssl.ca_file + use_ssl = cert_file or key_file + + if cert_file and not os.path.exists(cert_file): + raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) + + if ca_file and not os.path.exists(ca_file): + raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) + + if key_file and not os.path.exists(key_file): + raise RuntimeError(_("Unable to find key_file : %s") % key_file) + + if use_ssl and (not cert_file or not key_file): + raise RuntimeError(_("When running server in SSL mode, you must " + "specify both a cert_file and key_file " + "option value in your configuration file")) + + return use_ssl + + +def wrap(sock): + ssl_kwargs = { + 'server_side': True, + 'certfile': CONF.ssl.cert_file, + 'keyfile': CONF.ssl.key_file, + 'cert_reqs': ssl.CERT_NONE, + } + + if CONF.ssl.ca_file: + ssl_kwargs['ca_certs'] = CONF.ssl.ca_file + ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED + + return ssl.wrap_socket(sock, **ssl_kwargs) diff --git a/reddwarf/openstack/common/wsgi.py b/reddwarf/openstack/common/wsgi.py index f3ff899013..7f9b716689 100644 --- a/reddwarf/openstack/common/wsgi.py +++ b/reddwarf/openstack/common/wsgi.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -17,15 +17,22 @@ """Utility methods for working with WSGI servers.""" -import datetime -import eventlet -import eventlet.wsgi +from __future__ import print_function +import eventlet eventlet.patcher.monkey_patch(all=False, socket=True) +import datetime +import errno +import socket +import sys +import time + +import eventlet.wsgi +from oslo.config import cfg import routes import routes.middleware -import sys +#import six import webob.dec import webob.exc from xml.dom import minidom @@ -36,15 +43,29 @@ from reddwarf.openstack.common.gettextutils import _ from reddwarf.openstack.common import jsonutils from reddwarf.openstack.common import log as logging from reddwarf.openstack.common import service +from reddwarf.openstack.common import sslutils +from reddwarf.openstack.common import xmlutils +socket_opts = [ + cfg.IntOpt('backlog', + default=4096, + help="Number of backlog requests to configure the socket with"), + cfg.IntOpt('tcp_keepidle', + default=600, + help="Sets the value of TCP_KEEPIDLE in seconds for each " + "server socket. Not supported on OS X."), +] + +CONF = cfg.CONF +CONF.register_opts(socket_opts) LOG = logging.getLogger(__name__) -def run_server(application, port): +def run_server(application, port, **kwargs): """Run a WSGI server with the given application.""" sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, application) + eventlet.wsgi.server(sock, application, **kwargs) class Service(service.Service): @@ -56,13 +77,55 @@ class Service(service.Service): """ def __init__(self, application, port, - host='0.0.0.0', backlog=128, threads=1000): + host='0.0.0.0', backlog=4096, threads=1000): self.application = application self._port = port self._host = host - self.backlog = backlog + self._backlog = backlog if backlog else CONF.backlog + self._socket = self._get_socket(host, port, self._backlog) super(Service, self).__init__(threads) + def _get_socket(self, host, port, backlog): + # TODO(dims): eventlet's green dns/socket module does not actually + # support IPv6 in getaddrinfo(). We need to get around this in the + # future or monitor upstream for a fix + info = socket.getaddrinfo(host, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM)[0] + family = info[0] + bind_addr = info[-1] + + sock = None + retry_until = time.time() + 30 + while not sock and time.time() < retry_until: + try: + sock = eventlet.listen(bind_addr, + backlog=backlog, + family=family) + if sslutils.is_enabled(): + sock = sslutils.wrap(sock) + + except socket.error as err: + if err.args[0] != errno.EADDRINUSE: + raise + eventlet.sleep(0.1) + if not sock: + raise RuntimeError(_("Could not bind to %(host)s:%(port)s " + "after trying for 30 seconds") % + {'host': host, 'port': port}) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + # This option isn't available in the OS X version of eventlet + if hasattr(socket, 'TCP_KEEPIDLE'): + sock.setsockopt(socket.IPPROTO_TCP, + socket.TCP_KEEPIDLE, + CONF.tcp_keepidle) + + return sock + def start(self): """Start serving this service using the provided server instance. @@ -70,10 +133,12 @@ class Service(service.Service): """ super(Service, self).start() - self._socket = eventlet.listen((self._host, self._port), - backlog=self.backlog) self.tg.add_thread(self._run, self.application, self._socket) + @property + def backlog(self): + return self._backlog + @property def host(self): return self._socket.getsockname()[0] if self._socket else self._host @@ -93,7 +158,9 @@ class Service(service.Service): def _run(self, application, socket): """Start a WSGI server in a new green thread.""" logger = logging.getLogger('eventlet.wsgi') - eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool, + eventlet.wsgi.server(socket, + application, + custom_pool=self.tg.pool, log=logging.WritableLogger(logger)) @@ -139,16 +206,16 @@ class Debug(Middleware): @webob.dec.wsgify def __call__(self, req): - print ("*" * 40) + " REQUEST ENVIRON" + print(("*" * 40) + " REQUEST ENVIRON") for key, value in req.environ.items(): - print key, "=", value - print + print(key, "=", value) + print() resp = req.get_response(self.application) - print ("*" * 40) + " RESPONSE HEADERS" + print(("*" * 40) + " RESPONSE HEADERS") for (key, value) in resp.headers.iteritems(): - print key, "=", value - print + print(key, "=", value) + print() resp.app_iter = self.print_generator(resp.app_iter) @@ -160,12 +227,12 @@ class Debug(Middleware): Iterator that prints the contents of a wrapper string iterator when iterated. """ - print ("*" * 40) + " BODY" + print(("*" * 40) + " BODY") for part in app_iter: sys.stdout.write(part) sys.stdout.flush() yield part - print + print() class Router(object): @@ -257,7 +324,7 @@ class Request(webob.Request): Does not do any body introspection, only checks header """ - if not "Content-Type" in self.headers: + if "Content-Type" not in self.headers: return None content_type = self.content_type @@ -388,6 +455,7 @@ class JSONDictSerializer(DictSerializer): _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) return _dtime.isoformat() return obj +# return six.text_type(obj) return jsonutils.dumps(data, default=sanitizer) @@ -680,7 +748,7 @@ class XMLDeserializer(TextDeserializer): plurals = set(self.metadata.get('plurals', {})) try: - node = minidom.parseString(datastring).childNodes[0] + node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0] return {node.nodeName: self._from_xml_node(node, plurals)} except expat.ExpatError: msg = _("cannot understand XML") diff --git a/reddwarf/openstack/common/xmlutils.py b/reddwarf/openstack/common/xmlutils.py new file mode 100644 index 0000000000..b131d3e2e9 --- /dev/null +++ b/reddwarf/openstack/common/xmlutils.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from xml.dom import minidom +from xml.parsers import expat +from xml import sax +from xml.sax import expatreader + + +class ProtectedExpatParser(expatreader.ExpatParser): + """An expat parser which disables DTD's and entities by default.""" + + def __init__(self, forbid_dtd=True, forbid_entities=True, + *args, **kwargs): + # Python 2.x old style class + expatreader.ExpatParser.__init__(self, *args, **kwargs) + self.forbid_dtd = forbid_dtd + self.forbid_entities = forbid_entities + + def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): + raise ValueError("Inline DTD forbidden") + + def entity_decl(self, entityName, is_parameter_entity, value, base, + systemId, publicId, notationName): + raise ValueError(" entity declaration forbidden") + + def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): + # expat 1.2 + raise ValueError(" unparsed entity forbidden") + + def external_entity_ref(self, context, base, systemId, publicId): + raise ValueError(" external entity forbidden") + + def notation_decl(self, name, base, sysid, pubid): + raise ValueError(" notation forbidden") + + def reset(self): + expatreader.ExpatParser.reset(self) + if self.forbid_dtd: + self._parser.StartDoctypeDeclHandler = self.start_doctype_decl + self._parser.EndDoctypeDeclHandler = None + if self.forbid_entities: + self._parser.EntityDeclHandler = self.entity_decl + self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl + self._parser.ExternalEntityRefHandler = self.external_entity_ref + self._parser.NotationDeclHandler = self.notation_decl + try: + self._parser.SkippedEntityHandler = None + except AttributeError: + # some pyexpat versions do not support SkippedEntity + pass + + +def safe_minidom_parse_string(xml_string): + """Parse an XML string using minidom safely. + + """ + try: + return minidom.parseString(xml_string, parser=ProtectedExpatParser()) + except sax.SAXParseException: + raise expat.ExpatError() diff --git a/reddwarf/tests/api/instances_actions.py b/reddwarf/tests/api/instances_actions.py index 4e71c55a08..acb4385f46 100644 --- a/reddwarf/tests/api/instances_actions.py +++ b/reddwarf/tests/api/instances_actions.py @@ -597,7 +597,7 @@ class UpdateGuest(object): @test(enabled=UPDATE_GUEST_CONF is not None) def upload_update_to_repo(self): cmds = UPDATE_GUEST_CONF["install-repo-cmd"] - utils.execute(*cmds, run_as_root=True) + utils.execute(*cmds, run_as_root=True, root_helper="sudo") @test(enabled=UPDATE_GUEST_CONF is not None, depends_on=[upload_update_to_repo]) diff --git a/reddwarf/tests/util/client.py b/reddwarf/tests/util/client.py index 563ddea898..0c73b32f73 100644 --- a/reddwarf/tests/util/client.py +++ b/reddwarf/tests/util/client.py @@ -130,7 +130,7 @@ def call_xmllint(name, body): if CONFIG.get('xml_xsd', None): args += ["--schema", CONFIG.xml_xsd] output = processutils.execute(CONFIG.xmllint_bin, *args, - check_exit_code=0, with_shell=False) + check_exit_code=0, shell=False) except processutils.ProcessExecutionError as pe: fail("Error validating XML! %s" % pe)