Refresh Oslo code - add support for ssl

We want to add ssl support for the reddwarf api server.
For this, updating the openstack/common/wsgi module, and
all it's dependencies as a result of using:
$> python update.py --base reddwarf --dest-dir ../reddwarf --modules wsgi
(details:https://wiki.openstack.org/wiki/Oslo)

I'm commenting out the "six" update, since it seems to break
our fake mode testing with regard to reddwarf-cli.

I'll open a separate bug for it.

Change-Id: I763d06658488bd1ca1eac5a2ba1ffde5629f842f
Fixes: Bug #1178421

etc/reddwarf/reddwarf.conf.sample

@@ -103,3 +103,16 @@ notifier_queue_transport = memory
 #log_dir = /integration/report
 #log_file = reddwarf-api.log
 
+
+# ============ SSL configuration (and enablement) =============================
+# In order to enable SSL for the reddwarf api server, uncomment
+# the cert_file and key_file - and of course have those files
+# accessible. The existance of those setting and files will
+# enable SSL.
+
+[ssl]
+
+#cert_file = /path/to/server.crt
+#key_file = /path/to/server.key
+#optional:
+#ca_file = /path/to/ca_file

reddwarf/guestagent/manager/mysql.py

@@ -184,7 +184,7 @@ def get_engine():
 def load_mysqld_options():
     try:
         out, err = utils.execute("/usr/sbin/mysqld", "--print-defaults",
-                                 run_as_root=True)
+                                 run_as_root=True, root_helper="sudo")
         arglist = re.split("\n", out)[1].split()
         args = {}
         for item in arglist:
@@ -256,7 +256,7 @@ class MySqlAppStatus(object):
         try:
             out, err = utils.execute_with_timeout(
                 "/usr/bin/mysqladmin",
-                "ping", run_as_root=True)
+                "ping", run_as_root=True, root_helper="sudo")
             LOG.info("Service Status is RUNNING.")
             return rd_models.ServiceStatuses.RUNNING
         except ProcessExecutionError as e:
@@ -791,7 +791,7 @@ class MySqlApp(object):
             command = command % locals()
         else:
             command = "sudo update-rc.d mysql enable"
-        utils.execute_with_timeout(command, with_shell=True)
+        utils.execute_with_timeout(command, shell=True)
 
     def _disable_mysql_on_boot(self):
         '''
@@ -809,7 +809,7 @@ class MySqlApp(object):
             command = command % locals()
         else:
             command = "sudo update-rc.d mysql disable"
-        utils.execute_with_timeout(command, with_shell=True)
+        utils.execute_with_timeout(command, shell=True)
 
     def stop_db(self, update_db=False, do_not_start_on_reboot=False):
         LOG.info(_("Stopping mysql..."))

reddwarf/openstack/common/eventlet_backdoor.py

@@ -16,6 +16,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import print_function
+
 import gc
 import pprint
 import sys
@@ -37,7 +39,7 @@ CONF.register_opts(eventlet_backdoor_opts)
 
 
 def _dont_use_this():
-    print "Don't use this, just disconnect instead"
+    print("Don't use this, just disconnect instead")
 
 
 def _find_objects(t):
@@ -46,16 +48,16 @@ def _find_objects(t):
 
 def _print_greenthreads():
     for i, gt in enumerate(_find_objects(greenlet.greenlet)):
-        print i, gt
+        print(i, gt)
         traceback.print_stack(gt.gr_frame)
-        print
+        print()
 
 
 def _print_nativethreads():
     for threadId, stack in sys._current_frames().items():
-        print threadId
+        print(threadId)
         traceback.print_stack(stack)
-        print
+        print()
 
 
 def initialize_if_enabled():

reddwarf/openstack/common/exception.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2011 OpenStack LLC.
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -23,6 +23,8 @@ import logging
 
 from reddwarf.openstack.common.gettextutils import _
 
+_FATAL_EXCEPTION_FORMAT_ERRORS = False
+
 
 class Error(Exception):
     def __init__(self, message=None):
@@ -96,7 +98,7 @@ def wrap_exception(f):
     def _wrap(*args, **kw):
         try:
             return f(*args, **kw)
-        except Exception, e:
+        except Exception as e:
             if not isinstance(e, Error):
                 #exc_type, exc_value, exc_traceback = sys.exc_info()
                 logging.exception(_('Uncaught exception'))
@@ -121,9 +123,12 @@ class OpenstackException(Exception):
         try:
             self._error_string = self.message % kwargs
 
-        except Exception:
+        except Exception as e:
-            # at least get the core message out if something happened
+            if _FATAL_EXCEPTION_FORMAT_ERRORS:
-            self._error_string = self.message
+                raise e
+            else:
+                # at least get the core message out if something happened
+                self._error_string = self.message
 
     def __str__(self):
         return self._error_string

reddwarf/openstack/common/jsonutils.py

@@ -41,6 +41,8 @@ import json
 import types
 import xmlrpclib
 
+#import six
+
 from reddwarf.openstack.common import timeutils
 
 
@@ -94,6 +96,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
     # and results in infinite loop when list(value) is called.
     if type(value) == itertools.count:
         return unicode(value)
+#        return six.text_type(value)
 
     # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
     #              tests that raise an exception in a mocked method that
@@ -138,11 +141,13 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
         else:
             if any(test(value) for test in _nasty_type_tests):
                 return unicode(value)
+#                return six.text_type(value)
             return value
     except TypeError:
         # Class objects are tricky since they may define something like
         # __iter__ defined but it isn't callable as list().
         return unicode(value)
+#        return six.text_type(value)
 
 
 def dumps(value, default=to_primitive, **kwargs):

reddwarf/openstack/common/log.py

@@ -37,7 +37,6 @@ import logging
 import logging.config
 import logging.handlers
 import os
-import stat
 import sys
 import traceback
 
@@ -49,7 +48,6 @@ from reddwarf.openstack.common import local
 from reddwarf.openstack.common import notifier
 
 
-_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
 common_cli_opts = [
@@ -74,11 +72,13 @@ logging_cli_opts = [
                     'documentation for details on logging configuration '
                     'files.'),
     cfg.StrOpt('log-format',
-               default=_DEFAULT_LOG_FORMAT,
+               default=None,
                metavar='FORMAT',
                help='A logging.Formatter log message format string which may '
                     'use any of the available logging.LogRecord attributes. '
-                    'Default: %(default)s'),
+                    'This option is deprecated.  Please use '
+                    'logging_context_format_string and '
+                    'logging_default_format_string instead.'),
     cfg.StrOpt('log-date-format',
                default=_DEFAULT_LOG_DATE_FORMAT,
                metavar='DATE_FORMAT',
@@ -104,10 +104,7 @@ logging_cli_opts = [
 generic_log_opts = [
     cfg.BoolOpt('use_stderr',
                 default=True,
-                help='Log output to standard error'),
+                help='Log output to standard error')
-    cfg.StrOpt('logfile_mode',
-               default='0644',
-               help='Default file mode used when creating log files'),
 ]
 
 log_opts = [
@@ -211,7 +208,27 @@ def _get_log_file_path(binary=None):
         return '%s.log' % (os.path.join(logdir, binary),)
 
 
-class ContextAdapter(logging.LoggerAdapter):
+class BaseLoggerAdapter(logging.LoggerAdapter):
+
+    def audit(self, msg, *args, **kwargs):
+        self.log(logging.AUDIT, msg, *args, **kwargs)
+
+
+class LazyAdapter(BaseLoggerAdapter):
+    def __init__(self, name='unknown', version='unknown'):
+        self._logger = None
+        self.extra = {}
+        self.name = name
+        self.version = version
+
+    @property
+    def logger(self):
+        if not self._logger:
+            self._logger = getLogger(self.name, self.version)
+        return self._logger
+
+
+class ContextAdapter(BaseLoggerAdapter):
     warn = logging.LoggerAdapter.warning
 
     def __init__(self, logger, project_name, version_string):
@@ -219,8 +236,9 @@ class ContextAdapter(logging.LoggerAdapter):
         self.project = project_name
         self.version = version_string
 
-    def audit(self, msg, *args, **kwargs):
+    @property
-        self.log(logging.AUDIT, msg, *args, **kwargs)
+    def handlers(self):
+        return self.logger.handlers
 
     def deprecated(self, msg, *args, **kwargs):
         stdmsg = _("Deprecated: %s") % msg
@@ -399,11 +417,6 @@ def _setup_logging_from_conf():
         filelog = logging.handlers.WatchedFileHandler(logpath)
         log_root.addHandler(filelog)
 
-        mode = int(CONF.logfile_mode, 8)
-        st = os.stat(logpath)
-        if st.st_mode != (stat.S_IFREG | mode):
-            os.chmod(logpath, mode)
-
     if CONF.use_stderr:
         streamlog = ColorHandler()
         log_root.addHandler(streamlog)
@@ -417,13 +430,17 @@ def _setup_logging_from_conf():
     if CONF.publish_errors:
         log_root.addHandler(PublishErrorsHandler(logging.ERROR))
 
+    datefmt = CONF.log_date_format
     for handler in log_root.handlers:
-        datefmt = CONF.log_date_format
+        # NOTE(alaski): CONF.log_format overrides everything currently.  This
+        # should be deprecated in favor of context aware formatting.
         if CONF.log_format:
             handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
                                                    datefmt=datefmt))
+            log_root.info('Deprecated: log_format is now deprecated and will '
+                          'be removed in the next release')
         else:
-            handler.setFormatter(LegacyFormatter(datefmt=datefmt))
+            handler.setFormatter(ContextFormatter(datefmt=datefmt))
 
     if CONF.debug:
         log_root.setLevel(logging.DEBUG)
@@ -449,6 +466,15 @@ def getLogger(name='unknown', version='unknown'):
     return _loggers[name]
 
 
+def getLazyLogger(name='unknown', version='unknown'):
+    """
+    create a pass-through logger that does not create the real logger
+    until it is really needed and delegates all calls to the real logger
+    once it is created
+    """
+    return LazyAdapter(name, version)
+
+
 class WritableLogger(object):
     """A thin wrapper that responds to `write` and logs."""
 
@@ -460,7 +486,7 @@ class WritableLogger(object):
         self.logger.log(self.level, msg)
 
 
-class LegacyFormatter(logging.Formatter):
+class ContextFormatter(logging.Formatter):
     """A context.RequestContext aware formatter configured through flags.
 
     The flags used to set format strings are: logging_context_format_string

reddwarf/openstack/common/notifier/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/notifier/api.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/notifier/log_notifier.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/notifier/no_op_notifier.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/notifier/rpc_notifier.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/notifier/test_notifier.py

@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack Foundation
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

reddwarf/openstack/common/processutils.py

@@ -19,8 +19,10 @@
 System-level utilities and helper functions.
 """
 
+import os
 import random
 import shlex
+import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
@@ -40,6 +42,12 @@ class UnknownArgumentError(Exception):
 class ProcessExecutionError(Exception):
     def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
                  description=None):
+        self.exit_code = exit_code
+        self.stderr = stderr
+        self.stdout = stdout
+        self.cmd = cmd
+        self.description = description
+
         if description is None:
             description = "Unexpected error while running command."
         if exit_code is None:
@@ -49,6 +57,17 @@ class ProcessExecutionError(Exception):
         super(ProcessExecutionError, self).__init__(message)
 
 
+class NoRootWrapSpecified(Exception):
+    def __init__(self, message=None):
+        super(NoRootWrapSpecified, self).__init__(message)
+
+
+def _subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+
 def execute(*cmd, **kwargs):
     """
     Helper method to shell out and execute a command through subprocess with
@@ -58,11 +77,11 @@ def execute(*cmd, **kwargs):
     :type cmd:              string
     :param process_input:   Send to opened process.
     :type proces_input:     string
-    :param check_exit_code: Defaults to 0. Will raise
+    :param check_exit_code: Single bool, int, or list of allowed exit
-                            :class:`ProcessExecutionError`
+                            codes.  Defaults to [0].  Raise
-                            if the command exits without returning this value
+                            :class:`ProcessExecutionError` unless
-                            as a returncode
+                            program exits with one of these code.
-    :type check_exit_code:  int
+    :type check_exit_code:  boolean, int, or [int]
     :param delay_on_retry:  True | False. Defaults to True. If set to True,
                             wait a short amount of time before retrying.
     :type delay_on_retry:   boolean
@@ -72,8 +91,12 @@ def execute(*cmd, **kwargs):
                             the command is prefixed by the command specified
                             in the root_helper kwarg.
     :type run_as_root:      boolean
-    :param root_helper:     command to prefix all cmd's with
+    :param root_helper:     command to prefix to commands called with
+                            run_as_root=True
     :type root_helper:      string
+    :param shell:           whether or not there should be a shell used to
+                            execute this command. Defaults to false.
+    :type shell:            boolean
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -81,17 +104,31 @@ def execute(*cmd, **kwargs):
     """
 
     process_input = kwargs.pop('process_input', None)
-    check_exit_code = kwargs.pop('check_exit_code', 0)
+    check_exit_code = kwargs.pop('check_exit_code', [0])
+    ignore_exit_code = False
     delay_on_retry = kwargs.pop('delay_on_retry', True)
     attempts = kwargs.pop('attempts', 1)
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
-    with_shell = kwargs.pop('with_shell', False)
+    shell = kwargs.pop('shell', False)
+
+    if isinstance(check_exit_code, bool):
+        ignore_exit_code = not check_exit_code
+        check_exit_code = [0]
+    elif isinstance(check_exit_code, int):
+        check_exit_code = [check_exit_code]
+
     if len(kwargs):
         raise UnknownArgumentError(_('Got unknown keyword args '
                                      'to utils.execute: %r') % kwargs)
-    if run_as_root:
+
+    if run_as_root and os.geteuid() != 0:
+        if not root_helper:
+            raise NoRootWrapSpecified(
+                message=('Command requested root, but did not specify a root '
+                         'helper.'))
         cmd = shlex.split(root_helper) + list(cmd)
+
     cmd = map(str, cmd)
 
     while attempts > 0:
@@ -99,12 +136,21 @@ def execute(*cmd, **kwargs):
         try:
             LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
             _PIPE = subprocess.PIPE  # pylint: disable=E1101
+
+            if os.name == 'nt':
+                preexec_fn = None
+                close_fds = False
+            else:
+                preexec_fn = _subprocess_setup
+                close_fds = True
+
             obj = subprocess.Popen(cmd,
                                    stdin=_PIPE,
                                    stdout=_PIPE,
                                    stderr=_PIPE,
-                                   close_fds=True,
+                                   close_fds=close_fds,
-                                   shell=with_shell)
+                                   preexec_fn=preexec_fn,
+                                   shell=shell)
             result = None
             if process_input is not None:
                 result = obj.communicate(process_input)
@@ -114,9 +160,7 @@ def execute(*cmd, **kwargs):
             _returncode = obj.returncode  # pylint: disable=E1101
             if _returncode:
                 LOG.debug(_('Result was %s') % _returncode)
-                if (isinstance(check_exit_code, int) and
+                if not ignore_exit_code and _returncode not in check_exit_code:
-                    not isinstance(check_exit_code, bool) and
-                        _returncode != check_exit_code):
                     (stdout, stderr) = result
                     raise ProcessExecutionError(exit_code=_returncode,
                                                 stdout=stdout,

reddwarf/openstack/common/rpc/common.py

@@ -22,6 +22,7 @@ import sys
 import traceback
 
 from oslo.config import cfg
+#import six
 
 from reddwarf.openstack.common.gettextutils import _
 from reddwarf.openstack.common import importutils
@@ -300,6 +301,8 @@ def serialize_remote_exception(failure_info, log_failure=True):
     failure = failure_info[1]
     if log_failure:
         LOG.error(_("Returning exception %s to caller"), unicode(failure))
+#        LOG.error(_("Returning exception %s to caller"),
+#                  six.text_type(failure))
         LOG.error(tb)
 
     kwargs = {}
@@ -310,6 +313,7 @@ def serialize_remote_exception(failure_info, log_failure=True):
         'class': str(failure.__class__.__name__),
         'module': str(failure.__class__.__module__),
         'message': unicode(failure),
+#        'message': six.text_type(failure),
         'tb': tb,
         'args': failure.args,
         'kwargs': kwargs

reddwarf/openstack/common/rpc/impl_qpid.py

@@ -331,15 +331,16 @@ class Connection(object):
 
     def reconnect(self):
         """Handles reconnecting and re-establishing sessions and queues"""
-        if self.connection.opened():
-            try:
-                self.connection.close()
-            except qpid_exceptions.ConnectionError:
-                pass
-
         attempt = 0
         delay = 1
         while True:
+            # Close the session if necessary
+            if self.connection.opened():
+                try:
+                    self.connection.close()
+                except qpid_exceptions.ConnectionError:
+                    pass
+
             broker = self.brokers[attempt % len(self.brokers)]
             attempt += 1
 

reddwarf/openstack/common/sslutils.py

@@ -0,0 +1,80 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import ssl
+
+from oslo.config import cfg
+
+from reddwarf.openstack.common.gettextutils import _
+
+
+ssl_opts = [
+    cfg.StrOpt('ca_file',
+               default=None,
+               help="CA certificate file to use to verify "
+                    "connecting clients"),
+    cfg.StrOpt('cert_file',
+               default=None,
+               help="Certificate file to use when starting "
+                    "the server securely"),
+    cfg.StrOpt('key_file',
+               default=None,
+               help="Private key file to use when starting "
+                    "the server securely"),
+]
+
+
+CONF = cfg.CONF
+CONF.register_opts(ssl_opts, "ssl")
+
+
+def is_enabled():
+    cert_file = CONF.ssl.cert_file
+    key_file = CONF.ssl.key_file
+    ca_file = CONF.ssl.ca_file
+    use_ssl = cert_file or key_file
+
+    if cert_file and not os.path.exists(cert_file):
+        raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
+
+    if ca_file and not os.path.exists(ca_file):
+        raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)
+
+    if key_file and not os.path.exists(key_file):
+        raise RuntimeError(_("Unable to find key_file : %s") % key_file)
+
+    if use_ssl and (not cert_file or not key_file):
+        raise RuntimeError(_("When running server in SSL mode, you must "
+                             "specify both a cert_file and key_file "
+                             "option value in your configuration file"))
+
+    return use_ssl
+
+
+def wrap(sock):
+    ssl_kwargs = {
+        'server_side': True,
+        'certfile': CONF.ssl.cert_file,
+        'keyfile': CONF.ssl.key_file,
+        'cert_reqs': ssl.CERT_NONE,
+    }
+
+    if CONF.ssl.ca_file:
+        ssl_kwargs['ca_certs'] = CONF.ssl.ca_file
+        ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
+
+    return ssl.wrap_socket(sock, **ssl_kwargs)

reddwarf/openstack/common/wsgi.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2011 OpenStack LLC.
+# Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -17,15 +17,22 @@
 
 """Utility methods for working with WSGI servers."""
 
-import datetime
+from __future__ import print_function
-import eventlet
-import eventlet.wsgi
 
+import eventlet
 eventlet.patcher.monkey_patch(all=False, socket=True)
 
+import datetime
+import errno
+import socket
+import sys
+import time
+
+import eventlet.wsgi
+from oslo.config import cfg
 import routes
 import routes.middleware
-import sys
+#import six
 import webob.dec
 import webob.exc
 from xml.dom import minidom
@@ -36,15 +43,29 @@ from reddwarf.openstack.common.gettextutils import _
 from reddwarf.openstack.common import jsonutils
 from reddwarf.openstack.common import log as logging
 from reddwarf.openstack.common import service
+from reddwarf.openstack.common import sslutils
+from reddwarf.openstack.common import xmlutils
 
+socket_opts = [
+    cfg.IntOpt('backlog',
+               default=4096,
+               help="Number of backlog requests to configure the socket with"),
+    cfg.IntOpt('tcp_keepidle',
+               default=600,
+               help="Sets the value of TCP_KEEPIDLE in seconds for each "
+                    "server socket. Not supported on OS X."),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(socket_opts)
 
 LOG = logging.getLogger(__name__)
 
 
-def run_server(application, port):
+def run_server(application, port, **kwargs):
     """Run a WSGI server with the given application."""
     sock = eventlet.listen(('0.0.0.0', port))
-    eventlet.wsgi.server(sock, application)
+    eventlet.wsgi.server(sock, application, **kwargs)
 
 
 class Service(service.Service):
@@ -56,13 +77,55 @@ class Service(service.Service):
     """
 
     def __init__(self, application, port,
-                 host='0.0.0.0', backlog=128, threads=1000):
+                 host='0.0.0.0', backlog=4096, threads=1000):
         self.application = application
         self._port = port
         self._host = host
-        self.backlog = backlog
+        self._backlog = backlog if backlog else CONF.backlog
+        self._socket = self._get_socket(host, port, self._backlog)
         super(Service, self).__init__(threads)
 
+    def _get_socket(self, host, port, backlog):
+        # TODO(dims): eventlet's green dns/socket module does not actually
+        # support IPv6 in getaddrinfo(). We need to get around this in the
+        # future or monitor upstream for a fix
+        info = socket.getaddrinfo(host,
+                                  port,
+                                  socket.AF_UNSPEC,
+                                  socket.SOCK_STREAM)[0]
+        family = info[0]
+        bind_addr = info[-1]
+
+        sock = None
+        retry_until = time.time() + 30
+        while not sock and time.time() < retry_until:
+            try:
+                sock = eventlet.listen(bind_addr,
+                                       backlog=backlog,
+                                       family=family)
+                if sslutils.is_enabled():
+                    sock = sslutils.wrap(sock)
+
+            except socket.error as err:
+                if err.args[0] != errno.EADDRINUSE:
+                    raise
+                eventlet.sleep(0.1)
+        if not sock:
+            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
+                               "after trying for 30 seconds") %
+                               {'host': host, 'port': port})
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        # sockets can hang around forever without keepalive
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+        # This option isn't available in the OS X version of eventlet
+        if hasattr(socket, 'TCP_KEEPIDLE'):
+            sock.setsockopt(socket.IPPROTO_TCP,
+                            socket.TCP_KEEPIDLE,
+                            CONF.tcp_keepidle)
+
+        return sock
+
     def start(self):
         """Start serving this service using the provided server instance.
 
@@ -70,10 +133,12 @@ class Service(service.Service):
 
         """
         super(Service, self).start()
-        self._socket = eventlet.listen((self._host, self._port),
-                                       backlog=self.backlog)
         self.tg.add_thread(self._run, self.application, self._socket)
 
+    @property
+    def backlog(self):
+        return self._backlog
+
     @property
     def host(self):
         return self._socket.getsockname()[0] if self._socket else self._host
@@ -93,7 +158,9 @@ class Service(service.Service):
     def _run(self, application, socket):
         """Start a WSGI server in a new green thread."""
         logger = logging.getLogger('eventlet.wsgi')
-        eventlet.wsgi.server(socket, application, custom_pool=self.tg.pool,
+        eventlet.wsgi.server(socket,
+                             application,
+                             custom_pool=self.tg.pool,
                              log=logging.WritableLogger(logger))
 
 
@@ -139,16 +206,16 @@ class Debug(Middleware):
 
     @webob.dec.wsgify
     def __call__(self, req):
-        print ("*" * 40) + " REQUEST ENVIRON"
+        print(("*" * 40) + " REQUEST ENVIRON")
         for key, value in req.environ.items():
-            print key, "=", value
+            print(key, "=", value)
-        print
+        print()
         resp = req.get_response(self.application)
 
-        print ("*" * 40) + " RESPONSE HEADERS"
+        print(("*" * 40) + " RESPONSE HEADERS")
         for (key, value) in resp.headers.iteritems():
-            print key, "=", value
+            print(key, "=", value)
-        print
+        print()
 
         resp.app_iter = self.print_generator(resp.app_iter)
 
@@ -160,12 +227,12 @@ class Debug(Middleware):
         Iterator that prints the contents of a wrapper string iterator
         when iterated.
         """
-        print ("*" * 40) + " BODY"
+        print(("*" * 40) + " BODY")
         for part in app_iter:
             sys.stdout.write(part)
             sys.stdout.flush()
             yield part
-        print
+        print()
 
 
 class Router(object):
@@ -257,7 +324,7 @@ class Request(webob.Request):
         Does not do any body introspection, only checks header
 
         """
-        if not "Content-Type" in self.headers:
+        if "Content-Type" not in self.headers:
             return None
 
         content_type = self.content_type
@@ -388,6 +455,7 @@ class JSONDictSerializer(DictSerializer):
                 _dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
                 return _dtime.isoformat()
             return obj
+#            return six.text_type(obj)
         return jsonutils.dumps(data, default=sanitizer)
 
 
@@ -680,7 +748,7 @@ class XMLDeserializer(TextDeserializer):
         plurals = set(self.metadata.get('plurals', {}))
 
         try:
-            node = minidom.parseString(datastring).childNodes[0]
+            node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0]
             return {node.nodeName: self._from_xml_node(node, plurals)}
         except expat.ExpatError:
             msg = _("cannot understand XML")

reddwarf/openstack/common/xmlutils.py

@@ -0,0 +1,74 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from xml.dom import minidom
+from xml.parsers import expat
+from xml import sax
+from xml.sax import expatreader
+
+
+class ProtectedExpatParser(expatreader.ExpatParser):
+    """An expat parser which disables DTD's and entities by default."""
+
+    def __init__(self, forbid_dtd=True, forbid_entities=True,
+                 *args, **kwargs):
+        # Python 2.x old style class
+        expatreader.ExpatParser.__init__(self, *args, **kwargs)
+        self.forbid_dtd = forbid_dtd
+        self.forbid_entities = forbid_entities
+
+    def start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
+        raise ValueError("Inline DTD forbidden")
+
+    def entity_decl(self, entityName, is_parameter_entity, value, base,
+                    systemId, publicId, notationName):
+        raise ValueError("<!ENTITY> entity declaration forbidden")
+
+    def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
+        # expat 1.2
+        raise ValueError("<!ENTITY> unparsed entity forbidden")
+
+    def external_entity_ref(self, context, base, systemId, publicId):
+        raise ValueError("<!ENTITY> external entity forbidden")
+
+    def notation_decl(self, name, base, sysid, pubid):
+        raise ValueError("<!ENTITY> notation forbidden")
+
+    def reset(self):
+        expatreader.ExpatParser.reset(self)
+        if self.forbid_dtd:
+            self._parser.StartDoctypeDeclHandler = self.start_doctype_decl
+            self._parser.EndDoctypeDeclHandler = None
+        if self.forbid_entities:
+            self._parser.EntityDeclHandler = self.entity_decl
+            self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl
+            self._parser.ExternalEntityRefHandler = self.external_entity_ref
+            self._parser.NotationDeclHandler = self.notation_decl
+            try:
+                self._parser.SkippedEntityHandler = None
+            except AttributeError:
+                # some pyexpat versions do not support SkippedEntity
+                pass
+
+
+def safe_minidom_parse_string(xml_string):
+    """Parse an XML string using minidom safely.
+
+    """
+    try:
+        return minidom.parseString(xml_string, parser=ProtectedExpatParser())
+    except sax.SAXParseException:
+        raise expat.ExpatError()

reddwarf/tests/api/instances_actions.py

@@ -597,7 +597,7 @@ class UpdateGuest(object):
     @test(enabled=UPDATE_GUEST_CONF is not None)
     def upload_update_to_repo(self):
         cmds = UPDATE_GUEST_CONF["install-repo-cmd"]
-        utils.execute(*cmds, run_as_root=True)
+        utils.execute(*cmds, run_as_root=True, root_helper="sudo")
 
     @test(enabled=UPDATE_GUEST_CONF is not None,
           depends_on=[upload_update_to_repo])

reddwarf/tests/util/client.py

@@ -130,7 +130,7 @@ def call_xmllint(name, body):
         if CONFIG.get('xml_xsd', None):
             args += ["--schema", CONFIG.xml_xsd]
         output = processutils.execute(CONFIG.xmllint_bin, *args,
-                                      check_exit_code=0, with_shell=False)
+                                      check_exit_code=0, shell=False)
     except processutils.ProcessExecutionError as pe:
         fail("Error validating XML! %s" % pe)