Add possibility to run 'manila-api' with wsgi web servers

One of the goals for Pike [1] is to make each API service be able to
run under web servers that support WSGI applications,
such as Apache (+mod-wsgi) and Nginx (+uWSGI).

Do following to address governance requirements:
- Split existing manila/wsgi.py module into 3 modules:
  First (manila/wsgi/eventlet_server.py) is used by
  eventlet-based WSGI application approach.
  Second (manila/wsgi/wsgi.py) is used for WSGI web servers.
  And third (manila/wsgi/common.py) is common code for both.
  All three are made in cinder-like way to have alike-approach.
- Reuse common code from "oslo_service/wsgi.py" module that
  allows us to remove code duplication.
- Delete config opts that are defined by newly reused common code.
- Register new entry point that will be manila wsgi app: "manila-wsgi".
- Fix "manila/api/openstack/wsgi.py" module to be compatible
  with str/bytes handling approach used by Apache mod-wsgi plugin using
  different python versions (2/3).
- Add web server config template "devstack/apache-manila.template"
- Add devstack support where usage of this feature can be
  enabled or disabled using "MANILA_USE_MOD_WSGI" env var.
  It is set to "True" by default, because it is requirement for Pike
  release - to have it running in all CI jobs.
  Disable it only for one CI job that uses dummy driver and tests
  various manila core features that are not covered by other CI jobs.

[1] https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html

Partially-Implements BluePrint wsgi-web-servers-support
DocImpact
Change-Id: Ibdef3c6810b65a5d6f3611e2d0079c635ee523ab

contrib/ci/pre_test_hook.sh

@@ -85,6 +85,12 @@ elif [[ "$DRIVER" == "dummy" ]]; then
     driver_path="manila.tests.share.drivers.dummy.DummyDriver"
     DEFAULT_EXTRA_SPECS="'snapshot_support=True create_share_from_snapshot_support=True revert_to_snapshot_support=True mount_snapshot_support=True'"
     echo "MANILA_SERVICE_IMAGE_ENABLED=False" >> $localconf
+
+    # Run dummy driver CI job using standalone approach for running
+    # manila API service just because we need to test this approach too,
+    # that is very useful for development needs.
+    echo "MANILA_USE_MOD_WSGI=False" >> $localconf
+
     echo "SHARE_DRIVER=$driver_path" >> $localconf
     echo "SUPPRESS_ERRORS_IN_CLEANUP=False" >> $localconf
     echo "MANILA_REPLICA_STATE_UPDATE_INTERVAL=10" >> $localconf

devstack/apache-manila.template

@@ -0,0 +1,25 @@
+Listen %PORT%
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" manila_combined
+
+<VirtualHost *:%PORT%>
+    WSGIDaemonProcess manila-api processes=%APIWORKERS% threads=2 user=%USER% display-name=%{GROUP}
+    WSGIProcessGroup manila-api
+    WSGIScriptAlias / %MANILA_BIN_DIR%/manila-wsgi
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/%APACHE_NAME%/manila_api.log
+    CustomLog /var/log/%APACHE_NAME%/manila_api_access.log manila_combined
+
+    <Directory %MANILA_BIN_DIR%>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>

devstack/plugin.sh

@@ -59,6 +59,22 @@ function cleanup_manila {
     _clean_zfsonlinux_data
 }
 
+# _config_manila_apache_wsgi() - Configure manila-api wsgi application.
+function _config_manila_apache_wsgi {
+    local manila_api_apache_conf
+    local venv_path=""
+    manila_api_apache_conf=$(apache_site_config_for manila-api)
+
+    sudo cp $MANILA_DIR/devstack/apache-manila.template $manila_api_apache_conf
+    sudo sed -e "
+        s|%APACHE_NAME%|$APACHE_NAME|g;
+        s|%MANILA_BIN_DIR%|$MANILA_BIN_DIR|g;
+        s|%PORT%|$MANILA_SERVICE_PORT|g;
+        s|%APIWORKERS%|$API_WORKERS|g;
+        s|%USER%|$STACK_USER|g;
+    " -i $manila_api_apache_conf
+}
+
 # configure_default_backends - configures default Manila backends with generic driver.
 function configure_default_backends {
     # Configure two default backends with generic drivers onboard
@@ -257,6 +273,10 @@ function configure_manila {
     MANILA_CONFIGURE_GROUPS=${MANILA_CONFIGURE_GROUPS:-"$MANILA_ENABLED_BACKENDS"}
     set_config_opts $MANILA_CONFIGURE_GROUPS
     set_config_opts DEFAULT
+
+    if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then
+        _config_manila_apache_wsgi
+    fi
 }
 
 
@@ -759,7 +779,14 @@ function configure_samba {
 
 # start_manila_api - starts manila API services and checks its availability
 function start_manila_api {
+    if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then
+        install_apache_wsgi
+        enable_apache_site manila-api
+        restart_apache_server
+        tail_log m-api /var/log/$APACHE_NAME/manila_api.log
+    else
         run_process m-api "$MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF"
+    fi
 
     echo "Waiting for Manila API to start..."
     if ! wait_for_service $SERVICE_TIMEOUT $MANILA_SERVICE_PROTOCOL://$MANILA_SERVICE_HOST:$MANILA_SERVICE_PORT; then
@@ -788,8 +815,16 @@ function start_manila {
 
 # stop_manila - Stop running processes
 function stop_manila {
-    # Kill the manila processes
+    # Disable manila api service
-    for serv in m-api m-sch m-shr m-dat; do
+    if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then
+        disable_apache_site manila-api
+        restart_apache_server
+    else
+        stop_process m-api
+    fi
+
+    # Kill all other manila processes
+    for serv in m-sch m-shr m-dat; do
         stop_process $serv
     done
 }

devstack/settings

@@ -73,10 +73,9 @@ MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS=${MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS:-'
 # Public facing bits
 MANILA_SERVICE_HOST=${MANILA_SERVICE_HOST:-$SERVICE_HOST}
 MANILA_SERVICE_PORT=${MANILA_SERVICE_PORT:-8786}
-MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18776}
+MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18786}
 MANILA_SERVICE_PROTOCOL=${MANILA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-
 # Support entry points installation of console scripts
 if [[ -d $MANILA_DIR/bin ]]; then
     MANILA_BIN_DIR=$MANILA_DIR/bin
@@ -84,7 +83,6 @@ else
     MANILA_BIN_DIR=$(get_python_exec_prefix)
 fi
 
-
 # Common opts
 SHARE_NAME_PREFIX=${SHARE_NAME_PREFIX:-share-}
 MANILA_ENABLED_SHARE_PROTOCOLS=${ENABLED_SHARE_PROTOCOLS:-"NFS,CIFS"}
@@ -97,6 +95,14 @@ MANILA_SERVICE_SECGROUP="manila-service"
 # migrations again.
 MANILA_USE_DOWNGRADE_MIGRATIONS=${MANILA_USE_DOWNGRADE_MIGRATIONS:-"False"}
 
+# Toggle for deploying manila-api service under Apache web server with enabled 'mod_wsgi' plugin.
+# Disabled by default, which means running manila-api service as standalone
+# eventlet-based WSGI application.
+# Set it as True, because starting with Pike it is requirement from
+# 'governance' project. See:
+# https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html#completion-criteria
+MANILA_USE_MOD_WSGI=${MANILA_USE_MOD_WSGI:-True}
+
 # Common info for Generic driver(s)
 SHARE_DRIVER=${SHARE_DRIVER:-manila.share.drivers.generic.GenericShareDriver}
 

manila/api/middleware/auth.py

@@ -27,7 +27,7 @@ import webob.exc
 from manila.api.openstack import wsgi
 from manila import context
 from manila.i18n import _
-from manila import wsgi as base_wsgi
+from manila.wsgi import common as base_wsgi
 
 use_forwarded_for_opt = cfg.BoolOpt(
     'use_forwarded_for',

manila/api/middleware/fault.py

@@ -21,7 +21,7 @@ import webob.exc
 
 from manila.api.openstack import wsgi
 from manila import utils
-from manila import wsgi as base_wsgi
+from manila.wsgi import common as base_wsgi
 
 LOG = log.getLogger(__name__)
 

manila/api/openstack/__init__.py

@@ -19,11 +19,11 @@ WSGI middleware for OpenStack API controllers.
 """
 
 from oslo_log import log
+from oslo_service import wsgi as base_wsgi
 import routes
 
 from manila.api.openstack import wsgi
 from manila.i18n import _
-from manila import wsgi as base_wsgi
 
 LOG = log.getLogger(__name__)
 
@@ -117,13 +117,3 @@ class APIRouter(base_wsgi.Router):
 
     def _setup_routes(self, mapper, ext_mgr):
         raise NotImplementedError
-
-
-class FaultWrapper(base_wsgi.Middleware):
-    def __init__(self, application):
-        LOG.warning('manila.api.openstack:FaultWrapper is deprecated. '
-                    'Please use '
-                    'manila.api.middleware.fault:FaultWrapper instead.')
-        # Avoid circular imports from here.
-        from manila.api.middleware import fault
-        super(FaultWrapper, self).__init__(fault.FaultWrapper(application))

manila/api/openstack/wsgi.py

@@ -31,7 +31,8 @@ from manila.common import constants
 from manila import exception
 from manila.i18n import _
 from manila import policy
-from manila import wsgi
+from manila import utils
+from manila.wsgi import common as wsgi
 
 LOG = log.getLogger(__name__)
 
@@ -860,15 +861,20 @@ class Resource(wsgi.Application):
 
         if hasattr(response, 'headers'):
             for hdr, val in response.headers.items():
-                # Headers must be utf-8 strings
+                val = utils.convert_str(val)
-                response.headers[hdr] = six.text_type(val)
+                response.headers[hdr] = val
 
             if not request.api_version_request.is_null():
                 response.headers[API_VERSION_REQUEST_HEADER] = (
                     request.api_version_request.get_string())
                 if request.api_version_request.experimental:
+                    # NOTE(vponomaryov): Translate our boolean header
+                    # to string explicitly to avoid 'TypeError' failure
+                    # running manila API under Apache + mod-wsgi.
+                    # It is safe to do so, because all headers are returned as
+                    # strings anyway.
                     response.headers[EXPERIMENTAL_API_REQUEST_HEADER] = (
-                        request.api_version_request.experimental)
+                        '%s' % request.api_version_request.experimental)
                 response.headers['Vary'] = API_VERSION_REQUEST_HEADER
 
         return response
@@ -1280,14 +1286,19 @@ class Fault(webob.exc.HTTPException):
                 'message': self.wrapped_exc.explanation}}
         if code == 413:
             retry = self.wrapped_exc.headers['Retry-After']
-            fault_data[fault_name]['retryAfter'] = retry
+            fault_data[fault_name]['retryAfter'] = '%s' % retry
 
         if not req.api_version_request.is_null():
             self.wrapped_exc.headers[API_VERSION_REQUEST_HEADER] = (
                 req.api_version_request.get_string())
             if req.api_version_request.experimental:
+                # NOTE(vponomaryov): Translate our boolean header
+                # to string explicitly to avoid 'TypeError' failure
+                # running manila API under Apache + mod-wsgi.
+                # It is safe to do so, because all headers are returned as
+                # strings anyway.
                 self.wrapped_exc.headers[EXPERIMENTAL_API_REQUEST_HEADER] = (
-                    req.api_version_request.experimental)
+                    '%s' % req.api_version_request.experimental)
             self.wrapped_exc.headers['Vary'] = API_VERSION_REQUEST_HEADER
 
         content_type = req.best_match_content_type()
@@ -1330,7 +1341,7 @@ class OverLimitFault(webob.exc.HTTPException):
     def _retry_after(retry_time):
         delay = int(math.ceil(retry_time - time.time()))
         retry_after = delay if delay > 0 else 0
-        headers = {'Retry-After': '%d' % retry_after}
+        headers = {'Retry-After': '%s' % retry_after}
         return headers
 
     @webob.dec.wsgify(RequestClass=Request)

manila/api/v1/limits.py

@@ -33,7 +33,7 @@ from manila.api.openstack import wsgi
 from manila.api.views import limits as limits_views
 from manila.i18n import _
 from manila import quota
-from manila import wsgi as base_wsgi
+from manila.wsgi import common as base_wsgi
 
 QUOTAS = quota.QUOTAS
 

manila/common/config.py

@@ -41,9 +41,6 @@ log.register_options(CONF)
 
 
 core_opts = [
-    cfg.StrOpt('api_paste_config',
-               default="api-paste.ini",
-               help='File name for the paste.deploy config for manila-api.'),
     cfg.StrOpt('state_path',
                default='/var/lib/manila',
                help="Top-level directory for maintaining manila's state."),

manila/exception.py

@@ -386,7 +386,7 @@ class WillNotSchedule(ManilaException):
 class QuotaError(ManilaException):
     message = _("Quota exceeded: code=%(code)s.")
     code = 413
-    headers = {'Retry-After': 0}
+    headers = {'Retry-After': '0'}
     safe = True
 
 

manila/opts.py

@@ -85,7 +85,7 @@ import manila.share.hook
 import manila.share.manager
 import manila.volume
 import manila.volume.cinder
-import manila.wsgi
+import manila.wsgi.eventlet_server
 
 
 # List of *all* options in [DEFAULT] namespace of manila.
@@ -165,8 +165,7 @@ _global_opt_lists = [
     manila.share.hook.hook_options,
     manila.share.manager.share_manager_opts,
     manila.volume._volume_opts,
-    manila.wsgi.eventlet_opts,
+    manila.wsgi.eventlet_server.socket_opts,
-    manila.wsgi.socket_opts,
 ]
 
 _opts = [

manila/service.py

@@ -26,6 +26,7 @@ from oslo_log import log
 import oslo_messaging as messaging
 from oslo_service import loopingcall
 from oslo_service import service
+from oslo_service import wsgi
 from oslo_utils import importutils
 
 from manila import context
@@ -34,7 +35,6 @@ from manila import db
 from manila import exception
 from manila import rpc
 from manila import version
-from manila import wsgi
 
 LOG = log.getLogger(__name__)
 
@@ -283,7 +283,7 @@ class WSGIService(service.ServiceBase):
         """
         self.name = name
         self.manager = self._get_manager()
-        self.loader = loader or wsgi.Loader()
+        self.loader = loader or wsgi.Loader(CONF)
         if not rpc.initialized():
             rpc.init(CONF)
         self.app = self.loader.load_app(name)
@@ -296,10 +296,13 @@ class WSGIService(service.ServiceBase):
                 "greater than 1.  Input value ignored." % {'name': name})
             # Reset workers to default
             self.workers = None
-        self.server = wsgi.Server(name,
+        self.server = wsgi.Server(
+            CONF,
+            name,
             self.app,
             host=self.host,
-                                  port=self.port)
+            port=self.port,
+        )
 
     def _get_manager(self):
         """Initialize a Manager object appropriate for this service.

manila/tests/api/fakes.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_service import wsgi
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import routes
@@ -33,7 +34,6 @@ from manila.api import versions
 from manila.common import constants
 from manila import context
 from manila import exception
-from manila import wsgi
 
 
 FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'

manila/tests/api/middleware/test_faults.py

@@ -19,7 +19,9 @@ import webob
 import webob.dec
 import webob.exc
 
+from manila.api.middleware import fault
 from manila.api.openstack import wsgi
+from manila import exception
 from manila import test
 
 
@@ -72,7 +74,7 @@ class TestFaults(test.TestCase):
                 "overLimit": {
                     "message": "sorry",
                     "code": 413,
-                    "retryAfter": 4,
+                    "retryAfter": '4',
                 },
             }
             actual = jsonutils.loads(response.body)
@@ -109,3 +111,76 @@ class TestFaults(test.TestCase):
         """Ensure the status_int is set correctly on faults."""
         fault = wsgi.Fault(webob.exc.HTTPBadRequest(explanation='what?'))
         self.assertEqual(400, fault.status_int)
+
+
+class ExceptionTest(test.TestCase):
+
+    def _wsgi_app(self, inner_app):
+        return fault.FaultWrapper(inner_app)
+
+    def _do_test_exception_safety_reflected_in_faults(self, expose):
+        class ExceptionWithSafety(exception.ManilaException):
+            safe = expose
+
+        @webob.dec.wsgify
+        def fail(req):
+            raise ExceptionWithSafety('some explanation')
+
+        api = self._wsgi_app(fail)
+        resp = webob.Request.blank('/').get_response(api)
+        self.assertIn('{"computeFault', six.text_type(resp.body), resp.body)
+        expected = ('ExceptionWithSafety: some explanation' if expose else
+                    'The server has either erred or is incapable '
+                    'of performing the requested operation.')
+        self.assertIn(expected, six.text_type(resp.body), resp.body)
+        self.assertEqual(500, resp.status_int, resp.body)
+
+    def test_safe_exceptions_are_described_in_faults(self):
+        self._do_test_exception_safety_reflected_in_faults(True)
+
+    def test_unsafe_exceptions_are_not_described_in_faults(self):
+        self._do_test_exception_safety_reflected_in_faults(False)
+
+    def _do_test_exception_mapping(self, exception_type, msg):
+        @webob.dec.wsgify
+        def fail(req):
+            raise exception_type(msg)
+
+        api = self._wsgi_app(fail)
+        resp = webob.Request.blank('/').get_response(api)
+        self.assertIn(msg, six.text_type(resp.body), resp.body)
+        self.assertEqual(exception_type.code, resp.status_int, resp.body)
+
+        if hasattr(exception_type, 'headers'):
+            for (key, value) in exception_type.headers.items():
+                self.assertIn(key, resp.headers)
+                self.assertEqual(value, resp.headers[key])
+
+    def test_quota_error_mapping(self):
+        self._do_test_exception_mapping(exception.QuotaError, 'too many used')
+
+    def test_non_manila_notfound_exception_mapping(self):
+        class ExceptionWithCode(Exception):
+            code = 404
+
+        self._do_test_exception_mapping(ExceptionWithCode,
+                                        'NotFound')
+
+    def test_non_manila_exception_mapping(self):
+        class ExceptionWithCode(Exception):
+            code = 417
+
+        self._do_test_exception_mapping(ExceptionWithCode,
+                                        'Expectation failed')
+
+    def test_exception_with_none_code_throws_500(self):
+        class ExceptionWithNoneCode(Exception):
+            code = None
+
+        @webob.dec.wsgify
+        def fail(req):
+            raise ExceptionWithNoneCode()
+
+        api = self._wsgi_app(fail)
+        resp = webob.Request.blank('/').get_response(api)
+        self.assertEqual(500, resp.status_int)

manila/tests/api/test_versions.py

@@ -185,7 +185,7 @@ class ExperimentalAPITestCase(test.TestCase):
         self.assertEqual('2.0', response.headers[version_header_name])
 
         if experimental:
-            self.assertEqual(experimental,
+            self.assertEqual('%s' % experimental,
                              response.headers.get(experimental_header_name))
         else:
             self.assertNotIn(experimental_header_name, response.headers)

manila/tests/api/test_wsgi.py

@@ -19,33 +19,19 @@
 Test WSGI basics and provide some helper functions for other WSGI tests.
 """
 
-from manila import test
+from oslo_service import wsgi
-
 import routes
-import six
 import webob
 
-from manila import wsgi
+from manila import test
+from manila.wsgi import common as common_wsgi
 
 
 class Test(test.TestCase):
 
-    def test_debug(self):
-
-        class Application(wsgi.Application):
-            """Dummy application to test debug."""
-
-            def __call__(self, environ, start_response):
-                start_response("200", [("X-Test", "checking")])
-                return [six.b('Test result')]
-
-        application = wsgi.Debug(Application())
-        result = webob.Request.blank('/').get_response(application)
-        self.assertEqual(six.b("Test result"), result.body)
-
     def test_router(self):
 
-        class Application(wsgi.Application):
+        class Application(common_wsgi.Application):
             """Test application to call from router."""
 
             def __call__(self, environ, start_response):

manila/tests/conf_fixture.py

@@ -17,6 +17,7 @@
 import os
 
 from oslo_policy import opts
+from oslo_service import wsgi
 
 from manila.common import config
 
@@ -38,6 +39,7 @@ def set_defaults(conf):
     _safe_set_of_opts(conf, 'service_instance_user', 'fake_user')
     _API_PASTE_PATH = os.path.abspath(os.path.join(CONF.state_path,
                                                    'etc/manila/api-paste.ini'))
+    wsgi.register_opts(conf)
     _safe_set_of_opts(conf, 'api_paste_config', _API_PASTE_PATH)
     _safe_set_of_opts(conf, 'share_driver',
                       'manila.tests.fake_driver.FakeShareDriver')

manila/tests/test_service.py

@@ -24,6 +24,7 @@ Unit Tests for remote procedure calls using queue
 import ddt
 import mock
 from oslo_config import cfg
+from oslo_service import wsgi
 
 from manila import context
 from manila import db
@@ -32,7 +33,6 @@ from manila import manager
 from manila import service
 from manila import test
 from manila import utils
-from manila import wsgi
 
 test_service_opts = [
     cfg.StrOpt("fake_manager",
@@ -224,5 +224,5 @@ class TestWSGIService(test.TestCase):
         # Resetting pool size to default
         self.test_service.reset()
         self.test_service.start()
-        self.assertEqual(1000, self.test_service.server._pool.size)
+        self.assertGreater(self.test_service.server._pool.size, 0)
         wsgi.Loader.load_app.assert_called_once_with("test_service")

manila/tests/test_utils.py

@@ -25,6 +25,7 @@ from oslo_config import cfg
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import paramiko
+import six
 from webob import exc
 
 import manila
@@ -744,3 +745,39 @@ class ShareMigrationHelperTestCase(test.TestCase):
         self.assertRaises(expected_exc,
                           utils.wait_for_access_update, self.context,
                           db, fake_instance, 1)
+
+
+@ddt.ddt
+class ConvertStrTestCase(test.TestCase):
+
+    def test_convert_str_str_input(self):
+        self.mock_object(utils.encodeutils, 'safe_encode')
+        input_value = six.text_type("string_input")
+
+        output_value = utils.convert_str(input_value)
+
+        if six.PY2:
+            utils.encodeutils.safe_encode.assert_called_once_with(input_value)
+            self.assertEqual(
+                utils.encodeutils.safe_encode.return_value, output_value)
+        else:
+            self.assertEqual(0, utils.encodeutils.safe_encode.call_count)
+            self.assertEqual(input_value, output_value)
+
+    def test_convert_str_bytes_input(self):
+        self.mock_object(utils.encodeutils, 'safe_encode')
+        if six.PY2:
+            input_value = six.binary_type("binary_input")
+        else:
+            input_value = six.binary_type("binary_input", "utf-8")
+
+        output_value = utils.convert_str(input_value)
+
+        if six.PY2:
+            utils.encodeutils.safe_encode.assert_called_once_with(input_value)
+            self.assertEqual(
+                utils.encodeutils.safe_encode.return_value, output_value)
+        else:
+            self.assertEqual(0, utils.encodeutils.safe_encode.call_count)
+            self.assertIsInstance(output_value, six.string_types)
+            self.assertEqual(six.text_type("binary_input"), output_value)

manila/tests/test_wsgi.py

@@ -1,334 +0,0 @@
-# Copyright 2011 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""Unit tests for `manila.wsgi`."""
-
-import os.path
-import ssl
-import tempfile
-
-import ddt
-import eventlet
-import mock
-from oslo_config import cfg
-from oslo_utils import netutils
-import six
-from six.moves import urllib
-import testtools
-import webob
-import webob.dec
-
-from manila.api.middleware import fault
-from manila import exception
-from manila import test
-from manila import utils
-import manila.wsgi
-
-CONF = cfg.CONF
-
-TEST_VAR_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                               'var'))
-
-
-class TestLoaderNothingExists(test.TestCase):
-    """Loader tests where os.path.exists always returns False."""
-
-    def test_config_not_found(self):
-        self.assertRaises(
-            manila.exception.ConfigNotFound,
-            manila.wsgi.Loader,
-            'nonexistent_file.ini',
-        )
-
-
-class TestLoaderNormalFilesystem(test.TestCase):
-    """Loader tests with normal filesystem (unmodified os.path module)."""
-
-    _paste_config = """
-[app:test_app]
-use = egg:Paste#static
-document_root = /tmp
-    """
-
-    def setUp(self):
-        super(TestLoaderNormalFilesystem, self).setUp()
-        self.config = tempfile.NamedTemporaryFile(mode="w+t")
-        self.config.write(self._paste_config.lstrip())
-        self.config.seek(0)
-        self.config.flush()
-        self.loader = manila.wsgi.Loader(self.config.name)
-        self.addCleanup(self.config.close)
-
-    def test_config_found(self):
-        self.assertEqual(self.config.name, self.loader.config_path)
-
-    def test_app_not_found(self):
-        self.assertRaises(
-            manila.exception.PasteAppNotFound,
-            self.loader.load_app,
-            "non-existent app",
-        )
-
-    def test_app_found(self):
-        url_parser = self.loader.load_app("test_app")
-        self.assertEqual("/tmp", url_parser.directory)
-
-
-@ddt.ddt
-class TestWSGIServer(test.TestCase):
-    """WSGI server tests."""
-
-    def test_no_app(self):
-        server = manila.wsgi.Server("test_app", None, host="127.0.0.1", port=0)
-        self.assertEqual("test_app", server.name)
-
-    def test_start_random_port(self):
-        server = manila.wsgi.Server("test_random_port", None, host="127.0.0.1")
-        server.start()
-        self.assertNotEqual(0, server.port)
-        server.stop()
-        server.wait()
-
-    @testtools.skipIf(not netutils.is_ipv6_enabled(),
-                      "Test requires an IPV6 configured interface")
-    @testtools.skipIf(utils.is_eventlet_bug105(),
-                      'Eventlet bug #105 affect test results.')
-    def test_start_random_port_with_ipv6(self):
-        server = manila.wsgi.Server("test_random_port",
-                                    None,
-                                    host="::1")
-        server.start()
-        self.assertEqual("::1", server.host)
-        self.assertNotEqual(0, server.port)
-        server.stop()
-        server.wait()
-
-    def test_start_with_default_tcp_options(self):
-        server = manila.wsgi.Server("test_tcp_options",
-                                    None,
-                                    host="127.0.0.1")
-        self.mock_object(
-            netutils, 'set_tcp_keepalive')
-        server.start()
-        netutils.set_tcp_keepalive.assert_called_once_with(
-            mock.ANY, tcp_keepalive=True, tcp_keepalive_count=None,
-            tcp_keepalive_interval=None, tcp_keepidle=600)
-
-    def test_start_with_custom_tcp_options(self):
-        CONF.set_default("tcp_keepalive", False)
-        CONF.set_default("tcp_keepalive_count", 33)
-        CONF.set_default("tcp_keepalive_interval", 22)
-        CONF.set_default("tcp_keepidle", 11)
-        server = manila.wsgi.Server("test_tcp_options",
-                                    None,
-                                    host="127.0.0.1")
-        self.mock_object(
-            netutils, 'set_tcp_keepalive')
-        server.start()
-        netutils.set_tcp_keepalive.assert_called_once_with(
-            mock.ANY, tcp_keepalive=False, tcp_keepalive_count=33,
-            tcp_keepalive_interval=22, tcp_keepidle=11)
-
-    def test_app(self):
-        self.mock_object(
-            eventlet, 'spawn', mock.Mock(side_effect=eventlet.spawn))
-        greetings = 'Hello, World!!!'
-
-        def hello_world(env, start_response):
-            if env['PATH_INFO'] != '/':
-                start_response('404 Not Found',
-                               [('Content-Type', 'text/plain')])
-                return ['Not Found\r\n']
-            start_response('200 OK', [('Content-Type', 'text/plain')])
-            return [greetings]
-
-        server = manila.wsgi.Server(
-            "test_app", hello_world, host="127.0.0.1", port=0)
-        server.start()
-
-        response = urllib.request.urlopen('http://127.0.0.1:%d/' % server.port)
-        self.assertEqual(six.b(greetings), response.read())
-
-        # Verify provided parameters to eventlet.spawn func
-        eventlet.spawn.assert_called_once_with(
-            func=eventlet.wsgi.server,
-            sock=mock.ANY,
-            site=server.app,
-            protocol=server._protocol,
-            custom_pool=server._pool,
-            log=server._logger,
-            socket_timeout=server.client_socket_timeout,
-            keepalive=manila.wsgi.CONF.wsgi_keep_alive,
-        )
-
-        server.stop()
-
-    @ddt.data(0, 0.1, 1, None)
-    def test_init_server_with_socket_timeout(self, client_socket_timeout):
-        CONF.set_default("client_socket_timeout", client_socket_timeout)
-        server = manila.wsgi.Server(
-            "test_app", lambda *args, **kwargs: None, host="127.0.0.1", port=0)
-        self.assertEqual(client_socket_timeout, server.client_socket_timeout)
-
-    @testtools.skipIf(six.PY3, "bug/1482633")
-    def test_app_using_ssl(self):
-        CONF.set_default("ssl_cert_file",
-                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
-        CONF.set_default("ssl_key_file",
-                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))
-
-        greetings = 'Hello, World!!!'
-
-        @webob.dec.wsgify
-        def hello_world(req):
-            return greetings
-
-        server = manila.wsgi.Server(
-            "test_app", hello_world, host="127.0.0.1", port=0)
-        server.start()
-
-        if hasattr(ssl, '_create_unverified_context'):
-            response = urllib.request.urlopen(
-                'https://127.0.0.1:%d/' % server.port,
-                context=ssl._create_unverified_context())
-        else:
-            response = urllib.request.urlopen(
-                'https://127.0.0.1:%d/' % server.port)
-
-        self.assertEqual(greetings, response.read())
-
-        server.stop()
-
-    @testtools.skipIf(not netutils.is_ipv6_enabled(),
-                      "Test requires an IPV6 configured interface")
-    @testtools.skipIf(utils.is_eventlet_bug105(),
-                      'Eventlet bug #105 affect test results.')
-    @testtools.skipIf(six.PY3, "bug/1482633")
-    def test_app_using_ipv6_and_ssl(self):
-        CONF.set_default("ssl_cert_file",
-                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
-        CONF.set_default("ssl_key_file",
-                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))
-
-        greetings = 'Hello, World!!!'
-
-        @webob.dec.wsgify
-        def hello_world(req):
-            return greetings
-
-        server = manila.wsgi.Server("test_app",
-                                    hello_world,
-                                    host="::1",
-                                    port=0)
-        server.start()
-
-        if hasattr(ssl, '_create_unverified_context'):
-            response = urllib.request.urlopen(
-                'https://[::1]:%d/' % server.port,
-                context=ssl._create_unverified_context())
-        else:
-            response = urllib.request.urlopen(
-                'https://[::1]:%d/' % server.port)
-
-        self.assertEqual(greetings, response.read())
-
-        server.stop()
-
-    def test_reset_pool_size_to_default(self):
-        server = manila.wsgi.Server("test_resize", None, host="127.0.0.1")
-        server.start()
-
-        # Stopping the server, which in turn sets pool size to 0
-        server.stop()
-        self.assertEqual(0, server._pool.size)
-
-        # Resetting pool size to default
-        server.reset()
-        server.start()
-        self.assertEqual(1000, server._pool.size)
-
-
-class ExceptionTest(test.TestCase):
-
-    def _wsgi_app(self, inner_app):
-        return fault.FaultWrapper(inner_app)
-
-    def _do_test_exception_safety_reflected_in_faults(self, expose):
-        class ExceptionWithSafety(exception.ManilaException):
-            safe = expose
-
-        @webob.dec.wsgify
-        def fail(req):
-            raise ExceptionWithSafety('some explanation')
-
-        api = self._wsgi_app(fail)
-        resp = webob.Request.blank('/').get_response(api)
-        self.assertIn('{"computeFault', six.text_type(resp.body), resp.body)
-        expected = ('ExceptionWithSafety: some explanation' if expose else
-                    'The server has either erred or is incapable '
-                    'of performing the requested operation.')
-        self.assertIn(expected, six.text_type(resp.body), resp.body)
-        self.assertEqual(500, resp.status_int, resp.body)
-
-    def test_safe_exceptions_are_described_in_faults(self):
-        self._do_test_exception_safety_reflected_in_faults(True)
-
-    def test_unsafe_exceptions_are_not_described_in_faults(self):
-        self._do_test_exception_safety_reflected_in_faults(False)
-
-    def _do_test_exception_mapping(self, exception_type, msg):
-        @webob.dec.wsgify
-        def fail(req):
-            raise exception_type(msg)
-
-        api = self._wsgi_app(fail)
-        resp = webob.Request.blank('/').get_response(api)
-        self.assertIn(msg, six.text_type(resp.body), resp.body)
-        self.assertEqual(exception_type.code, resp.status_int, resp.body)
-
-        if hasattr(exception_type, 'headers'):
-            for (key, value) in exception_type.headers.items():
-                self.assertIn(key, resp.headers)
-                self.assertEqual(value, resp.headers[key])
-
-    def test_quota_error_mapping(self):
-        self._do_test_exception_mapping(exception.QuotaError, 'too many used')
-
-    def test_non_manila_notfound_exception_mapping(self):
-        class ExceptionWithCode(Exception):
-            code = 404
-
-        self._do_test_exception_mapping(ExceptionWithCode,
-                                        'NotFound')
-
-    def test_non_manila_exception_mapping(self):
-        class ExceptionWithCode(Exception):
-            code = 417
-
-        self._do_test_exception_mapping(ExceptionWithCode,
-                                        'Expectation failed')
-
-    def test_exception_with_none_code_throws_500(self):
-        class ExceptionWithNoneCode(Exception):
-            code = None
-
-        @webob.dec.wsgify
-        def fail(req):
-            raise ExceptionWithNoneCode()
-
-        api = self._wsgi_app(fail)
-        resp = webob.Request.blank('/').get_response(api)
-        self.assertEqual(500, resp.status_int)

manila/tests/wsgi/test_common.py

@@ -0,0 +1,45 @@
+# Copyright 2017 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from manila import test
+from manila.wsgi import common
+
+
+class FakeApp(common.Application):
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
+class WSGICommonTestCase(test.TestCase):
+
+    def test_application_factory(self):
+        fake_global_config = mock.Mock()
+        kwargs = {"k1": "v1", "k2": "v2"}
+
+        result = FakeApp.factory(fake_global_config, **kwargs)
+
+        fake_global_config.assert_not_called()
+        self.assertIsInstance(result, FakeApp)
+        for k, v in kwargs.items():
+            self.assertTrue(hasattr(result, k))
+            self.assertEqual(getattr(result, k), v)
+
+    def test_application___call__(self):
+        self.assertRaises(
+            NotImplementedError,
+            common.Application(), 'fake_environ', 'fake_start_response')

manila/tests/wsgi/test_wsgi.py

@@ -0,0 +1,45 @@
+# Copyright 2017 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from manila import test
+from manila.wsgi import wsgi
+
+
+class WSGITestCase(test.TestCase):
+
+    def test_initialize_application(self):
+        self.mock_object(wsgi.log, 'register_options')
+        self.mock_object(wsgi.cfg.ConfigOpts, '__call__')
+        self.mock_object(wsgi.config, 'verify_share_protocols')
+        self.mock_object(wsgi.log, 'setup')
+        self.mock_object(wsgi.rpc, 'init')
+        self.mock_object(wsgi.wsgi, 'Loader')
+        wsgi.sys.argv = ['--verbose', '--debug']
+
+        result = wsgi.initialize_application()
+
+        self.assertEqual(
+            wsgi.wsgi.Loader.return_value.load_app.return_value, result)
+        wsgi.log.register_options.assert_called_once_with(mock.ANY)
+        wsgi.cfg.ConfigOpts.__call__.assert_called_once_with(
+            mock.ANY, project="manila", version=wsgi.version.version_string())
+        wsgi.config.verify_share_protocols.assert_called_once_with()
+        wsgi.log.setup.assert_called_once_with(mock.ANY, "manila")
+        wsgi.rpc.init.assert_called_once_with(mock.ANY)
+        wsgi.wsgi.Loader.assert_called_once_with(mock.ANY)
+        wsgi.wsgi.Loader.return_value.load_app.assert_called_once_with(
+            name='osapi_share')

manila/utils.py

@@ -36,6 +36,7 @@ from oslo_concurrency import lockutils
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log
+from oslo_utils import encodeutils
 from oslo_utils import importutils
 from oslo_utils import netutils
 from oslo_utils import strutils
@@ -520,6 +521,24 @@ def require_driver_initialized(func):
     return wrapper
 
 
+def convert_str(text):
+    """Convert to native string.
+
+    Convert bytes and Unicode strings to native strings:
+
+    * convert to bytes on Python 2:
+      encode Unicode using encodeutils.safe_encode()
+    * convert to Unicode on Python 3: decode bytes from UTF-8
+    """
+    if six.PY2:
+        return encodeutils.safe_encode(text)
+    else:
+        if isinstance(text, bytes):
+            return text.decode('utf-8')
+        else:
+            return text
+
+
 def translate_string_size_to_float(string, multiplier='G'):
     """Translates human-readable storage size to float value.
 

manila/wsgi.py

@@ -1,551 +0,0 @@
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# Copyright 2010 OpenStack LLC.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""Utility methods for working with WSGI servers."""
-
-from __future__ import print_function
-
-import errno
-import os
-import socket
-import ssl
-import sys
-import time
-
-import eventlet
-import eventlet.wsgi
-import greenlet
-from oslo_config import cfg
-from oslo_log import log
-from oslo_service import service
-from oslo_utils import excutils
-from oslo_utils import netutils
-from paste import deploy
-import routes.middleware
-import webob.dec
-import webob.exc
-
-from manila.common import config
-from manila import exception
-from manila.i18n import _
-
-socket_opts = [
-    cfg.IntOpt('backlog',
-               default=4096,
-               help="Number of backlog requests to configure the socket "
-                    "with."),
-    cfg.BoolOpt('tcp_keepalive',
-                default=True,
-                help="Sets the value of TCP_KEEPALIVE (True/False) for each "
-                     "server socket."),
-    cfg.IntOpt('tcp_keepidle',
-               default=600,
-               help="Sets the value of TCP_KEEPIDLE in seconds for each "
-                    "server socket. Not supported on OS X."),
-    cfg.IntOpt('tcp_keepalive_interval',
-               help="Sets the value of TCP_KEEPINTVL in seconds for each "
-                    "server socket. Not supported on OS X."),
-    cfg.IntOpt('tcp_keepalive_count',
-               help="Sets the value of TCP_KEEPCNT for each "
-                    "server socket. Not supported on OS X."),
-    cfg.StrOpt('ssl_ca_file',
-               help="CA certificate file to use to verify "
-                    "connecting clients."),
-    cfg.StrOpt('ssl_cert_file',
-               help="Certificate file to use when starting "
-                    "the server securely."),
-    cfg.StrOpt('ssl_key_file',
-               help="Private key file to use when starting "
-                    "the server securely."),
-]
-
-eventlet_opts = [
-    cfg.IntOpt('max_header_line',
-               default=16384,
-               help="Maximum line size of message headers to be accepted. "
-                    "Option max_header_line may need to be increased when "
-                    "using large tokens (typically those generated by the "
-                    "Keystone v3 API with big service catalogs)."),
-    cfg.IntOpt('client_socket_timeout',
-               default=900,
-               help="Timeout for client connections socket operations. "
-                    "If an incoming connection is idle for this number of "
-                    "seconds it will be closed. A value of '0' means "
-                    "wait forever."),
-    cfg.BoolOpt('wsgi_keep_alive',
-                default=True,
-                help='If False, closes the client socket connection '
-                     'explicitly. Setting it to True to maintain backward '
-                     'compatibility. Recommended setting is set it to False.'),
-]
-
-CONF = cfg.CONF
-CONF.register_opts(socket_opts)
-CONF.register_opts(eventlet_opts)
-
-LOG = log.getLogger(__name__)
-
-
-class Server(service.ServiceBase):
-    """Server class to manage a WSGI server, serving a WSGI application."""
-
-    default_pool_size = 1000
-
-    def __init__(self, name, app, host=None, port=None, pool_size=None,
-                 protocol=eventlet.wsgi.HttpProtocol, backlog=128):
-        """Initialize, but do not start, a WSGI server.
-
-        :param name: Pretty name for logging.
-        :param app: The WSGI application to serve.
-        :param host: IP address to serve the application.
-        :param port: Port number to server the application.
-        :param pool_size: Maximum number of eventlets to spawn concurrently.
-        :returns: None
-
-        """
-        eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
-        self.client_socket_timeout = CONF.client_socket_timeout
-        self.name = name
-        self.app = app
-        self._host = host or "0.0.0.0"
-        self._port = port or 0
-        self._server = None
-        self._socket = None
-        self._protocol = protocol
-        self.pool_size = pool_size or self.default_pool_size
-        self._pool = eventlet.GreenPool(self.pool_size)
-        self._logger = log.getLogger("eventlet.wsgi.server")
-
-        if backlog < 1:
-            raise exception.InvalidInput(
-                reason='The backlog must be more than 1')
-
-        bind_addr = (host, port)
-        # TODO(dims): eventlet's green dns/socket module does not actually
-        # support IPv6 in getaddrinfo(). We need to get around this in the
-        # future or monitor upstream for a fix
-        try:
-            info = socket.getaddrinfo(bind_addr[0],
-                                      bind_addr[1],
-                                      socket.AF_UNSPEC,
-                                      socket.SOCK_STREAM)[0]
-            family = info[0]
-            bind_addr = info[-1]
-        except Exception:
-            family = socket.AF_INET
-
-        cert_file = CONF.ssl_cert_file
-        key_file = CONF.ssl_key_file
-        ca_file = CONF.ssl_ca_file
-        self._use_ssl = cert_file or key_file
-
-        if cert_file and not os.path.exists(cert_file):
-            raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
-
-        if ca_file and not os.path.exists(ca_file):
-            raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)
-
-        if key_file and not os.path.exists(key_file):
-            raise RuntimeError(_("Unable to find key_file : %s") % key_file)
-
-        if self._use_ssl and (not cert_file or not key_file):
-            raise RuntimeError(_("When running server in SSL mode, you must "
-                                 "specify both a cert_file and key_file "
-                                 "option value in your configuration file"))
-
-        retry_until = time.time() + 30
-        while not self._socket and time.time() < retry_until:
-            try:
-                self._socket = eventlet.listen(
-                    bind_addr, backlog=backlog, family=family)
-            except socket.error as err:
-                if err.args[0] != errno.EADDRINUSE:
-                    raise
-                eventlet.sleep(0.1)
-
-        if not self._socket:
-            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
-                               "after trying for 30 seconds") %
-                               {'host': host, 'port': port})
-
-        (self._host, self._port) = self._socket.getsockname()[0:2]
-        LOG.info("%(name)s listening on %(_host)s:%(_port)s",
-                 {'name': self.name, '_host': self._host, '_port': self._port})
-
-    def start(self):
-        """Start serving a WSGI application.
-
-        :returns: None
-        :raises: manila.exception.InvalidInput
-
-        """
-        # The server socket object will be closed after server exits,
-        # but the underlying file descriptor will remain open, and will
-        # give bad file descriptor error. So duplicating the socket object,
-        # to keep file descriptor usable.
-
-        config.set_middleware_defaults()
-        dup_socket = self._socket.dup()
-
-        netutils.set_tcp_keepalive(
-            dup_socket,
-            tcp_keepalive=CONF.tcp_keepalive,
-            tcp_keepidle=CONF.tcp_keepidle,
-            tcp_keepalive_interval=CONF.tcp_keepalive_interval,
-            tcp_keepalive_count=CONF.tcp_keepalive_count
-        )
-
-        if self._use_ssl:
-            try:
-                ssl_kwargs = {
-                    'server_side': True,
-                    'certfile': CONF.ssl_cert_file,
-                    'keyfile': CONF.ssl_key_file,
-                    'cert_reqs': ssl.CERT_NONE,
-                }
-
-                if CONF.ssl_ca_file:
-                    ssl_kwargs['ca_certs'] = CONF.ssl_ca_file
-                    ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
-
-                dup_socket = ssl.wrap_socket(dup_socket,
-                                             **ssl_kwargs)
-
-                dup_socket.setsockopt(socket.SOL_SOCKET,
-                                      socket.SO_REUSEADDR, 1)
-
-            except Exception:
-                with excutils.save_and_reraise_exception():
-                    LOG.error(
-                        ("Failed to start %(name)s on %(_host)s:%(_port)s "
-                         "with SSL support."),
-                        {"name": self.name, "_host": self._host,
-                         "_port": self._port}
-                    )
-
-        wsgi_kwargs = {
-            'func': eventlet.wsgi.server,
-            'sock': dup_socket,
-            'site': self.app,
-            'protocol': self._protocol,
-            'custom_pool': self._pool,
-            'log': self._logger,
-            'socket_timeout': self.client_socket_timeout,
-            'keepalive': CONF.wsgi_keep_alive,
-        }
-
-        self._server = eventlet.spawn(**wsgi_kwargs)
-
-    @property
-    def host(self):
-        return self._host
-
-    @property
-    def port(self):
-        return self._port
-
-    def stop(self):
-        """Stop this server.
-
-        This is not a very nice action, as currently the method by which a
-        server is stopped is by killing its eventlet.
-
-        :returns: None
-
-        """
-        LOG.info("Stopping WSGI server.")
-        if self._server is not None:
-            # Resize pool to stop new requests from being processed
-            self._pool.resize(0)
-            self._server.kill()
-
-    def wait(self):
-        """Block, until the server has stopped.
-
-        Waits on the server's eventlet to finish, then returns.
-
-        :returns: None
-
-        """
-        try:
-            if self._server is not None:
-                self._pool.waitall()
-                self._server.wait()
-        except greenlet.GreenletExit:
-            LOG.info("WSGI server has stopped.")
-
-    def reset(self):
-        """Reset server greenpool size to default.
-
-        :returns: None
-        """
-        self._pool.resize(self.pool_size)
-
-
-class Request(webob.Request):
-    pass
-
-
-class Application(object):
-    """Base WSGI application wrapper. Subclasses need to implement __call__."""
-
-    @classmethod
-    def factory(cls, global_config, **local_config):
-        """Used for paste app factories in paste.deploy config files.
-
-        Any local configuration (that is, values under the [app:APPNAME]
-        section of the paste config) will be passed into the `__init__` method
-        as kwargs.
-
-        A hypothetical configuration would look like:
-
-            [app:wadl]
-            latest_version = 1.3
-            paste.app_factory = manila.api.fancy_api:Wadl.factory
-
-        which would result in a call to the `Wadl` class as
-
-            import manila.api.fancy_api
-            fancy_api.Wadl(latest_version='1.3')
-
-        You could of course re-implement the `factory` method in subclasses,
-        but using the kwarg passing it shouldn't be necessary.
-
-        """
-        return cls(**local_config)
-
-    def __call__(self, environ, start_response):
-        r"""Subclasses will probably want to implement __call__ like this:
-
-        @webob.dec.wsgify(RequestClass=Request)
-        def __call__(self, req):
-          # Any of the following objects work as responses:
-
-          # Option 1: simple string
-          res = 'message\n'
-
-          # Option 2: a nicely formatted HTTP exception page
-          res = exc.HTTPForbidden(detail='Nice try')
-
-          # Option 3: a webob Response object (in case you need to play with
-          # headers, or you want to be treated like an iterable, or or or)
-          res = Response();
-          res.app_iter = open('somefile')
-
-          # Option 4: any wsgi app to be run next
-          res = self.application
-
-          # Option 5: you can get a Response object for a wsgi app, too, to
-          # play with headers etc
-          res = req.get_response(self.application)
-
-          # You can then just return your response...
-          return res
-          # ... or set req.response and return None.
-          req.response = res
-
-        See the end of http://pythonpaste.org/webob/modules/dec.html
-        for more info.
-
-        """
-        raise NotImplementedError(_('You must implement __call__'))
-
-
-class Middleware(Application):
-    """Base WSGI middleware.
-
-    These classes require an application to be
-    initialized that will be called next.  By default the middleware will
-    simply call its wrapped app, or you can override __call__ to customize its
-    behavior.
-
-    """
-
-    @classmethod
-    def factory(cls, global_config, **local_config):
-        """Used for paste app factories in paste.deploy config files.
-
-        Any local configuration (that is, values under the [filter:APPNAME]
-        section of the paste config) will be passed into the `__init__` method
-        as kwargs.
-
-        A hypothetical configuration would look like:
-
-            [filter:analytics]
-            redis_host = 127.0.0.1
-            paste.filter_factory = manila.api.analytics:Analytics.factory
-
-        which would result in a call to the `Analytics` class as
-
-            import manila.api.analytics
-            analytics.Analytics(app_from_paste, redis_host='127.0.0.1')
-
-        You could of course re-implement the `factory` method in subclasses,
-        but using the kwarg passing it shouldn't be necessary.
-
-        """
-        def _factory(app):
-            return cls(app, **local_config)
-        return _factory
-
-    def __init__(self, application):
-        self.application = application
-
-    def process_request(self, req):
-        """Called on each request.
-
-        If this returns None, the next application down the stack will be
-        executed. If it returns a response then that response will be returned
-        and execution will stop here.
-
-        """
-        return None
-
-    def process_response(self, response):
-        """Do whatever you'd like to the response."""
-        return response
-
-    @webob.dec.wsgify(RequestClass=Request)
-    def __call__(self, req):
-        response = self.process_request(req)
-        if response:
-            return response
-        response = req.get_response(self.application)
-        return self.process_response(response)
-
-
-class Debug(Middleware):
-    """Helper class for debugging a WSGI application.
-
-    Can be inserted into any WSGI application chain to get information
-    about the request and response.
-
-    """
-
-    @webob.dec.wsgify(RequestClass=Request)
-    def __call__(self, req):
-        print(('*' * 40) + ' REQUEST ENVIRON')
-        for key, value in req.environ.items():
-            print(key, '=', value)
-        print()
-        resp = req.get_response(self.application)
-
-        print(('*' * 40) + ' RESPONSE HEADERS')
-        for (key, value) in resp.headers.items():
-            print(key, '=', value)
-        print()
-
-        resp.app_iter = self.print_generator(resp.app_iter)
-
-        return resp
-
-    @staticmethod
-    def print_generator(app_iter):
-        """Iterator that prints the contents of a wrapper string."""
-        print(('*' * 40) + ' BODY')
-        for part in app_iter:
-            sys.stdout.write(part.decode())
-            sys.stdout.flush()
-            yield part
-        print()
-
-
-class Router(object):
-    """WSGI middleware that maps incoming requests to WSGI apps."""
-
-    def __init__(self, mapper):
-        """Create a router for the given routes.Mapper.
-
-        Each route in `mapper` must specify a 'controller', which is a
-        WSGI app to call.  You'll probably want to specify an 'action' as
-        well and have your controller be an object that can route
-        the request to the action-specific method.
-
-        Examples:
-          mapper = routes.Mapper()
-          sc = ServerController()
-
-          # Explicit mapping of one route to a controller+action
-          mapper.connect(None, '/svrlist', controller=sc, action='list')
-
-          # Actions are all implicitly defined
-          mapper.resource('server', 'servers', controller=sc)
-
-          # Pointing to an arbitrary WSGI app.  You can specify the
-          # {path_info:.*} parameter so the target app can be handed just that
-          # section of the URL.
-          mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp())
-
-        """
-        self.map = mapper
-        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
-                                                          self.map)
-
-    @webob.dec.wsgify(RequestClass=Request)
-    def __call__(self, req):
-        """Route the incoming request to a controller based on self.map.
-
-        If no match, return a 404.
-
-        """
-        return self._router
-
-    @staticmethod
-    @webob.dec.wsgify(RequestClass=Request)
-    def _dispatch(req):
-        """Dispatch the request to the appropriate controller.
-
-        Called by self._router after matching the incoming request to a route
-        and putting the information into req.environ.  Either returns 404
-        or the routed WSGI app's response.
-
-        """
-        match = req.environ['wsgiorg.routing_args'][1]
-        if not match:
-            return webob.exc.HTTPNotFound()
-        app = match['controller']
-        return app
-
-
-class Loader(object):
-    """Used to load WSGI applications from paste configurations."""
-
-    def __init__(self, config_path=None):
-        """Initialize the loader, and attempt to find the config.
-
-        :param config_path: Full or relative path to the paste config.
-        :returns: None
-
-        """
-        config_path = config_path or CONF.api_paste_config
-        self.config_path = CONF.find_file(config_path)
-        if not self.config_path:
-            raise exception.ConfigNotFound(path=config_path)
-
-    def load_app(self, name):
-        """Return the paste URLMap wrapped WSGI application.
-
-        :param name: Name of the application to load.
-        :returns: Paste URLMap object wrapping the requested application.
-        :raises: `manila.exception.PasteAppNotFound`
-
-        """
-        try:
-            return deploy.loadapp("config:%s" % self.config_path, name=name)
-        except LookupError as err:
-            LOG.error(err)
-            raise exception.PasteAppNotFound(name=name, path=self.config_path)

manila/wsgi/common.py

@@ -0,0 +1,155 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2010 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utility methods for working with WSGI servers."""
+
+import webob.dec
+import webob.exc
+
+from manila.i18n import _
+
+
+class Request(webob.Request):
+    pass
+
+
+class Application(object):
+    """Base WSGI application wrapper. Subclasses need to implement __call__."""
+
+    @classmethod
+    def factory(cls, global_config, **local_config):
+        """Used for paste app factories in paste.deploy config files.
+
+        Any local configuration (that is, values under the [app:APPNAME]
+        section of the paste config) will be passed into the `__init__` method
+        as kwargs.
+
+        A hypothetical configuration would look like:
+
+            [app:wadl]
+            latest_version = 1.3
+            paste.app_factory = manila.api.fancy_api:Wadl.factory
+
+        which would result in a call to the `Wadl` class as
+
+            import manila.api.fancy_api
+            fancy_api.Wadl(latest_version='1.3')
+
+        You could of course re-implement the `factory` method in subclasses,
+        but using the kwarg passing it shouldn't be necessary.
+
+        """
+        return cls(**local_config)
+
+    def __call__(self, environ, start_response):
+        r"""Subclasses will probably want to implement __call__ like this:
+
+        @webob.dec.wsgify(RequestClass=Request)
+        def __call__(self, req):
+          # Any of the following objects work as responses:
+
+          # Option 1: simple string
+          res = 'message\n'
+
+          # Option 2: a nicely formatted HTTP exception page
+          res = exc.HTTPForbidden(detail='Nice try')
+
+          # Option 3: a webob Response object (in case you need to play with
+          # headers, or you want to be treated like an iterable, or or or)
+          res = Response();
+          res.app_iter = open('somefile')
+
+          # Option 4: any wsgi app to be run next
+          res = self.application
+
+          # Option 5: you can get a Response object for a wsgi app, too, to
+          # play with headers etc
+          res = req.get_response(self.application)
+
+          # You can then just return your response...
+          return res
+          # ... or set req.response and return None.
+          req.response = res
+
+        See the end of http://pythonpaste.org/webob/modules/dec.html
+        for more info.
+
+        """
+        raise NotImplementedError(_('You must implement __call__'))
+
+
+class Middleware(Application):
+    """Base WSGI middleware.
+
+    These classes require an application to be
+    initialized that will be called next.  By default the middleware will
+    simply call its wrapped app, or you can override __call__ to customize its
+    behavior.
+
+    """
+
+    @classmethod
+    def factory(cls, global_config, **local_config):
+        """Used for paste app factories in paste.deploy config files.
+
+        Any local configuration (that is, values under the [filter:APPNAME]
+        section of the paste config) will be passed into the `__init__` method
+        as kwargs.
+
+        A hypothetical configuration would look like:
+
+            [filter:analytics]
+            redis_host = 127.0.0.1
+            paste.filter_factory = manila.api.analytics:Analytics.factory
+
+        which would result in a call to the `Analytics` class as
+
+            import manila.api.analytics
+            analytics.Analytics(app_from_paste, redis_host='127.0.0.1')
+
+        You could of course re-implement the `factory` method in subclasses,
+        but using the kwarg passing it shouldn't be necessary.
+
+        """
+        def _factory(app):
+            return cls(app, **local_config)
+        return _factory
+
+    def __init__(self, application):
+        self.application = application
+
+    def process_request(self, req):
+        """Called on each request.
+
+        If this returns None, the next application down the stack will be
+        executed. If it returns a response then that response will be returned
+        and execution will stop here.
+
+        """
+        return None
+
+    def process_response(self, response):
+        """Do whatever you'd like to the response."""
+        return response
+
+    @webob.dec.wsgify(RequestClass=Request)
+    def __call__(self, req):
+        response = self.process_request(req)
+        if response:
+            return response
+        response = req.get_response(self.application)
+        return self.process_response(response)

manila/wsgi/eventlet_server.py

@@ -0,0 +1,59 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2010 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utility methods for working with WSGI servers."""
+
+import socket
+
+from oslo_config import cfg
+from oslo_service import wsgi
+from oslo_utils import netutils
+
+socket_opts = [
+    cfg.BoolOpt('tcp_keepalive',
+                default=True,
+                help="Sets the value of TCP_KEEPALIVE (True/False) for each "
+                     "server socket."),
+    cfg.IntOpt('tcp_keepalive_interval',
+               help="Sets the value of TCP_KEEPINTVL in seconds for each "
+                    "server socket. Not supported on OS X."),
+    cfg.IntOpt('tcp_keepalive_count',
+               help="Sets the value of TCP_KEEPCNT for each "
+                    "server socket. Not supported on OS X."),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(socket_opts)
+
+
+class Server(wsgi.Server):
+    """Server class to manage a WSGI server, serving a WSGI application."""
+
+    def _set_socket_opts(self, _socket):
+        _socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        # NOTE(praneshp): Call set_tcp_keepalive in oslo to set
+        # tcp keepalive parameters. Sockets can hang around forever
+        # without keepalive
+        netutils.set_tcp_keepalive(
+            _socket,
+            self.conf.tcp_keepalive,
+            self.conf.tcp_keepidle,
+            self.conf.tcp_keepalive_count,
+            self.conf.tcp_keepalive_interval,
+        )
+        return _socket

manila/wsgi/wsgi.py

@@ -0,0 +1,39 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Manila OS API WSGI application."""
+
+
+import sys
+
+from oslo_config import cfg
+from oslo_log import log
+from oslo_service import wsgi
+
+from manila import i18n
+i18n.enable_lazy()
+
+# Need to register global_opts
+from manila.common import config
+from manila import rpc
+from manila import version
+
+CONF = cfg.CONF
+
+
+def initialize_application():
+    log.register_options(CONF)
+    CONF(sys.argv[1:], project="manila", version=version.version_string())
+    config.verify_share_protocols()
+    log.setup(CONF, "manila")
+    rpc.init(CONF)
+    return wsgi.Loader(CONF).load_app(name='osapi_share')

releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - Manila API service now can be run using web servers that support
+    WSGI applications.
+upgrade:
+  - Deprecated path 'manila.api.openstack:FaultWrapper' to 'FaultWrapper'
+    was removed and now only current path is available, which
+    is 'manila.api.middleware.fault:FaultWrapper'.

setup.cfg

@@ -34,6 +34,8 @@ console_scripts =
     manila-rootwrap = oslo_rootwrap.cmd:main
     manila-scheduler = manila.cmd.scheduler:main
     manila-share = manila.cmd.share:main
+wsgi_scripts =
+    manila-wsgi = manila.wsgi.wsgi:initialize_application
 manila.scheduler.filters =
     AvailabilityZoneFilter = manila.scheduler.filters.availability_zone:AvailabilityZoneFilter
     CapabilitiesFilter = manila.scheduler.filters.capabilities:CapabilitiesFilter