From ea8845a48f3def7a53407aadc14d45bf0609cc7d Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sun, 31 Aug 2014 11:31:58 -0500 Subject: [PATCH] Refactor keystone-all and http/keystone There was a lot of code in keystone-all and httpd/keystone. Functions in keystone-all and http/keystone can't be called from the tests, so this code should be in the keystone package. This also allows some sharing of common code between keystone-all, httpd/keystone, and the tests. bp refactor-keystone-all-http Change-Id: I1c4e59e253b1816ccfb4d5bf1d2aa40b49221b4f --- bin/keystone-all | 133 +------------------------------- httpd/keystone.py | 39 +--------- keystone/server/__init__.py | 0 keystone/server/common.py | 41 ++++++++++ keystone/server/eventlet.py | 146 ++++++++++++++++++++++++++++++++++++ keystone/server/wsgi.py | 51 +++++++++++++ keystone/tests/core.py | 9 +-- 7 files changed, 245 insertions(+), 174 deletions(-) create mode 100644 keystone/server/__init__.py create mode 100644 keystone/server/common.py create mode 100644 keystone/server/eventlet.py create mode 100644 keystone/server/wsgi.py diff --git a/bin/keystone-all b/bin/keystone-all index 56f4cafb0..3b0038207 100755 --- a/bin/keystone-all +++ b/bin/keystone-all @@ -14,15 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import os -import socket import sys -from oslo import i18n -from oslo_concurrency import processutils -import pbr.version - # If ../keystone/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -34,132 +28,9 @@ if os.path.exists(os.path.join(possible_topdir, '__init__.py')): sys.path.insert(0, possible_topdir) -# NOTE(dstanek): i18n.enable_lazy() must be called before -# keystone.i18n._() is called to ensure it has the desired lazy lookup -# behavior. This includes cases, like keystone.exceptions, where -# keystone.i18n._() is called at import time. -i18n.enable_lazy() - -from keystone import backends -from keystone.common import dependency -from keystone.common import environment -from keystone.common import sql -from keystone.common import utils -from keystone import config -from keystone.i18n import _ -from keystone.openstack.common import service -from keystone.openstack.common import systemd -from keystone import service as keystone_service - - -CONF = config.CONF - - -class ServerWrapper(object): - """Wraps a Server with some launching info & capabilities.""" - - def __init__(self, server, workers): - self.server = server - self.workers = workers - - def launch_with(self, launcher): - self.server.listen() - if self.workers > 1: - # Use multi-process launcher - launcher.launch_service(self.server, self.workers) - else: - # Use single process launcher - launcher.launch_service(self.server) - - -def create_server(conf, name, host, port, workers): - app = keystone_service.loadapp('config:%s' % conf, name) - server = environment.Server(app, host=host, port=port, - keepalive=CONF.tcp_keepalive, - keepidle=CONF.tcp_keepidle) - if CONF.ssl.enable: - server.set_ssl(CONF.ssl.certfile, CONF.ssl.keyfile, - CONF.ssl.ca_certs, CONF.ssl.cert_required) - return name, ServerWrapper(server, workers) - - -def serve(*servers): - if max([server[1].workers for server in servers]) > 1: - launcher = service.ProcessLauncher() - else: - launcher = service.ServiceLauncher() - - for name, server in servers: - try: - server.launch_with(launcher) - except socket.error: - logging.exception(_('Failed to start the %(name)s server') % { - 'name': name}) - raise - - # notify calling process we are ready to serve - systemd.notify_once() - - for name, server in servers: - launcher.wait() - - -def _get_workers(worker_type_config_opt): - # Get the value from config, if the config value is None (not set), return - # the number of cpus with a minimum of 2. - worker_count = CONF.get(worker_type_config_opt) - if not worker_count: - worker_count = max(2, processutils.get_worker_count()) - return worker_count +from keystone.server import eventlet as eventlet_server if __name__ == '__main__': - dev_conf = os.path.join(possible_topdir, - 'etc', - 'keystone.conf') - config_files = None - if os.path.exists(dev_conf): - config_files = [dev_conf] - - config.configure() - sql.initialize() - config.set_default_for_default_log_levels() - - CONF(project='keystone', - version=pbr.version.VersionInfo('keystone').version_string(), - default_config_files=config_files) - - config.setup_logging() - - paste_config = config.find_paste_config() - - monkeypatch_thread = not CONF.standard_threads - pydev_debug_url = utils.setup_remote_pydev_debug() - if pydev_debug_url: - # in order to work around errors caused by monkey patching we have to - # set the thread to False. An explanation is here: - # http://lists.openstack.org/pipermail/openstack-dev/2012-August/ - # 000794.html - monkeypatch_thread = False - environment.use_eventlet(monkeypatch_thread) - - backends.load_backends() - - admin_worker_count = _get_workers('admin_workers') - public_worker_count = _get_workers('public_workers') - - servers = [] - servers.append(create_server(paste_config, - 'admin', - CONF.admin_bind_host, - int(CONF.admin_port), - admin_worker_count)) - servers.append(create_server(paste_config, - 'main', - CONF.public_bind_host, - int(CONF.public_port), - public_worker_count)) - - dependency.resolve_future_dependencies() - serve(*servers) + eventlet_server.run(possible_topdir) diff --git a/httpd/keystone.py b/httpd/keystone.py index f5ce498c5..0c7018ff6 100644 --- a/httpd/keystone.py +++ b/httpd/keystone.py @@ -12,49 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import os -from oslo import i18n +from keystone.server import wsgi as wsgi_server -# NOTE(dstanek): i18n.enable_lazy() must be called before -# keystone.i18n._() is called to ensure it has the desired lazy lookup -# behavior. This includes cases, like keystone.exceptions, where -# keystone.i18n._() is called at import time. -i18n.enable_lazy() - - -from keystone import backends -from keystone.common import dependency -from keystone.common import environment -from keystone.common import sql -from keystone import config -from keystone.openstack.common import log -from keystone import service - - -CONF = config.CONF - -config.configure() -sql.initialize() -config.set_default_for_default_log_levels() - -CONF(project='keystone') -config.setup_logging() - -environment.use_stdlib() name = os.path.basename(__file__) -if CONF.debug: - CONF.log_opt_values(log.getLogger(CONF.prog), logging.DEBUG) - - -drivers = backends.load_backends() - # NOTE(ldbragst): 'application' is required in this context by WSGI spec. # The following is a reference to Python Paste Deploy documentation # http://pythonpaste.org/deploy/ -application = service.loadapp('config:%s' % config.find_paste_config(), name) - -dependency.resolve_future_dependencies() +application = wsgi_server.initialize_application(name) diff --git a/keystone/server/__init__.py b/keystone/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/keystone/server/common.py b/keystone/server/common.py new file mode 100644 index 000000000..b0ad43e9c --- /dev/null +++ b/keystone/server/common.py @@ -0,0 +1,41 @@ + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from keystone import backends +from keystone.common import dependency +from keystone.common import sql +from keystone import config + + +CONF = config.CONF + + +def configure(version=None, config_files=None): + config.configure() + sql.initialize() + config.set_default_for_default_log_levels() + + CONF(project='keystone', version=version, + default_config_files=config_files) + + config.setup_logging() + + +def setup_backends(load_extra_backends_fn=lambda: {}, + startup_application_fn=lambda: None): + drivers = backends.load_backends() + drivers.update(load_extra_backends_fn()) + res = startup_application_fn() + drivers.update(dependency.resolve_future_dependencies()) + return drivers, res diff --git a/keystone/server/eventlet.py b/keystone/server/eventlet.py new file mode 100644 index 000000000..f29d6e34c --- /dev/null +++ b/keystone/server/eventlet.py @@ -0,0 +1,146 @@ + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import socket + +from oslo import i18n +from oslo_concurrency import processutils +import pbr.version + + +# NOTE(dstanek): i18n.enable_lazy() must be called before +# keystone.i18n._() is called to ensure it has the desired lazy lookup +# behavior. This includes cases, like keystone.exceptions, where +# keystone.i18n._() is called at import time. +i18n.enable_lazy() + + +from keystone.common import environment +from keystone.common import utils +from keystone import config +from keystone.i18n import _ +from keystone.openstack.common import service +from keystone.openstack.common import systemd +from keystone.server import common +from keystone import service as keystone_service + + +CONF = config.CONF + + +class ServerWrapper(object): + """Wraps a Server with some launching info & capabilities.""" + + def __init__(self, server, workers): + self.server = server + self.workers = workers + + def launch_with(self, launcher): + self.server.listen() + if self.workers > 1: + # Use multi-process launcher + launcher.launch_service(self.server, self.workers) + else: + # Use single process launcher + launcher.launch_service(self.server) + + +def create_server(conf, name, host, port, workers): + app = keystone_service.loadapp('config:%s' % conf, name) + server = environment.Server(app, host=host, port=port, + keepalive=CONF.tcp_keepalive, + keepidle=CONF.tcp_keepidle) + if CONF.ssl.enable: + server.set_ssl(CONF.ssl.certfile, CONF.ssl.keyfile, + CONF.ssl.ca_certs, CONF.ssl.cert_required) + return name, ServerWrapper(server, workers) + + +def serve(*servers): + if max([server[1].workers for server in servers]) > 1: + launcher = service.ProcessLauncher() + else: + launcher = service.ServiceLauncher() + + for name, server in servers: + try: + server.launch_with(launcher) + except socket.error: + logging.exception(_('Failed to start the %(name)s server') % { + 'name': name}) + raise + + # notify calling process we are ready to serve + systemd.notify_once() + + for name, server in servers: + launcher.wait() + + +def _get_workers(worker_type_config_opt): + # Get the value from config, if the config value is None (not set), return + # the number of cpus with a minimum of 2. + worker_count = CONF.get(worker_type_config_opt) + if not worker_count: + worker_count = max(2, processutils.get_worker_count()) + return worker_count + + +def run(possible_topdir): + dev_conf = os.path.join(possible_topdir, + 'etc', + 'keystone.conf') + config_files = None + if os.path.exists(dev_conf): + config_files = [dev_conf] + + common.configure( + version=pbr.version.VersionInfo('keystone').version_string(), + config_files=config_files) + + paste_config = config.find_paste_config() + + monkeypatch_thread = not CONF.standard_threads + pydev_debug_url = utils.setup_remote_pydev_debug() + if pydev_debug_url: + # in order to work around errors caused by monkey patching we have to + # set the thread to False. An explanation is here: + # http://lists.openstack.org/pipermail/openstack-dev/2012-August/ + # 000794.html + monkeypatch_thread = False + environment.use_eventlet(monkeypatch_thread) + + def create_servers(): + admin_worker_count = _get_workers('admin_workers') + public_worker_count = _get_workers('public_workers') + + servers = [] + servers.append(create_server(paste_config, + 'admin', + CONF.admin_bind_host, + int(CONF.admin_port), + admin_worker_count)) + servers.append(create_server(paste_config, + 'main', + CONF.public_bind_host, + int(CONF.public_port), + public_worker_count)) + return servers + + _unused, servers = common.setup_backends( + startup_application_fn=create_servers) + serve(*servers) diff --git a/keystone/server/wsgi.py b/keystone/server/wsgi.py new file mode 100644 index 000000000..2d15c08e1 --- /dev/null +++ b/keystone/server/wsgi.py @@ -0,0 +1,51 @@ +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from oslo import i18n + + +# NOTE(dstanek): i18n.enable_lazy() must be called before +# keystone.i18n._() is called to ensure it has the desired lazy lookup +# behavior. This includes cases, like keystone.exceptions, where +# keystone.i18n._() is called at import time. +i18n.enable_lazy() + + +from keystone.common import environment +from keystone import config +from keystone.server import common +from keystone import service as keystone_service + + +CONF = config.CONF + + +def initialize_application(name): + common.configure() + + # Log the options used when starting if we're in debug mode... + if CONF.debug: + CONF.log_opt_values(logging.getLogger(CONF.prog), logging.DEBUG) + + environment.use_stdlib() + + def loadapp(): + return keystone_service.loadapp( + 'config:%s' % config.find_paste_config(), name) + + _unused, application = common.setup_backends( + startup_application_fn=loadapp) + return application diff --git a/keystone/tests/core.py b/keystone/tests/core.py index e36a79cca..0febccea7 100644 --- a/keystone/tests/core.py +++ b/keystone/tests/core.py @@ -41,7 +41,6 @@ from keystone.common import environment # noqa environment.use_eventlet() from keystone import auth -from keystone import backends from keystone.common import config as common_cfg from keystone.common import dependency from keystone.common import kvs @@ -52,6 +51,7 @@ from keystone import exception from keystone.i18n import _LW from keystone import notifications from keystone.openstack.common import log +from keystone.server import common from keystone import service from keystone.tests import ksfixtures from keystone.tests import utils @@ -517,11 +517,8 @@ class TestCase(BaseTestCase): kvs_core.KEY_VALUE_STORE_REGISTRY.clear() self.clear_auth_plugin_registry() - drivers = backends.load_backends() - - drivers.update(self.load_extra_backends()) - - drivers.update(dependency.resolve_future_dependencies()) + drivers, _unused = common.setup_backends( + load_extra_backends_fn=self.load_extra_backends) for manager_name, manager in six.iteritems(drivers): setattr(self, manager_name, manager)