openstack
/
panko
Code Issues Proposed changes

Stop relying on global CONF object 

The conf object is now prepared on demand by panko.service.prepare_service.

Change-Id: I3526e73f8acbcb089d42850c0762305a08726db7
This commit is contained in:
Julien Danjou 2016-06-23 11:20:15 +02:00
parent ab59ea4406
commit 95abeb8b06
23 changed files with 141 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
panko/api/__init__.py
View File

@ -28,9 +28,3 @@ OPTS = [
help='The listen IP for the Panko API server.',
),
]
CONF = cfg.CONF
opt_group = cfg.OptGroup(name='api',
title='Options for the panko-api service')
CONF.register_group(opt_group)
CONF.register_opts(OPTS, opt_group)

60
panko/api/app.py
View File

@ -14,6 +14,7 @@
# under the License.
import os
import uuid
from oslo_config import cfg
from oslo_log import log
@ -27,7 +28,6 @@ from panko.i18n import _LI, _LW
LOG = log.getLogger(__name__)
CONF = cfg.CONF
OPTS = [
cfg.StrOpt('api_paste_config',
@ -54,14 +54,13 @@ API_OPTS = [
help='Number of workers for api, default value is 1.'),
]
CONF.register_opts(OPTS)
CONF.register_opts(API_OPTS, group='api')
def setup_app(pecan_config=None):
def setup_app(pecan_config=None, conf=None):
if conf is None:
raise RuntimeError("No configuration passed")
# FIXME: Replace DBHook with a hooks.TransactionHook
app_hooks = [hooks.ConfigHook(),
hooks.DBHook(),
app_hooks = [hooks.ConfigHook(conf),
hooks.DBHook(conf),
hooks.TranslationHook()]
pecan_config = pecan_config or {
@ -74,8 +73,8 @@ def setup_app(pecan_config=None):
pecan.configuration.set_config(dict(pecan_config), overwrite=True)
# NOTE(sileht): pecan debug won't work in multi-process environment
pecan_debug = CONF.api.pecan_debug
if CONF.api.workers and CONF.api.workers != 1 and pecan_debug:
pecan_debug = conf.api.pecan_debug
if conf.api.workers and conf.api.workers != 1 and pecan_debug:
pecan_debug = False
LOG.warning(_LW('pecan_debug cannot be enabled, if workers is > 1, '
'the value is overrided with False'))
@ -91,29 +90,46 @@ def setup_app(pecan_config=None):
return app
def load_app():
# NOTE(sileht): pastedeploy uses ConfigParser to handle
# global_conf, since python 3 ConfigParser doesn't
# allow to store object as config value, only strings are
# permit, so to be able to pass an object created before paste load
# the app, we store them into a global var. But the each loaded app
# store it's configuration in unique key to be concurrency safe.
global APPCONFIGS
APPCONFIGS = {}
def load_app(conf):
global APPCONFIGS
# Build the WSGI app
cfg_file = None
cfg_path = cfg.CONF.api_paste_config
cfg_path = conf.api_paste_config
if not os.path.isabs(cfg_path):
cfg_file = CONF.find_file(cfg_path)
cfg_file = conf.find_file(cfg_path)
elif os.path.exists(cfg_path):
cfg_file = cfg_path
if not cfg_file:
raise cfg.ConfigFilesNotFoundError([cfg.CONF.api_paste_config])
raise cfg.ConfigFilesNotFoundError([conf.api_paste_config])
configkey = str(uuid.uuid4())
APPCONFIGS[configkey] = conf
LOG.info("Full WSGI config used: %s" % cfg_file)
return deploy.loadapp("config:" + cfg_file)
return deploy.loadapp("config:" + cfg_file,
global_conf={'configkey': configkey})
def build_server():
app = load_app()
def build_server(conf):
app = load_app(conf)
# Create the WSGI server and start it
host, port = cfg.CONF.api.host, cfg.CONF.api.port
host, port = conf.api.host, conf.api.port
LOG.info(_LI('Starting server in PID %s') % os.getpid())
LOG.info(_LI("Configuration:"))
cfg.CONF.log_opt_values(LOG, log.INFO)
conf.log_opt_values(LOG, log.INFO)
if host == '0.0.0.0':
LOG.info(_LI(
@ -123,9 +139,11 @@ def build_server():
LOG.info(_LI("serving on http://%(host)s:%(port)s") % (
{'host': host, 'port': port}))
serving.run_simple(cfg.CONF.api.host, cfg.CONF.api.port,
app, processes=CONF.api.workers)
serving.run_simple(conf.api.host, conf.api.port,
app, processes=conf.api.workers)
def app_factory(global_config, **local_conf):
return setup_app()
global APPCONFIGS
conf = APPCONFIGS.get(global_config.get('configkey'))
return setup_app(conf=conf)

4
panko/api/app.wsgi
View File

@ -21,5 +21,5 @@ from panko import service
from panko.api import app
# Initialize the oslo configuration library and logging
service.prepare_service([])
application = app.load_app()
conf = service.prepare_service([])
application = app.load_app(conf)

9
panko/api/controllers/v2/events.py
View File

@ -33,7 +33,7 @@ from panko.api.controllers.v2 import utils as v2_utils
from panko.api import rbac
from panko.event import storage
from panko.event.storage import models as event_models
from panko.i18n import _
from panko.i18n import _, _LI
LOG = log.getLogger(__name__)
@ -272,7 +272,12 @@ class EventsController(rest.RestController):
"""
rbac.enforce("events:index", pecan.request)
q = q or []
limit = v2_utils.enforce_limit(limit)
if limit is None:
limit = pecan.request.cfg.api.default_api_return_limit
LOG.info(_LI('No limit value provided, result set will be'
' limited to %(limit)d.'), {'limit': limit})
if not limit or limit <= 0:
raise base.ClientSideError(_("Limit must be positive"))
event_filter = _event_query_to_event_filter(q)
return [Event(message_id=event.message_id,
event_type=event.event_type,

15
panko/api/controllers/v2/utils.py
View File

@ -20,28 +20,13 @@
import functools
from oslo_config import cfg
from oslo_log import log
import pecan
from panko.api.controllers.v2 import base
from panko.api import rbac
from panko.i18n import _, _LI
LOG = log.getLogger(__name__)
cfg.CONF.import_opt('default_api_return_limit', 'panko.api.app',
group='api')
def enforce_limit(limit):
"""Ensure limit is defined and is valid. if not, set a default."""
if limit is None:
limit = cfg.CONF.api.default_api_return_limit
LOG.info(_LI('No limit value provided, result set will be'
' limited to %(limit)d.'), {'limit': limit})
if not limit or limit <= 0:
raise base.ClientSideError(_("Limit must be positive"))
return limit
def get_auth_project(on_behalf_of=None):

15
panko/api/hooks.py
View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from pecan import hooks
from panko import storage
@ -26,16 +24,19 @@ class ConfigHook(hooks.PecanHook):
That allows controllers to get it.
"""
@staticmethod
def before(state):
state.request.cfg = cfg.CONF
def __init__(self, conf):
super(ConfigHook, self).__init__()
self.conf = conf
def before(self, state):
state.request.cfg = self.conf
class DBHook(hooks.PecanHook):
def __init__(self):
def __init__(self, conf):
self.event_storage_connection = storage.get_connection_from_config(
cfg.CONF)
conf)
def before(self, state):
state.request.event_storage_conn = self.event_storage_connection

7
panko/api/rbac.py
View File

@ -16,14 +16,11 @@
"""Access Control Lists (ACL's) control access the API server."""
from oslo_config import cfg
from oslo_policy import policy
import pecan
_ENFORCER = None
CONF = cfg.CONF
def reset():
global _ENFORCER
@ -46,7 +43,7 @@ def enforce(policy_name, request):
"""
global _ENFORCER
if not _ENFORCER:
_ENFORCER = policy.Enforcer(CONF)
_ENFORCER = policy.Enforcer(pecan.request.cfg)
_ENFORCER.load_rules()
rule_method = "telemetry:" + policy_name
@ -77,7 +74,7 @@ def get_limited_to(headers):
"""
global _ENFORCER
if not _ENFORCER:
_ENFORCER = policy.Enforcer(CONF)
_ENFORCER = policy.Enforcer(pecan.request.cfg)
_ENFORCER.load_rules()
policy_dict = dict()

3
panko/cmd/api.py
View File

@ -19,5 +19,4 @@ from panko import service
def main():
service.prepare_service()
app.build_server()
app.build_server(service.prepare_service())

13
panko/cmd/storage.py
View File

@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log
from panko.i18n import _LI
@ -26,18 +25,18 @@ LOG = log.getLogger(__name__)
def dbsync():
service.prepare_service()
storage.get_connection_from_config(cfg.CONF).upgrade()
conf = service.prepare_service()
storage.get_connection_from_config(conf).upgrade()
def expirer():
service.prepare_service()
conf = service.prepare_service()
if cfg.CONF.database.event_time_to_live > 0:
if conf.database.event_time_to_live > 0:
LOG.debug("Clearing expired event data")
event_conn = storage.get_connection_from_config(cfg.CONF)
event_conn = storage.get_connection_from_config(conf)
event_conn.clear_expired_event_data(
cfg.CONF.database.event_time_to_live)
conf.database.event_time_to_live)
else:
LOG.info(_LI("Nothing to clean, database event time to live "
"is disabled"))

1
panko/dispatcher/__init__.py
View File

@ -27,4 +27,3 @@ STORAGE_OPTS = [
deprecated_group='database',
help='Interval (in seconds) between retries of connection.'),
]
cfg.CONF.register_opts(STORAGE_OPTS, group='storage')

18
panko/service.py
View File

@ -15,28 +15,36 @@
import sys
from oslo_config import cfg
from oslo_db import options as db_options
import oslo_i18n
from oslo_log import log
from oslo_reports import guru_meditation_report as gmr
from panko.conf import defaults
from panko import opts
from panko import version
def prepare_service(argv=None, config_files=None):
conf = cfg.ConfigOpts()
oslo_i18n.enable_lazy()
log.register_options(cfg.CONF)
for group, options in opts.list_opts():
conf.register_opts(list(options),
group=None if group == "DEFAULT" else group)
defaults.set_cors_middleware_defaults()
db_options.set_defaults(conf)
log.register_options(conf)
if argv is None:
argv = sys.argv
cfg.CONF(argv[1:], project='panko', validate_default_values=True,
version=version.version_info.version_string(),
default_config_files=config_files)
conf(argv[1:], project='panko', validate_default_values=True,
version=version.version_info.version_string(),
default_config_files=config_files)
log.setup(cfg.CONF, 'panko')
log.setup(conf, 'panko')
# NOTE(liusheng): guru cannot run with service under apache daemon, so when
# panko-api running with mod_wsgi, the argv is [], we don't start
# guru.
if argv:
gmr.TextGuruMeditation.setup_autorun(version)
return conf

5
panko/storage/__init__.py
View File

@ -16,7 +16,6 @@
"""
from oslo_config import cfg
from oslo_db import options as db_options
from oslo_log import log
import retrying
import six.moves.urllib.parse as urlparse
@ -37,10 +36,6 @@ OPTS = [
'database. (if unset, connection is used)'),
]
cfg.CONF.register_opts(OPTS, group='database')
db_options.set_defaults(cfg.CONF)
class StorageUnknownWriteError(Exception):
"""Error raised when an unknown error occurs while recording."""

5
panko/tests/db.py
View File

@ -21,13 +21,13 @@ import warnings
import fixtures
import mock
from oslo_config import fixture as fixture_config
from oslotest import mockpatch
import six
from six.moves.urllib import parse as urlparse
import sqlalchemy
from testtools import testcase
from panko import service
from panko import storage
from panko.tests import base as test_base
try:
@ -189,8 +189,7 @@ class TestBase(test_base.BaseTestCase):
raise testcase.TestSkipped(
'Test is not applicable for %s' % engine)
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF([], project='panko', validate_default_values=True)
self.CONF = service.prepare_service([], [])
manager = self.DRIVER_MANAGERS.get(engine)
if not manager:

13
panko/tests/functional/api/__init__.py
View File

@ -15,17 +15,14 @@
"""Base classes for API tests.
"""
from oslo_config import cfg
from oslo_config import fixture as fixture_config
from oslo_policy import opts
import pecan
import pecan.testing
from panko.api import rbac
from panko import service
from panko.tests import db as db_test_base
cfg.CONF.import_group('api', 'panko.api.controllers.v2.root')
class FunctionalTest(db_test_base.TestBase):
"""Used for functional tests of Pecan controllers.
@ -38,16 +35,16 @@ class FunctionalTest(db_test_base.TestBase):
def setUp(self):
super(FunctionalTest, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF = service.prepare_service([], [])
opts.set_defaults(self.CONF)
self.CONF.set_override("policy_file",
self.path_get('etc/panko/policy.json'),
group='oslo_policy')
self.app = self._make_app()
self.app = self._make_app(self.CONF)
def _make_app(self, enable_acl=False):
def _make_app(self, conf, enable_acl=False):
self.config = {
'app': {
'root': 'panko.api.controllers.root.RootController',
@ -59,7 +56,7 @@ class FunctionalTest(db_test_base.TestBase):
},
}
return pecan.testing.load_test_app(self.config)
return pecan.testing.load_test_app(self.config, conf=conf)
def tearDown(self):
super(FunctionalTest, self).tearDown()

10
panko/tests/functional/api/v2/test_acl_scenarios.py
View File

@ -63,10 +63,10 @@ class TestAPIACL(v2.FunctionalTest):
q=q or [],
**params)
def _make_app(self):
def _make_app(self, conf):
file_name = self.path_get('etc/panko/api_paste.ini')
self.CONF.set_override("api_paste_config", file_name)
return webtest.TestApp(app.load_app())
conf.set_override("api_paste_config", file_name)
return webtest.TestApp(app.load_app(conf=conf))
class TestAPIEventACL(TestAPIACL):
@ -128,7 +128,7 @@ class TestBaseApiEventRBAC(v2.FunctionalTest):
class TestApiEventAdminRBAC(TestBaseApiEventRBAC):
def _make_app(self, enable_acl=False):
def _make_app(self, conf, enable_acl=False):
content = ('{"context_is_admin": "role:admin",'
'"telemetry:events:index": "rule:context_is_admin",'
'"telemetry:events:show": "rule:context_is_admin"}')
@ -140,7 +140,7 @@ class TestApiEventAdminRBAC(TestBaseApiEventRBAC):
self.CONF.set_override("policy_file", self.tempfile,
group='oslo_policy')
return super(TestApiEventAdminRBAC, self)._make_app()
return super(TestApiEventAdminRBAC, self)._make_app(conf)
def tearDown(self):
os.remove(self.tempfile)

40
panko/tests/functional/gabbi/fixtures.py
View File

@ -22,15 +22,28 @@ import uuid
from gabbi import fixture
from oslo_config import cfg
from oslo_config import fixture as fixture_config
from oslo_policy import opts
from oslo_utils import fileutils
import six
from six.moves.urllib import parse as urlparse
from panko.api import app
from panko.event.storage import models
from panko import service
from panko import storage
# NOTE(chdent): Hack to restore semblance of global configuration to
# pass to the WSGI app used per test suite. LOAD_APP_KWARGS are the olso
# configuration, and the pecan application configuration of
# which the critical part is a reference to the current indexer.
LOAD_APP_KWARGS = None
def setup_app():
global LOAD_APP_KWARGS
return app.load_app(**LOAD_APP_KWARGS)
# TODO(chdent): For now only MongoDB is supported, because of easy
# database name handling and intentional focus on the API, not the
# data store.
@ -43,6 +56,8 @@ class ConfigFixture(fixture.GabbiFixture):
def start_fixture(self):
"""Set up config."""
global LOAD_APP_KWARGS
self.conf = None
# Determine the database connection.
@ -55,11 +70,8 @@ class ConfigFixture(fixture.GabbiFixture):
if engine not in ENGINES:
raise case.SkipTest('Database engine not supported')
conf = fixture_config.Config().conf
self.conf = conf
self.conf([], project='panko', validate_default_values=True)
conf = self.conf = service.prepare_service([], [])
opts.set_defaults(self.conf)
conf.import_group('api', 'panko.api.controllers.v2.root')
content = ('{"default": ""}')
if six.PY3:
@ -80,22 +92,26 @@ class ConfigFixture(fixture.GabbiFixture):
conf.set_override('connection', database_name, group='database')
conf.set_override('event_connection', '', group='database')
conf.set_override('pecan_debug', True, group='api')
LOAD_APP_KWARGS = {
'conf': conf,
}
def stop_fixture(self):
"""Reset the config and remove data."""
if self.conf:
storage.get_connection_from_config(self.conf).clear()
self.conf.reset()
class EventDataFixture(fixture.GabbiFixture):
class EventDataFixture(ConfigFixture):
"""Instantiate some sample event data for use in testing."""
def start_fixture(self):
"""Create some events."""
conf = fixture_config.Config().conf
self.conn = storage.get_connection_from_config(conf)
super(EventDataFixture, self).start_fixture()
self.conn = None
if not self.conf:
return
self.conn = storage.get_connection_from_config(self.conf)
events = []
name_list = ['chocolate.chip', 'peanut.butter', 'sugar']
for ix, name in enumerate(name_list):
@ -112,7 +128,9 @@ class EventDataFixture(fixture.GabbiFixture):
def stop_fixture(self):
"""Destroy the events."""
self.conn.db.event.remove({'event_type': '/^cookies_/'})
if self.conn:
self.conn.db.event.remove({'event_type': '/^cookies_/'})
super(EventDataFixture, self).stop_fixture()
class CORSConfigFixture(fixture.GabbiFixture):

8
panko/tests/functional/gabbi/test_gabbi.py
View File

@ -22,8 +22,8 @@ import os
from gabbi import driver
from panko.api import app
from panko.tests.functional.gabbi import fixtures as fixture_module
from panko.tests.functional.gabbi import fixtures
TESTS_DIR = 'gabbits'
@ -32,5 +32,5 @@ def load_tests(loader, tests, pattern):
"""Provide a TestSuite to the discovery process."""
test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
return driver.build_tests(test_dir, loader, host=None,
intercept=app.load_app,
fixture_module=fixture_module)
intercept=fixtures.setup_app,
fixture_module=fixtures)

7
panko/tests/functional/gabbi/test_gabbi_prefix.py
View File

@ -19,8 +19,7 @@ import os
from gabbi import driver
from panko.api import app
from panko.tests.functional.gabbi import fixtures as fixture_module
from panko.tests.functional.gabbi import fixtures
TESTS_DIR = 'gabbits_prefix'
@ -30,5 +29,5 @@ def load_tests(loader, tests, pattern):
test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
return driver.build_tests(test_dir, loader, host=None,
prefix='/telemetry',
intercept=app.setup_app,
fixture_module=fixture_module)
intercept=fixtures.setup_app,
fixture_module=fixtures)

11
panko/tests/unit/api/test_app.py
View File

@ -15,10 +15,9 @@
import mock
from oslo_config import cfg
from oslo_config import fixture as fixture_config
from oslo_log import log
from panko.api import app
from panko import service
from panko.tests import base
@ -26,14 +25,14 @@ class TestApp(base.BaseTestCase):
def setUp(self):
super(TestApp, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
log.register_options(cfg.CONF)
self.CONF = service.prepare_service([], [])
def test_api_paste_file_not_exist(self):
self.CONF.set_override('api_paste_config', 'non-existent-file')
with mock.patch.object(self.CONF, 'find_file') as ff:
ff.return_value = None
self.assertRaises(cfg.ConfigFilesNotFoundError, app.load_app)
self.assertRaises(cfg.ConfigFilesNotFoundError, app.load_app,
self.CONF)
@mock.patch('panko.storage.get_connection_from_config',
mock.MagicMock())
@ -44,7 +43,7 @@ class TestApp(base.BaseTestCase):
if p_debug is not None:
self.CONF.set_override('pecan_debug', p_debug, group='api')
self.CONF.set_override('workers', workers, group='api')
app.setup_app()
app.setup_app(conf=self.CONF)
args, kwargs = mocked.call_args
self.assertEqual(expected, kwargs.get('debug'))

4
panko/tests/unit/dispatcher/test_db.py
View File

@ -16,18 +16,18 @@ import datetime
import uuid
import mock
from oslo_config import fixture as fixture_config
from oslotest import base
from panko.dispatcher import database
from panko.event.storage import models as event_models
from panko import service
class TestDispatcherDB(base.BaseTestCase):
def setUp(self):
super(TestDispatcherDB, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF = service.prepare_service([], [])
self.CONF.set_override('connection', 'sqlite://', group='database')
self.dispatcher = database.DatabaseDispatcher(self.CONF)
self.ctx = None

6
panko/tests/unit/storage/test_get_connection.py
View File

@ -16,12 +16,12 @@
"""
import mock
from oslo_config import fixture as fixture_config
from oslotest import base
import retrying
from panko.event.storage import impl_log
from panko.event.storage import impl_sqlalchemy
from panko import service
from panko import storage
import six
@ -42,7 +42,7 @@ class EngineTest(base.BaseTestCase):
class ConnectionRetryTest(base.BaseTestCase):
def setUp(self):
super(ConnectionRetryTest, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF = service.prepare_service([], config_files=[])
def test_retries(self):
with mock.patch.object(
@ -61,7 +61,7 @@ class ConnectionRetryTest(base.BaseTestCase):
class ConnectionConfigTest(base.BaseTestCase):
def setUp(self):
super(ConnectionConfigTest, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF = service.prepare_service([], config_files=[])
def test_only_default_url(self):
self.CONF.set_override("connection", "log://", group="database")

6
tools/make_test_event_data.py
View File

@ -26,10 +26,10 @@ import datetime
import random
import uuid
from oslo_config import cfg
from oslo_utils import timeutils
from panko.event.storage import models
from panko import service
from panko import storage
@ -67,7 +67,7 @@ def make_test_data(conn, start, end, interval, event_types):
def main():
cfg.CONF([], project='panko')
conf = service.prepare_service()
parser = argparse.ArgumentParser(
description='generate event data',
@ -99,7 +99,7 @@ def main():
args = parser.parse_args()
# Connect to the event database
conn = storage.get_connection_from_config(cfg.CONF)
conn = storage.get_connection_from_config(conf)
# Compute the correct time span
start = datetime.datetime.utcnow() - datetime.timedelta(days=args.start)

6
tools/test_hbase_table_utils.py
View File

@ -15,18 +15,16 @@
import os
import sys
from oslo_config import cfg
from panko import service
from panko import storage
def main(argv):
cfg.CONF([], project='panko')
if os.getenv("PANKO_TEST_STORAGE_URL", "").startswith("hbase://"):
url = ("%s?table_prefix=%s" %
(os.getenv("PANKO_TEST_STORAGE_URL"),
os.getenv("PANKO_TEST_HBASE_TABLE_PREFIX", "test")))
event_conn = storage.get_connection(url, None)
event_conn = storage.get_connection(url, service.prepare_service())
for arg in argv:
if arg == "--upgrade":
event_conn.upgrade()