From 039e899d8bdc727cea86c7cb20c4123409cb2a7d Mon Sep 17 00:00:00 2001 From: Renat Akhmerov Date: Wed, 27 Nov 2013 16:55:08 +0700 Subject: [PATCH] Adding REST API application skeleton based on pecan/wsme * Adding required dependencies to use and test pecan/wsme app * Adding additional modules from oslo-incubator * Adding configuration files mistral.conf and specific file loggin.conf for logging * Adding dependencies for testing: fixtures, testtools, mock * Updating tox.ini to run nosetests for py26 and py27 environments Change-Id: I4fd63820aaaf3b50fb1c981031f60faa68a6d307 --- etc/logging.conf | 32 +++ mistral/api/app.py | 37 +++ mistral/api/config.py | 28 ++ mistral/api/controllers/common_types.py | 22 ++ mistral/api/controllers/root.py | 45 +++ mistral/api/controllers/v1/__init__.py | 0 mistral/api/controllers/v1/listener.py | 119 ++++++++ mistral/api/controllers/v1/root.py | 40 +++ mistral/api/controllers/v1/workbook.py | 110 +++++++ mistral/cmd/api.py | 55 ++-- mistral/common/config.py | 220 ++++++++++++++ mistral/config.py | 61 ++++ .../openstack/common/apiclient/__init__.py | 2 - mistral/openstack/common/apiclient/auth.py | 2 - mistral/openstack/common/apiclient/base.py | 2 - mistral/openstack/common/apiclient/client.py | 2 - .../openstack/common/apiclient/exceptions.py | 2 - .../openstack/common/apiclient/fake_client.py | 6 +- mistral/openstack/common/cliutils.py | 5 +- mistral/openstack/common/config/__init__.py | 0 mistral/openstack/common/config/generator.py | 268 ++++++++++++++++++ mistral/openstack/common/db/__init__.py | 2 - mistral/openstack/common/db/api.py | 2 - mistral/openstack/common/db/exception.py | 7 +- .../common/db/sqlalchemy/__init__.py | 2 - .../openstack/common/db/sqlalchemy/models.py | 2 - .../common/db/sqlalchemy/provision.py | 187 ++++++++++++ .../openstack/common/db/sqlalchemy/session.py | 2 - .../common/db/sqlalchemy/test_migrations.py | 36 ++- .../openstack/common/db/sqlalchemy/utils.py | 2 - mistral/openstack/common/excutils.py | 2 - mistral/openstack/common/fileutils.py | 2 - mistral/openstack/common/gettextutils.py | 2 - mistral/openstack/common/importutils.py | 2 - mistral/openstack/common/jsonutils.py | 2 - mistral/openstack/common/local.py | 2 - mistral/openstack/common/lockutils.py | 2 - mistral/openstack/common/log.py | 4 +- mistral/openstack/common/processutils.py | 248 ++++++++++++++++ .../openstack/common/py3kcompat/__init__.py | 16 ++ .../openstack/common/py3kcompat/urlutils.py | 63 ++++ mistral/openstack/common/strutils.py | 2 - mistral/openstack/common/test.py | 2 - mistral/openstack/common/timeutils.py | 11 +- mistral/tests/api/__init__.py | 0 mistral/tests/api/base.py | 43 +++ mistral/tests/api/test_app.py | 27 ++ mistral/tests/api/v1/__init__.py | 0 mistral/tests/api/v1/controllers/__init__.py | 0 .../api/v1/controllers/test_workbooks.py | 34 +++ mistral/tests/api/v1/test_root.py | 33 +++ mistral/tests/base.py | 30 ++ mistral/version.py | 1 + openstack-common.conf | 4 + requirements.txt | 1 + test-requirements.txt | 6 + tools/config/generate_sample.sh | 99 +++++++ tox.ini | 1 + 58 files changed, 1855 insertions(+), 84 deletions(-) create mode 100644 etc/logging.conf create mode 100644 mistral/api/app.py create mode 100644 mistral/api/config.py create mode 100644 mistral/api/controllers/common_types.py create mode 100644 mistral/api/controllers/root.py create mode 100644 mistral/api/controllers/v1/__init__.py create mode 100644 mistral/api/controllers/v1/listener.py create mode 100644 mistral/api/controllers/v1/root.py create mode 100644 mistral/api/controllers/v1/workbook.py create mode 100644 mistral/common/config.py create mode 100644 mistral/config.py create mode 100644 mistral/openstack/common/config/__init__.py create mode 100644 mistral/openstack/common/config/generator.py create mode 100644 mistral/openstack/common/db/sqlalchemy/provision.py create mode 100644 mistral/openstack/common/processutils.py create mode 100644 mistral/openstack/common/py3kcompat/__init__.py create mode 100644 mistral/openstack/common/py3kcompat/urlutils.py create mode 100644 mistral/tests/api/__init__.py create mode 100644 mistral/tests/api/base.py create mode 100644 mistral/tests/api/test_app.py create mode 100644 mistral/tests/api/v1/__init__.py create mode 100644 mistral/tests/api/v1/controllers/__init__.py create mode 100644 mistral/tests/api/v1/controllers/test_workbooks.py create mode 100644 mistral/tests/api/v1/test_root.py create mode 100644 mistral/tests/base.py create mode 100755 tools/config/generate_sample.sh diff --git a/etc/logging.conf b/etc/logging.conf new file mode 100644 index 000000000..dcedf9f75 --- /dev/null +++ b/etc/logging.conf @@ -0,0 +1,32 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler + +[formatters] +keys=verboseFormatter, simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler, fileHandler + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=simpleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=FileHandler +level=INFO +formatter=verboseFormatter +args=("/tmp/mistral.log",) + +[formatter_verboseFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= + +[formatter_simpleFormatter] +format=%(asctime)s - %(levelname)s - %(message)s +datefmt= diff --git a/mistral/api/app.py b/mistral/api/app.py new file mode 100644 index 000000000..1693b2b59 --- /dev/null +++ b/mistral/api/app.py @@ -0,0 +1,37 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan + +from mistral.api import config as api_config + + +def get_pecan_config(): + # Set up the pecan configuration + filename = api_config.__file__.replace('.pyc', '.py') + + return pecan.configuration.conf_from_file(filename) + + +def setup_app(config=None): + if not config: + config = get_pecan_config() + + app_conf = dict(config.app) + + return pecan.make_app( + app_conf.pop('root'), + logging=getattr(config, 'logging', {}), + **app_conf + ) diff --git a/mistral/api/config.py b/mistral/api/config.py new file mode 100644 index 000000000..723f10160 --- /dev/null +++ b/mistral/api/config.py @@ -0,0 +1,28 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pecan Application Configurations + +app = { + 'root': 'mistral.api.controllers.root.RootController', + 'modules': ['mistral.api'], + 'debug': True, +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/mistral/api/controllers/common_types.py b/mistral/api/controllers/common_types.py new file mode 100644 index 000000000..1bdd8b9c5 --- /dev/null +++ b/mistral/api/controllers/common_types.py @@ -0,0 +1,22 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from wsme import types as wtypes + + +class Link(wtypes.Base): + """Web link.""" + + href = wtypes.text + target = wtypes.text diff --git a/mistral/api/controllers/root.py b/mistral/api/controllers/root.py new file mode 100644 index 000000000..7927e77fb --- /dev/null +++ b/mistral/api/controllers/root.py @@ -0,0 +1,45 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import common_types +from mistral.api.controllers.v1 import root as v1_root + + +API_STATUS = wtypes.Enum(str, 'SUPPORTED', 'CURRENT', 'DEPRECATED') + + +class APIVersion(wtypes.Base): + """API Version.""" + + id = wtypes.text + status = API_STATUS + link = common_types.Link + + +class RootController(object): + + v1 = v1_root.Controller() + + @wsme_pecan.wsexpose([APIVersion]) + def index(self): + host_url = '%s/%s' % (pecan.request.host_url, 'v1') + api_v1 = APIVersion(id='v1.0', + status='CURRENT', + link=common_types.Link(href=host_url, target='v1')) + + return [api_v1] diff --git a/mistral/api/controllers/v1/__init__.py b/mistral/api/controllers/v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/api/controllers/v1/listener.py b/mistral/api/controllers/v1/listener.py new file mode 100644 index 000000000..a79a523a8 --- /dev/null +++ b/mistral/api/controllers/v1/listener.py @@ -0,0 +1,119 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class Event(wtypes.Base): + """Event descriptor.""" + pass + + +class TaskEvent(Event): + type = "TASK_STATE" + task = wtypes.text + + +class ExecutionEvent(Event): + type = "EXECUTION_STATE" + workbook_name = wtypes.text + + +class Listener(wtypes.Base): + """Workbook resource.""" + + id = wtypes.text + description = wtypes.text + workbook_name = wtypes.text + webhook = wtypes.text + events = [Event] + + +class Listeners(wtypes.Base): + """A collection of Listeners.""" + + listeners = [Listener] + + def __str__(self): + return "Listeners [listeners=%s]" % self.listeners + + +class ListenersController(rest.RestController): + """Operations on collection of listeners.""" + + @wsme_pecan.wsexpose(Listener, wtypes.text, wtypes.text) + def get(self, workbook_name, id): + LOG.debug("Fetch listener [workbook_name=%s, id=%s]" % + (workbook_name, id)) + + # TODO: fetch the listener from DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Listener, wtypes.text, wtypes.text, body=Listener) + def put(self, workbook_name, id, listener): + LOG.debug("Update listener [workbook_name=%s, id=%s, listener=%s]" % + (workbook_name, id, listener)) + + # TODO: modify the listener in DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(None, wtypes.text, wtypes.text, status_code=204) + def delete(self, workbook_name, id): + LOG.debug("Delete listener [workbook_name=%s, id=%s]" % + (workbook_name, id)) + + # TODO: delete the listener from DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Listener, wtypes.text, body=Listener, status_code=201) + def post(self, workbook_name, listener): + LOG.debug("Create listener [workbook_name=%s, listener=%s]" % + (workbook_name, listener)) + + # TODO: create listener in DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Listeners, wtypes.text) + def get_all(self, workbook_name): + LOG.debug("Fetch listeners [workbook_name=%s]" % workbook_name) + + listeners = [] + # TODO: fetch listeners from DB + + return Listeners(listeners=listeners) diff --git a/mistral/api/controllers/v1/root.py b/mistral/api/controllers/v1/root.py new file mode 100644 index 000000000..5c7fc6443 --- /dev/null +++ b/mistral/api/controllers/v1/root.py @@ -0,0 +1,40 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers.v1 import workbook + + +class RootResource(wtypes.Base): + """Root resource for API version 1. + + It references all other resources belonging to the API. + """ + + uri = wtypes.text + + # TODO: what else do we need here? + + +class Controller(object): + """API root controller for version 1.""" + + workbooks = workbook.WorkbooksController() + + @wsme_pecan.wsexpose(RootResource) + def index(self): + return RootResource(uri='%s/%s' % (pecan.request.host_url, 'v1')) diff --git a/mistral/api/controllers/v1/workbook.py b/mistral/api/controllers/v1/workbook.py new file mode 100644 index 000000000..355e43444 --- /dev/null +++ b/mistral/api/controllers/v1/workbook.py @@ -0,0 +1,110 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers.v1 import listener +from mistral.openstack.common import log as logging + + +LOG = logging.getLogger("%s" % __name__) + + +class Workbook(wtypes.Base): + """Workbook resource.""" + + name = wtypes.text + description = wtypes.text + tags = [wtypes.text] + + def __str__(self): + return "Workbook [name='%s', description='%s', tags='%s']" % \ + (self.name, self.description, self.tags) + + +class Workbooks(wtypes.Base): + """A collection of Workbooks.""" + + workbooks = [Workbook] + + def __str__(self): + return "Workbooks [workbooks=%s]" % self.workbooks + + +class WorkbooksController(rest.RestController): + """Operations on collection of workbooks.""" + + listeners = listener.ListenersController() + + #@pecan.expose() + #def _lookup(self, workbook_name, *remainder): + # # Standard Pecan delegation. + # return WorkbookController(workbook_name), remainder + + @wsme_pecan.wsexpose(Workbook, wtypes.text) + def get(self, name): + LOG.debug("Fetch workbook [name=%s]" % name) + + # TODO: fetch the workbook from the DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Workbook, wtypes.text, body=Workbook) + def put(self, name, workbook): + LOG.debug("Update workbook [name=%s, workbook=%s]" % (name, workbook)) + + # TODO: modify the workbook in DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + LOG.debug("Delete workbook [name=%s]" % name) + + # TODO: delete the workbook from DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Workbook, body=Workbook, status_code=201) + def post(self, workbook): + LOG.debug("Create workbook [workbook=%s]" % workbook) + + # TODO: create the listener in DB + + error = "Not implemented" + pecan.response.translatable_error = error + + raise wsme.exc.ClientSideError(unicode(error)) + + @wsme_pecan.wsexpose(Workbooks) + def get_all(self): + LOG.debug("Fetch workbooks.") + + workbooks = [] + # TODO: fetch workbooks from DB + + return Workbooks(workbooks=workbooks) diff --git a/mistral/cmd/api.py b/mistral/cmd/api.py index e16dd9302..398d61de0 100644 --- a/mistral/cmd/api.py +++ b/mistral/cmd/api.py @@ -1,35 +1,50 @@ -#!/usr/bin/env python +# Copyright 2013 - Mirantis, Inc. # -# Copyright (c) 2013 Mirantis, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to start Mistral API service.""" import os import sys +from wsgiref import simple_server -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(__file__), - os.pardir, - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'mistral-api', '__init__.py')): - sys.path.insert(0, possible_topdir) +from oslo.config import cfg -#from mistral import config -#from mistral.openstack.common import log +from mistral.api import app +from mistral import config +from mistral.openstack.common import log as logging + + +LOG = logging.getLogger('mistral.cmd.api') def main(): - raise NotImplemented('Mistral API is not implemented yet.') + try: + config.parse_args() + logging.setup('Mistral') + + host = cfg.CONF.api.host + port = cfg.CONF.api.port + + server = simple_server.make_server(host, port, app.setup_app()) + + LOG.info("Mistral API is serving on http://%s:%s (PID=%s)" % + (host, port, os.getpid())) + + server.serve_forever() + except RuntimeError, e: + sys.stderr.write("ERROR: %s\n" % e) + sys.exit(1) if __name__ == '__main__': diff --git a/mistral/common/config.py b/mistral/common/config.py new file mode 100644 index 000000000..375124f85 --- /dev/null +++ b/mistral/common/config.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Routines for configuring Glance +""" + +import logging +import logging.config +import logging.handlers +import os +import sys + +from oslo.config import cfg +from paste import deploy + +from muranoapi.openstack.common import log +from muranoapi import __version__ as version + +paste_deploy_opts = [ + cfg.StrOpt('flavor'), + cfg.StrOpt('config_file'), +] + +bind_opts = [ + cfg.StrOpt('bind-host', default='0.0.0.0'), + cfg.IntOpt('bind-port', default='8082'), +] + +reports_opts = [ + cfg.StrOpt('results_exchange', default='task-results'), + cfg.StrOpt('results_queue', default='task-results'), + cfg.StrOpt('reports_exchange', default='task-reports'), + cfg.StrOpt('reports_queue', default='task-reports') +] + +rabbit_opts = [ + cfg.StrOpt('host', default='localhost'), + cfg.IntOpt('port', default=5672), + cfg.StrOpt('login', default='guest'), + cfg.StrOpt('password', default='guest'), + cfg.StrOpt('virtual_host', default='/'), + cfg.BoolOpt('ssl', default=False), + cfg.StrOpt('ca_certs', default='') +] + +db_opts = [ + cfg.BoolOpt('auto_create', default=False, + help=_('A boolean that determines if the database will be ' + 'automatically created.')), +] + +CONF = cfg.CONF +CONF.register_opts(paste_deploy_opts, group='paste_deploy') +CONF.register_cli_opts(bind_opts) +CONF.register_opts(reports_opts, group='reports') +CONF.register_opts(rabbit_opts, group='rabbitmq') +CONF.register_opts(db_opts, group='database') + + +CONF.import_opt('verbose', 'muranoapi.openstack.common.log') +CONF.import_opt('debug', 'muranoapi.openstack.common.log') +CONF.import_opt('log_dir', 'muranoapi.openstack.common.log') +CONF.import_opt('log_file', 'muranoapi.openstack.common.log') +CONF.import_opt('log_config', 'muranoapi.openstack.common.log') +CONF.import_opt('log_format', 'muranoapi.openstack.common.log') +CONF.import_opt('log_date_format', 'muranoapi.openstack.common.log') +CONF.import_opt('use_syslog', 'muranoapi.openstack.common.log') +CONF.import_opt('syslog_log_facility', 'muranoapi.openstack.common.log') + + +cfg.set_defaults(log.log_opts, + default_log_levels=['qpid.messaging=INFO', + 'sqlalchemy=WARN', + 'keystoneclient=INFO', + 'eventlet.wsgi.server=WARN']) + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='muranoapi', + version=version, + usage=usage, + default_config_files=default_config_files) + + +def setup_logging(): + """ + Sets up the logging options for a log with supplied name + """ + + if CONF.log_config: + # Use a logging configuration file for all settings... + if os.path.exists(CONF.log_config): + logging.config.fileConfig(CONF.log_config) + return + else: + raise RuntimeError("Unable to locate specified logging " + "config file: %s" % CONF.log_config) + + root_logger = logging.root + if CONF.debug: + root_logger.setLevel(logging.DEBUG) + elif CONF.verbose: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + + formatter = logging.Formatter(CONF.log_format, CONF.log_date_format) + + if CONF.use_syslog: + try: + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility) + except AttributeError: + raise ValueError(_("Invalid syslog facility")) + + handler = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + elif CONF.log_file: + logfile = CONF.log_file + if CONF.log_dir: + logfile = os.path.join(CONF.log_dir, logfile) + handler = logging.handlers.WatchedFileHandler(logfile) + else: + handler = logging.StreamHandler(sys.stdout) + + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + +def _get_deployment_flavor(): + """ + Retrieve the paste_deploy.flavor config item, formatted appropriately + for appending to the application name. + """ + flavor = CONF.paste_deploy.flavor + return '' if not flavor else ('-' + flavor) + + +def _get_paste_config_path(): + paste_suffix = '-paste.ini' + conf_suffix = '.conf' + if CONF.config_file: + # Assume paste config is in a paste.ini file corresponding + # to the last config file + path = CONF.config_file[-1].replace(conf_suffix, paste_suffix) + else: + path = CONF.prog + '-paste.ini' + return CONF.find_file(os.path.basename(path)) + + +def _get_deployment_config_file(): + """ + Retrieve the deployment_config_file config item, formatted as an + absolute pathname. + """ + path = CONF.paste_deploy.config_file + if not path: + path = _get_paste_config_path() + if not path: + msg = "Unable to locate paste config file for %s." % CONF.prog + raise RuntimeError(msg) + return os.path.abspath(path) + + +def load_paste_app(app_name=None): + """ + Builds and returns a WSGI app from a paste config file. + + We assume the last config file specified in the supplied ConfigOpts + object is the paste config file. + + :param app_name: name of the application to load + + :raises RuntimeError when config file cannot be located or application + cannot be loaded from config file + """ + if app_name is None: + app_name = CONF.prog + + # append the deployment flavor to the application name, + # in order to identify the appropriate paste pipeline + app_name += _get_deployment_flavor() + + conf_file = _get_deployment_config_file() + + try: + logger = logging.getLogger(__name__) + logger.debug(_("Loading %(app_name)s from %(conf_file)s"), + {'conf_file': conf_file, 'app_name': app_name}) + + app = deploy.loadapp("config:%s" % conf_file, name=app_name) + + # Log the options used when starting if we're in debug mode... + if CONF.debug: + CONF.log_opt_values(logger, logging.DEBUG) + + return app + except (LookupError, ImportError), e: + msg = _("Unable to load %(app_name)s from " + "configuration file %(conf_file)s." + "\nGot: %(e)r") % locals() + logger.error(msg) + raise RuntimeError(msg) diff --git a/mistral/config.py b/mistral/config.py new file mode 100644 index 000000000..d58ee27e0 --- /dev/null +++ b/mistral/config.py @@ -0,0 +1,61 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Configuration options registration and useful routines. +""" + +from oslo.config import cfg + +from mistral.openstack.common import log +from mistral import version + + +api_opts = [ + cfg.StrOpt('host', default='0.0.0.0', help='Mistral API server host'), + cfg.IntOpt('port', default=8989, help='Mistral API server port') +] + +db_opts = [ + # TODO: add DB properties. +] + +CONF = cfg.CONF + +CONF.register_opts(api_opts, group='api') +CONF.register_opts(db_opts, group='database') + + +CONF.import_opt('verbose', 'mistral.openstack.common.log') +CONF.import_opt('debug', 'mistral.openstack.common.log') +CONF.import_opt('log_dir', 'mistral.openstack.common.log') +CONF.import_opt('log_file', 'mistral.openstack.common.log') +CONF.import_opt('log_config_append', 'mistral.openstack.common.log') +CONF.import_opt('log_format', 'mistral.openstack.common.log') +CONF.import_opt('log_date_format', 'mistral.openstack.common.log') +CONF.import_opt('use_syslog', 'mistral.openstack.common.log') +CONF.import_opt('syslog_log_facility', 'mistral.openstack.common.log') + + +cfg.set_defaults(log.log_opts, + default_log_levels=['sqlalchemy=WARN', + 'eventlet.wsgi.server=WARN']) + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='mistral', + version=version, + usage=usage, + default_config_files=default_config_files) diff --git a/mistral/openstack/common/apiclient/__init__.py b/mistral/openstack/common/apiclient/__init__.py index d5d002224..f3d0cdefd 100644 --- a/mistral/openstack/common/apiclient/__init__.py +++ b/mistral/openstack/common/apiclient/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 OpenStack Foundation # All Rights Reserved. # diff --git a/mistral/openstack/common/apiclient/auth.py b/mistral/openstack/common/apiclient/auth.py index f1df136c4..58fc6b6b8 100644 --- a/mistral/openstack/common/apiclient/auth.py +++ b/mistral/openstack/common/apiclient/auth.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 OpenStack Foundation # Copyright 2013 Spanish National Research Council. # All Rights Reserved. diff --git a/mistral/openstack/common/apiclient/base.py b/mistral/openstack/common/apiclient/base.py index 75dae495a..61551fdff 100644 --- a/mistral/openstack/common/apiclient/base.py +++ b/mistral/openstack/common/apiclient/base.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 Jacob Kaplan-Moss # Copyright 2011 OpenStack Foundation # Copyright 2012 Grid Dynamics diff --git a/mistral/openstack/common/apiclient/client.py b/mistral/openstack/common/apiclient/client.py index 814f7f926..1e9808790 100644 --- a/mistral/openstack/common/apiclient/client.py +++ b/mistral/openstack/common/apiclient/client.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 Jacob Kaplan-Moss # Copyright 2011 OpenStack Foundation # Copyright 2011 Piston Cloud Computing, Inc. diff --git a/mistral/openstack/common/apiclient/exceptions.py b/mistral/openstack/common/apiclient/exceptions.py index fec8ade15..b364d60dc 100644 --- a/mistral/openstack/common/apiclient/exceptions.py +++ b/mistral/openstack/common/apiclient/exceptions.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 Jacob Kaplan-Moss # Copyright 2011 Nebula, Inc. # Copyright 2013 Alessio Ababilov diff --git a/mistral/openstack/common/apiclient/fake_client.py b/mistral/openstack/common/apiclient/fake_client.py index 52b9866e7..7164c1803 100644 --- a/mistral/openstack/common/apiclient/fake_client.py +++ b/mistral/openstack/common/apiclient/fake_client.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 OpenStack Foundation # All Rights Reserved. # @@ -27,11 +25,11 @@ places where actual behavior differs from the spec. # pylint: disable=W0102 import json -import urlparse import requests from mistral.openstack.common.apiclient import client +from mistral.openstack.common.py3kcompat import urlutils def assert_has_keys(dct, required=[], optional=[]): @@ -146,7 +144,7 @@ class FakeHTTPClient(client.HTTPClient): "text": fixture[1]}) # Call the method - args = urlparse.parse_qsl(urlparse.urlparse(url)[4]) + args = urlutils.parse_qsl(urlutils.urlparse(url)[4]) kwargs.update(args) munged_url = url.rsplit('?', 1)[0] munged_url = munged_url.strip('/').replace('/', '_').replace('.', '_') diff --git a/mistral/openstack/common/cliutils.py b/mistral/openstack/common/cliutils.py index 74c129651..b69f17516 100644 --- a/mistral/openstack/common/cliutils.py +++ b/mistral/openstack/common/cliutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -26,6 +24,7 @@ import textwrap import prettytable import six +from six import moves from mistral.openstack.common.apiclient import exceptions from mistral.openstack.common import strutils @@ -200,7 +199,7 @@ def get_password(max_password_prompts=3): if hasattr(sys.stdin, "isatty") and sys.stdin.isatty(): # Check for Ctrl-D try: - for _ in xrange(max_password_prompts): + for _ in moves.range(max_password_prompts): pw1 = getpass.getpass("OS Password: ") if verify: pw2 = getpass.getpass("Please verify: ") diff --git a/mistral/openstack/common/config/__init__.py b/mistral/openstack/common/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/openstack/common/config/generator.py b/mistral/openstack/common/config/generator.py new file mode 100644 index 000000000..4c11dcebf --- /dev/null +++ b/mistral/openstack/common/config/generator.py @@ -0,0 +1,268 @@ +# Copyright 2012 SINA Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Extracts OpenStack config option info from module(s).""" + +from __future__ import print_function + +import imp +import os +import re +import socket +import sys +import textwrap + +from oslo.config import cfg +import six + +from mistral.openstack.common import gettextutils +from mistral.openstack.common import importutils + +gettextutils.install('mistral') + +STROPT = "StrOpt" +BOOLOPT = "BoolOpt" +INTOPT = "IntOpt" +FLOATOPT = "FloatOpt" +LISTOPT = "ListOpt" +MULTISTROPT = "MultiStrOpt" + +OPT_TYPES = { + STROPT: 'string value', + BOOLOPT: 'boolean value', + INTOPT: 'integer value', + FLOATOPT: 'floating point value', + LISTOPT: 'list value', + MULTISTROPT: 'multi valued', +} + +OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, + FLOATOPT, LISTOPT, + MULTISTROPT])) + +PY_EXT = ".py" +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + "../../../../")) +WORDWRAP_WIDTH = 60 + + +def generate(srcfiles): + mods_by_pkg = dict() + for filepath in srcfiles: + pkg_name = filepath.split(os.sep)[1] + mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), + os.path.basename(filepath).split('.')[0]]) + mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) + # NOTE(lzyeval): place top level modules before packages + pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys()) + pkg_names.sort() + ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys()) + ext_names.sort() + pkg_names.extend(ext_names) + + # opts_by_group is a mapping of group name to an options list + # The options list is a list of (module, options) tuples + opts_by_group = {'DEFAULT': []} + + extra_modules = os.getenv("MISTRAL_CONFIG_GENERATOR_EXTRA_MODULES", "") + if extra_modules: + for module_name in extra_modules.split(','): + module_name = module_name.strip() + module = _import_module(module_name) + if module: + for group, opts in _list_opts(module): + opts_by_group.setdefault(group, []).append((module_name, + opts)) + + for pkg_name in pkg_names: + mods = mods_by_pkg.get(pkg_name) + mods.sort() + for mod_str in mods: + if mod_str.endswith('.__init__'): + mod_str = mod_str[:mod_str.rfind(".")] + + mod_obj = _import_module(mod_str) + if not mod_obj: + raise RuntimeError("Unable to import module %s" % mod_str) + + for group, opts in _list_opts(mod_obj): + opts_by_group.setdefault(group, []).append((mod_str, opts)) + + print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) + for group, opts in opts_by_group.items(): + print_group_opts(group, opts) + + +def _import_module(mod_str): + try: + if mod_str.startswith('bin.'): + imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) + return sys.modules[mod_str[4:]] + else: + return importutils.import_module(mod_str) + except Exception as e: + sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) + return None + + +def _is_in_group(opt, group): + "Check if opt is in group." + for key, value in group._opts.items(): + if value['opt'] == opt: + return True + return False + + +def _guess_groups(opt, mod_obj): + # is it in the DEFAULT group? + if _is_in_group(opt, cfg.CONF): + return 'DEFAULT' + + # what other groups is it in? + for key, value in cfg.CONF.items(): + if isinstance(value, cfg.CONF.GroupAttr): + if _is_in_group(opt, value._group): + return value._group.name + + raise RuntimeError( + "Unable to find group for option %s, " + "maybe it's defined twice in the same group?" + % opt.name + ) + + +def _list_opts(obj): + def is_opt(o): + return (isinstance(o, cfg.Opt) and + not isinstance(o, cfg.SubCommandOpt)) + + opts = list() + for attr_str in dir(obj): + attr_obj = getattr(obj, attr_str) + if is_opt(attr_obj): + opts.append(attr_obj) + elif (isinstance(attr_obj, list) and + all(map(lambda x: is_opt(x), attr_obj))): + opts.extend(attr_obj) + + ret = {} + for opt in opts: + ret.setdefault(_guess_groups(opt, obj), []).append(opt) + return ret.items() + + +def print_group_opts(group, opts_by_module): + print("[%s]" % group) + print('') + for mod, opts in opts_by_module: + print('#') + print('# Options defined in %s' % mod) + print('#') + print('') + for opt in opts: + _print_opt(opt) + print('') + + +def _get_my_ip(): + try: + csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + csock.connect(('8.8.8.8', 80)) + (addr, port) = csock.getsockname() + csock.close() + return addr + except socket.error: + return None + + +def _sanitize_default(name, value): + """Set up a reasonably sensible default for pybasedir, my_ip and host.""" + if value.startswith(sys.prefix): + # NOTE(jd) Don't use os.path.join, because it is likely to think the + # second part is an absolute pathname and therefore drop the first + # part. + value = os.path.normpath("/usr/" + value[len(sys.prefix):]) + elif value.startswith(BASEDIR): + return value.replace(BASEDIR, '/usr/lib/python/site-packages') + elif BASEDIR in value: + return value.replace(BASEDIR, '') + elif value == _get_my_ip(): + return '10.0.0.1' + elif value == socket.gethostname() and 'host' in name: + return 'mistral' + elif value.strip() != value: + return '"%s"' % value + return value + + +def _print_opt(opt): + opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help + if not opt_help: + sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) + opt_help = "" + opt_type = None + try: + opt_type = OPTION_REGEX.search(str(type(opt))).group(0) + except (ValueError, AttributeError) as err: + sys.stderr.write("%s\n" % str(err)) + sys.exit(1) + opt_help += ' (' + OPT_TYPES[opt_type] + ')' + print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) + if opt.deprecated_opts: + for deprecated_opt in opt.deprecated_opts: + if deprecated_opt.name: + deprecated_group = (deprecated_opt.group if + deprecated_opt.group else "DEFAULT") + print('# Deprecated group/name - [%s]/%s' % + (deprecated_group, + deprecated_opt.name)) + try: + if opt_default is None: + print('#%s=' % opt_name) + elif opt_type == STROPT: + assert(isinstance(opt_default, six.string_types)) + print('#%s=%s' % (opt_name, _sanitize_default(opt_name, + opt_default))) + elif opt_type == BOOLOPT: + assert(isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, str(opt_default).lower())) + elif opt_type == INTOPT: + assert(isinstance(opt_default, int) and + not isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == FLOATOPT: + assert(isinstance(opt_default, float)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == LISTOPT: + assert(isinstance(opt_default, list)) + print('#%s=%s' % (opt_name, ','.join(opt_default))) + elif opt_type == MULTISTROPT: + assert(isinstance(opt_default, list)) + if not opt_default: + opt_default = [''] + for default in opt_default: + print('#%s=%s' % (opt_name, default)) + print('') + except Exception: + sys.stderr.write('Error in option "%s"\n' % opt_name) + sys.exit(1) + + +def main(): + generate(sys.argv[1:]) + +if __name__ == '__main__': + main() diff --git a/mistral/openstack/common/db/__init__.py b/mistral/openstack/common/db/__init__.py index 1b9b60dec..5f5273f3e 100644 --- a/mistral/openstack/common/db/__init__.py +++ b/mistral/openstack/common/db/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Cloudscaling Group, Inc # All Rights Reserved. # diff --git a/mistral/openstack/common/db/api.py b/mistral/openstack/common/db/api.py index 458e67eb6..c0df82d30 100644 --- a/mistral/openstack/common/db/api.py +++ b/mistral/openstack/common/db/api.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 Rackspace Hosting # All Rights Reserved. # diff --git a/mistral/openstack/common/db/exception.py b/mistral/openstack/common/db/exception.py index d372ea820..69eecb7d3 100644 --- a/mistral/openstack/common/db/exception.py +++ b/mistral/openstack/common/db/exception.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -49,3 +47,8 @@ class DbMigrationError(DBError): """Wraps migration specific exception.""" def __init__(self, message=None): super(DbMigrationError, self).__init__(str(message)) + + +class DBConnectionError(DBError): + """Wraps connection specific exception.""" + pass diff --git a/mistral/openstack/common/db/sqlalchemy/__init__.py b/mistral/openstack/common/db/sqlalchemy/__init__.py index 1b9b60dec..5f5273f3e 100644 --- a/mistral/openstack/common/db/sqlalchemy/__init__.py +++ b/mistral/openstack/common/db/sqlalchemy/__init__.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Cloudscaling Group, Inc # All Rights Reserved. # diff --git a/mistral/openstack/common/db/sqlalchemy/models.py b/mistral/openstack/common/db/sqlalchemy/models.py index 883eb4187..84dbda0f7 100644 --- a/mistral/openstack/common/db/sqlalchemy/models.py +++ b/mistral/openstack/common/db/sqlalchemy/models.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2011 X.commerce, a business unit of eBay Inc. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. diff --git a/mistral/openstack/common/db/sqlalchemy/provision.py b/mistral/openstack/common/db/sqlalchemy/provision.py new file mode 100644 index 000000000..33b466fc3 --- /dev/null +++ b/mistral/openstack/common/db/sqlalchemy/provision.py @@ -0,0 +1,187 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Mirantis.inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provision test environment for specific DB backends""" + +import argparse +import os +import random +import string + +import sqlalchemy + +from mistral.openstack.common.db import exception as exc + + +SQL_CONNECTION = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', 'sqlite://') + + +def _gen_credentials(*names): + """Generate credentials.""" + auth_dict = {} + for name in names: + val = ''.join(random.choice(string.lowercase) for i in xrange(10)) + auth_dict[name] = val + return auth_dict + + +def _get_engine(uri=SQL_CONNECTION): + """Engine creation + + By default the uri is SQL_CONNECTION which is admin credentials. + Call the function without arguments to get admin connection. Admin + connection required to create temporary user and database for each + particular test. Otherwise use existing connection to recreate connection + to the temporary database. + """ + return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool) + + +def _execute_sql(engine, sql, driver): + """Initialize connection, execute sql query and close it.""" + try: + with engine.connect() as conn: + if driver == 'postgresql': + conn.connection.set_isolation_level(0) + for s in sql: + conn.execute(s) + except sqlalchemy.exc.OperationalError: + msg = ('%s does not match database admin ' + 'credentials or database does not exist.') + raise exc.DBConnectionError(msg % SQL_CONNECTION) + + +def create_database(engine): + """Provide temporary user and database for each particular test.""" + driver = engine.name + + auth = _gen_credentials('database', 'user', 'passwd') + + sqls = { + 'mysql': [ + "drop database if exists %(database)s;", + "grant all on %(database)s.* to '%(user)s'@'localhost'" + " identified by '%(passwd)s';", + "create database %(database)s;", + ], + 'postgresql': [ + "drop database if exists %(database)s;", + "drop user if exists %(user)s;", + "create user %(user)s with password '%(passwd)s';", + "create database %(database)s owner %(user)s;", + ] + } + + if driver == 'sqlite': + return 'sqlite:////tmp/%s' % auth['database'] + + try: + sql_rows = sqls[driver] + except KeyError: + raise ValueError('Unsupported RDBMS %s' % driver) + sql_query = map(lambda x: x % auth, sql_rows) + + _execute_sql(engine, sql_query, driver) + + params = auth.copy() + params['backend'] = driver + return "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" % params + + +def drop_database(engine, current_uri): + """Drop temporary database and user after each particular test.""" + engine = _get_engine(current_uri) + admin_engine = _get_engine() + driver = engine.name + auth = {'database': engine.url.database, 'user': engine.url.username} + + if driver == 'sqlite': + try: + os.remove(auth['database']) + except OSError: + pass + return + + sqls = { + 'mysql': [ + "drop database if exists %(database)s;", + "drop user '%(user)s'@'localhost';", + ], + 'postgresql': [ + "drop database if exists %(database)s;", + "drop user if exists %(user)s;", + ] + } + + try: + sql_rows = sqls[driver] + except KeyError: + raise ValueError('Unsupported RDBMS %s' % driver) + sql_query = map(lambda x: x % auth, sql_rows) + + _execute_sql(admin_engine, sql_query, driver) + + +def main(): + """Controller to handle commands + + ::create: Create test user and database with random names. + ::drop: Drop user and database created by previous command. + """ + parser = argparse.ArgumentParser( + description='Controller to handle database creation and dropping' + ' commands.', + epilog='Under normal circumstances is not used directly.' + ' Used in .testr.conf to automate test database creation' + ' and dropping processes.') + subparsers = parser.add_subparsers( + help='Subcommands to manipulate temporary test databases.') + + create = subparsers.add_parser( + 'create', + help='Create temporary test ' + 'databases and users.') + create.set_defaults(which='create') + create.add_argument( + 'instances_count', + type=int, + help='Number of databases to create.') + + drop = subparsers.add_parser( + 'drop', + help='Drop temporary test databases and users.') + drop.set_defaults(which='drop') + drop.add_argument( + 'instances', + nargs='+', + help='List of databases uri to be dropped.') + + args = parser.parse_args() + + engine = _get_engine() + which = args.which + + if which == "create": + for i in range(int(args.instances_count)): + print(create_database(engine)) + elif which == "drop": + for db in args.instances: + drop_database(engine, db) + + +if __name__ == "__main__": + main() diff --git a/mistral/openstack/common/db/sqlalchemy/session.py b/mistral/openstack/common/db/sqlalchemy/session.py index 7aec94eb6..ae2c0bc5c 100644 --- a/mistral/openstack/common/db/sqlalchemy/session.py +++ b/mistral/openstack/common/db/sqlalchemy/session.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/mistral/openstack/common/db/sqlalchemy/test_migrations.py b/mistral/openstack/common/db/sqlalchemy/test_migrations.py index 2f60f3b59..f0caf1891 100644 --- a/mistral/openstack/common/db/sqlalchemy/test_migrations.py +++ b/mistral/openstack/common/db/sqlalchemy/test_migrations.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010-2011 OpenStack Foundation # Copyright 2012-2013 IBM Corp. # All Rights Reserved. @@ -16,17 +14,18 @@ # License for the specific language governing permissions and limitations # under the License. - -import commands import ConfigParser +import functools import os -import urlparse +import lockfile import sqlalchemy import sqlalchemy.exc -from mistral.openstack.common import lockutils +from mistral.openstack.common.gettextutils import _ from mistral.openstack.common import log as logging +from mistral.openstack.common import processutils +from mistral.openstack.common.py3kcompat import urlutils from mistral.openstack.common import test LOG = logging.getLogger(__name__) @@ -93,6 +92,22 @@ def get_db_connection_info(conn_pieces): return (user, password, database, host) +def _set_db_lock(lock_path=None, lock_prefix=None): + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + path = lock_path or os.environ.get("MISTRAL_LOCK_PATH") + lock = lockfile.FileLock(os.path.join(path, lock_prefix)) + with lock: + LOG.debug(_('Got lock "%s"') % f.__name__) + return f(*args, **kwargs) + finally: + LOG.debug(_('Lock released "%s"') % f.__name__) + return wrapper + return decorator + + class BaseMigrationTestCase(test.BaseTestCase): """Base class fort testing of migration utils.""" @@ -143,12 +158,13 @@ class BaseMigrationTestCase(test.BaseTestCase): super(BaseMigrationTestCase, self).tearDown() def execute_cmd(self, cmd=None): - status, output = commands.getstatusoutput(cmd) + out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True) + output = out or err LOG.debug(output) - self.assertEqual(0, status, + self.assertEqual('', err, "Failed to run: %s\n%s" % (cmd, output)) - @lockutils.synchronized('pgadmin', 'tests-', external=True) + @_set_db_lock('pgadmin', 'tests-') def _reset_pg(self, conn_pieces): (user, password, database, host) = get_db_connection_info(conn_pieces) os.environ['PGPASSWORD'] = password @@ -173,7 +189,7 @@ class BaseMigrationTestCase(test.BaseTestCase): def _reset_databases(self): for key, engine in self.engines.items(): conn_string = self.test_databases[key] - conn_pieces = urlparse.urlparse(conn_string) + conn_pieces = urlutils.urlparse(conn_string) engine.dispose() if conn_string.startswith('sqlite'): # We can just delete the SQLite database, which is diff --git a/mistral/openstack/common/db/sqlalchemy/utils.py b/mistral/openstack/common/db/sqlalchemy/utils.py index b8aafb1bc..f25ade90f 100644 --- a/mistral/openstack/common/db/sqlalchemy/utils.py +++ b/mistral/openstack/common/db/sqlalchemy/utils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2010-2011 OpenStack Foundation. diff --git a/mistral/openstack/common/excutils.py b/mistral/openstack/common/excutils.py index 923261513..3aa232d30 100644 --- a/mistral/openstack/common/excutils.py +++ b/mistral/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # diff --git a/mistral/openstack/common/fileutils.py b/mistral/openstack/common/fileutils.py index 96104d01a..c6515166e 100644 --- a/mistral/openstack/common/fileutils.py +++ b/mistral/openstack/common/fileutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/mistral/openstack/common/gettextutils.py b/mistral/openstack/common/gettextutils.py index f5108e972..293a47cc9 100644 --- a/mistral/openstack/common/gettextutils.py +++ b/mistral/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. diff --git a/mistral/openstack/common/importutils.py b/mistral/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/mistral/openstack/common/importutils.py +++ b/mistral/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/mistral/openstack/common/jsonutils.py b/mistral/openstack/common/jsonutils.py index 7222af5a6..47b9a9c05 100644 --- a/mistral/openstack/common/jsonutils.py +++ b/mistral/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara diff --git a/mistral/openstack/common/local.py b/mistral/openstack/common/local.py index e82f17d0f..0819d5b97 100644 --- a/mistral/openstack/common/local.py +++ b/mistral/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/mistral/openstack/common/lockutils.py b/mistral/openstack/common/lockutils.py index 132e4917e..4304dc2c3 100644 --- a/mistral/openstack/common/lockutils.py +++ b/mistral/openstack/common/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/mistral/openstack/common/log.py b/mistral/openstack/common/log.py index 9b369a3b3..a80540d17 100644 --- a/mistral/openstack/common/log.py +++ b/mistral/openstack/common/log.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -477,7 +475,7 @@ def _setup_logging_from_conf(): streamlog = ColorHandler() log_root.addHandler(streamlog) - elif not CONF.log_file: + elif not logpath: # pass sys.stdout as a positional argument # python2.6 calls the argument strm, in 2.7 it's stream streamlog = logging.StreamHandler(sys.stdout) diff --git a/mistral/openstack/common/processutils.py b/mistral/openstack/common/processutils.py new file mode 100644 index 000000000..6035631b5 --- /dev/null +++ b/mistral/openstack/common/processutils.py @@ -0,0 +1,248 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import logging as stdlib_logging +import os +import random +import shlex +import signal + +from eventlet.green import subprocess +from eventlet import greenthread + +from mistral.openstack.common.gettextutils import _ # noqa +from mistral.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + +class UnknownArgumentError(Exception): + def __init__(self, message=None): + super(UnknownArgumentError, self).__init__(message) + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" + % (description, cmd, exit_code, stdout, stderr)) + super(ProcessExecutionError, self).__init__(message) + + +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: string + :param process_input: Send to opened process. + :type proces_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + the command is prefixed by the command specified + in the root_helper kwarg. + :type run_as_root: boolean + :param root_helper: command to prefix to commands called with + run_as_root=True + :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) + :returns: (stdout, stderr) from process execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + root_helper = kwargs.pop('root_helper', '') + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise UnknownArgumentError(_('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=('Command requested root, but did not specify a root ' + 'helper.')) + cmd = shlex.split(root_helper) + list(cmd) + + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, _('Running cmd (subprocess): %s'), ' '.join(cmd)) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, _('Result was %s') % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.log(loglevel, _('%r failed. Retrying.'), cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + LOG.debug(_('Running cmd (SSH): %s'), cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + stderr = stderr_stream.read() + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug(_('Result was %s') % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=stdout, + stderr=stderr, + cmd=cmd) + + return (stdout, stderr) diff --git a/mistral/openstack/common/py3kcompat/__init__.py b/mistral/openstack/common/py3kcompat/__init__.py new file mode 100644 index 000000000..97ae4e34a --- /dev/null +++ b/mistral/openstack/common/py3kcompat/__init__.py @@ -0,0 +1,16 @@ +# +# Copyright 2013 Canonical Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# diff --git a/mistral/openstack/common/py3kcompat/urlutils.py b/mistral/openstack/common/py3kcompat/urlutils.py new file mode 100644 index 000000000..51e18111a --- /dev/null +++ b/mistral/openstack/common/py3kcompat/urlutils.py @@ -0,0 +1,63 @@ +# +# Copyright 2013 Canonical Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +""" +Python2/Python3 compatibility layer for OpenStack +""" + +import six + +if six.PY3: + # python3 + import urllib.error + import urllib.parse + import urllib.request + + urlencode = urllib.parse.urlencode + urljoin = urllib.parse.urljoin + quote = urllib.parse.quote + parse_qsl = urllib.parse.parse_qsl + unquote = urllib.parse.unquote + urlparse = urllib.parse.urlparse + urlsplit = urllib.parse.urlsplit + urlunsplit = urllib.parse.urlunsplit + SplitResult = urllib.parse.SplitResult + + urlopen = urllib.request.urlopen + URLError = urllib.error.URLError + pathname2url = urllib.request.pathname2url +else: + # python2 + import urllib + import urllib2 + import urlparse + + urlencode = urllib.urlencode + quote = urllib.quote + unquote = urllib.unquote + + parse = urlparse + parse_qsl = parse.parse_qsl + urljoin = parse.urljoin + urlparse = parse.urlparse + urlsplit = parse.urlsplit + urlunsplit = parse.urlunsplit + SplitResult = parse.SplitResult + + urlopen = urllib2.urlopen + URLError = urllib2.URLError + pathname2url = urllib.pathname2url diff --git a/mistral/openstack/common/strutils.py b/mistral/openstack/common/strutils.py index 102270811..c187dd5ae 100644 --- a/mistral/openstack/common/strutils.py +++ b/mistral/openstack/common/strutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/mistral/openstack/common/test.py b/mistral/openstack/common/test.py index a1a736f7d..6b6a5f913 100644 --- a/mistral/openstack/common/test.py +++ b/mistral/openstack/common/test.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # diff --git a/mistral/openstack/common/timeutils.py b/mistral/openstack/common/timeutils.py index b79ebf378..c8b0b1539 100644 --- a/mistral/openstack/common/timeutils.py +++ b/mistral/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -178,6 +176,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: diff --git a/mistral/tests/api/__init__.py b/mistral/tests/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/tests/api/base.py b/mistral/tests/api/base.py new file mode 100644 index 000000000..39b3b49f0 --- /dev/null +++ b/mistral/tests/api/base.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +import pecan.testing + +from mistral.tests.base import BaseTest + +__all__ = ['FunctionalTest'] + + +class FunctionalTest(BaseTest): + """Used for functional tests where you need to test your + literal application and its integration with the framework. + """ + + def setUp(self): + super(FunctionalTest, self).setUp() + + self.app = pecan.testing.load_test_app({ + 'app': { + 'root': 'mistral.api.controllers.root.RootController', + 'modules': ['mistral.api'], + 'debug': False + } + }) + + def tearDown(self): + super(FunctionalTest, self).tearDown() + pecan.set_config({}, overwrite=True) diff --git a/mistral/tests/api/test_app.py b/mistral/tests/api/test_app.py new file mode 100644 index 000000000..5941a44e0 --- /dev/null +++ b/mistral/tests/api/test_app.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.api import app as api_app +from mistral.api import config as api_config +from mistral.tests.api.base import FunctionalTest + + +class TestAppConfig(FunctionalTest): + + def test_get_pecan_config(self): + config = api_app.get_pecan_config() + + self.assertEqual(dict(config.app), api_config.app) diff --git a/mistral/tests/api/v1/__init__.py b/mistral/tests/api/v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/tests/api/v1/controllers/__init__.py b/mistral/tests/api/v1/controllers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/tests/api/v1/controllers/test_workbooks.py b/mistral/tests/api/v1/controllers/test_workbooks.py new file mode 100644 index 000000000..59726350a --- /dev/null +++ b/mistral/tests/api/v1/controllers/test_workbooks.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.openstack.common import jsonutils +from mistral.tests.api import base + + +class TestWorkbooksController(base.FunctionalTest): + + def test_get_all(self): + resp = self.app.get('/v1/workbooks', + headers={'Accept': 'application/json'}) + + self.assertEqual(resp.status_int, 200) + + data = jsonutils.loads(resp.body.decode()) + + print "json=%s" % data + + #self.assertEqual(data['name'], 'my_workbook') + #self.assertEqual(data['description'], 'My cool workbook') diff --git a/mistral/tests/api/v1/test_root.py b/mistral/tests/api/v1/test_root.py new file mode 100644 index 000000000..a2146dbff --- /dev/null +++ b/mistral/tests/api/v1/test_root.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.openstack.common import jsonutils +from mistral.tests.api import base + + +class TestRootController(base.FunctionalTest): + + def test_index(self): + resp = self.app.get('/', headers={'Accept': 'application/json'}) + + self.assertEqual(resp.status_int, 200) + + data = jsonutils.loads(resp.body.decode()) + + self.assertEqual(data[0]['id'], 'v1.0') + self.assertEqual(data[0]['status'], 'CURRENT') + self.assertEqual(data[0]['link'], {'href': 'http://localhost/v1', + 'target': 'v1'}) diff --git a/mistral/tests/base.py b/mistral/tests/base.py new file mode 100644 index 000000000..9c4074cad --- /dev/null +++ b/mistral/tests/base.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest2 + + +class BaseTest(unittest2.TestCase): + + def setUp(self): + super(BaseTest, self).setUp() + + # TODO: add whatever is needed for all Mistral tests in here + + def tearDown(self): + super(BaseTest, self).tearDown() + + # TODO: add whatever is needed for all Mistral tests in here diff --git a/mistral/version.py b/mistral/version.py index d57161af2..1cafedb7f 100644 --- a/mistral/version.py +++ b/mistral/version.py @@ -16,3 +16,4 @@ from pbr import version version_info = version.VersionInfo('Mistral') +version_string = version_info.version_string diff --git a/openstack-common.conf b/openstack-common.conf index fd81b62d2..054b5d13e 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,11 +1,15 @@ [DEFAULT] # The list of modules to copy from oslo-incubator.git +module=wsgi +module=config +module=exception module=cliutils module=db module=db.sqlalchemy module=log module=test +module=jsonutils # The base module to hold the copy of openstack.common base=mistral diff --git a/requirements.txt b/requirements.txt index 40d7e39ba..ba26b1d0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pecan>=0.2.0 WSME>=0.5b6 amqplib>=0.6.1 argparse +oslo.config>=1.2.0 diff --git a/test-requirements.txt b/test-requirements.txt index e6e70b25d..e73a995c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,9 @@ pyflakes>=0.7.2,<0.7.4 flake8==2.0 pylint==0.25.2 sphinx>=1.1.2 +unittest2 +fixtures>=0.3.14 +mock>=1.0 +nose +testtools>=0.9.32 +lockfile>=0.9.1 diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh new file mode 100755 index 000000000..29fb346b1 --- /dev/null +++ b/tools/config/generate_sample.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +print_hint() { + echo "Try \`${0##*/} --help' for more information." >&2 +} + +PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \ + --long help,base-dir:,package-name:,output-dir: -- "$@") + +if [ $? != 0 ] ; then print_hint ; exit 1 ; fi + +eval set -- "$PARSED_OPTIONS" + +while true; do + case "$1" in + -h|--help) + echo "${0##*/} [options]" + echo "" + echo "options:" + echo "-h, --help show brief help" + echo "-b, --base-dir=DIR project base directory" + echo "-p, --package-name=NAME project package name" + echo "-o, --output-dir=DIR file output directory" + exit 0 + ;; + -b|--base-dir) + shift + BASEDIR=`echo $1 | sed -e 's/\/*$//g'` + shift + ;; + -p|--package-name) + shift + PACKAGENAME=`echo $1` + shift + ;; + -o|--output-dir) + shift + OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` + shift + ;; + --) + break + ;; + esac +done + +BASEDIR=${BASEDIR:-`pwd`} +if ! [ -d $BASEDIR ] +then + echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 +elif [[ $BASEDIR != /* ]] +then + BASEDIR=$(cd "$BASEDIR" && pwd) +fi + +PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}} +TARGETDIR=$BASEDIR/$PACKAGENAME +if ! [ -d $TARGETDIR ] +then + echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 +fi + +OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} +# NOTE(bnemec): Some projects put their sample config in etc/, +# some in etc/$PACKAGENAME/ +if [ -d $OUTPUTDIR/$PACKAGENAME ] +then + OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME +elif ! [ -d $OUTPUTDIR ] +then + echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 + exit 1 +fi + +BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` +find $TARGETDIR -type f -name "*.pyc" -delete +FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ + -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) + +EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc" +if test -r "$EXTRA_MODULES_FILE" +then + source "$EXTRA_MODULES_FILE" +fi + +export EVENTLET_NO_GREENDNS=yes + +OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) +[ "$OS_VARS" ] && eval "unset \$OS_VARS" +DEFAULT_MODULEPATH=mistral.openstack.common.config.generator +MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} +OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample +python -m $MODULEPATH $FILES > $OUTPUTFILE + +# Hook to allow projects to append custom config file snippets +CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) +for CONCAT_FILE in $CONCAT_FILES; do + cat $CONCAT_FILE >> $OUTPUTFILE +done diff --git a/tox.ini b/tox.ini index 3763e757b..52d5cff8f 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ setenv = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +commands = nosetests [testenv:pep8] commands = flake8 {posargs}