Add mistral-db-manage script

Implements: blueprint mistral-manage-db-script

Co-Authored-By: Nikolay Mahotkin <nmakhotkin@mirantis.com>

Change-Id: If8465033e14af223bd5fea0b9ca9383e29db21c4

mistral/config.py

@@ -23,6 +23,7 @@ from oslo.config import cfg
 from mistral.openstack.common import log
 from mistral import version
 
+
 launch_opt = cfg.ListOpt(
     'server',
     default=['all'],

mistral/context.py

@@ -22,10 +22,8 @@ from pecan import hooks
 
 from mistral import exceptions as exc
 from mistral.openstack.common import jsonutils
-from mistral.openstack.common import log as logging
 from mistral import utils
 
-LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
 
@@ -103,8 +101,6 @@ def _wrapper(context, thread_desc, thread_group, func, *args, **kwargs):
         set_ctx(context)
         func(*args, **kwargs)
     except Exception as e:
-        LOG.exception("Thread '%s' fails with exception: '%s'"
-                      % (thread_desc, e))
         if thread_group and not thread_group.exc:
             thread_group.exc = e
             thread_group.failed_thread = thread_desc

mistral/db/sqlalchemy/migration/alembic.ini

@@ -0,0 +1,58 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = mistral/db/sqlalchemy/migration/alembic_migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+sqlalchemy.url =
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

mistral/db/sqlalchemy/migration/alembic_migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

mistral/db/sqlalchemy/migration/alembic_migrations/env.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from __future__ import with_statement
+from alembic import context
+from logging import config as c
+from oslo_utils import importutils
+from sqlalchemy import create_engine
+from sqlalchemy import pool
+
+from mistral.db.sqlalchemy import model_base
+
+
+importutils.try_import('mistral.db.v2.sqlalchemy.models')
+
+# This is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+mistral_config = config.mistral_config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+c.fileConfig(config.config_file_name)
+
+# Add your model's MetaData object here for 'autogenerate' support.
+target_metadata = model_base.MistralSecureModelBase.metadata
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(url=mistral_config.database.connection)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = create_engine(
+        mistral_config.database.connection,
+        poolclass=pool.NullPool
+    )
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

mistral/db/sqlalchemy/migration/alembic_migrations/script.py.mako

@@ -0,0 +1,34 @@
+# Copyright ${create_date.year} OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

mistral/db/sqlalchemy/migration/alembic_migrations/versions/001_kilo.py

@@ -0,0 +1,228 @@
+# Copyright 2015 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Kilo release
+
+Revision ID: 001
+Revises: None
+Create Date: 2015-03-31 12:02:51.935368
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+from mistral.db.sqlalchemy import types as st
+
+
+def upgrade():
+    op.create_table(
+        'workbooks_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=True),
+        sa.Column('definition', sa.Text(), nullable=True),
+        sa.Column('spec', st.JsonEncoded(), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name', 'project_id')
+    )
+    op.create_table(
+        'tasks',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=True),
+        sa.Column('requires', st.JsonEncoded(), nullable=True),
+        sa.Column('workbook_name', sa.String(length=80), nullable=True),
+        sa.Column('execution_id', sa.String(length=36), nullable=True),
+        sa.Column('description', sa.String(length=200), nullable=True),
+        sa.Column('task_spec', st.JsonEncoded(), nullable=True),
+        sa.Column('action_spec', st.JsonEncoded(), nullable=True),
+        sa.Column('state', sa.String(length=20), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.Column('in_context', st.JsonEncoded(), nullable=True),
+        sa.Column('parameters', st.JsonEncoded(), nullable=True),
+        sa.Column('output', st.JsonEncoded(), nullable=True),
+        sa.Column('task_runtime_context', st.JsonEncoded(), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'action_definitions_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=True),
+        sa.Column('definition', sa.Text(), nullable=True),
+        sa.Column('spec', st.JsonEncoded(), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('input', sa.Text(), nullable=True),
+        sa.Column('action_class', sa.String(length=200), nullable=True),
+        sa.Column('attributes', st.JsonEncoded(), nullable=True),
+        sa.Column('is_system', sa.Boolean(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name', 'project_id')
+    )
+    op.create_table(
+        'workflow_definitions_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=True),
+        sa.Column('definition', sa.Text(), nullable=True),
+        sa.Column('spec', st.JsonEncoded(), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name', 'project_id')
+    )
+    op.create_table(
+        'executions_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('type', sa.String(length=50), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=True),
+        sa.Column('workflow_name', sa.String(length=80), nullable=True),
+        sa.Column('spec', st.JsonEncoded(), nullable=True),
+        sa.Column('state', sa.String(length=20), nullable=True),
+        sa.Column('state_info', sa.String(length=1024), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.Column('accepted', sa.Boolean(), nullable=True),
+        sa.Column('input', st.JsonEncoded(), nullable=True),
+        sa.Column('output', st.JsonLongDictType(), nullable=True),
+        sa.Column('params', st.JsonEncoded(), nullable=True),
+        sa.Column('context', st.JsonEncoded(), nullable=True),
+        sa.Column('action_spec', st.JsonEncoded(), nullable=True),
+        sa.Column('processed', sa.BOOLEAN(), nullable=True),
+        sa.Column('in_context', st.JsonLongDictType(), nullable=True),
+        sa.Column('published', st.JsonEncoded(), nullable=True),
+        sa.Column('runtime_context', st.JsonEncoded(), nullable=True),
+        sa.Column('task_execution_id', sa.String(length=36), nullable=True),
+        sa.Column(
+            'workflow_execution_id', sa.String(length=36), nullable=True
+        ),
+        sa.ForeignKeyConstraint(
+            ['task_execution_id'], [u'executions_v2.id'],
+        ),
+        sa.ForeignKeyConstraint(
+            ['workflow_execution_id'], [u'executions_v2.id'],
+        ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'workbooks',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=False),
+        sa.Column('definition', sa.Text(), nullable=True),
+        sa.Column('description', sa.String(length=200), nullable=True),
+        sa.Column('tags', st.JsonEncoded(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('trust_id', sa.String(length=80), nullable=True),
+        sa.PrimaryKeyConstraint('id', 'name'),
+        sa.UniqueConstraint('name')
+    )
+    op.create_table(
+        'environments_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=200), nullable=True),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('variables', st.JsonEncoded(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name', 'project_id')
+    )
+    op.create_table(
+        'triggers',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=80), nullable=False),
+        sa.Column('pattern', sa.String(length=20), nullable=False),
+        sa.Column('next_execution_time', sa.DateTime(), nullable=False),
+        sa.Column('workbook_name', sa.String(length=80), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name')
+    )
+    op.create_table(
+        'delayed_calls_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column(
+            'factory_method_path', sa.String(length=200), nullable=True
+        ),
+        sa.Column('target_method_name', sa.String(length=80), nullable=False),
+        sa.Column('method_arguments', st.JsonEncoded(), nullable=True),
+        sa.Column('serializers', st.JsonEncoded(), nullable=True),
+        sa.Column('auth_context', st.JsonEncoded(), nullable=True),
+        sa.Column('execution_time', sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'workflow_executions',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('workbook_name', sa.String(length=80), nullable=True),
+        sa.Column('task', sa.String(length=80), nullable=True),
+        sa.Column('state', sa.String(length=20), nullable=True),
+        sa.Column('context', st.JsonEncoded(), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'cron_triggers_v2',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('scope', sa.String(length=80), nullable=True),
+        sa.Column('project_id', sa.String(length=80), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=200), nullable=True),
+        sa.Column('pattern', sa.String(length=100), nullable=True),
+        sa.Column('next_execution_time', sa.DateTime(), nullable=False),
+        sa.Column('workflow_name', sa.String(length=80), nullable=True),
+        sa.Column('remaining_executions', sa.Integer(), nullable=True),
+        sa.Column('workflow_id', sa.String(length=36), nullable=True),
+        sa.Column('workflow_input', st.JsonEncoded(), nullable=True),
+        sa.Column('workflow_input_hash', sa.CHAR(length=64), nullable=True),
+        sa.Column('trust_id', sa.String(length=80), nullable=True),
+        sa.ForeignKeyConstraint(
+            ['workflow_id'], [u'workflow_definitions_v2.id'],
+        ),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name', 'project_id'),
+        sa.UniqueConstraint(
+            'workflow_input_hash', 'workflow_name', 'pattern', 'project_id'
+        )
+    )

mistral/db/sqlalchemy/migration/cli.py

@@ -0,0 +1,132 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Starter script for mistral-db-manage."""
+
+import os
+
+from alembic import command as alembic_cmd
+from alembic import config as alembic_cfg
+from alembic import util as alembic_u
+from oslo.config import cfg
+import six
+
+from mistral.openstack.common import importutils
+from mistral.services import action_manager
+from mistral.services import workflows
+
+
+# We need to import mistral.api.app to
+# make sure we register all needed options.
+importutils.try_import('mistral.api.app')
+
+
+CONF = cfg.CONF
+
+
+def do_alembic_command(config, cmd, *args, **kwargs):
+    try:
+        getattr(alembic_cmd, cmd)(config, *args, **kwargs)
+    except alembic_u.CommandError as e:
+        alembic_u.err(six.text_type(e))
+
+
+def do_check_migration(config, _cmd):
+    do_alembic_command(config, 'branches')
+
+
+def do_upgrade_downgrade(config, cmd):
+    if not CONF.command.revision and not CONF.command.delta:
+        raise SystemExit('You must provide a revision or relative delta')
+
+    revision = CONF.command.revision
+
+    if CONF.command.delta:
+        sign = '+' if CONF.command.name == 'upgrade' else '-'
+        revision = sign + str(CONF.command.delta)
+
+    do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
+
+
+def do_stamp(config, cmd):
+    do_alembic_command(
+        config, cmd,
+        CONF.command.revision,
+        sql=CONF.command.sql
+    )
+
+
+def do_populate(config, cmd):
+    action_manager.sync_db()
+    workflows.sync_db()
+
+
+def do_revision(config, cmd):
+    do_alembic_command(
+        config, cmd,
+        message=CONF.command.message,
+        autogenerate=CONF.command.autogenerate,
+        sql=CONF.command.sql
+    )
+
+
+def add_command_parsers(subparsers):
+    for name in ['current', 'history', 'branches']:
+        parser = subparsers.add_parser(name)
+        parser.set_defaults(func=do_alembic_command)
+
+    parser = subparsers.add_parser('upgrade')
+    parser.add_argument('--delta', type=int)
+    parser.add_argument('--sql', action='store_true')
+    parser.add_argument('revision', nargs='?')
+    parser.set_defaults(func=do_upgrade_downgrade)
+
+    parser = subparsers.add_parser('populate')
+    parser.set_defaults(func=do_populate)
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('--sql', action='store_true')
+    parser.add_argument('revision', nargs='?')
+    parser.set_defaults(func=do_stamp)
+
+    parser = subparsers.add_parser('revision')
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.add_argument('--sql', action='store_true')
+    parser.set_defaults(func=do_revision)
+
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Command',
+                                help='Available commands',
+                                handler=add_command_parsers)
+
+CONF.register_cli_opt(command_opt)
+
+
+def main():
+    config = alembic_cfg.Config(
+        os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    )
+    config.set_main_option(
+        'script_location',
+        'mistral.db.sqlalchemy.migration:alembic_migrations'
+    )
+    # attach the Mistral conf to the Alembic conf
+    config.mistral_config = CONF
+
+    CONF(project='mistral')
+    CONF.command.func(config, CONF.command.name)
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -1,3 +1,4 @@
+alembic>=0.7.2
 pbr>=0.6,!=0.7,<1.0
 eventlet>=0.15.0
 PyYAML>=3.1.0

setup.cfg

@@ -32,6 +32,7 @@ upload-dir = doc/build/html
 [entry_points]
 console_scripts =
     mistral-server = mistral.cmd.launch:main
+    mistral-db-manage = mistral.db.sqlalchemy.migration.cli:main
 
 mistral.engine.drivers =
     default = mistral.engine.drivers.default.engine:DefaultEngine