From aa3b8ba7773da88c101cac0296a7b2b47f1fef82 Mon Sep 17 00:00:00 2001 From: Sam Betts Date: Tue, 15 Sep 2015 13:40:51 +0100 Subject: [PATCH] Add alembic migrations for the inspector database This patch adds a new command ironic-inspector-dbsync which can be used to sync the ironic inspector database using alembic migrations. It adds a migration to match the current required db schema. Change-Id: I21188b3f5003c8ab43d82903473e2a6ef7f755a0 Closes-Bug: #1495620 --- CONTRIBUTING.rst | 25 ++++++ README.rst | 35 ++++++++ devstack/plugin.sh | 6 ++ ironic_inspector/alembic.ini | 38 ++++++++ ironic_inspector/db.py | 3 - ironic_inspector/dbsync.py | 90 +++++++++++++++++++ ironic_inspector/migrations/env.py | 82 +++++++++++++++++ ironic_inspector/migrations/script.py.mako | 36 ++++++++ .../versions/578f84f38d_inital_db_schema.py | 63 +++++++++++++ .../versions/d588418040d_add_rules.py | 64 +++++++++++++ ironic_inspector/test/functional.py | 7 ++ setup.cfg | 1 + 12 files changed, 447 insertions(+), 3 deletions(-) create mode 100644 ironic_inspector/alembic.ini create mode 100644 ironic_inspector/dbsync.py create mode 100644 ironic_inspector/migrations/env.py create mode 100644 ironic_inspector/migrations/script.py.mako create mode 100644 ironic_inspector/migrations/versions/578f84f38d_inital_db_schema.py create mode 100644 ironic_inspector/migrations/versions/d588418040d_add_rules.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 20240e54b..165cf82ca 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -238,3 +238,28 @@ Writing a Plugin .. _ironic_inspector.plugins.base: https://github.com/openstack/ironic-inspector/blob/master/ironic_inspector/plugins/base.py .. _Introspection Rules: https://github.com/openstack/ironic-inspector#introspection-rules + +Adding migrations to the database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to make any changes to the database, you must add a new migration. +This can be done using alembic:: + + alembic --config ironic_inspector/alembic.ini revision -m "A short description" + +This will generate an empty migration file, with the correct revision +information already included. In this file there are two functions: + +* upgrade - The upgrade function is run when + ``ironic-inspector-dbsync upgrade`` is run, and should be populated with + code to bring the database up to its new state from the state it was in + after the last migration. + +* downgrade - The downgrade function should have code to undo the actions which + upgrade performs, returning the database to the state it would have been in + before the migration ran. + +For further information on creating a migration, refer to +`Create a Migration Script`_ from the alembic documentation. + +.. _Create a Migration Script: https://alembic.readthedocs.org/en/latest/tutorial.html#create-a-migration-script diff --git a/README.rst b/README.rst index b05ef8898..5821006c5 100644 --- a/README.rst +++ b/README.rst @@ -294,6 +294,41 @@ will be accessed by ramdisk on a booting machine). .. _ironic-discoverd-ramdisk element: https://github.com/openstack/diskimage-builder/tree/master/elements/ironic-discoverd-ramdisk .. _ironic-python-agent: https://github.com/openstack/ironic-python-agent +Managing the **ironic-inspector** database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**ironic-inspector** provides a command line client for managing its database, +this client can be used for upgrading, and downgrading the database using +alembic migrations. + +If this is your first time running **ironic-inspector** to migrate the +database simply run: +:: + + ironic-inspector-dbsync --config-file /etc/ironic-inspector/inspector.conf upgrade + +If you have previously run a version of **ironic-inspector** earlier than +2.2.0, to ensure your database will work with the migrations, you'll need to +run an extra step before upgrading the database. You only need to do this the +first time running version 2.2.0 or later. + +If you are upgrading from **ironic-inspector** version 2.1.0 or lower: +:: + + ironic-inspector-dbsync --config-file /etc/ironic-inspector/inspector.conf stamp --revision 578f84f38d + ironic-inspector-dbsync --config-file /etc/ironic-inspector/inspector.conf upgrade + +If you are upgrading from a git master install of **ironic-inspector** from +after `Introspection Rules`_ were introduced: +:: + + ironic-inspector-dbsync --config-file /etc/ironic-inspector/inspector.conf stamp --revision d588418040d + ironic-inspector-dbsync --config-file /etc/ironic-inspector/inspector.conf upgrade + +Other available commands can be discovered by running:: + + ironic-inspector-dbsync --help + Running ~~~~~~~ diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 05f192ee1..25ac2e657 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -2,6 +2,7 @@ IRONIC_INSPECTOR_DEBUG=${IRONIC_INSPECTOR_DEBUG:-false} IRONIC_INSPECTOR_DIR=$DEST/ironic-inspector IRONIC_INSPECTOR_BIN_DIR=$(get_python_exec_prefix) IRONIC_INSPECTOR_BIN_FILE=$IRONIC_INSPECTOR_BIN_DIR/ironic-inspector +IRONIC_INSPECTOR_DBSYNC_BIN_FILE=$IRONIC_INSPECTOR_BIN_DIR/ironic-inspector-dbsync IRONIC_INSPECTOR_CONF_DIR=${IRONIC_INSPECTOR_CONF_DIR:-/etc/ironic-inspector} IRONIC_INSPECTOR_CONF_FILE=$IRONIC_INSPECTOR_CONF_DIR/inspector.conf IRONIC_INSPECTOR_CMD="$IRONIC_INSPECTOR_BIN_FILE --config-file $IRONIC_INSPECTOR_CONF_FILE" @@ -220,6 +221,10 @@ function cleanup_inspector { sudo ovs-vsctl --if-exists del-port brbm-inspector } +function sync_inspector_database { + $IRONIC_INSPECTOR_DBSYNC_BIN_FILE --config-file $IRONIC_INSPECTOR_CONF_FILE upgrade +} + ### Entry points if [[ "$1" == "stack" && "$2" == "install" ]]; then @@ -236,6 +241,7 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then configure_inspector_dhcp fi configure_inspector + sync_inspector_database elif [[ "$1" == "stack" && "$2" == "extra" ]]; then echo_summary "Initializing ironic-inspector" prepare_environment diff --git a/ironic_inspector/alembic.ini b/ironic_inspector/alembic.ini new file mode 100644 index 000000000..d1831a656 --- /dev/null +++ b/ironic_inspector/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +# path to migration scripts +script_location = %(here)s/migrations + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ironic_inspector/db.py b/ironic_inspector/db.py index f716baef9..7300f827d 100644 --- a/ironic_inspector/db.py +++ b/ironic_inspector/db.py @@ -109,9 +109,6 @@ def init(): db_opts.set_defaults(CONF, connection='sqlite:///%s' % str(CONF.discoverd.database).strip()) - # TODO(yuikotakada) alembic migration - engine = get_engine() - Base.metadata.create_all(engine) return get_session() diff --git a/ironic_inspector/dbsync.py b/ironic_inspector/dbsync.py new file mode 100644 index 000000000..b3d197073 --- /dev/null +++ b/ironic_inspector/dbsync.py @@ -0,0 +1,90 @@ +# Copyright 2015 Cisco Systems +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import six +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import util as alembic_util + +from oslo_config import cfg +from oslo_db import options as db_opts +from oslo_log import log + +from ironic_inspector import conf # noqa + +CONF = cfg.CONF +db_opts.set_defaults(CONF) + + +def add_alembic_command(subparsers, name): + return subparsers.add_parser( + name, help=getattr(alembic_command, name).__doc__) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches', 'heads']: + parser = add_alembic_command(subparsers, name) + parser.set_defaults(func=do_alembic_command) + + for name in ['downgrade', 'stamp', 'show', 'edit']: + parser = add_alembic_command(subparsers, name) + parser.set_defaults(func=with_revision) + parser.add_argument('--revision', nargs='?', required=True) + + parser = add_alembic_command(subparsers, 'upgrade') + parser.set_defaults(func=with_revision) + parser.add_argument('--revision', nargs='?') + + parser = add_alembic_command(subparsers, 'revision') + parser.set_defaults(func=do_revision) + parser.add_argument('-m', '--message') + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def do_revision(config, cmd, *args, **kwargs): + do_alembic_command(config, cmd, message=CONF.command.message) + + +def with_revision(config, cmd, *args, **kwargs): + revision = CONF.command.revision or 'head' + do_alembic_command(config, cmd, revision) + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_command, cmd)(config, *args, **kwargs) + except alembic_util.CommandError as e: + alembic_util.err(six.text_type(e)) + + +def main(args=sys.argv[1:]): + log.register_options(CONF) + CONF(args, project='ironic-inspector') + config = alembic_config.Config(os.path.join(os.path.dirname(__file__), + 'alembic.ini')) + config.set_main_option('script_location', "ironic_inspector:migrations") + config.ironic_inspector_config = CONF + + CONF.command.func(config, CONF.command.name) diff --git a/ironic_inspector/migrations/env.py b/ironic_inspector/migrations/env.py new file mode 100644 index 000000000..c0796ef5b --- /dev/null +++ b/ironic_inspector/migrations/env.py @@ -0,0 +1,82 @@ +# Copyright 2015 Cisco Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import context +from logging.config import fileConfig +from sqlalchemy import create_engine + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +ironic_inspector_config = config.ironic_inspector_config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from ironic_inspector import db +target_metadata = db.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = ironic_inspector_config.database.connection + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = create_engine(ironic_inspector_config.database.connection) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ironic_inspector/migrations/script.py.mako b/ironic_inspector/migrations/script.py.mako new file mode 100644 index 000000000..d45a32ae0 --- /dev/null +++ b/ironic_inspector/migrations/script.py.mako @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ironic_inspector/migrations/versions/578f84f38d_inital_db_schema.py b/ironic_inspector/migrations/versions/578f84f38d_inital_db_schema.py new file mode 100644 index 000000000..dc7cf0b7f --- /dev/null +++ b/ironic_inspector/migrations/versions/578f84f38d_inital_db_schema.py @@ -0,0 +1,63 @@ +# Copyright 2015 Cisco Systems, Inc. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""inital_db_schema + +Revision ID: 578f84f38d +Revises: +Create Date: 2015-09-15 14:52:22.448944 + +""" + +# revision identifiers, used by Alembic. +revision = '578f84f38d' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'nodes', + sa.Column('uuid', sa.String(36), primary_key=True), + sa.Column('started_at', sa.Float, nullable=True), + sa.Column('finished_at', sa.Float, nullable=True), + sa.Column('error', sa.Text, nullable=True) + ) + + op.create_table( + 'attributes', + sa.Column('name', sa.Text, primary_key=True), + sa.Column('value', sa.Text, primary_key=True), + sa.Column('uuid', sa.String(36), sa.ForeignKey('nodes.uuid')) + ) + + op.create_table( + 'options', + sa.Column('uuid', sa.String(36), sa.ForeignKey('nodes.uuid'), + primary_key=True), + sa.Column('name', sa.Text, primary_key=True), + sa.Column('value', sa.Text) + ) + + +def downgrade(): + op.drop_table('nodes') + op.drop_table('attributes') + op.drop_table('options') diff --git a/ironic_inspector/migrations/versions/d588418040d_add_rules.py b/ironic_inspector/migrations/versions/d588418040d_add_rules.py new file mode 100644 index 000000000..4b93cdd71 --- /dev/null +++ b/ironic_inspector/migrations/versions/d588418040d_add_rules.py @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Add Rules + +Revision ID: d588418040d +Revises: 578f84f38d +Create Date: 2015-09-21 14:31:03.048455 + +""" + +# revision identifiers, used by Alembic. +revision = 'd588418040d' +down_revision = '578f84f38d' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +from oslo_db.sqlalchemy import types + + +def upgrade(): + op.create_table( + 'rules', + sa.Column('uuid', sa.String(36), primary_key=True), + sa.Column('created_at', sa.DateTime, nullable=False), + sa.Column('description', sa.Text), + sa.Column('disabled', sa.Boolean, default=False), + ) + + op.create_table( + 'rule_conditions', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('rule', sa.String(36), sa.ForeignKey('rules.uuid')), + sa.Column('op', sa.String(255), nullable=False), + sa.Column('multiple', sa.String(255), nullable=False), + sa.Column('field', sa.Text), + sa.Column('params', types.JsonEncodedDict) + ) + + op.create_table( + 'rule_actions', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('rule', sa.String(36), sa.ForeignKey('rules.uuid')), + sa.Column('action', sa.String(255), nullable=False), + sa.Column('params', types.JsonEncodedDict) + ) + + +def downgrade(): + op.drop_table('rules') + op.drop_table('rule_conditions') + op.drop_table('rule_actions') diff --git a/ironic_inspector/test/functional.py b/ironic_inspector/test/functional.py index 010d1b802..ea5ab3e00 100644 --- a/ironic_inspector/test/functional.py +++ b/ironic_inspector/test/functional.py @@ -22,9 +22,11 @@ import tempfile import unittest import mock +from oslo_config import cfg from oslo_utils import units import requests +from ironic_inspector import dbsync from ironic_inspector import main from ironic_inspector import rules from ironic_inspector.test import base @@ -364,6 +366,11 @@ def mocked_server(): with mock.patch.object(utils, 'check_auth'): with mock.patch.object(utils, 'get_client'): + dbsync.main(args=['--config-file', conf_file, 'upgrade']) + + cfg.CONF.reset() + cfg.CONF.unregister_opt(dbsync.command_opt) + eventlet.greenthread.spawn_n(main.main, args=['--config-file', conf_file], in_functional_test=True) diff --git a/setup.cfg b/setup.cfg index 74a4ccde2..0d44a1d1a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,7 @@ packages = [entry_points] console_scripts = ironic-inspector = ironic_inspector.main:main + ironic-inspector-dbsync = ironic_inspector.dbsync:main ironic-inspector-rootwrap = oslo_rootwrap.cmd:main ironic_inspector.hooks.processing = scheduler = ironic_inspector.plugins.standard:SchedulerHook