From dbf0b22abb5a55e3bab5eb335eb47cd843a7a43d Mon Sep 17 00:00:00 2001 From: Mark McClain Date: Tue, 18 Dec 2012 13:27:39 -0500 Subject: [PATCH] Add migration support to Quantum implements blueprint quantum-db-upgrades This changeset provide database migration capabilities to Quantum by wrapping the Alembic library. Change-Id: I8ba3a07f5a65e0fda9c0e85ed9c07c5978c53bc7 --- MANIFEST.in | 4 + bin/quantum-db-manage | 25 + quantum/db/migration/README | 94 +++ quantum/db/migration/__init__.py | 24 + quantum/db/migration/alembic.ini | 52 ++ .../migration/alembic_migrations/__init__.py | 17 + .../db/migration/alembic_migrations/env.py | 100 +++ .../alembic_migrations/script.py.mako | 54 ++ .../versions/5a875d0e5c_ryu.py | 75 +++ .../alembic_migrations/versions/README | 5 + .../versions/folsom_initial.py | 574 ++++++++++++++++++ quantum/db/migration/cli.py | 128 ++++ quantum/tests/unit/test_db_migration.py | 117 ++++ quantum/tests/unit/test_db_plugin.py | 1 + setup.py | 3 +- tools/pip-requires | 1 + 16 files changed, 1273 insertions(+), 1 deletion(-) create mode 100755 bin/quantum-db-manage create mode 100644 quantum/db/migration/README create mode 100644 quantum/db/migration/__init__.py create mode 100644 quantum/db/migration/alembic.ini create mode 100644 quantum/db/migration/alembic_migrations/__init__.py create mode 100644 quantum/db/migration/alembic_migrations/env.py create mode 100644 quantum/db/migration/alembic_migrations/script.py.mako create mode 100644 quantum/db/migration/alembic_migrations/versions/5a875d0e5c_ryu.py create mode 100644 quantum/db/migration/alembic_migrations/versions/README create mode 100644 quantum/db/migration/alembic_migrations/versions/folsom_initial.py create mode 100644 quantum/db/migration/cli.py create mode 100644 quantum/tests/unit/test_db_migration.py diff --git a/MANIFEST.in b/MANIFEST.in index 6725543f610..d9cbbdaaafa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,10 @@ include AUTHORS include ChangeLog include quantum/versioninfo +include quantum/db/migration/README +include quantum/db/migration/alembic.ini +include quantum/db/migration/alembic/script.py.mako +include quantum/db/migration/alembic/versions/README exclude .gitignore exclude .gitreview diff --git a/bin/quantum-db-manage b/bin/quantum-db-manage new file mode 100755 index 00000000000..5c06512e40e --- /dev/null +++ b/bin/quantum-db-manage @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 New Dream Network, LLC (DreamHost) +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys +sys.path.insert(0, os.getcwd()) +from quantum.cli import main + + +main() diff --git a/quantum/db/migration/README b/quantum/db/migration/README new file mode 100644 index 00000000000..2eb7a09abe1 --- /dev/null +++ b/quantum/db/migration/README @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author Mark McClain (DreamHost) + +The migrations in the alembic/versions contain the changes needed to migrate +from older Quantum releases to newer versions. A migration occurs by executing +a script that details the changes needed to upgrade/downgrade the database. The +migration scripts are ordered so that multiple scripts can run sequentially to +update the database. The scripts are executed by Quantum's migration wrapper +which uses the Alembic library to manage the migration. Quantum supports +migration from Folsom or later. + + +If you are a deployer or developer and want to migrate from Folsom to Grizzly +or later you must first add version tracking to the database: + +$ quantum-db-manage -config-file /path/to/quantum.conf \ + --config-file /path/to/plugin/config.ini stamp folsom + +You can then upgrade to the latest database version via: +$ quantum-db-manage --config-file /path/to/quantum.conf \ + --config-file /path/to/plugin/config.ini upgrade head + +To check the current database version: +$ quantum-db-manage --config-file /path/to/quantum.conf \ + --config-file /path/to/plugin/config.ini current + +To create a script to run the migration offline: +$ quantum-db-manage --config-file /path/to/quantum.conf \ + --config-file /path/to/plugin/config.ini upgrade head --sql + +To run the offline migration between specific migration versions: +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini upgrade \ +: --sql + +Upgrade the database incrementally: +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini upgrade --delta <# of revs> + +Downgrade the database by a certain number of revisions: +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini downgrade --delta <# of revs> + + +DEVELOPERS: +A database migration script is required when you submit a change to Quantum +that alters the database model definition. The migration script is a special +python file that includes code to update/downgrade the database to match the +changes in the model definition. Alembic will execute these scripts in order to +provide a linear migration path between revision. The quantum-db-manage command +can be used to generate migration template for you to complete. The operations +in the template are those supported by the Alembic migration library. + +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini revision \ +-m "description of revision" \ +--autogenerate + +This generates a prepopulated template with the changes needed to match the +database state with the models. You should inspect the autogenerated template +to ensure that the proper models have been altered. + +In rare circumstances, you may want to start with an empty migration template +and manually author the changes necessary for an upgrade/downgrade. You can +create a blank file via: + +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini revision \ +-m "description of revision" + +The migration timeline should remain linear so that there is a clear path when +upgrading/downgrading. To verify that the timeline does branch, you can run +this command: +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini check_migration + +If the migration path does branch, you can find the branch point via: +$ quantum-db-manage --config-file /path/to/quantum.conf \ +--config-file /path/to/plugin/config.ini history diff --git a/quantum/db/migration/__init__.py b/quantum/db/migration/__init__.py new file mode 100644 index 00000000000..ec8012693a5 --- /dev/null +++ b/quantum/db/migration/__init__.py @@ -0,0 +1,24 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Mark McClain, DreamHost + + +def should_run(active_plugin, migrate_plugins): + if '*' in migrate_plugins: + return True + else: + return active_plugin in migrate_plugins diff --git a/quantum/db/migration/alembic.ini b/quantum/db/migration/alembic.ini new file mode 100644 index 00000000000..3b390b78937 --- /dev/null +++ b/quantum/db/migration/alembic.ini @@ -0,0 +1,52 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# default to an empty string because the Quantum migration cli will +# extract the correct value and set it programatically before alemic is fully +# invoked. +sqlalchemy.url = + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/quantum/db/migration/alembic_migrations/__init__.py b/quantum/db/migration/alembic_migrations/__init__.py new file mode 100644 index 00000000000..6e2c062056b --- /dev/null +++ b/quantum/db/migration/alembic_migrations/__init__.py @@ -0,0 +1,17 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Mark McClain, DreamHost diff --git a/quantum/db/migration/alembic_migrations/env.py b/quantum/db/migration/alembic_migrations/env.py new file mode 100644 index 00000000000..32904977994 --- /dev/null +++ b/quantum/db/migration/alembic_migrations/env.py @@ -0,0 +1,100 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Mark McClain, DreamHost + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +from quantum.db import model_base +from quantum.openstack.common import importutils + + +DATABASE_QUOTA_DRIVER = 'quantum.extensions._quotav2_driver.DbQuotaDriver' + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +quantum_config = config.quantum_config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +plugin_klass = importutils.import_class(quantum_config.core_plugin) + +# set the target for 'autogenerate' support +target_metadata = model_base.BASEV2.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + context.configure(url=quantum_config.DATABASE.sql_connection) + + with context.begin_transaction(): + context.run_migrations(active_plugin=quantum_config.core_plugin, + options=build_options()) + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = create_engine( + quantum_config.DATABASE.sql_connection, + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + try: + with context.begin_transaction(): + context.run_migrations(active_plugin=quantum_config.core_plugin, + options=build_options()) + finally: + connection.close() + + +def build_options(): + return {'folsom_quota_db_enabled': is_db_quota_enabled()} + + +def is_db_quota_enabled(): + return quantum_config.QUOTAS.quota_driver == DATABASE_QUOTA_DRIVER + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/quantum/db/migration/alembic_migrations/script.py.mako b/quantum/db/migration/alembic_migrations/script.py.mako new file mode 100644 index 00000000000..21cc9dc0fb0 --- /dev/null +++ b/quantum/db/migration/alembic_migrations/script.py.mako @@ -0,0 +1,54 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright ${create_date.year} OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +# Change to ['*'] if this migration applies to all plugins + +migration_for_plugins = [ + '${config.quantum_config.core_plugin}' +] + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +from quantum.db import migration + + +def upgrade(active_plugin=None, enable_db_quota=False): + if not migration.should_run(active_plugin, migration_for_plugins): + return + + ${upgrades if upgrades else "pass"} + + +def downgrade(active_plugin=None, enable_db_quota=False): + if not migration.should_run(active_plugin, migration_for_plugins): + return + + ${downgrades if downgrades else "pass"} diff --git a/quantum/db/migration/alembic_migrations/versions/5a875d0e5c_ryu.py b/quantum/db/migration/alembic_migrations/versions/5a875d0e5c_ryu.py new file mode 100644 index 00000000000..651ce569f86 --- /dev/null +++ b/quantum/db/migration/alembic_migrations/versions/5a875d0e5c_ryu.py @@ -0,0 +1,75 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Mark McClain, DreamHost + +"""ryu + +This retroactively provides migration support for +https://review.openstack.org/#/c/11204/ + +Revision ID: 5a875d0e5c +Revises: folsom +Create Date: 2012-12-18 12:32:04.482477 + +""" + + +# revision identifiers, used by Alembic. +revision = '5a875d0e5c' +down_revision = 'folsom' + +# Change to ['*'] if this migration applies to all plugins + +migration_for_plugins = [ + 'quantum.plugins.ryu.ryu_quantum_plugin.RyuQuantumPluginV2' +] + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +from quantum.db import migration + + +def upgrade(active_plugin=None, options=None): + if not migration.should_run(active_plugin, migration_for_plugins): + return + + op.create_table( + 'tunnelkeys', + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('last_key', sa.Integer(), autoincrement=False, + nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('last_key') + ) + + op.create_table( + 'tunnelkeylasts', + sa.Column('last_key', sa.Integer(), autoincrement=False, + nullable=False), + sa.PrimaryKeyConstraint('last_key') + ) + + +def downgrade(active_plugin=None, options=None): + if not migration.should_run(active_plugin, migration_for_plugins): + return + + op.drop_table('tunnelkeylasts') + op.drop_table('tunnelkeys') diff --git a/quantum/db/migration/alembic_migrations/versions/README b/quantum/db/migration/alembic_migrations/versions/README new file mode 100644 index 00000000000..5bbc79703e9 --- /dev/null +++ b/quantum/db/migration/alembic_migrations/versions/README @@ -0,0 +1,5 @@ +This directory contains the migration scripts for the Quantum project. Please +see the README in quantum/db/migration on how to use and generate new +migrations. + + diff --git a/quantum/db/migration/alembic_migrations/versions/folsom_initial.py b/quantum/db/migration/alembic_migrations/versions/folsom_initial.py new file mode 100644 index 00000000000..6eb7ceab68f --- /dev/null +++ b/quantum/db/migration/alembic_migrations/versions/folsom_initial.py @@ -0,0 +1,574 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author Mark McClain (DreamHost) + +"""folsom initial database + +Revision ID: folsom +Revises: None +Create Date: 2012-12-03 09:14:50.579765 + +""" + +PLUGINS = { + 'bigswitch': 'quantum.plugins.bigswitch.plugin.QuantumRestProxyV2', + 'cisco': 'quantum.plugins.cisco.network_plugin.PluginV2', + 'lbr': 'quantum.plugins.linuxbridge.lb_quantum_plugin.LinuxBridgePluginV2', + 'meta': 'quantum.plugins.metaplugin.meta_quantum_plugin.MetaPluginV2', + 'nec': 'quantum.plugins.nec.nec_plugin.NECPluginV2', + 'nvp': 'quantum.plugins.nicira/nicira_nvp_plugin/QuantumPlugin', + 'ovs': 'quantum.plugins.openvswitch.ovs_quantum_plugin.OVSQuantumPluginV2', + 'ryu': 'quantum.plugins.ryu.ryu_quantum_plugin.RyuQuantumPluginV2', +} + +L3_CAPABLE = [ + PLUGINS['lbr'], + PLUGINS['meta'], + PLUGINS['nec'], + PLUGINS['ovs'], + PLUGINS['ryu'], +] + +FOLSOM_QUOTA = [ + PLUGINS['lbr'], + PLUGINS['nvp'], + PLUGINS['ovs'], +] + + +# revision identifiers, used by Alembic. +revision = 'folsom' +down_revision = None + +from alembic import op +import sqlalchemy as sa + +# NOTE: This is a special migration that creates a Folsom compatible database. + + +def upgrade(active_plugin=None, options=None): + # general model + upgrade_base() + + if active_plugin in L3_CAPABLE: + upgrade_l3() + + if active_plugin in FOLSOM_QUOTA: + upgrade_quota(options) + + if active_plugin == PLUGINS['lbr']: + upgrade_linuxbridge() + elif active_plugin == PLUGINS['ovs']: + upgrade_ovs() + elif active_plugin == PLUGINS['cisco']: + upgrade_cisco() + # Cisco plugin imports OVS models too + upgrade_ovs() + elif active_plugin == PLUGINS['meta']: + upgrade_meta() + elif active_plugin == PLUGINS['nec']: + upgrade_nec() + elif active_plugin == PLUGINS['ryu']: + upgrade_ryu() + + +def upgrade_base(): + op.create_table( + 'networks', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=16), nullable=True), + sa.Column('admin_state_up', sa.Boolean(), nullable=True), + sa.Column('shared', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'subnets', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('network_id', sa.String(length=36), nullable=True), + sa.Column('ip_version', sa.Integer(), nullable=False), + sa.Column('cidr', sa.String(length=64), nullable=False), + sa.Column('gateway_ip', sa.String(length=64), nullable=True), + sa.Column('enable_dhcp', sa.Boolean(), nullable=True), + sa.Column('shared', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'ports', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('mac_address', sa.String(length=32), nullable=False), + sa.Column('admin_state_up', sa.Boolean(), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('device_id', sa.String(length=255), nullable=False), + sa.Column('device_owner', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'dnsnameservers', + sa.Column('address', sa.String(length=128), nullable=False), + sa.Column('subnet_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('address', 'subnet_id') + ) + + op.create_table( + 'ipallocations', + sa.Column('port_id', sa.String(length=36), nullable=True), + sa.Column('ip_address', sa.String(length=64), nullable=False), + sa.Column('subnet_id', sa.String(length=36), nullable=False), + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('expiration', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint(['port_id'], ['ports.id'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('ip_address', 'subnet_id', 'network_id') + ) + + op.create_table( + 'routes', + sa.Column('destination', sa.String(length=64), nullable=False), + sa.Column('nexthop', sa.String(length=64), nullable=False), + sa.Column('subnet_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('destination', 'nexthop', 'subnet_id') + ) + + op.create_table( + 'ipallocationpools', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('subnet_id', sa.String(length=36), nullable=True), + sa.Column('first_ip', sa.String(length=64), nullable=False), + sa.Column('last_ip', sa.String(length=64), nullable=False), + sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'ipavailabilityranges', + sa.Column('allocation_pool_id', sa.String(length=36), nullable=True), + sa.Column('first_ip', sa.String(length=64), nullable=False), + sa.Column('last_ip', sa.String(length=64), nullable=False), + sa.ForeignKeyConstraint(['allocation_pool_id'], + ['ipallocationpools.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('allocation_pool_id', 'first_ip', 'last_ip') + ) + + +def upgrade_l3(): + op.create_table( + 'routers', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=16), nullable=True), + sa.Column('admin_state_up', sa.Boolean(), nullable=True), + sa.Column('gw_port_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['gw_port_id'], ['ports.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'externalnetworks', + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('network_id') + ) + + op.create_table( + 'floatingips', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('floating_ip_address', sa.String(length=64), nullable=False), + sa.Column('floating_network_id', sa.String(length=36), nullable=False), + sa.Column('floating_port_id', sa.String(length=36), nullable=False), + sa.Column('fixed_port_id', sa.String(length=36), nullable=True), + sa.Column('fixed_ip_address', sa.String(length=64), nullable=True), + sa.Column('router_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['fixed_port_id'], ['ports.id'], ), + sa.ForeignKeyConstraint(['floating_port_id'], ['ports.id'], ), + sa.ForeignKeyConstraint(['router_id'], ['routers.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + +def upgrade_quota(options=None): + if not (options or {}).get('folsom_quota_db_enabled'): + return + + op.create_table( + 'quotas', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('tenant_id', sa.String(255), index=True), + sa.Column('resource', sa.String(255)), + sa.Column('limit', sa.Integer()), + sa.PrimaryKeyConstraint('id') + ) + + +def upgrade_linuxbridge(): + op.create_table( + 'network_states', + sa.Column('physical_network', sa.String(length=64), nullable=False), + sa.Column('vlan_id', sa.Integer(), autoincrement=False, + nullable=False), + sa.Column('allocated', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('physical_network', 'vlan_id') + ) + + op.create_table( + 'network_bindings', + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('physical_network', sa.String(length=64), nullable=True), + sa.Column('vlan_id', sa.Integer(), autoincrement=False, + nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('network_id') + ) + + +def upgrade_ovs(): + op.create_table( + 'ovs_tunnel_endpoints', + sa.Column('ip_address', sa.String(length=64), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('ip_address') + ) + + op.create_table( + 'ovs_tunnel_ips', + sa.Column('ip_address', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('ip_address') + ) + + op.create_table( + 'ovs_vlan_allocations', + sa.Column('physical_network', sa.String(length=64), nullable=False), + sa.Column('vlan_id', sa.Integer(), autoincrement=False, + nullable=False), + sa.Column('allocated', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('physical_network', 'vlan_id') + ) + + op.create_table( + 'ovs_tunnel_allocations', + sa.Column('tunnel_id', sa.Integer(), autoincrement=False, + nullable=False), + sa.Column('allocated', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('tunnel_id') + ) + + op.create_table( + 'ovs_network_bindings', + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('network_type', sa.String(length=32), nullable=False), + sa.Column('physical_network', sa.String(length=64), nullable=True), + sa.Column('segmentation_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('network_id') + ) + + +def upgrade_meta(): + op.create_table( + 'networkflavors', + sa.Column('flavor', sa.String(length=255)), + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('network_id') + ) + + op.create_table( + 'routerflavors', + sa.Column('flavor', sa.String(length=255)), + sa.Column('router_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['router_id'], ['routers.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('router_id') + ) + + +def upgrade_nec(): + op.create_table( + 'ofctenants', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('quantum_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'ofcnetworks', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('quantum_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'ofcports', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('quantum_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'ofcfilters', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('quantum_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'portinfos', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('datapath_id', sa.String(length=36), nullable=False), + sa.Column('port_no', sa.Integer(), nullable=False), + sa.Column('vlan_id', sa.Integer(), nullable=False), + sa.Column('mac', sa.String(length=32), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'packetfilters', + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('network_id', sa.String(length=36), nullable=True), + sa.Column('priority', sa.Integer(), nullable=False), + sa.Column('action', sa.String(16), nullable=False), + sa.Column('in_port', sa.String(36), nullable=False), + sa.Column('src_mac', sa.String(32), nullable=False), + sa.Column('dst_mac', sa.String(32), nullable=False), + sa.Column('eth_type', sa.Integer(), nullable=False), + sa.Column('src_cidr', sa.String(64), nullable=False), + sa.Column('dst_cidr', sa.String(64), nullable=False), + sa.Column('protocol', sa.String(16), nullable=False), + sa.Column('src_port', sa.Integer(), nullable=False), + sa.Column('dst_port', sa.Integer(), nullable=False), + sa.Column('admin_state_up', sa.Boolean(), nullable=False), + sa.Column('status', sa.String(16), nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + +def upgrade_ryu(): + op.create_table( + 'ofp_server', + sa.Column('id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('address', sa.String(255)), + sa.Column('host_type', sa.String(255)), + sa.PrimaryKeyConstraint('id') + ) + + +def upgrade_cisco(): + op.create_table( + 'cisco_vlan_ids', + sa.Column('vlan_id', sa.Integer(), autoincrement=True), + sa.Column('vlan_used', sa.Boolean()), + sa.PrimaryKeyConstraint('vlan_id') + ) + + op.create_table( + 'cisco_vlan_bindings', + sa.Column('vlan_id', sa.Integer(), autoincrement=True), + sa.Column('vlan_name', sa.String(255)), + sa.Column('network_id', sa.String(255), nullable=False), + sa.PrimaryKeyConstraint('vlan_id') + ) + + op.create_table( + 'portprofiles', + sa.Column('uuid', sa.String(255), nullable=False), + sa.Column('name', sa.String(255)), + sa.Column('vlan_id', sa.Integer()), + sa.Column('qos', sa.String(255)), + sa.PrimaryKeyConstraint('uuid') + ) + + op.create_table( + 'portprofile_bindings', + sa.Column('id', sa.Integer(), autoincrement=True), + sa.Column('tenant_id', sa.String(255)), + sa.Column('port_id', sa.String(255), nullable=False), + sa.Column('portprofile_id', sa.String(255), nullable=False), + sa.Column('default', sa.Boolean()), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['port_id'], ['ports.id'], ), + sa.ForeignKeyConstraint(['portprofile_id'], ['portprofiles.uuid'], ), + ) + + op.create_table( + 'qoss', # yes two S's + sa.Column('qos_id', sa.String(255)), + sa.Column('tenant_id', sa.String(255)), + sa.Column('qos_name', sa.String(255)), + sa.Column('qos_desc', sa.String(255)), + sa.PrimaryKeyConstraint('tenant_id', 'qos_name') + ) + + op.create_table( + 'credentials', + sa.Column('credential_id', sa.String(255)), + sa.Column('tenant_id', sa.String(255)), + sa.Column('credential_name', sa.String(255)), + sa.Column('user_name', sa.String(255)), + sa.Column('password', sa.String(255)), + sa.PrimaryKeyConstraint('tenant_id', 'credential_name') + ) + + op.create_table( + 'port_bindings', + sa.Column('id', sa.Integer(), autoincrement=True), + sa.Column('port_id', sa.String(255), nullable=False), + sa.Column('blade_intf_dn', sa.String(255), nullable=False), + sa.Column('portprofile_name', sa.String(255)), + sa.Column('vlan_name', sa.String(255)), + sa.Column('vlan_id', sa.Integer()), + sa.Column('qos', sa.String(255)), + sa.Column('tenant_id', sa.String(255)), + sa.Column('instance_id', sa.String(255)), + sa.Column('vif_id', sa.String(255)), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table( + 'nexusport_bindings', + sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True), + sa.Column('port_id', sa.String(255)), + sa.Column('vlan_id', sa.Integer(255)), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade(active_plugin=None, options=None): + if active_plugin == PLUGINS['lbr']: + downgrade_linuxbridge() + elif active_plugin == PLUGINS['ovs']: + downgrade_ovs() + elif active_plugin == PLUGINS['cisco']: + # Cisco plugin imports OVS models too + downgrade_ovs() + downgrade_cisco() + elif active_plugin == PLUGINS['meta']: + downgrade_meta() + elif active_plugin == PLUGINS['nec']: + downgrade_nec() + elif active_plugin == PLUGINS['ryu']: + downgrade_ryu() + + if active_plugin in FOLSOM_QUOTA: + downgrade_quota(options) + + if active_plugin in L3_CAPABLE: + downgrade_l3() + + downgrade_base() + + +def downgrade_base(): + drop_tables( + 'ipavailabilityranges', + 'ipallocationpools', + 'routes', + 'ipallocations', + 'dnsnameservers', + 'ports', + 'subnets', + 'networks' + ) + + +def downgrade_l3(): + drop_tables('floatingips', 'routers', 'externalnetworks') + + +def downgrade_quota(options=None): + if (options or {}).get('folsom_quota_db_enabled'): + drop_tables('quotas') + + +def downgrade_linuxbridge(): + drop_tables('network_bindings', 'network_states') + + +def downgrade_ovs(): + drop_tables( + 'ovs_network_bindings', + 'ovs_tunnel_allocations', + 'ovs_vlan_allocations', + 'ovs_tunnel_ips', + 'ovs_tunnel_endpoints' + ) + + +def downgrade_meta(): + drop_tables('routerflavors', 'networkflavors') + + +def downgrade_nec(): + drop_tables( + 'packetfilters', + 'portinfos', + 'ofcfilters', + 'ofcports', + 'ofcnetworks', + 'ofctenants' + ) + + +def downgrade_ryu(): + op.drop_table('ofp_server') + + +def downgrade_cisco(): + op.drop_tables( + 'nextport_bindings', + 'port_bindings', + 'credentials', + 'qoss', + 'portprofile_bindings', + 'portprofiles', + 'cisco_vlan_bindings', + 'cisco_vlan_ids' + ) + + +def drop_tables(*tables): + for table in tables: + op.drop_table(table) diff --git a/quantum/db/migration/cli.py b/quantum/db/migration/cli.py new file mode 100644 index 00000000000..50dd24ec1b5 --- /dev/null +++ b/quantum/db/migration/cli.py @@ -0,0 +1,128 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Mark McClain, DreamHost + +import os +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import util as alembic_util + +from quantum import manager +from quantum.openstack.common import cfg + +_core_opts = [ + cfg.StrOpt('core_plugin', + default='', + help='Quantum plugin provider module'), +] + +_quota_opts = [ + cfg.StrOpt('quota_driver', + default='', + help='Quantum quota driver class'), +] + +_db_opts = [ + cfg.StrOpt('sql_connection', + default='', + help='URL to database'), +] + +_cmd_opts = [ + cfg.StrOpt('message', + short='m', + default='', + help="Message string to use with 'revision'"), + cfg.BoolOpt('autogenerate', + default=False, + help=("Populate revision script with candidate " + "migration operations, based on comparison " + "of database to model.")), + cfg.BoolOpt('sql', + default=False, + help=("Don't emit SQL to database - dump to " + "standard output/file instead")), + cfg.IntOpt('delta', + default=0, + help='Number of relative migrations to upgrade/downgrade'), + +] + +CONF = cfg.CommonConfigOpts() +CONF.register_opts(_core_opts) +CONF.register_opts(_db_opts, 'DATABASE') +CONF.register_opts(_quota_opts, 'QUOTAS') +CONF.register_cli_opts(_cmd_opts) + + +def main(): + config = alembic_config.Config( + os.path.join(os.path.dirname(__file__), 'alembic.ini') + ) + config.set_main_option('script_location', + 'quantum.db.migration:alembic_migrations') + # attach the Quantum conf to the Alembic conf + config.quantum_config = CONF + + cmd, args, kwargs = process_argv(sys.argv) + + try: + getattr(alembic_command, cmd)(config, *args, **kwargs) + except alembic_util.CommandError, e: + alembic_util.err(str(e)) + + +def process_argv(argv): + positional = CONF(argv) + + if len(positional) > 1: + cmd = positional[1] + revision = positional[2:] and positional[2:][0] + + args = () + kwargs = {} + + if cmd == 'stamp': + args = (revision,) + kwargs = {'sql': CONF.sql} + elif cmd in ('current', 'history'): + pass # these commands do not require additional args + elif cmd in ('upgrade', 'downgrade'): + if CONF.delta: + revision = '%s%d' % ({'upgrade': '+', 'downgrade': '-'}[cmd], + CONF.delta) + elif not revision: + raise SystemExit( + _('You must provide a revision or relative delta') + ) + args = (revision,) + kwargs = {'sql': CONF.sql} + elif cmd == 'revision': + kwargs = { + 'message': CONF.message, + 'autogenerate': CONF.autogenerate, + 'sql': CONF.sql} + elif cmd == 'check_migration': + cmd = 'branches' + else: + raise SystemExit(_('Unrecognized Command: %s') % cmd) + + return cmd, args, kwargs + else: + raise SystemExit(_('You must provide a sub-command')) diff --git a/quantum/tests/unit/test_db_migration.py b/quantum/tests/unit/test_db_migration.py new file mode 100644 index 00000000000..9c54115b2c2 --- /dev/null +++ b/quantum/tests/unit/test_db_migration.py @@ -0,0 +1,117 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2012 New Dream Network, LLC (DreamHost) +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# @author Mark McClain (DreamHost) + +import sys + +import mock +import unittest2 as unittest + +from quantum.db import migration +from quantum.db.migration import cli + + +class TestDbMigration(unittest.TestCase): + def test_should_run_plugin_in_list(self): + self.assertTrue(migration.should_run('foo', ['foo', 'bar'])) + self.assertFalse(migration.should_run('foo', ['bar'])) + + def test_should_run_plugin_wildcard(self): + self.assertTrue(migration.should_run('foo', ['*'])) + + +class TestMain(unittest.TestCase): + def setUp(self): + self.process_argv_p = mock.patch.object(cli, 'process_argv') + self.process_argv = self.process_argv_p.start() + + self.alembic_cmd_p = mock.patch.object(cli, 'alembic_command') + self.alembic_cmd = self.alembic_cmd_p.start() + + def tearDown(self): + self.alembic_cmd_p.stop() + self.process_argv_p.stop() + + def test_main(self): + self.process_argv.return_value = ('foo', ('bar', ), {'baz': 1}) + cli.main() + + self.process_argv.assert_called_once_with(sys.argv) + self.alembic_cmd.foo.assert_called_once_with(mock.ANY, 'bar', baz=1) + + +class TestDatabaseSync(unittest.TestCase): + def test_process_argv_stamp(self): + self.assertEqual( + ('stamp', ('foo',), {'sql': False}), + cli.process_argv(['prog', 'stamp', 'foo'])) + + self.assertEqual( + ('stamp', ('foo',), {'sql': True}), + cli.process_argv(['prog', 'stamp', '--sql', 'foo'])) + + def test_process_argv_current(self): + self.assertEqual( + ('current', (), {}), + cli.process_argv(['prog', 'current'])) + + def test_process_argv_history(self): + self.assertEqual( + ('history', (), {}), + cli.process_argv(['prog', 'history'])) + + def test_process_argv_check_migration(self): + self.assertEqual( + ('branches', (), {}), + cli.process_argv(['prog', 'check_migration'])) + + def test_database_sync_revision(self): + expected = ( + 'revision', + (), + {'message': 'message', 'sql': False, 'autogenerate': True} + ) + + self.assertEqual( + cli.process_argv( + ['prog', 'revision', '-m', 'message', '--autogenerate'] + ), + expected + ) + + def test_database_sync_upgrade(self): + self.assertEqual( + cli.process_argv(['prog', 'upgrade', 'head']), + ('upgrade', ('head', ), {'sql': False}) + ) + + self.assertEqual( + cli.process_argv(['prog', 'upgrade', '--delta', '3']), + ('upgrade', ('+3', ), {'sql': False}) + ) + + def test_database_sync_downgrade(self): + self.assertEqual( + cli.process_argv(['prog', 'downgrade', 'folsom']), + ('downgrade', ('folsom', ), {'sql': False}) + ) + + self.assertEqual( + cli.process_argv(['prog', 'downgrade', '--delta', '2']), + ('downgrade', ('-2', ), {'sql': False}) + ) diff --git a/quantum/tests/unit/test_db_plugin.py b/quantum/tests/unit/test_db_plugin.py index 822f280a7b8..4f31b18da5f 100644 --- a/quantum/tests/unit/test_db_plugin.py +++ b/quantum/tests/unit/test_db_plugin.py @@ -23,6 +23,7 @@ import random import mock import unittest2 +import sqlalchemy as sa import webob.exc import quantum diff --git a/setup.py b/setup.py index d9c23547c2b..56d2452bbd8 100644 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ setuptools.setup( scripts=ProjectScripts, install_requires=requires, dependency_links=depend_links, - include_package_data=False, + include_package_data=True, setup_requires=['setuptools_git>=0.4'], packages=setuptools.find_packages('.'), cmdclass=setup.get_cmdclass(), @@ -140,6 +140,7 @@ setuptools.setup( 'quantum-server = quantum.server:main', 'quantum-debug = quantum.debug.shell:main', 'quantum-ovs-cleanup = quantum.agent.ovs_cleanup_util:main', + 'quantum-db-manage = quantum.db.migration.cli:main', ] }, ) diff --git a/tools/pip-requires b/tools/pip-requires index e3df85e2123..35ae81e4408 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -14,3 +14,4 @@ pyudev sqlalchemy==0.7.9 webob==1.2.3 python-keystoneclient>=0.2.0 +alembic>=0.4.1