Add migration support to Quantum

implements blueprint quantum-db-upgrades

This changeset provide database migration capabilities to Quantum by
wrapping the Alembic library.

Change-Id: I8ba3a07f5a65e0fda9c0e85ed9c07c5978c53bc7

MANIFEST.in

@@ -1,6 +1,10 @@
 include AUTHORS
 include ChangeLog
 include quantum/versioninfo
+include quantum/db/migration/README
+include quantum/db/migration/alembic.ini
+include quantum/db/migration/alembic/script.py.mako
+include quantum/db/migration/alembic/versions/README
 
 exclude .gitignore
 exclude .gitreview

bin/quantum-db-manage

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+from quantum.cli import main
+
+
+main()

quantum/db/migration/README

@@ -0,0 +1,94 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author Mark McClain (DreamHost)
+
+The migrations in the alembic/versions contain the changes needed to migrate
+from older Quantum releases to newer versions. A migration occurs by executing
+a script that details the changes needed to upgrade/downgrade the database. The
+migration scripts are ordered so that multiple scripts can run sequentially to
+update the database. The scripts are executed by Quantum's migration wrapper
+which uses the Alembic library to manage the migration.  Quantum supports
+migration from Folsom or later.
+
+
+If you are a deployer or developer and want to migrate from Folsom to Grizzly
+or later you must first add version tracking to the database:
+
+$ quantum-db-manage -config-file /path/to/quantum.conf \
+ --config-file /path/to/plugin/config.ini stamp folsom
+
+You can then upgrade to the latest database version via:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+ --config-file /path/to/plugin/config.ini upgrade head
+
+To check the current database version:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+ --config-file /path/to/plugin/config.ini current
+
+To create a script to run the migration offline:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+ --config-file /path/to/plugin/config.ini upgrade head --sql
+
+To run the offline migration between specific migration versions:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini upgrade \
+<start version>:<end version> --sql
+
+Upgrade the database incrementally:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini upgrade --delta <# of revs>
+
+Downgrade the database by a certain number of revisions:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini downgrade --delta <# of revs>
+
+
+DEVELOPERS:
+A database migration script is required when you submit a change to Quantum
+that alters the database model definition.  The migration script is a special
+python file that includes code to update/downgrade the database to match the
+changes in the model definition. Alembic will execute these scripts in order to
+provide a linear migration path between revision. The quantum-db-manage command
+can be used to generate migration template for you to complete.  The operations
+in the template are those supported by the Alembic migration library.
+
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini revision \
+-m "description of revision" \
+--autogenerate
+
+This generates a prepopulated template with the changes needed to match the
+database state with the models.  You should inspect the autogenerated template
+to ensure that the proper models have been altered.
+
+In rare circumstances, you may want to start with an empty migration template
+and manually author the changes necessary for an upgrade/downgrade.  You can
+create a blank file via:
+
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini revision \
+-m "description of revision"
+
+The migration timeline should remain linear so that there is a clear path when
+upgrading/downgrading.  To verify that the timeline does branch, you can run
+this command:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini check_migration
+
+If the migration path does branch, you can find the branch point via:
+$ quantum-db-manage --config-file /path/to/quantum.conf \
+--config-file /path/to/plugin/config.ini history

quantum/db/migration/__init__.py

@@ -0,0 +1,24 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Mark McClain, DreamHost
+
+
+def should_run(active_plugin, migrate_plugins):
+    if '*' in migrate_plugins:
+        return True
+    else:
+        return active_plugin in migrate_plugins

quantum/db/migration/alembic.ini

@@ -0,0 +1,52 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# default to an empty string because the Quantum migration cli will
+# extract the correct value and set it programatically before alemic is fully
+# invoked.
+sqlalchemy.url =
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

quantum/db/migration/alembic_migrations/__init__.py

@@ -0,0 +1,17 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Mark McClain, DreamHost

quantum/db/migration/alembic_migrations/env.py

@@ -0,0 +1,100 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Mark McClain, DreamHost
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import create_engine, pool
+
+from quantum.db import model_base
+from quantum.openstack.common import importutils
+
+
+DATABASE_QUOTA_DRIVER = 'quantum.extensions._quotav2_driver.DbQuotaDriver'
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+quantum_config = config.quantum_config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+plugin_klass = importutils.import_class(quantum_config.core_plugin)
+
+# set the target for 'autogenerate' support
+target_metadata = model_base.BASEV2.metadata
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(url=quantum_config.DATABASE.sql_connection)
+
+    with context.begin_transaction():
+        context.run_migrations(active_plugin=quantum_config.core_plugin,
+                               options=build_options())
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = create_engine(
+        quantum_config.DATABASE.sql_connection,
+        poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations(active_plugin=quantum_config.core_plugin,
+                                   options=build_options())
+    finally:
+        connection.close()
+
+
+def build_options():
+    return {'folsom_quota_db_enabled': is_db_quota_enabled()}
+
+
+def is_db_quota_enabled():
+    return quantum_config.QUOTAS.quota_driver == DATABASE_QUOTA_DRIVER
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

quantum/db/migration/alembic_migrations/script.py.mako

@@ -0,0 +1,54 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright ${create_date.year} OpenStack LLC
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+# Change to ['*'] if this migration applies to all plugins
+
+migration_for_plugins = [
+    '${config.quantum_config.core_plugin}'
+]
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+from quantum.db import migration
+
+
+def upgrade(active_plugin=None, enable_db_quota=False):
+    if not migration.should_run(active_plugin, migration_for_plugins):
+        return
+
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade(active_plugin=None, enable_db_quota=False):
+    if not migration.should_run(active_plugin, migration_for_plugins):
+        return
+
+    ${downgrades if downgrades else "pass"}

quantum/db/migration/alembic_migrations/versions/5a875d0e5c_ryu.py

@@ -0,0 +1,75 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Mark McClain, DreamHost
+
+"""ryu
+
+This retroactively provides migration support for
+https://review.openstack.org/#/c/11204/
+
+Revision ID: 5a875d0e5c
+Revises: folsom
+Create Date: 2012-12-18 12:32:04.482477
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = '5a875d0e5c'
+down_revision = 'folsom'
+
+# Change to ['*'] if this migration applies to all plugins
+
+migration_for_plugins = [
+    'quantum.plugins.ryu.ryu_quantum_plugin.RyuQuantumPluginV2'
+]
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+from quantum.db import migration
+
+
+def upgrade(active_plugin=None, options=None):
+    if not migration.should_run(active_plugin, migration_for_plugins):
+        return
+
+    op.create_table(
+        'tunnelkeys',
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('last_key', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('last_key')
+    )
+
+    op.create_table(
+        'tunnelkeylasts',
+        sa.Column('last_key', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.PrimaryKeyConstraint('last_key')
+    )
+
+
+def downgrade(active_plugin=None, options=None):
+    if not migration.should_run(active_plugin, migration_for_plugins):
+        return
+
+    op.drop_table('tunnelkeylasts')
+    op.drop_table('tunnelkeys')

quantum/db/migration/alembic_migrations/versions/README

@@ -0,0 +1,5 @@
+This directory contains the migration scripts for the Quantum project.  Please
+see the README in quantum/db/migration on how to use and generate new
+migrations.
+
+

quantum/db/migration/alembic_migrations/versions/folsom_initial.py

@@ -0,0 +1,574 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author Mark McClain (DreamHost)
+
+"""folsom initial database
+
+Revision ID: folsom
+Revises: None
+Create Date: 2012-12-03 09:14:50.579765
+
+"""
+
+PLUGINS = {
+    'bigswitch': 'quantum.plugins.bigswitch.plugin.QuantumRestProxyV2',
+    'cisco': 'quantum.plugins.cisco.network_plugin.PluginV2',
+    'lbr': 'quantum.plugins.linuxbridge.lb_quantum_plugin.LinuxBridgePluginV2',
+    'meta': 'quantum.plugins.metaplugin.meta_quantum_plugin.MetaPluginV2',
+    'nec': 'quantum.plugins.nec.nec_plugin.NECPluginV2',
+    'nvp': 'quantum.plugins.nicira/nicira_nvp_plugin/QuantumPlugin',
+    'ovs': 'quantum.plugins.openvswitch.ovs_quantum_plugin.OVSQuantumPluginV2',
+    'ryu': 'quantum.plugins.ryu.ryu_quantum_plugin.RyuQuantumPluginV2',
+}
+
+L3_CAPABLE = [
+    PLUGINS['lbr'],
+    PLUGINS['meta'],
+    PLUGINS['nec'],
+    PLUGINS['ovs'],
+    PLUGINS['ryu'],
+]
+
+FOLSOM_QUOTA = [
+    PLUGINS['lbr'],
+    PLUGINS['nvp'],
+    PLUGINS['ovs'],
+]
+
+
+# revision identifiers, used by Alembic.
+revision = 'folsom'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+# NOTE: This is a special migration that creates a Folsom compatible database.
+
+
+def upgrade(active_plugin=None, options=None):
+    # general model
+    upgrade_base()
+
+    if active_plugin in L3_CAPABLE:
+        upgrade_l3()
+
+    if active_plugin in FOLSOM_QUOTA:
+        upgrade_quota(options)
+
+    if active_plugin == PLUGINS['lbr']:
+        upgrade_linuxbridge()
+    elif active_plugin == PLUGINS['ovs']:
+        upgrade_ovs()
+    elif active_plugin == PLUGINS['cisco']:
+        upgrade_cisco()
+        # Cisco plugin imports OVS models too
+        upgrade_ovs()
+    elif active_plugin == PLUGINS['meta']:
+        upgrade_meta()
+    elif active_plugin == PLUGINS['nec']:
+        upgrade_nec()
+    elif active_plugin == PLUGINS['ryu']:
+        upgrade_ryu()
+
+
+def upgrade_base():
+    op.create_table(
+        'networks',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True),
+        sa.Column('status', sa.String(length=16), nullable=True),
+        sa.Column('admin_state_up', sa.Boolean(), nullable=True),
+        sa.Column('shared', sa.Boolean(), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'subnets',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True),
+        sa.Column('network_id', sa.String(length=36), nullable=True),
+        sa.Column('ip_version', sa.Integer(), nullable=False),
+        sa.Column('cidr', sa.String(length=64), nullable=False),
+        sa.Column('gateway_ip', sa.String(length=64), nullable=True),
+        sa.Column('enable_dhcp', sa.Boolean(), nullable=True),
+        sa.Column('shared', sa.Boolean(), nullable=True),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'ports',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True),
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('mac_address', sa.String(length=32), nullable=False),
+        sa.Column('admin_state_up', sa.Boolean(), nullable=False),
+        sa.Column('status', sa.String(length=16), nullable=False),
+        sa.Column('device_id', sa.String(length=255), nullable=False),
+        sa.Column('device_owner', sa.String(length=255), nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'dnsnameservers',
+        sa.Column('address', sa.String(length=128), nullable=False),
+        sa.Column('subnet_id', sa.String(length=36), nullable=False),
+        sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('address', 'subnet_id')
+    )
+
+    op.create_table(
+        'ipallocations',
+        sa.Column('port_id', sa.String(length=36), nullable=True),
+        sa.Column('ip_address', sa.String(length=64), nullable=False),
+        sa.Column('subnet_id', sa.String(length=36), nullable=False),
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('expiration', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.ForeignKeyConstraint(['port_id'], ['ports.id'],
+                                ondelete='CASCADE'),
+        sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('ip_address', 'subnet_id', 'network_id')
+    )
+
+    op.create_table(
+        'routes',
+        sa.Column('destination', sa.String(length=64), nullable=False),
+        sa.Column('nexthop', sa.String(length=64), nullable=False),
+        sa.Column('subnet_id', sa.String(length=36), nullable=False),
+        sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('destination', 'nexthop', 'subnet_id')
+    )
+
+    op.create_table(
+        'ipallocationpools',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('subnet_id', sa.String(length=36), nullable=True),
+        sa.Column('first_ip', sa.String(length=64), nullable=False),
+        sa.Column('last_ip', sa.String(length=64), nullable=False),
+        sa.ForeignKeyConstraint(['subnet_id'], ['subnets.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'ipavailabilityranges',
+        sa.Column('allocation_pool_id', sa.String(length=36), nullable=True),
+        sa.Column('first_ip', sa.String(length=64), nullable=False),
+        sa.Column('last_ip', sa.String(length=64), nullable=False),
+        sa.ForeignKeyConstraint(['allocation_pool_id'],
+                                ['ipallocationpools.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('allocation_pool_id', 'first_ip', 'last_ip')
+    )
+
+
+def upgrade_l3():
+    op.create_table(
+        'routers',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True),
+        sa.Column('status', sa.String(length=16), nullable=True),
+        sa.Column('admin_state_up', sa.Boolean(), nullable=True),
+        sa.Column('gw_port_id', sa.String(length=36), nullable=True),
+        sa.ForeignKeyConstraint(['gw_port_id'], ['ports.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'externalnetworks',
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('network_id')
+    )
+
+    op.create_table(
+        'floatingips',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('floating_ip_address', sa.String(length=64), nullable=False),
+        sa.Column('floating_network_id', sa.String(length=36), nullable=False),
+        sa.Column('floating_port_id', sa.String(length=36), nullable=False),
+        sa.Column('fixed_port_id', sa.String(length=36), nullable=True),
+        sa.Column('fixed_ip_address', sa.String(length=64), nullable=True),
+        sa.Column('router_id', sa.String(length=36), nullable=True),
+        sa.ForeignKeyConstraint(['fixed_port_id'], ['ports.id'], ),
+        sa.ForeignKeyConstraint(['floating_port_id'], ['ports.id'], ),
+        sa.ForeignKeyConstraint(['router_id'], ['routers.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def upgrade_quota(options=None):
+    if not (options or {}).get('folsom_quota_db_enabled'):
+        return
+
+    op.create_table(
+        'quotas',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('tenant_id', sa.String(255), index=True),
+        sa.Column('resource', sa.String(255)),
+        sa.Column('limit', sa.Integer()),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def upgrade_linuxbridge():
+    op.create_table(
+        'network_states',
+        sa.Column('physical_network', sa.String(length=64), nullable=False),
+        sa.Column('vlan_id', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.Column('allocated', sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint('physical_network', 'vlan_id')
+    )
+
+    op.create_table(
+        'network_bindings',
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('physical_network', sa.String(length=64), nullable=True),
+        sa.Column('vlan_id', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('network_id')
+    )
+
+
+def upgrade_ovs():
+    op.create_table(
+        'ovs_tunnel_endpoints',
+        sa.Column('ip_address', sa.String(length=64), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('ip_address')
+    )
+
+    op.create_table(
+        'ovs_tunnel_ips',
+        sa.Column('ip_address', sa.String(length=255), nullable=False),
+        sa.PrimaryKeyConstraint('ip_address')
+    )
+
+    op.create_table(
+        'ovs_vlan_allocations',
+        sa.Column('physical_network', sa.String(length=64), nullable=False),
+        sa.Column('vlan_id', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.Column('allocated', sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint('physical_network', 'vlan_id')
+    )
+
+    op.create_table(
+        'ovs_tunnel_allocations',
+        sa.Column('tunnel_id', sa.Integer(), autoincrement=False,
+                  nullable=False),
+        sa.Column('allocated', sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint('tunnel_id')
+    )
+
+    op.create_table(
+        'ovs_network_bindings',
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.Column('network_type', sa.String(length=32), nullable=False),
+        sa.Column('physical_network', sa.String(length=64), nullable=True),
+        sa.Column('segmentation_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('network_id')
+    )
+
+
+def upgrade_meta():
+    op.create_table(
+        'networkflavors',
+        sa.Column('flavor', sa.String(length=255)),
+        sa.Column('network_id', sa.String(length=36), nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('network_id')
+    )
+
+    op.create_table(
+        'routerflavors',
+        sa.Column('flavor', sa.String(length=255)),
+        sa.Column('router_id', sa.String(length=36), nullable=False),
+        sa.ForeignKeyConstraint(['router_id'], ['routers.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('router_id')
+    )
+
+
+def upgrade_nec():
+    op.create_table(
+        'ofctenants',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('quantum_id', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'ofcnetworks',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('quantum_id', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'ofcports',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('quantum_id', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'ofcfilters',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('quantum_id', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'portinfos',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('datapath_id', sa.String(length=36), nullable=False),
+        sa.Column('port_no', sa.Integer(), nullable=False),
+        sa.Column('vlan_id', sa.Integer(), nullable=False),
+        sa.Column('mac', sa.String(length=32), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'packetfilters',
+        sa.Column('tenant_id', sa.String(length=255), nullable=True),
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('network_id', sa.String(length=36), nullable=True),
+        sa.Column('priority', sa.Integer(), nullable=False),
+        sa.Column('action', sa.String(16), nullable=False),
+        sa.Column('in_port', sa.String(36), nullable=False),
+        sa.Column('src_mac', sa.String(32), nullable=False),
+        sa.Column('dst_mac', sa.String(32), nullable=False),
+        sa.Column('eth_type', sa.Integer(), nullable=False),
+        sa.Column('src_cidr', sa.String(64), nullable=False),
+        sa.Column('dst_cidr', sa.String(64), nullable=False),
+        sa.Column('protocol', sa.String(16), nullable=False),
+        sa.Column('src_port', sa.Integer(), nullable=False),
+        sa.Column('dst_port', sa.Integer(), nullable=False),
+        sa.Column('admin_state_up', sa.Boolean(), nullable=False),
+        sa.Column('status', sa.String(16), nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def upgrade_ryu():
+    op.create_table(
+        'ofp_server',
+        sa.Column('id', sa.Integer(), autoincrement=False, nullable=False),
+        sa.Column('address', sa.String(255)),
+        sa.Column('host_type', sa.String(255)),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def upgrade_cisco():
+    op.create_table(
+        'cisco_vlan_ids',
+        sa.Column('vlan_id', sa.Integer(), autoincrement=True),
+        sa.Column('vlan_used', sa.Boolean()),
+        sa.PrimaryKeyConstraint('vlan_id')
+    )
+
+    op.create_table(
+        'cisco_vlan_bindings',
+        sa.Column('vlan_id', sa.Integer(), autoincrement=True),
+        sa.Column('vlan_name', sa.String(255)),
+        sa.Column('network_id', sa.String(255), nullable=False),
+        sa.PrimaryKeyConstraint('vlan_id')
+    )
+
+    op.create_table(
+        'portprofiles',
+        sa.Column('uuid', sa.String(255), nullable=False),
+        sa.Column('name', sa.String(255)),
+        sa.Column('vlan_id', sa.Integer()),
+        sa.Column('qos', sa.String(255)),
+        sa.PrimaryKeyConstraint('uuid')
+    )
+
+    op.create_table(
+        'portprofile_bindings',
+        sa.Column('id', sa.Integer(), autoincrement=True),
+        sa.Column('tenant_id', sa.String(255)),
+        sa.Column('port_id', sa.String(255), nullable=False),
+        sa.Column('portprofile_id', sa.String(255), nullable=False),
+        sa.Column('default', sa.Boolean()),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['port_id'], ['ports.id'], ),
+        sa.ForeignKeyConstraint(['portprofile_id'], ['portprofiles.uuid'], ),
+    )
+
+    op.create_table(
+        'qoss',  # yes two S's
+        sa.Column('qos_id', sa.String(255)),
+        sa.Column('tenant_id', sa.String(255)),
+        sa.Column('qos_name', sa.String(255)),
+        sa.Column('qos_desc', sa.String(255)),
+        sa.PrimaryKeyConstraint('tenant_id', 'qos_name')
+    )
+
+    op.create_table(
+        'credentials',
+        sa.Column('credential_id', sa.String(255)),
+        sa.Column('tenant_id', sa.String(255)),
+        sa.Column('credential_name', sa.String(255)),
+        sa.Column('user_name', sa.String(255)),
+        sa.Column('password', sa.String(255)),
+        sa.PrimaryKeyConstraint('tenant_id', 'credential_name')
+    )
+
+    op.create_table(
+        'port_bindings',
+        sa.Column('id', sa.Integer(), autoincrement=True),
+        sa.Column('port_id', sa.String(255), nullable=False),
+        sa.Column('blade_intf_dn', sa.String(255), nullable=False),
+        sa.Column('portprofile_name', sa.String(255)),
+        sa.Column('vlan_name', sa.String(255)),
+        sa.Column('vlan_id', sa.Integer()),
+        sa.Column('qos', sa.String(255)),
+        sa.Column('tenant_id', sa.String(255)),
+        sa.Column('instance_id', sa.String(255)),
+        sa.Column('vif_id', sa.String(255)),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    op.create_table(
+        'nexusport_bindings',
+        sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
+        sa.Column('port_id', sa.String(255)),
+        sa.Column('vlan_id', sa.Integer(255)),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def downgrade(active_plugin=None, options=None):
+    if active_plugin == PLUGINS['lbr']:
+        downgrade_linuxbridge()
+    elif active_plugin == PLUGINS['ovs']:
+        downgrade_ovs()
+    elif active_plugin == PLUGINS['cisco']:
+        # Cisco plugin imports OVS models too
+        downgrade_ovs()
+        downgrade_cisco()
+    elif active_plugin == PLUGINS['meta']:
+        downgrade_meta()
+    elif active_plugin == PLUGINS['nec']:
+        downgrade_nec()
+    elif active_plugin == PLUGINS['ryu']:
+        downgrade_ryu()
+
+    if active_plugin in FOLSOM_QUOTA:
+        downgrade_quota(options)
+
+    if active_plugin in L3_CAPABLE:
+        downgrade_l3()
+
+    downgrade_base()
+
+
+def downgrade_base():
+    drop_tables(
+        'ipavailabilityranges',
+        'ipallocationpools',
+        'routes',
+        'ipallocations',
+        'dnsnameservers',
+        'ports',
+        'subnets',
+        'networks'
+    )
+
+
+def downgrade_l3():
+    drop_tables('floatingips', 'routers', 'externalnetworks')
+
+
+def downgrade_quota(options=None):
+    if (options or {}).get('folsom_quota_db_enabled'):
+        drop_tables('quotas')
+
+
+def downgrade_linuxbridge():
+    drop_tables('network_bindings', 'network_states')
+
+
+def downgrade_ovs():
+    drop_tables(
+        'ovs_network_bindings',
+        'ovs_tunnel_allocations',
+        'ovs_vlan_allocations',
+        'ovs_tunnel_ips',
+        'ovs_tunnel_endpoints'
+    )
+
+
+def downgrade_meta():
+    drop_tables('routerflavors', 'networkflavors')
+
+
+def downgrade_nec():
+    drop_tables(
+        'packetfilters',
+        'portinfos',
+        'ofcfilters',
+        'ofcports',
+        'ofcnetworks',
+        'ofctenants'
+    )
+
+
+def downgrade_ryu():
+    op.drop_table('ofp_server')
+
+
+def downgrade_cisco():
+    op.drop_tables(
+        'nextport_bindings',
+        'port_bindings',
+        'credentials',
+        'qoss',
+        'portprofile_bindings',
+        'portprofiles',
+        'cisco_vlan_bindings',
+        'cisco_vlan_ids'
+    )
+
+
+def drop_tables(*tables):
+    for table in tables:
+        op.drop_table(table)

quantum/db/migration/cli.py

@@ -0,0 +1,128 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Mark McClain, DreamHost
+
+import os
+import sys
+
+from alembic import command as alembic_command
+from alembic import config as alembic_config
+from alembic import util as alembic_util
+
+from quantum import manager
+from quantum.openstack.common import cfg
+
+_core_opts = [
+    cfg.StrOpt('core_plugin',
+               default='',
+               help='Quantum plugin provider module'),
+]
+
+_quota_opts = [
+    cfg.StrOpt('quota_driver',
+               default='',
+               help='Quantum quota driver class'),
+]
+
+_db_opts = [
+    cfg.StrOpt('sql_connection',
+               default='',
+               help='URL to database'),
+]
+
+_cmd_opts = [
+    cfg.StrOpt('message',
+               short='m',
+               default='',
+               help="Message string to use with 'revision'"),
+    cfg.BoolOpt('autogenerate',
+                default=False,
+                help=("Populate revision script with candidate "
+                      "migration operations, based on comparison "
+                      "of database to model.")),
+    cfg.BoolOpt('sql',
+                default=False,
+                help=("Don't emit SQL to database - dump to "
+                      "standard output/file instead")),
+    cfg.IntOpt('delta',
+               default=0,
+               help='Number of relative migrations to upgrade/downgrade'),
+
+]
+
+CONF = cfg.CommonConfigOpts()
+CONF.register_opts(_core_opts)
+CONF.register_opts(_db_opts, 'DATABASE')
+CONF.register_opts(_quota_opts, 'QUOTAS')
+CONF.register_cli_opts(_cmd_opts)
+
+
+def main():
+    config = alembic_config.Config(
+        os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    )
+    config.set_main_option('script_location',
+                           'quantum.db.migration:alembic_migrations')
+    # attach the Quantum conf to the Alembic conf
+    config.quantum_config = CONF
+
+    cmd, args, kwargs = process_argv(sys.argv)
+
+    try:
+        getattr(alembic_command, cmd)(config, *args, **kwargs)
+    except alembic_util.CommandError, e:
+        alembic_util.err(str(e))
+
+
+def process_argv(argv):
+    positional = CONF(argv)
+
+    if len(positional) > 1:
+        cmd = positional[1]
+        revision = positional[2:] and positional[2:][0]
+
+        args = ()
+        kwargs = {}
+
+        if cmd == 'stamp':
+            args = (revision,)
+            kwargs = {'sql': CONF.sql}
+        elif cmd in ('current', 'history'):
+            pass  # these commands do not require additional args
+        elif cmd in ('upgrade', 'downgrade'):
+            if CONF.delta:
+                revision = '%s%d' % ({'upgrade': '+', 'downgrade': '-'}[cmd],
+                                     CONF.delta)
+            elif not revision:
+                raise SystemExit(
+                    _('You must provide a revision or relative delta')
+                )
+            args = (revision,)
+            kwargs = {'sql': CONF.sql}
+        elif cmd == 'revision':
+            kwargs = {
+                'message': CONF.message,
+                'autogenerate': CONF.autogenerate,
+                'sql': CONF.sql}
+        elif cmd == 'check_migration':
+            cmd = 'branches'
+        else:
+            raise SystemExit(_('Unrecognized Command: %s') % cmd)
+
+        return cmd, args, kwargs
+    else:
+        raise SystemExit(_('You must provide a sub-command'))

quantum/tests/unit/test_db_migration.py

@@ -0,0 +1,117 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# @author Mark McClain (DreamHost)
+
+import sys
+
+import mock
+import unittest2 as unittest
+
+from quantum.db import migration
+from quantum.db.migration import cli
+
+
+class TestDbMigration(unittest.TestCase):
+    def test_should_run_plugin_in_list(self):
+        self.assertTrue(migration.should_run('foo', ['foo', 'bar']))
+        self.assertFalse(migration.should_run('foo', ['bar']))
+
+    def test_should_run_plugin_wildcard(self):
+        self.assertTrue(migration.should_run('foo', ['*']))
+
+
+class TestMain(unittest.TestCase):
+    def setUp(self):
+        self.process_argv_p = mock.patch.object(cli, 'process_argv')
+        self.process_argv = self.process_argv_p.start()
+
+        self.alembic_cmd_p = mock.patch.object(cli, 'alembic_command')
+        self.alembic_cmd = self.alembic_cmd_p.start()
+
+    def tearDown(self):
+        self.alembic_cmd_p.stop()
+        self.process_argv_p.stop()
+
+    def test_main(self):
+        self.process_argv.return_value = ('foo', ('bar', ), {'baz': 1})
+        cli.main()
+
+        self.process_argv.assert_called_once_with(sys.argv)
+        self.alembic_cmd.foo.assert_called_once_with(mock.ANY, 'bar', baz=1)
+
+
+class TestDatabaseSync(unittest.TestCase):
+    def test_process_argv_stamp(self):
+        self.assertEqual(
+            ('stamp', ('foo',), {'sql': False}),
+            cli.process_argv(['prog', 'stamp', 'foo']))
+
+        self.assertEqual(
+            ('stamp', ('foo',), {'sql': True}),
+            cli.process_argv(['prog', 'stamp', '--sql', 'foo']))
+
+    def test_process_argv_current(self):
+        self.assertEqual(
+            ('current', (), {}),
+            cli.process_argv(['prog', 'current']))
+
+    def test_process_argv_history(self):
+        self.assertEqual(
+            ('history', (), {}),
+            cli.process_argv(['prog', 'history']))
+
+    def test_process_argv_check_migration(self):
+        self.assertEqual(
+            ('branches', (), {}),
+            cli.process_argv(['prog', 'check_migration']))
+
+    def test_database_sync_revision(self):
+        expected = (
+            'revision',
+            (),
+            {'message': 'message', 'sql': False, 'autogenerate': True}
+        )
+
+        self.assertEqual(
+            cli.process_argv(
+                ['prog', 'revision', '-m', 'message', '--autogenerate']
+            ),
+            expected
+        )
+
+    def test_database_sync_upgrade(self):
+        self.assertEqual(
+            cli.process_argv(['prog', 'upgrade', 'head']),
+            ('upgrade', ('head', ), {'sql': False})
+        )
+
+        self.assertEqual(
+            cli.process_argv(['prog', 'upgrade', '--delta', '3']),
+            ('upgrade', ('+3', ), {'sql': False})
+        )
+
+    def test_database_sync_downgrade(self):
+        self.assertEqual(
+            cli.process_argv(['prog', 'downgrade', 'folsom']),
+            ('downgrade', ('folsom', ), {'sql': False})
+        )
+
+        self.assertEqual(
+            cli.process_argv(['prog', 'downgrade', '--delta', '2']),
+            ('downgrade', ('-2', ), {'sql': False})
+        )

quantum/tests/unit/test_db_plugin.py

@@ -23,6 +23,7 @@ import random
 
 import mock
 import unittest2
+import sqlalchemy as sa
 import webob.exc
 
 import quantum

setup.py

@@ -112,7 +112,7 @@ setuptools.setup(
     scripts=ProjectScripts,
     install_requires=requires,
     dependency_links=depend_links,
-    include_package_data=False,
+    include_package_data=True,
     setup_requires=['setuptools_git>=0.4'],
     packages=setuptools.find_packages('.'),
     cmdclass=setup.get_cmdclass(),
@@ -140,6 +140,7 @@ setuptools.setup(
             'quantum-server = quantum.server:main',
             'quantum-debug = quantum.debug.shell:main',
             'quantum-ovs-cleanup = quantum.agent.ovs_cleanup_util:main',
+            'quantum-db-manage = quantum.db.migration.cli:main',
         ]
     },
 )

tools/pip-requires

@@ -14,3 +14,4 @@ pyudev
 sqlalchemy==0.7.9
 webob==1.2.3
 python-keystoneclient>=0.2.0
+alembic>=0.4.1