Add Database Migrations Classifications

- Introducing database migrations for the neutron common
   classification framework.

Change-Id: Iceca7c10aaec7a8566d994ed1acd057818e54d5d
Co-Authored-By: David Shaughnessy <david.shaughnessy@intel.com>

neutron_classifier/db/migration/alembic_migrations/env.py

@@ -0,0 +1,118 @@
+# Copyright 2017 OpenStack Foundation. All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Derived from neutron/db/migration/alembic_migrations/env.py
+
+from alembic import context
+from neutron_lib.db import model_base
+from oslo_config import cfg
+import sqlalchemy as sa
+from sqlalchemy import event
+
+from neutron.db.migration.alembic_migrations import external
+from neutron.db.migration import autogen
+from neutron.db.migration.connection import DBConnection
+
+
+try:
+    # NOTE(mriedem): This is to register the DB2 alembic code which
+    # is an optional runtime dependency.
+    from ibm_db_alembic.ibm_db import IbmDbImpl  # noqa # pylint: disable=unused-import
+except ImportError:
+    pass
+
+
+NAME_PREFIX = "idx_autoinc_"
+MYSQL_ENGINE = None
+CLASSIFIER_VERSION_TABLE = 'alembic_version_classifier'
+config = context.config
+neutron_config = config.neutron_config
+target_metadata = model_base.BASEV2.metadata
+
+
+def set_mysql_engine():
+    try:
+        mysql_engine = neutron_config.command.mysql_engine
+    except cfg.NoSuchOptError:
+        mysql_engine = None
+
+    global MYSQL_ENGINE
+    MYSQL_ENGINE = (mysql_engine or
+                    model_base.BASEV2.__table_args__['mysql_engine'])
+
+
+def include_object(object_, name, type_, reflected, compare_to):
+    if type_ == 'table' and name in external.TABLES:
+        return False
+    elif type_ == 'index' and reflected and name.startswith(NAME_PREFIX):
+        # skip indexes created by SQLAlchemy autoincrement=True
+        # on composite PK integer columns
+        return False
+    else:
+        return True
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with either a URL
+    or an Engine.
+    Calls to context.execute() here emit the given string to the
+    script output.
+    """
+    set_mysql_engine()
+
+    kwargs = dict()
+    if neutron_config.database.connection:
+        kwargs['url'] = neutron_config.database.connection
+    else:
+        kwargs['dialect_name'] = neutron_config.database.engine
+    kwargs['include_object'] = include_object
+    kwargs['version_table'] = CLASSIFIER_VERSION_TABLE
+    context.configure(**kwargs)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+@event.listens_for(sa.Table, 'after_parent_attach')
+def set_storage_engine(target, parent):
+    if MYSQL_ENGINE:
+        target.kwargs['mysql_engine'] = MYSQL_ENGINE
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+    set_mysql_engine()
+    connection = config.attributes.get('connection')
+    with DBConnection(neutron_config.database.connection, connection) as conn:
+        context.configure(
+            connection=conn,
+            target_metadata=target_metadata,
+            version_table=CLASSIFIER_VERSION_TABLE,
+            include_object=include_object,
+            process_revision_directives=autogen.process_revision_directives
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

neutron_classifier/db/migration/alembic_migrations/script.py.mako

@@ -0,0 +1,39 @@
+# Copyright 2017 OpenStack Foundation. All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+# Derived from neutron/db/migration/alembic_migrations/script.py.mako
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+% if branch_labels:
+branch_labels = ${repr(branch_labels)}
+%endif
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

neutron_classifier/db/migration/alembic_migrations/versions/CONTRACT_HEAD

@@ -0,0 +1 @@
+None

neutron_classifier/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -0,0 +1 @@
+4e97d48da530

neutron_classifier/db/migration/alembic_migrations/versions/queens/expand/4e97d48da530_initial_ccf_database_.py

@@ -0,0 +1,130 @@
+# Copyright 2017 Intel Corporation.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""initial CCF database expansion
+
+Revision ID: 4e97d48da530
+Revises: None
+Create Date: 2017-08-28 14:14:31.181166
+
+"""
+
+# revision identifiers, used by Alembic.
+from alembic import op
+import sqlalchemy as sa
+
+revision = '4e97d48da530'
+down_revision = 'start_neutron_classifier'
+
+
+def upgrade():
+    op.create_table(
+        'classification_groups',
+        sa.Column('id', sa.String(length=36), primary_key=True),
+        sa.Column('name', sa.String(length=255)),
+        sa.Column('description', sa.String(length=255)),
+        sa.Column('project_id', sa.String(length=255),
+                  index=True),
+        sa.Column('shared', sa.Boolean(), nullable=False),
+        sa.Column('operator', sa.Enum("AND", "OR"), nullable=False))
+
+    op.create_table(
+        'classifications',
+        sa.Column('id', sa.String(length=36), primary_key=True),
+        sa.Column('c_type', sa.String(length=36)),
+        sa.Column('name', sa.String(length=255)),
+        sa.Column('description', sa.String(length=255)),
+        sa.Column('negated', sa.Boolean(), nullable=True),
+        sa.Column('shared', sa.Boolean(), nullable=True),
+        sa.Column('project_id', sa.String(length=255),
+                  index=True))
+
+    op.create_table(
+        'classification_group_to_classification_mappings',
+        sa.Column('container_cg_id', sa.String(length=36), sa.ForeignKey(
+                  "classification_groups.id"), primary_key=True),
+        sa.Column('stored_classification_id', sa.String(length=36),
+                  sa.ForeignKey("classifications.id"), primary_key=True))
+
+    op.create_table(
+        'classification_group_to_cg_mappings',
+        sa.Column('container_cg_id', sa.String(length=36), sa.ForeignKey(
+                  "classification_groups.id"), primary_key=True),
+        sa.Column('stored_cg_id', sa.String(length=36), sa.ForeignKey(
+                  "classification_groups.id"), primary_key=True))
+
+    op.create_table(
+        'ipv4_classifications',
+        sa.Column('id', sa.String(length=36), sa.ForeignKey(
+                  "classifications.id"), primary_key=True),
+        sa.Column('dscp', sa.Integer()),
+        sa.Column('dscp_mask', sa.Integer()),
+        sa.Column('ecn', sa.Enum("0", "1", "2", "3")),
+        sa.Column('length_min', sa.Integer()),
+        sa.Column('length_max', sa.Integer()),
+        sa.Column('flags', sa.Integer()),
+        sa.Column('flags_mask', sa.Integer()),
+        sa.Column('ttl_min', sa.SmallInteger()),
+        sa.Column('ttl_max', sa.SmallInteger()),
+        sa.Column('protocol', sa.Integer()),
+        sa.Column('src_addr', sa.String(length=19)),
+        sa.Column('dst_addr', sa.String(length=19)))
+
+    op.create_table(
+        'ipv6_classifications',
+        sa.Column('id', sa.String(length=36), sa.ForeignKey(
+                  "classifications.id"), primary_key=True),
+        sa.Column('dscp', sa.Integer()),
+        sa.Column('dscp_mask', sa.Integer()),
+        sa.Column('ecn', sa.Enum("0", "1", "2", "3")),
+        sa.Column('length_min', sa.Integer()),
+        sa.Column('length_max', sa.Integer()),
+        sa.Column('next_header', sa.Integer()),
+        sa.Column('hops_min', sa.SmallInteger()),
+        sa.Column('hops_max', sa.SmallInteger()),
+        sa.Column('src_addr', sa.String(length=49)),
+        sa.Column('dst_addr', sa.String(length=49)))
+
+    op.create_table(
+        'ethernet_classifications',
+        sa.Column('id', sa.String(length=36), sa.ForeignKey(
+                  "classifications.id"), primary_key=True),
+        sa.Column('ethertype', sa.Integer()),
+        sa.Column('src_addr', sa.String(length=17)),
+        sa.Column('dst_addr', sa.String(length=17)))
+
+    op.create_table(
+        'udp_classifications',
+        sa.Column('id', sa.String(length=36), sa.ForeignKey(
+                  "classifications.id"), primary_key=True),
+        sa.Column('src_port_min', sa.Integer()),
+        sa.Column('src_port_max', sa.Integer()),
+        sa.Column('dst_port_min', sa.Integer()),
+        sa.Column('dst_port_max', sa.Integer()),
+        sa.Column('length_min', sa.Integer()),
+        sa.Column('length_max', sa.Integer()))
+
+    op.create_table(
+        'tcp_classifications',
+        sa.Column('id', sa.String(length=36), sa.ForeignKey(
+                  "classifications.id"), primary_key=True),
+        sa.Column('src_port_min', sa.Integer()),
+        sa.Column('src_port_max', sa.Integer()),
+        sa.Column('dst_port_min', sa.Integer()),
+        sa.Column('dst_port_max', sa.Integer()),
+        sa.Column('window_min', sa.Integer()),
+        sa.Column('window_max', sa.Integer()),
+        sa.Column('flags', sa.Integer()),
+        sa.Column('flags_mask', sa.Integer()))

neutron_classifier/db/migration/alembic_migrations/versions/start_neutron_classifier.py

@@ -0,0 +1,28 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""start neutron-classifier chain
+
+Revision ID: start_neutron_classifier
+Revises: None
+Create Date: 2017-08-28 18:42:08.262632
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'start_neutron_classifier'
+down_revision = None
+
+
+def upgrade():
+    pass

neutron_classifier/db/migration/models/head.py

@@ -0,0 +1,21 @@
+# Copyright 2017 Intel Corporation.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron_lib.db import model_base
+
+from neutron_classifier.db import models  # noqa
+
+
+def get_metadata():
+    return model_base.BASEV2.metadata

setup.cfg

@@ -18,11 +18,16 @@ classifier =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.3
     Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
 
 [files]
 packages =
     neutron_classifier
 
+[entry_points]
+neutron.db.alembic_migrations =
+    neutron-classifier = neutron_classifier.db.migration:alembic_migrations
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build