From d73e2bafa426dc51b37ed12aff552fff835527e8 Mon Sep 17 00:00:00 2001 From: Thaynara Silva Date: Mon, 28 Aug 2017 12:16:14 +0000 Subject: [PATCH] Add Database Migrations Classifications - Introducing database migrations for the neutron common classification framework. Change-Id: Iceca7c10aaec7a8566d994ed1acd057818e54d5d Co-Authored-By: David Shaughnessy --- neutron_classifier/db/migration/__init__.py | 0 .../migration/alembic_migrations/__init__.py | 0 .../db/migration/alembic_migrations/env.py | 118 ++++++++++++++++ .../alembic_migrations/script.py.mako | 39 ++++++ .../alembic_migrations/versions/CONTRACT_HEAD | 1 + .../alembic_migrations/versions/EXPAND_HEAD | 1 + .../4e97d48da530_initial_ccf_database_.py | 130 ++++++++++++++++++ .../versions/start_neutron_classifier.py | 28 ++++ .../db/migration/models/__init__.py | 0 .../db/migration/models/head.py | 21 +++ setup.cfg | 5 + 11 files changed, 343 insertions(+) create mode 100644 neutron_classifier/db/migration/__init__.py create mode 100644 neutron_classifier/db/migration/alembic_migrations/__init__.py create mode 100644 neutron_classifier/db/migration/alembic_migrations/env.py create mode 100644 neutron_classifier/db/migration/alembic_migrations/script.py.mako create mode 100644 neutron_classifier/db/migration/alembic_migrations/versions/CONTRACT_HEAD create mode 100644 neutron_classifier/db/migration/alembic_migrations/versions/EXPAND_HEAD create mode 100644 neutron_classifier/db/migration/alembic_migrations/versions/queens/expand/4e97d48da530_initial_ccf_database_.py create mode 100644 neutron_classifier/db/migration/alembic_migrations/versions/start_neutron_classifier.py create mode 100644 neutron_classifier/db/migration/models/__init__.py create mode 100644 neutron_classifier/db/migration/models/head.py diff --git a/neutron_classifier/db/migration/__init__.py b/neutron_classifier/db/migration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neutron_classifier/db/migration/alembic_migrations/__init__.py b/neutron_classifier/db/migration/alembic_migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neutron_classifier/db/migration/alembic_migrations/env.py b/neutron_classifier/db/migration/alembic_migrations/env.py new file mode 100644 index 0000000..c75757b --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/env.py @@ -0,0 +1,118 @@ +# Copyright 2017 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Derived from neutron/db/migration/alembic_migrations/env.py + +from alembic import context +from neutron_lib.db import model_base +from oslo_config import cfg +import sqlalchemy as sa +from sqlalchemy import event + +from neutron.db.migration.alembic_migrations import external +from neutron.db.migration import autogen +from neutron.db.migration.connection import DBConnection + + +try: + # NOTE(mriedem): This is to register the DB2 alembic code which + # is an optional runtime dependency. + from ibm_db_alembic.ibm_db import IbmDbImpl # noqa # pylint: disable=unused-import +except ImportError: + pass + + +NAME_PREFIX = "idx_autoinc_" +MYSQL_ENGINE = None +CLASSIFIER_VERSION_TABLE = 'alembic_version_classifier' +config = context.config +neutron_config = config.neutron_config +target_metadata = model_base.BASEV2.metadata + + +def set_mysql_engine(): + try: + mysql_engine = neutron_config.command.mysql_engine + except cfg.NoSuchOptError: + mysql_engine = None + + global MYSQL_ENGINE + MYSQL_ENGINE = (mysql_engine or + model_base.BASEV2.__table_args__['mysql_engine']) + + +def include_object(object_, name, type_, reflected, compare_to): + if type_ == 'table' and name in external.TABLES: + return False + elif type_ == 'index' and reflected and name.startswith(NAME_PREFIX): + # skip indexes created by SQLAlchemy autoincrement=True + # on composite PK integer columns + return False + else: + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with either a URL + or an Engine. + Calls to context.execute() here emit the given string to the + script output. + """ + set_mysql_engine() + + kwargs = dict() + if neutron_config.database.connection: + kwargs['url'] = neutron_config.database.connection + else: + kwargs['dialect_name'] = neutron_config.database.engine + kwargs['include_object'] = include_object + kwargs['version_table'] = CLASSIFIER_VERSION_TABLE + context.configure(**kwargs) + + with context.begin_transaction(): + context.run_migrations() + + +@event.listens_for(sa.Table, 'after_parent_attach') +def set_storage_engine(target, parent): + if MYSQL_ENGINE: + target.kwargs['mysql_engine'] = MYSQL_ENGINE + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + """ + set_mysql_engine() + connection = config.attributes.get('connection') + with DBConnection(neutron_config.database.connection, connection) as conn: + context.configure( + connection=conn, + target_metadata=target_metadata, + version_table=CLASSIFIER_VERSION_TABLE, + include_object=include_object, + process_revision_directives=autogen.process_revision_directives + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/neutron_classifier/db/migration/alembic_migrations/script.py.mako b/neutron_classifier/db/migration/alembic_migrations/script.py.mako new file mode 100644 index 0000000..abf8cc9 --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/script.py.mako @@ -0,0 +1,39 @@ +# Copyright 2017 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +# Derived from neutron/db/migration/alembic_migrations/script.py.mako + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +% if branch_labels: +branch_labels = ${repr(branch_labels)} +%endif + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/neutron_classifier/db/migration/alembic_migrations/versions/CONTRACT_HEAD b/neutron_classifier/db/migration/alembic_migrations/versions/CONTRACT_HEAD new file mode 100644 index 0000000..b0047fa --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/versions/CONTRACT_HEAD @@ -0,0 +1 @@ +None diff --git a/neutron_classifier/db/migration/alembic_migrations/versions/EXPAND_HEAD b/neutron_classifier/db/migration/alembic_migrations/versions/EXPAND_HEAD new file mode 100644 index 0000000..1650022 --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/versions/EXPAND_HEAD @@ -0,0 +1 @@ +4e97d48da530 diff --git a/neutron_classifier/db/migration/alembic_migrations/versions/queens/expand/4e97d48da530_initial_ccf_database_.py b/neutron_classifier/db/migration/alembic_migrations/versions/queens/expand/4e97d48da530_initial_ccf_database_.py new file mode 100644 index 0000000..653aeea --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/versions/queens/expand/4e97d48da530_initial_ccf_database_.py @@ -0,0 +1,130 @@ +# Copyright 2017 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""initial CCF database expansion + +Revision ID: 4e97d48da530 +Revises: None +Create Date: 2017-08-28 14:14:31.181166 + +""" + +# revision identifiers, used by Alembic. +from alembic import op +import sqlalchemy as sa + +revision = '4e97d48da530' +down_revision = 'start_neutron_classifier' + + +def upgrade(): + op.create_table( + 'classification_groups', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('name', sa.String(length=255)), + sa.Column('description', sa.String(length=255)), + sa.Column('project_id', sa.String(length=255), + index=True), + sa.Column('shared', sa.Boolean(), nullable=False), + sa.Column('operator', sa.Enum("AND", "OR"), nullable=False)) + + op.create_table( + 'classifications', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('c_type', sa.String(length=36)), + sa.Column('name', sa.String(length=255)), + sa.Column('description', sa.String(length=255)), + sa.Column('negated', sa.Boolean(), nullable=True), + sa.Column('shared', sa.Boolean(), nullable=True), + sa.Column('project_id', sa.String(length=255), + index=True)) + + op.create_table( + 'classification_group_to_classification_mappings', + sa.Column('container_cg_id', sa.String(length=36), sa.ForeignKey( + "classification_groups.id"), primary_key=True), + sa.Column('stored_classification_id', sa.String(length=36), + sa.ForeignKey("classifications.id"), primary_key=True)) + + op.create_table( + 'classification_group_to_cg_mappings', + sa.Column('container_cg_id', sa.String(length=36), sa.ForeignKey( + "classification_groups.id"), primary_key=True), + sa.Column('stored_cg_id', sa.String(length=36), sa.ForeignKey( + "classification_groups.id"), primary_key=True)) + + op.create_table( + 'ipv4_classifications', + sa.Column('id', sa.String(length=36), sa.ForeignKey( + "classifications.id"), primary_key=True), + sa.Column('dscp', sa.Integer()), + sa.Column('dscp_mask', sa.Integer()), + sa.Column('ecn', sa.Enum("0", "1", "2", "3")), + sa.Column('length_min', sa.Integer()), + sa.Column('length_max', sa.Integer()), + sa.Column('flags', sa.Integer()), + sa.Column('flags_mask', sa.Integer()), + sa.Column('ttl_min', sa.SmallInteger()), + sa.Column('ttl_max', sa.SmallInteger()), + sa.Column('protocol', sa.Integer()), + sa.Column('src_addr', sa.String(length=19)), + sa.Column('dst_addr', sa.String(length=19))) + + op.create_table( + 'ipv6_classifications', + sa.Column('id', sa.String(length=36), sa.ForeignKey( + "classifications.id"), primary_key=True), + sa.Column('dscp', sa.Integer()), + sa.Column('dscp_mask', sa.Integer()), + sa.Column('ecn', sa.Enum("0", "1", "2", "3")), + sa.Column('length_min', sa.Integer()), + sa.Column('length_max', sa.Integer()), + sa.Column('next_header', sa.Integer()), + sa.Column('hops_min', sa.SmallInteger()), + sa.Column('hops_max', sa.SmallInteger()), + sa.Column('src_addr', sa.String(length=49)), + sa.Column('dst_addr', sa.String(length=49))) + + op.create_table( + 'ethernet_classifications', + sa.Column('id', sa.String(length=36), sa.ForeignKey( + "classifications.id"), primary_key=True), + sa.Column('ethertype', sa.Integer()), + sa.Column('src_addr', sa.String(length=17)), + sa.Column('dst_addr', sa.String(length=17))) + + op.create_table( + 'udp_classifications', + sa.Column('id', sa.String(length=36), sa.ForeignKey( + "classifications.id"), primary_key=True), + sa.Column('src_port_min', sa.Integer()), + sa.Column('src_port_max', sa.Integer()), + sa.Column('dst_port_min', sa.Integer()), + sa.Column('dst_port_max', sa.Integer()), + sa.Column('length_min', sa.Integer()), + sa.Column('length_max', sa.Integer())) + + op.create_table( + 'tcp_classifications', + sa.Column('id', sa.String(length=36), sa.ForeignKey( + "classifications.id"), primary_key=True), + sa.Column('src_port_min', sa.Integer()), + sa.Column('src_port_max', sa.Integer()), + sa.Column('dst_port_min', sa.Integer()), + sa.Column('dst_port_max', sa.Integer()), + sa.Column('window_min', sa.Integer()), + sa.Column('window_max', sa.Integer()), + sa.Column('flags', sa.Integer()), + sa.Column('flags_mask', sa.Integer())) diff --git a/neutron_classifier/db/migration/alembic_migrations/versions/start_neutron_classifier.py b/neutron_classifier/db/migration/alembic_migrations/versions/start_neutron_classifier.py new file mode 100644 index 0000000..f7f5c01 --- /dev/null +++ b/neutron_classifier/db/migration/alembic_migrations/versions/start_neutron_classifier.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""start neutron-classifier chain + +Revision ID: start_neutron_classifier +Revises: None +Create Date: 2017-08-28 18:42:08.262632 + +""" + +# revision identifiers, used by Alembic. +revision = 'start_neutron_classifier' +down_revision = None + + +def upgrade(): + pass diff --git a/neutron_classifier/db/migration/models/__init__.py b/neutron_classifier/db/migration/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neutron_classifier/db/migration/models/head.py b/neutron_classifier/db/migration/models/head.py new file mode 100644 index 0000000..2e7c55a --- /dev/null +++ b/neutron_classifier/db/migration/models/head.py @@ -0,0 +1,21 @@ +# Copyright 2017 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.db import model_base + +from neutron_classifier.db import models # noqa + + +def get_metadata(): + return model_base.BASEV2.metadata diff --git a/setup.cfg b/setup.cfg index 42ac032..18302fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,11 +18,16 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 [files] packages = neutron_classifier +[entry_points] +neutron.db.alembic_migrations = + neutron-classifier = neutron_classifier.db.migration:alembic_migrations + [build_sphinx] source-dir = doc/source build-dir = doc/build