Bring models in sync with migrations, add test

Bring the sqlalchemy models in sync with the alembic migrations.

Add the neutron standard model-migration sync test. The test is added
to the unit test env since l2gw does not have a functional job
yet. This requires that l2gw uses the python-db-jobs template in the
gate.

(Also remove downgrades since they are not supported.)

Closes-Bug: #1605675

Depends-On: I3a8ea02b86846e9a8a7d7f2aee9700ca952b0248

Change-Id: Ib73f554d5f0bbfbe25a0f6ad33b23e4584800e21

networking_l2gw/db/l2gateway/head.py

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from neutron.db.migration.models import head
+
+import networking_l2gw.db.l2gateway.l2gateway_models  # noqa
+import networking_l2gw.db.l2gateway.ovsdb.models  # noqa
+
+
+def get_metadata():
+    return head.model_base.BASEV2.metadata

networking_l2gw/db/l2gateway/l2gateway_models.py

@@ -27,7 +27,8 @@ class L2GatewayConnection(model_base.BASEV2, models_v2.HasTenant,
                               sa.ForeignKey('l2gateways.id',
                                             ondelete='CASCADE'))
     network_id = sa.Column(sa.String(36),
-                           sa.ForeignKey('networks.id', ondelete='CASCADE'))
+                           sa.ForeignKey('networks.id', ondelete='CASCADE'),
+                           nullable=False)
     segmentation_id = sa.Column(sa.Integer)
     __table_args__ = (sa.UniqueConstraint(l2_gateway_id,
                                           network_id),)
@@ -38,19 +39,21 @@ class L2GatewayInterface(model_base.BASEV2, models_v2.HasId):
     interface_name = sa.Column(sa.String(255))
     device_id = sa.Column(sa.String(36),
                           sa.ForeignKey('l2gatewaydevices.id',
-                                        ondelete='CASCADE'))
+                                        ondelete='CASCADE'),
+                          nullable=False)
     segmentation_id = sa.Column(sa.Integer)
 
 
 class L2GatewayDevice(model_base.BASEV2, models_v2.HasId):
     """Define an l2 gateway device."""
-    device_name = sa.Column(sa.String(255))
+    device_name = sa.Column(sa.String(255), nullable=False)
     interfaces = orm.relationship(L2GatewayInterface,
                                   backref='l2gatewaydevices',
                                   cascade='all,delete')
     l2_gateway_id = sa.Column(sa.String(36),
                               sa.ForeignKey('l2gateways.id',
-                                            ondelete='CASCADE'))
+                                            ondelete='CASCADE'),
+                              nullable=False)
 
 
 class L2Gateway(model_base.BASEV2, models_v2.HasId,

networking_l2gw/db/l2gateway/ovsdb/models.py

@@ -22,72 +22,60 @@ from neutron.db import models_v2
 class PhysicalLocators(model_base.BASEV2):
     __tablename__ = 'physical_locators'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    dst_ip = sa.Column(sa.String(64), nullable=False)
+    dst_ip = sa.Column(sa.String(64), nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class PhysicalSwitches(model_base.BASEV2):
     __tablename__ = 'physical_switches'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    name = sa.Column(sa.String(255), nullable=False)
-    tunnel_ip = sa.Column(sa.String(64), nullable=False)
+    name = sa.Column(sa.String(255), nullable=True)
+    tunnel_ip = sa.Column(sa.String(64), nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
     switch_fault_status = sa.Column(sa.String(length=32), nullable=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class PhysicalPorts(model_base.BASEV2):
     __tablename__ = 'physical_ports'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    name = sa.Column(sa.String(255), nullable=False)
-    physical_switch_id = sa.Column(sa.String(36), nullable=False)
+    name = sa.Column(sa.String(255), nullable=True)
+    physical_switch_id = sa.Column(sa.String(36), nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
     port_fault_status = sa.Column(sa.String(length=32), nullable=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class LogicalSwitches(model_base.BASEV2):
     __tablename__ = 'logical_switches'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    name = sa.Column(sa.String(255), nullable=False)
-    key = sa.Column(sa.Integer, nullable=False)
+    name = sa.Column(sa.String(255), nullable=True)
+    key = sa.Column(sa.Integer, nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class UcastMacsLocals(model_base.BASEV2):
     __tablename__ = 'ucast_macs_locals'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    mac = sa.Column(sa.String(32), nullable=False)
-    logical_switch_id = sa.Column(sa.String(36), nullable=False)
-    physical_locator_id = sa.Column(sa.String(36), nullable=False)
-    ip_address = sa.Column(sa.String(64), nullable=False)
+    mac = sa.Column(sa.String(32), nullable=True)
+    logical_switch_id = sa.Column(sa.String(36), nullable=True)
+    physical_locator_id = sa.Column(sa.String(36), nullable=True)
+    ip_address = sa.Column(sa.String(64), nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class UcastMacsRemotes(model_base.BASEV2):
     __tablename__ = 'ucast_macs_remotes'
     uuid = sa.Column(sa.String(36), nullable=False, primary_key=True)
-    mac = sa.Column(sa.String(32), nullable=False)
-    logical_switch_id = sa.Column(sa.String(36), nullable=False)
-    physical_locator_id = sa.Column(sa.String(36), nullable=False)
-    ip_address = sa.Column(sa.String(64), nullable=False)
+    mac = sa.Column(sa.String(32), nullable=True)
+    logical_switch_id = sa.Column(sa.String(36), nullable=True)
+    physical_locator_id = sa.Column(sa.String(36), nullable=True)
+    ip_address = sa.Column(sa.String(64), nullable=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
-    __table_args__ = (sa.UniqueConstraint(uuid,
-                                          ovsdb_identifier),)
 
 
 class VlanBindings(model_base.BASEV2):
@@ -98,16 +86,13 @@ class VlanBindings(model_base.BASEV2):
                                     primary_key=True)
     ovsdb_identifier = sa.Column(sa.String(64), nullable=False,
                                  primary_key=True)
-    __table_args__ = (sa.UniqueConstraint(port_uuid, vlan,
-                                          logical_switch_uuid,
-                                          ovsdb_identifier),)
 
 
 class PendingUcastMacsRemote(model_base.BASEV2, models_v2.HasId):
     __tablename__ = 'pending_ucast_macs_remotes'
     uuid = sa.Column(sa.String(36), nullable=True)
     mac = sa.Column(sa.String(32), nullable=False)
-    logical_switch_uuid = sa.Column(sa.String(36), nullable=True)
+    logical_switch_uuid = sa.Column(sa.String(36), nullable=False)
     locator_uuid = sa.Column(sa.String(36), nullable=True)
     dst_ip = sa.Column(sa.String(64))
     vm_ip = sa.Column(sa.String(64))

networking_l2gw/db/migration/alembic_migrations/versions/42438454c556_l2gateway_models.py

@@ -94,11 +94,3 @@ def upgrade():
                               nullable=False),
                     sa.Column('operation', sa.String(8), nullable=False),
                     sa.Column('timestamp', sa.DateTime, nullable=False))
-
-
-def downgrade():
-    op.drop_table('l2gatewayconnections')
-    op.drop_table('l2gatewayinterfaces')
-    op.drop_table('l2gatewaydevices')
-    op.drop_table('l2gateways')
-    op.drop_table('pending_ucast_macs_remotes')

networking_l2gw/db/migration/alembic_migrations/versions/54c9c8fe22bf_db_models_for_ovsdb_hardware_vtep_schema.py

@@ -103,13 +103,3 @@ def upgrade():
                               nullable=False),
                     sa.PrimaryKeyConstraint('port_uuid', 'ovsdb_identifier',
                                             'vlan', 'logical_switch_uuid'))
-
-
-def downgrade():
-    op.drop_table('physical_locators')
-    op.drop_table('physical_switches')
-    op.drop_table('physical_ports')
-    op.drop_table('logical_switches')
-    op.drop_table('ucast_macs_locals')
-    op.drop_table('ucast_macs_remotes')
-    op.drop_table('vlan_bindings')

networking_l2gw/db/migration/alembic_migrations/versions/CONTRACT_HEAD

@@ -1,2 +1 @@
-60019185aa99
 79919185aa99

networking_l2gw/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -0,0 +1 @@
+49ce408ac349

networking_l2gw/db/migration/alembic_migrations/versions/newton/expand/49ce408ac349_add_indexes_to_tenant_id.py

@@ -0,0 +1,31 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""add indexes to tenant_id
+
+Revision ID: 49ce408ac349
+Create Date: 2016-07-22 10:42:14.495451
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '49ce408ac349'
+down_revision = '60019185aa99'
+
+
+def upgrade():
+    for table in ['l2gateways', 'l2gatewayconnections']:
+        op.create_index(op.f('ix_%s_tenant_id' % table),
+                        table, ['tenant_id'], unique=False)

networking_l2gw/db/migration/alembic_migrations/versions/start_networking_l2gw.py

@@ -28,7 +28,3 @@ down_revision = None
 
 def upgrade():
     pass
-
-
-def downgrade():
-    pass

networking_l2gw/tests/unit/db/test_migrations.py

@@ -0,0 +1,59 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from neutron.db.migration.alembic_migrations import external
+from neutron.db.migration import cli as migration
+from neutron.tests.functional.db import test_migrations
+from neutron.tests.unit import testlib_api
+
+from networking_l2gw.db.l2gateway import head
+
+# EXTERNAL_TABLES should contain all names of tables that are not related to
+# current repo.
+EXTERNAL_TABLES = set(external.TABLES)
+VERSION_TABLE = 'l2gw_alembic_version'
+
+
+class _TestModelsMigrationsL2GW(test_migrations._TestModelsMigrations):
+
+    def db_sync(self, engine):
+        cfg.CONF.set_override('connection', engine.url, group='database')
+        for conf in migration.get_alembic_configs():
+            self.alembic_config = conf
+            self.alembic_config.neutron_config = cfg.CONF
+            migration.do_alembic_command(conf, 'upgrade', 'heads')
+
+    def get_metadata(self):
+        return head.get_metadata()
+
+    def include_object(self, object_, name, type_, reflected, compare_to):
+        if type_ == 'table' and (name.startswith('alembic') or
+                                 name == VERSION_TABLE or
+                                 name in EXTERNAL_TABLES):
+            return False
+        if type_ == 'index' and reflected and name.startswith("idx_autoinc_"):
+            return False
+        return True
+
+
+class TestModelsMigrationsMysql(testlib_api.MySQLTestCaseMixin,
+                                _TestModelsMigrationsL2GW,
+                                testlib_api.SqlTestCaseLight):
+    pass
+
+
+class TestModelsMigrationsPostgresql(testlib_api.PostgreSQLTestCaseMixin,
+                                     _TestModelsMigrationsL2GW,
+                                     testlib_api.SqlTestCaseLight):
+    pass

test-requirements.txt

@@ -8,6 +8,8 @@ coverage>=3.6 # Apache-2.0
 discover # BSD
 python-subunit>=0.0.18 # Apache-2.0/BSD
 sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+psycopg2>=2.5 # LGPL/ZPL
+PyMySQL>=0.6.2 # MIT License
 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD

tox.ini

@@ -22,6 +22,10 @@ setenv = OS_TEST_PATH=./networking_l2gw/tests/fullstack
 commands =
   flake8
   neutron-db-manage --subproject networking-l2gw check_migration
+
+[testenv:py27]
+setenv = OS_FAIL_ON_MISSING_DEPS=1
+
 [testenv:venv]
 commands = {posargs}