ironic-inspector/ironic_inspector/test/unit/db/test_migrations.py

#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.


"""
Tests for database migrations. There are "opportunistic" tests here, supported
backends are: sqlite (used in test environment by default), mysql and
postgresql, which are required properly configured unit test environment.

For the opportunistic testing you need to set up a db named 'openstack_citest'
with user 'openstack_citest' and password 'openstack_citest' on localhost.
The test will then use that db and u/p combo to run the tests.

"""


import contextlib
import datetime
from unittest import mock

import alembic
from alembic import script
from oslo_config import cfg
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy.migration_cli import ext_alembic
from oslo_db.sqlalchemy import orm
from oslo_db.sqlalchemy import test_fixtures
from oslo_db.sqlalchemy import test_migrations
from oslo_db.sqlalchemy import utils as db_utils
from oslo_log import log as logging
from oslo_utils import timeutils
from oslo_utils import uuidutils
from oslotest import base as test_base
import sqlalchemy

from ironic_inspector.cmd import dbsync
from ironic_inspector.db import api as db
from ironic_inspector.db import model as db_model
from ironic_inspector import introspection_state as istate
from ironic_inspector.test import base

CONF = cfg.CONF
LOG = logging.getLogger(__name__)


@contextlib.contextmanager
def patch_with_engine(engine):
    with mock.patch.object(db, 'get_writer_session',
                           autospec=True) as patch_w_sess:
        patch_w_sess.return_value = (
            orm.get_maker(engine)())
        yield


class WalkVersionsMixin(object):
    def _walk_versions(self, engine=None, alembic_cfg=None):
        # Determine latest version script from the repo, then
        # upgrade from 1 through to the latest, with no data
        # in the databases. This just checks that the schema itself
        # upgrades successfully.

        with patch_with_engine(engine):
            script_directory = script.ScriptDirectory.from_config(alembic_cfg)

            self.assertIsNone(self.migration_ext.version())

            versions = [ver for ver in script_directory.walk_revisions()]

            for version in reversed(versions):
                self._migrate_up(engine, alembic_cfg,
                                 version.revision, with_data=True)

    def _migrate_up(self, engine, config, version, with_data=False):
        """migrate up to a new version of the db.

        We allow for data insertion and post checks at every
        migration version with special _pre_upgrade_### and
        _check_### functions in the main test.
        """
        # NOTE(sdague): try block is here because it's impossible to debug
        # where a failed data migration happens otherwise
        try:
            if with_data:
                data = None
                pre_upgrade = getattr(
                    self, "_pre_upgrade_%s" % version, None)
                if pre_upgrade:
                    data = pre_upgrade(engine)

            self.migration_ext.upgrade(version)
            self.assertEqual(version, self.migration_ext.version())
            if with_data:
                check = getattr(self, "_check_%s" % version, None)
                if check:
                    check(engine, data)
        except Exception:
            LOG.error("Failed to migrate to version %(version)s on engine "
                      "%(engine)s",
                      {'version': version, 'engine': engine})
            raise


class TestWalkVersions(base.BaseTest, WalkVersionsMixin):
    def setUp(self):
        super(TestWalkVersions, self).setUp()
        self.engine = mock.MagicMock()
        self.migration_ext = mock.MagicMock()
        self.config = mock.MagicMock()
        self.versions = [mock.Mock(revision='2b2'), mock.Mock(revision='1a1')]

    def test_migrate_up(self):
        self.migration_ext.version.return_value = 'dsa123'

        self._migrate_up(self.engine, self.config, 'dsa123')

        self.migration_ext.version.assert_called_with()

    def test_migrate_up_with_data(self):
        test_value = {"a": 1, "b": 2}
        self.migration_ext.version.return_value = '141'
        self._pre_upgrade_141 = mock.MagicMock()
        self._pre_upgrade_141.return_value = test_value
        self._check_141 = mock.MagicMock()

        self._migrate_up(self.engine, self.config, '141', True)

        self._pre_upgrade_141.assert_called_with(self.engine)
        self._check_141.assert_called_with(self.engine, test_value)

    @mock.patch.object(script, 'ScriptDirectory', autospec=True)
    @mock.patch.object(WalkVersionsMixin, '_migrate_up', autospec=True)
    def test_walk_versions_all_default(self, _migrate_up, script_directory):
        fc = script_directory.from_config.return_value
        fc.walk_revisions.return_value = self.versions
        self.migration_ext.version.return_value = None

        self._walk_versions(self.engine, self.config)

        self.migration_ext.version.assert_called_with()

        upgraded = [mock.call(self, self.engine, self.config, v.revision,
                    with_data=True) for v in reversed(self.versions)]
        self.assertEqual(self._migrate_up.call_args_list, upgraded)

    @mock.patch.object(script, 'ScriptDirectory', autospec=True)
    @mock.patch.object(WalkVersionsMixin, '_migrate_up', autospec=True)
    def test_walk_versions_all_false(self, _migrate_up, script_directory):
        fc = script_directory.from_config.return_value
        fc.walk_revisions.return_value = self.versions
        self.migration_ext.version.return_value = None

        self._walk_versions(self.engine, self.config)

        upgraded = [mock.call(self, self.engine, self.config, v.revision,
                    with_data=True) for v in reversed(self.versions)]
        self.assertEqual(upgraded, self._migrate_up.call_args_list)


class MigrationCheckersMixin(object):
    def setUp(self):
        super(MigrationCheckersMixin, self).setUp()
        self.engine = enginefacade.writer.get_engine()
        self.config = dbsync._get_alembic_config()
        self.config.ironic_inspector_config = CONF
        # create AlembicExtension with fake config and replace
        # with real one.
        self.migration_ext = ext_alembic.AlembicExtension(
            self.engine, {'alembic_ini_path': ''})
        self.migration_ext.config = self.config

    def test_walk_versions(self):
        self._walk_versions(self.engine, self.config)

    def _check_578f84f38d(self, engine, data):
        nodes = db_utils.get_table(engine, 'nodes')
        col_names = [column.name for column in nodes.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(nodes.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('started_at', col_names)
        self.assertIsInstance(nodes.c.started_at.type, sqlalchemy.types.Float)
        self.assertIn('finished_at', col_names)
        self.assertIsInstance(nodes.c.started_at.type, sqlalchemy.types.Float)
        self.assertIn('error', col_names)
        self.assertIsInstance(nodes.c.error.type, sqlalchemy.types.Text)

        attributes = db_utils.get_table(engine, 'attributes')
        col_names = [column.name for column in attributes.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(attributes.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('name', col_names)
        self.assertIsInstance(attributes.c.name.type, sqlalchemy.types.String)
        self.assertIn('value', col_names)
        self.assertIsInstance(attributes.c.value.type, sqlalchemy.types.String)

        options = db_utils.get_table(engine, 'options')
        col_names = [column.name for column in options.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(options.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('name', col_names)
        self.assertIsInstance(options.c.name.type, sqlalchemy.types.String)
        self.assertIn('value', col_names)
        self.assertIsInstance(options.c.value.type, sqlalchemy.types.Text)

    def _check_d588418040d(self, engine, data):
        rules = db_utils.get_table(engine, 'rules')
        col_names = [column.name for column in rules.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(rules.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('created_at', col_names)
        self.assertIsInstance(rules.c.created_at.type,
                              sqlalchemy.types.DateTime)
        self.assertIn('description', col_names)
        self.assertIsInstance(rules.c.description.type, sqlalchemy.types.Text)
        self.assertIn('disabled', col_names)
        # in some backends bool type is integer
        self.assertIsInstance(rules.c.disabled.type,
                              (sqlalchemy.types.Boolean,
                               sqlalchemy.types.Integer))

        conditions = db_utils.get_table(engine, 'rule_conditions')
        col_names = [column.name for column in conditions.c]
        self.assertIn('id', col_names)
        self.assertIsInstance(conditions.c.id.type, sqlalchemy.types.Integer)
        self.assertIn('rule', col_names)
        self.assertIsInstance(conditions.c.rule.type, sqlalchemy.types.String)
        self.assertIn('op', col_names)
        self.assertIsInstance(conditions.c.op.type, sqlalchemy.types.String)
        self.assertIn('multiple', col_names)
        self.assertIsInstance(conditions.c.multiple.type,
                              sqlalchemy.types.String)
        self.assertIn('field', col_names)
        self.assertIsInstance(conditions.c.field.type, sqlalchemy.types.Text)
        self.assertIn('params', col_names)
        self.assertIsInstance(conditions.c.params.type, sqlalchemy.types.Text)

        actions = db_utils.get_table(engine, 'rule_actions')
        col_names = [column.name for column in actions.c]
        self.assertIn('id', col_names)
        self.assertIsInstance(actions.c.id.type, sqlalchemy.types.Integer)
        self.assertIn('rule', col_names)
        self.assertIsInstance(actions.c.rule.type, sqlalchemy.types.String)
        self.assertIn('action', col_names)
        self.assertIsInstance(actions.c.action.type, sqlalchemy.types.String)
        self.assertIn('params', col_names)
        self.assertIsInstance(actions.c.params.type, sqlalchemy.types.Text)

    def _check_e169a4a81d88(self, engine, data):
        rule_conditions = db_utils.get_table(engine, 'rule_conditions')
        with engine.begin() as connection:
            # set invert with default value - False
            data = {'id': 1, 'op': 'eq', 'multiple': 'all'}
            insert_rc = rule_conditions.insert()
            connection.execute(insert_rc, data)
            rc_stmt = sqlalchemy.select(
                rule_conditions.c.invert
            ).where(rule_conditions.c.id == 1)
            conds = connection.execute(rc_stmt).first()
            self.assertFalse(conds[0])

            # set invert with - True
            data2 = {'id': 2, 'op': 'eq', 'multiple': 'all', 'invert': True}
            connection.execute(insert_rc, data2)

            rc_stmt2 = sqlalchemy.select(
                rule_conditions.c.invert
            ).where(rule_conditions.c.id == 2)

            conds2 = connection.execute(rc_stmt2).first()
            self.assertTrue(conds2[0])

    def _pre_upgrade_d2e48801c8ef(self, engine):
        ok_node_id = uuidutils.generate_uuid()
        err_node_id = uuidutils.generate_uuid()
        data = [
            {
                'uuid': ok_node_id,
                'error': None,
                'finished_at': 0.0,
                'started_at': 0.0
            },
            {
                'uuid': err_node_id,
                'error': 'Oops!',
                'finished_at': 0.0,
                'started_at': 0.0
            }
        ]
        nodes = db_utils.get_table(engine, 'nodes')
        with engine.begin() as connection:
            for node in data:
                insert_node = nodes.insert()
                connection.execute(insert_node, node)
        return {'err_node_id': err_node_id, 'ok_node_id': ok_node_id}

    def _check_d2e48801c8ef(self, engine, data):
        nodes = db_utils.get_table(engine, 'nodes')
        col_names = [column.name for column in nodes.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(nodes.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('version_id', col_names)
        self.assertIsInstance(nodes.c.version_id.type, sqlalchemy.types.String)
        self.assertIn('state', col_names)
        self.assertIsInstance(nodes.c.state.type, sqlalchemy.types.String)
        self.assertIn('started_at', col_names)
        self.assertIsInstance(nodes.c.started_at.type, sqlalchemy.types.Float)
        self.assertIn('finished_at', col_names)
        self.assertIsInstance(nodes.c.started_at.type, sqlalchemy.types.Float)
        self.assertIn('error', col_names)
        self.assertIsInstance(nodes.c.error.type, sqlalchemy.types.Text)

        ok_node_id = data['ok_node_id']
        err_node_id = data['err_node_id']
        with engine.begin() as connection:
            # assert the ok node is in the (default) finished state
            ok_node_stmt = sqlalchemy.select(
                nodes.c.state
            ).where(nodes.c.uuid == ok_node_id)
            ok_node = connection.execute(ok_node_stmt).first()
            self.assertEqual(istate.States.finished, ok_node[0])
            # assert err node state is error after the migration
            # even though the default state is finished
            err_node_stmt = sqlalchemy.select(
                nodes.c.state
            ).where(nodes.c.uuid == err_node_id)
            err_node = connection.execute(err_node_stmt).first()
            self.assertEqual(istate.States.error, err_node[0])

    def _pre_upgrade_d00d6e3f38c4(self, engine):
        nodes = db_utils.get_table(engine, 'nodes')
        data = []
        with engine.begin() as connection:
            for finished_at in (None, 1234.0):
                node = {'uuid': uuidutils.generate_uuid(),
                        'started_at': 1232.0,
                        'finished_at': finished_at,
                        'error': None}
                insert_node = nodes.insert().values(node)
                connection.execute(insert_node)
                data.append(node)
        return data

    def _check_d00d6e3f38c4(self, engine, data):
        nodes = db_utils.get_table(engine, 'nodes')
        col_names = [column.name for column in nodes.c]

        self.assertIn('started_at', col_names)
        self.assertIn('finished_at', col_names)
        self.assertIsInstance(nodes.c.started_at.type,
                              sqlalchemy.types.DateTime)
        self.assertIsInstance(nodes.c.finished_at.type,
                              sqlalchemy.types.DateTime)

        with engine.begin() as connection:
            for node in data:
                finished_at = datetime.datetime.utcfromtimestamp(
                    node['finished_at']) if node['finished_at'] else None

                stmt = sqlalchemy.select(
                    nodes.c.started_at,
                    nodes.c.finished_at
                ).where(nodes.c.uuid == node['uuid'])
                row = connection.execute(stmt).first()
                self.assertEqual(
                    datetime.datetime.utcfromtimestamp(node['started_at']),
                    row[0])
                self.assertEqual(
                    finished_at,
                    row[1])

    def _pre_upgrade_882b2d84cb1b(self, engine):
        attributes = db_utils.get_table(engine, 'attributes')
        nodes = db_utils.get_table(engine, 'nodes')
        self.node_uuid = uuidutils.generate_uuid()
        with engine.begin() as connection:
            node = {
                'uuid': self.node_uuid,
                'started_at': timeutils.utcnow(),
                'finished_at': None,
                'error': None,
                'state': istate.States.starting
            }
            insert_node = nodes.insert().values(node)
            connection.execute(insert_node)
            data = {
                'uuid': self.node_uuid,
                'name': 'foo',
                'value': 'bar'
            }
            insert_att = attributes.insert().values(data)
            connection.execute(insert_att)

    def _check_882b2d84cb1b(self, engine, data):
        attributes = db_utils.get_table(engine, 'attributes')
        col_names = [column.name for column in attributes.c]
        self.assertIn('uuid', col_names)
        self.assertIsInstance(attributes.c.uuid.type, sqlalchemy.types.String)
        self.assertIn('node_uuid', col_names)
        self.assertIsInstance(attributes.c.node_uuid.type,
                              sqlalchemy.types.String)
        self.assertIn('name', col_names)
        self.assertIsInstance(attributes.c.name.type, sqlalchemy.types.String)
        self.assertIn('value', col_names)
        self.assertIsInstance(attributes.c.value.type, sqlalchemy.types.String)

        with engine.begin() as connection:
            stmt = sqlalchemy.select(
                attributes.c.node_uuid,
                attributes.c.uuid,
                attributes.c.name,
                attributes.c.value
            ).where(attributes.c.node_uuid == self.node_uuid)
            row = connection.execute(stmt).first()
            self.assertEqual(self.node_uuid, row[0])
            self.assertNotEqual(self.node_uuid, row[1])
            self.assertIsNotNone(row[1])
            self.assertEqual('foo', row[2])
            self.assertEqual('bar', row[3])

    def _check_2970d2d44edc(self, engine, data):
        nodes = db_utils.get_table(engine, 'nodes')
        data = {'uuid': 'abcd'}

        with engine.begin() as connection:
            insert_node = nodes.insert()
            connection.execute(insert_node, data)
            n_stmt = sqlalchemy.select(
                nodes.c.manage_boot
            ).where(nodes.c.uuid == 'abcd')
            n = connection.execute(n_stmt).first()
            self.assertIsNone(n[0])

    def _check_bf8dec16023c(self, engine, data):
        introspection_data = db_utils.get_table(engine, 'introspection_data')
        col_names = [column.name for column in introspection_data.c]
        self.assertIn('uuid', col_names)
        self.assertIn('processed', col_names)
        self.assertIn('data', col_names)
        self.assertIsInstance(introspection_data.c.uuid.type,
                              sqlalchemy.types.String)
        self.assertIsInstance(introspection_data.c.processed.type,
                              sqlalchemy.types.Boolean)
        self.assertIsInstance(introspection_data.c.data.type,
                              sqlalchemy.types.Text)

    def test_upgrade_and_version(self):
        with patch_with_engine(self.engine):
            self.migration_ext.upgrade('head')
            self.assertIsNotNone(self.migration_ext.version())

    def test_upgrade_twice(self):
        with patch_with_engine(self.engine):
            self.migration_ext.upgrade('578f84f38d')
            v1 = self.migration_ext.version()
            self.migration_ext.upgrade('d588418040d')
            v2 = self.migration_ext.version()
            self.assertNotEqual(v1, v2)


class TestMigrationsMySQL(MigrationCheckersMixin,
                          WalkVersionsMixin,
                          test_fixtures.OpportunisticDBTestMixin,
                          test_base.BaseTestCase):
    FIXTURE = test_fixtures.MySQLOpportunisticFixture


class TestMigrationsPostgreSQL(MigrationCheckersMixin,
                               WalkVersionsMixin,
                               test_fixtures.OpportunisticDBTestMixin,
                               test_base.BaseTestCase):
    FIXTURE = test_fixtures.PostgresqlOpportunisticFixture


class TestMigrationSqlite(MigrationCheckersMixin,
                          WalkVersionsMixin,
                          test_fixtures.OpportunisticDBTestMixin,
                          test_base.BaseTestCase):
    FIXTURE = test_fixtures.OpportunisticDbFixture


class ModelsMigrationSyncMixin(object):

    def setUp(self):
        super(ModelsMigrationSyncMixin, self).setUp()
        self.engine = enginefacade.writer.get_engine()

    def get_metadata(self):
        return db_model.Base.metadata

    def get_engine(self):
        return self.engine

    def db_sync(self, engine):
        config = dbsync._get_alembic_config()
        config.ironic_inspector_config = CONF
        with patch_with_engine(engine):
            alembic.command.upgrade(config, 'head')


class ModelsMigrationsSyncMysql(ModelsMigrationSyncMixin,
                                test_migrations.ModelsMigrationsSync,
                                test_fixtures.OpportunisticDBTestMixin,
                                test_base.BaseTestCase):
    FIXTURE = test_fixtures.MySQLOpportunisticFixture


class ModelsMigrationsSyncPostgres(ModelsMigrationSyncMixin,
                                   test_migrations.ModelsMigrationsSync,
                                   test_fixtures.OpportunisticDBTestMixin,
                                   test_base.BaseTestCase):
    FIXTURE = test_fixtures.PostgresqlOpportunisticFixture


# NOTE(TheJulia): Sqlite database testing is known to encounter race
# conditions as the default always falls to the same database which
# means a different test runner an collide with the test and fail as
# a result. Commenting out in case we figure out a solid way to
# prevent this. It should be noted we only test actual databases
# in ironic's unit tests. Here we're testing databases and sqlite.
# class ModelsMigrationsSyncSqlite(ModelsMigrationSyncMixin,
#                                  test_migrations.ModelsMigrationsSync,
#                                  test_fixtures.OpportunisticDBTestMixin,
#                                  test_base.BaseTestCase):
#     FIXTURE = test_fixtures.OpportunisticDbFixture