From 5df8eaf1ae4a421f7fb3ebfbed1fa44e30f42ec2 Mon Sep 17 00:00:00 2001 From: Tobias Henkel Date: Tue, 10 Apr 2018 08:29:41 +0200 Subject: [PATCH] Prefix existing indexes In some database engines the index names are scoped within the database instead of the table name. Thus we need to also prefix the index names. This change adds a migration which renames already existing indexes. Change-Id: I08b6ba4d54a53a6f8d4ef1ed113cb24946b9f3e9 --- tests/unit/test_connection.py | 32 ++++++ .../versions/f181b33958c6_prefix_indexes.py | 100 ++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 zuul/driver/sql/alembic/versions/f181b33958c6_prefix_indexes.py diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index bacdf8fdb5..2692a66fae 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -77,6 +77,38 @@ class TestSQLConnection(ZuulDBTestCase): self.assertEqual(14, len(insp.get_columns(buildset_table))) self.assertEqual(10, len(insp.get_columns(build_table))) + def test_sql_indexes_created(self): + "Test the indexes are created properly" + + connection = self.connections.connections['resultsdb'] + insp = sa.engine.reflection.Inspector(connection.engine) + + table_prefix = connection.table_prefix + self.assertEqual(self.expected_table_prefix, table_prefix) + + buildset_table = table_prefix + 'zuul_buildset' + build_table = table_prefix + 'zuul_build' + + indexes_buildset = insp.get_indexes(buildset_table) + indexes_build = insp.get_indexes(build_table) + + # Remove implicitly generated indexes by the foreign key. + # MySQL creates an implicit index with the name if the column (which + # is not a problem as in MySQL the index names are scoped within the + # table). This is an implementation detail of the db engine so don't + # check this. + indexes_build = [x for x in indexes_build + if x['name'] != 'buildset_id'] + + self.assertEqual(3, len(indexes_buildset)) + self.assertEqual(1, len(indexes_build)) + + # check if all indexes are prefixed + if table_prefix: + indexes = indexes_buildset + indexes_build + for index in indexes: + self.assertTrue(index['name'].startswith(table_prefix)) + def test_sql_results(self): "Test results are entered into an sql table" self.executor_server.hold_jobs_in_build = True diff --git a/zuul/driver/sql/alembic/versions/f181b33958c6_prefix_indexes.py b/zuul/driver/sql/alembic/versions/f181b33958c6_prefix_indexes.py new file mode 100644 index 0000000000..827c0415af --- /dev/null +++ b/zuul/driver/sql/alembic/versions/f181b33958c6_prefix_indexes.py @@ -0,0 +1,100 @@ +# Copyright 2018 BMW Car IT GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""prefix-indexes + +Revision ID: f181b33958c6 +Revises: defa75d297bf +Create Date: 2018-04-10 07:51:48.918303 + +""" + +# revision identifiers, used by Alembic. +revision = 'f181b33958c6' +down_revision = 'defa75d297bf' +branch_labels = None +depends_on = None + +from alembic import op +from sqlalchemy import inspect + +BUILDSET_TABLE = 'zuul_buildset' +BUILD_TABLE = 'zuul_build' + + +def index_exists(index_name, table_name, inspector): + indexes = inspector.get_indexes(table_name) + for index in indexes: + if index.get('name') == index_name: + return True + return False + + +def upgrade(table_prefix=''): + if not table_prefix: + return + + inspector = inspect(op.get_bind()) + + prefixed_buildset = table_prefix + BUILDSET_TABLE + prefixed_build = table_prefix + BUILD_TABLE + + # We need to prefix any non-prefixed index if needed. Do this by + # creating the new indexes and then dropping the old ones. + + # To allow a dashboard to show a per-project view, optionally filtered + # by pipeline. + if not index_exists(table_prefix + 'project_pipeline_idx', + prefixed_buildset, inspector): + op.create_index( + table_prefix + 'project_pipeline_idx', + prefixed_buildset, ['project', 'pipeline']) + + # To allow a dashboard to show a per-project-change view + if not index_exists(table_prefix + 'project_change_idx', + prefixed_buildset, inspector): + op.create_index( + table_prefix + 'project_change_idx', + prefixed_buildset, ['project', 'change']) + + # To allow a dashboard to show a per-change view + if not index_exists(table_prefix + 'change_idx', + prefixed_buildset, inspector): + op.create_index(table_prefix + 'change_idx', + prefixed_buildset, ['change']) + + # To allow a dashboard to show a job lib view. buildset_id is included + # so that it's a covering index and can satisfy the join back to + # buildset without an additional lookup. + if not index_exists(table_prefix + 'job_name_buildset_id_idx', + prefixed_build, inspector): + op.create_index( + table_prefix + 'job_name_buildset_id_idx', prefixed_build, + ['job_name', 'buildset_id']) + + if index_exists('project_pipeline_idx', prefixed_buildset, inspector): + op.drop_index('project_pipeline_idx', prefixed_buildset) + + if index_exists('project_change_idx', prefixed_buildset, inspector): + op.drop_index('project_change_idx', prefixed_buildset) + + if index_exists('change_idx', prefixed_buildset, inspector): + op.drop_index('change_idx', prefixed_buildset) + + if index_exists('job_name_buildset_id_idx', prefixed_build, inspector): + op.drop_index('job_name_buildset_id_idx', prefixed_build) + + +def downgrade(): + raise Exception("Downgrades not supported")