Improve SQL query performance in some cases

The query for builds-joined-with-buildsets is currently optimized
for the case where little additional filtering is performed.  E.g.,
the case where a user browses to the builds tab and does not enter
any search terms.  In that case, mysql needs a hint supplied in
order to choose the right index.

When search terms are entered which can, due to the presense of
other indexes, greatly reduce the working set, it's better to let
the query planner off the leash and it will make the right choices.

This change stops adding the hint in the cases where a user supplies
a search term that matches one of the indexes on the build or
buildset table (notable exception: job_name because it is difficult
to generalize about that one).

It also adds an additional index for build and buildset uuids,
which should provide excellent performance when searching for
only those terms.

Change-Id: I0277be8cc4ba7555c5e6a9a7eb3eed988a24469c

tests/unit/test_connection.py

@@ -103,8 +103,8 @@ class TestSQLConnection(ZuulDBTestCase):
         indexes_build = [x for x in indexes_build
                          if x['name'] != 'buildset_id']
 
-        self.assertEqual(3, len(indexes_buildset))
-        self.assertEqual(1, len(indexes_build))
+        self.assertEqual(4, len(indexes_buildset))
+        self.assertEqual(2, len(indexes_build))
 
         # check if all indexes are prefixed
         if table_prefix:

zuul/driver/sql/alembic/versions/e0eda5d09eae_add_uuid_indexes.py

@@ -0,0 +1,49 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""add_uuid_indexes
+
+Revision ID: e0eda5d09eae
+Revises: c18b1277dfb5
+Create Date: 2019-07-24 16:04:18.042209
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e0eda5d09eae'
+down_revision = 'c18b1277dfb5'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+
+BUILDSET_TABLE = 'zuul_buildset'
+BUILD_TABLE = 'zuul_build'
+
+
+def upgrade(table_prefix=''):
+    prefixed_buildset = table_prefix + BUILDSET_TABLE
+    prefixed_build = table_prefix + BUILD_TABLE
+
+    # To look up a build by uuid. buildset_id is included
+    # so that it's a covering index and can satisfy the join back to buildset
+    # without an additional lookup.
+    op.create_index(
+        table_prefix + 'uuid_buildset_id_idx', prefixed_build,
+        ['uuid', 'buildset_id'])
+
+    # To look up a buildset by uuid.
+    op.create_index(table_prefix + 'uuid_idx', prefixed_buildset, ['uuid'])
+
+
+def downgrade():
+    raise Exception("Downgrades not supported")

zuul/driver/sql/sqlconnection.py

@@ -74,8 +74,19 @@ class DatabaseSession(object):
             outerjoin(self.connection.providesModel).\
             options(orm.contains_eager(self.connection.buildModel.buildset),
                     orm.selectinload(self.connection.buildModel.provides),
-                    orm.selectinload(self.connection.buildModel.artifacts)).\
-            with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
+                    orm.selectinload(self.connection.buildModel.artifacts))
+        # If the query planner isn't able to reduce either the number
+        # of rows returned by the buildset or build tables, then it
+        # tends to produce a very slow query.  This hint produces
+        # better results, but only in those cases.  When we can narrow
+        # things down with indexes, it's better to omit the hint.
+        # job_name is a tricky one.  It is indexed, but if there are a
+        # lot of rows, it is better to include the hint, but if there
+        # are few, it is better to not include it.  We include the hint
+        # regardless of whether job_name is specified (optimizing for
+        # the more common case).
+        if not (project or change or uuid):
+            q = q.with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
 
         q = self.listFilter(q, buildset_table.c.tenant, tenant)
         q = self.listFilter(q, buildset_table.c.project, project)
@@ -114,8 +125,10 @@ class DatabaseSession(object):
 
         buildset_table = self.connection.zuul_buildset_table
 
-        q = self.session().query(self.connection.buildSetModel).\
-            with_hint(buildset_table, 'USE INDEX (PRIMARY)', 'mysql')
+        # See note above about the hint.
+        q = self.session().query(self.connection.buildSetModel)
+        if not (project or change or uuid):
+            q = q.with_hint(buildset_table, 'USE INDEX (PRIMARY)', 'mysql')
 
         q = self.listFilter(q, buildset_table.c.tenant, tenant)
         q = self.listFilter(q, buildset_table.c.project, project)
@@ -146,8 +159,7 @@ class DatabaseSession(object):
             options(orm.joinedload(self.connection.buildSetModel.builds).
                     subqueryload(self.connection.buildModel.artifacts)).\
             options(orm.joinedload(self.connection.buildSetModel.builds).
-                    subqueryload(self.connection.buildModel.provides)).\
-            with_hint(buildset_table, 'USE INDEX (PRIMARY)', 'mysql')
+                    subqueryload(self.connection.buildModel.provides))
 
         q = self.listFilter(q, buildset_table.c.tenant, tenant)
         q = self.listFilter(q, buildset_table.c.uuid, uuid)