zuul/zuul
Code Issues Proposed changes

Add buildset event db table

Browse Source 
This adds a new database table, zuul_buildset_event, which stores
event information related to the buildset.  At the moment, the
only event information stored is a description of the trigger event
which originally caused the item to be enqueued.  Later changes may
add other events such as the reason the item was dequeued or
superceded, or the reason a buildset was canceled due to a gate reset.

A particular goal is to be able to inform users who see unsolicited
reports on their changes (which can happen due to a different change
in a dependency cycle being enqueued) understand why the report was
made.

The new event information is exposed via the rest api and a later
change will add it to the web ui.

Change-Id: I9bcbb64faa3d499a26b90d20932009b6ce226061
This commit is contained in:
James E. Blair 2024-03-25 14:27:47 -07:00
parent 8bcfb8bc4a
commit 762a96e571
7 changed files with 192 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tests/unit/test_connection.py
View File

@ -78,6 +78,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
artifact_table = table_prefix + 'zuul_artifact'
provides_table = table_prefix + 'zuul_provides'
build_event_table = table_prefix + 'zuul_build_event'
buildset_event_table = table_prefix + 'zuul_buildset_event'
self.assertEqual(9, len(insp.get_columns(ref_table)))
self.assertEqual(11, len(insp.get_columns(buildset_table)))
@ -86,6 +87,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
self.assertEqual(5, len(insp.get_columns(artifact_table)))
self.assertEqual(3, len(insp.get_columns(provides_table)))
self.assertEqual(5, len(insp.get_columns(build_event_table)))
self.assertEqual(5, len(insp.get_columns(buildset_event_table)))
def test_sql_tables_created(self):
"Test the tables for storing results are created properly"
@ -105,6 +107,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
artifact_table = table_prefix + 'zuul_artifact'
provides_table = table_prefix + 'zuul_provides'
build_event_table = table_prefix + 'zuul_build_event'
buildset_event_table = table_prefix + 'zuul_buildset_event'
indexes_ref = insp.get_indexes(ref_table)
indexes_buildset = insp.get_indexes(buildset_table)
@ -113,6 +116,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
indexes_artifact = insp.get_indexes(artifact_table)
indexes_provides = insp.get_indexes(provides_table)
indexes_build_event = insp.get_indexes(build_event_table)
indexes_buildset_event = insp.get_indexes(buildset_event_table)
self.assertEqual(8, len(indexes_ref))
self.assertEqual(2, len(indexes_buildset))
@ -121,12 +125,13 @@ class TestSQLConnectionMysql(ZuulTestCase):
self.assertEqual(1, len(indexes_artifact))
self.assertEqual(1, len(indexes_provides))
self.assertEqual(1, len(indexes_build_event))
self.assertEqual(1, len(indexes_buildset_event))
# check if all indexes are prefixed
if table_prefix:
indexes = (indexes_ref + indexes_buildset + indexes_buildset_ref +
indexes_build + indexes_artifact + indexes_provides +
indexes_build_event)
indexes_build_event + indexes_buildset_event)
for index in indexes:
self.assertTrue(index['name'].startswith(table_prefix))
@ -353,6 +358,9 @@ class TestSQLConnectionMysql(ZuulTestCase):
f"delete from {self.expected_table_prefix}zuul_buildset_ref;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}zuul_build;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}"
"zuul_buildset_event;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}zuul_buildset;"))
result = conn.execute(sa.text("commit;"))

5
tests/unit/test_web.py
View File

@ -2040,6 +2040,11 @@ class TestBuildInfo(BaseTestWeb):
"api/tenant/tenant-one/buildset/%s" % project_bs['uuid']).json()
self.assertEqual(3, len(buildset["builds"]))
self.assertEqual(1, len(buildset["events"]))
self.assertEqual('triggered', buildset["events"][0]['event_type'])
self.assertEqual('Triggered by GerritChange org/project 1,1',
buildset["events"][0]['description'])
project_test1_build = [x for x in buildset["builds"]
if x["job_name"] == "project-test1"][0]
self.assertEqual('SUCCESS', project_test1_build['result'])

57
zuul/driver/sql/alembic/versions/6c1582c1d08c_buildset_event_table.py Normal file
View File

@ -0,0 +1,57 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""buildset_event_table
Revision ID: 6c1582c1d08c
Revises: ac1dad8c9434
Create Date: 2024-03-25 12:28:58.885794
"""
# revision identifiers, used by Alembic.
revision = '6c1582c1d08c'
down_revision = 'ac1dad8c9434'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
BUILDSET_EVENT_TABLE = "zuul_buildset_event"
BUILDSET_TABLE = "zuul_buildset"
def upgrade(table_prefix=''):
prefixed_buildset_event = table_prefix + BUILDSET_EVENT_TABLE
prefixed_buildset = table_prefix + BUILDSET_TABLE
op.create_table(
prefixed_buildset_event,
sa.Column("id", sa.Integer, primary_key=True),
sa.Column("buildset_id", sa.Integer,
sa.ForeignKey(
f'{prefixed_buildset}.id',
name=f'{prefixed_buildset_event}_buildset_id_fkey',
)),
sa.Column("event_time", sa.DateTime),
sa.Column("event_type", sa.String(255)),
sa.Column("description", sa.TEXT()),
)
op.create_index(
f'{prefixed_buildset_event}_buildset_id_idx',
prefixed_buildset_event,
['buildset_id'])
def downgrade():
raise Exception("Downgrades not supported")

40
zuul/driver/sql/sqlconnection.py
View File

@ -33,8 +33,9 @@ from zuul.zk.locks import CONNECTION_LOCK_ROOT, locked, SessionAwareLock
BUILDSET_TABLE = 'zuul_buildset'
REF_TABLE = 'zuul_ref'
BUILDSET_REF_TABLE = 'zuul_buildset_ref'
BUILDSET_EVENT_TABLE = 'zuul_buildset_event'
BUILD_TABLE = 'zuul_build'
BUILD_EVENTS_TABLE = 'zuul_build_event'
BUILD_EVENT_TABLE = 'zuul_build_event'
ARTIFACT_TABLE = 'zuul_artifact'
PROVIDES_TABLE = 'zuul_provides'
@ -475,6 +476,8 @@ class DatabaseSession(object):
q = self.session().query(self.connection.buildSetModel).\
options(orm.joinedload(self.connection.buildSetModel.refs)).\
options(orm.joinedload(
self.connection.buildSetModel.buildset_events)).\
options(orm.joinedload(self.connection.buildSetModel.builds).
subqueryload(self.connection.buildModel.artifacts)).\
options(orm.joinedload(self.connection.buildSetModel.builds).
@ -505,6 +508,9 @@ class DatabaseSession(object):
orm.selectinload(
self.connection.buildSetModel.refs,
),
orm.selectinload(
self.connection.buildSetModel.buildset_events,
),
orm.selectinload(
self.connection.buildSetModel.builds,
self.connection.buildModel.provides,
@ -701,6 +707,15 @@ class SQLConnection(BaseConnection):
session.flush()
return b
def createBuildSetEvent(self, *args, **kw):
session = orm.session.Session.object_session(self)
e = BuildSetEventModel(*args, **kw)
e.buildset_id = self.id
self.buildset_events.append(e)
session.add(e)
session.flush()
return e
class BuildSetRefModel(Base):
__tablename__ = self.table_prefix + BUILDSET_REF_TABLE
__table_args__ = (
@ -719,6 +734,24 @@ class SQLConnection(BaseConnection):
sa.Index(self.table_prefix + 'zuul_buildset_ref_ref_id_idx',
ref_id)
class BuildSetEventModel(Base):
__tablename__ = self.table_prefix + BUILDSET_EVENT_TABLE
id = sa.Column(sa.Integer, primary_key=True)
buildset_id = sa.Column(sa.Integer, sa.ForeignKey(
self.table_prefix + BUILDSET_TABLE + ".id",
name=(self.table_prefix +
'zuul_buildset_event_buildset_id_fkey'),
))
event_time = sa.Column(sa.DateTime)
event_type = sa.Column(sa.String(255))
description = sa.Column(sa.TEXT())
buildset = orm.relationship(BuildSetModel,
backref=orm.backref(
"buildset_events",
cascade="all, delete-orphan"))
sa.Index(self.table_prefix + 'zuul_buildset_event_buildset_id_idx',
buildset_id)
class BuildModel(Base):
__tablename__ = self.table_prefix + BUILD_TABLE
id = sa.Column(sa.Integer, primary_key=True)
@ -837,7 +870,7 @@ class SQLConnection(BaseConnection):
build_id)
class BuildEventModel(Base):
__tablename__ = self.table_prefix + BUILD_EVENTS_TABLE
__tablename__ = self.table_prefix + BUILD_EVENT_TABLE
id = sa.Column(sa.Integer, primary_key=True)
build_id = sa.Column(sa.Integer, sa.ForeignKey(
self.table_prefix + BUILD_TABLE + ".id",
@ -856,6 +889,9 @@ class SQLConnection(BaseConnection):
self.buildEventModel = BuildEventModel
self.zuul_build_event_table = self.buildEventModel.__table__
self.buildSetEventModel = BuildSetEventModel
self.zuul_buildset_event_table = self.buildSetEventModel.__table__
self.providesModel = ProvidesModel
self.zuul_provides_table = self.providesModel.__table__

8
zuul/driver/sql/sqlreporter.py
View File

@ -75,6 +75,14 @@ class SQLReporter(BaseReporter):
branch=getattr(change, 'branch', ''),
)
db_buildset.refs.append(ref)
event_change = item.getEventChange()
if event_change:
db_buildset.createBuildSetEvent(
event_time=datetime.datetime.fromtimestamp(
item.event.timestamp, tz=datetime.timezone.utc),
event_type='triggered',
description=f'Triggered by {event_change.toString()}',
)
return db_buildset
def reportBuildsetStart(self, buildset):

38
zuul/model.py
View File

@ -6043,6 +6043,16 @@ class QueueItem(zkobject.ZKObject):
keys.add(secret['blob'])
return keys
def getEventChange(self):
if not self.event:
return None
if not self.event.ref:
return None
sched = self.pipeline.manager.sched
key = ChangeKey.fromReference(self.event.ref)
source = sched.connections.getSource(key.connection_name)
return source.getChange(key)
# Cache info of a ref
CacheStat = namedtuple("CacheStat",
@ -6113,6 +6123,28 @@ class Ref(object):
self.ref, self.oldrev, self.newrev)
return rep
def toString(self):
# Not using __str__ because of prevalence in log lines and we
# prefer the repr syntax.
rep = None
pname = None
if self.project and self.project.name:
pname = self.project.name
if self.newrev == '0000000000000000000000000000000000000000':
rep = '%s %s deletes %s from %s' % (
type(self).__name__, pname,
self.ref, self.oldrev)
elif self.oldrev == '0000000000000000000000000000000000000000':
rep = '%s %s creates %s on %s' % (
type(self).__name__, pname,
self.ref, self.newrev)
else:
# Catch all
rep = '%s %s %s updated %s..%s' % (
type(self).__name__, pname,
self.ref, self.oldrev, self.newrev)
return rep
def equals(self, other):
if (self.project == other.project
and self.ref == other.ref
@ -6345,6 +6377,12 @@ class Change(Branch):
pname = self.project.name
return '<Change 0x%x %s %s>' % (id(self), pname, self._id())
def toString(self):
pname = None
if self.project and self.project.name:
pname = self.project.name
return '%s %s %s' % (type(self).__name__, pname, self._id())
def equals(self, other):
if (super().equals(other) and
isinstance(other, Change) and

43
zuul/web/__init__.py
View File

@ -296,10 +296,31 @@ class BuildConverter:
return ret
class BuildsetEventConverter:
# A class to encapsulate the conversion of database BuildsetEvent
# objects to API output.
def toDict(event):
event_time = _datetimeToString(event.event_time)
ret = {
'event_time': event_time,
'event_type': event.event_type,
'description': event.description,
}
return ret
def schema(builds=False):
ret = {
'event_time': str,
'event_type': str,
'description': str,
}
return Prop('The buildset event', ret)
class BuildsetConverter:
# A class to encapsulate the conversion of database Buildset
# objects to API output.
def toDict(buildset, builds=[]):
def toDict(buildset, builds=None, events=None):
event_timestamp = _datetimeToString(buildset.event_timestamp)
start = _datetimeToString(buildset.first_build_start_time)
end = _datetimeToString(buildset.last_build_end_time)
@ -319,12 +340,12 @@ class BuildsetConverter:
],
}
if builds:
ret['builds'] = []
for build in builds:
ret['builds'].append(BuildConverter.toDict(build))
ret['builds'] = [BuildConverter.toDict(b) for b in builds]
if events:
ret['events'] = [BuildsetEventConverter.toDict(e) for e in events]
return ret
def schema(builds=False):
def schema(builds=False, events=False):
ret = {
'_id': str,
'uuid': str,
@ -341,6 +362,8 @@ class BuildsetConverter:
}
if builds:
ret['builds'] = [BuildConverter.schema()]
if events:
ret['events'] = [BuildsetEventConverter.schema()]
return Prop('The buildset', ret)
@ -2007,13 +2030,21 @@ class ZuulWebAPI(object):
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
@cherrypy.tools.handle_options()
@cherrypy.tools.check_tenant_auth()
@openapi_response(
code=200,
content_type='application/json',
description='Returns a buildset',
schema=BuildsetConverter.schema(builds=True, events=True),
)
@openapi_response(404, 'Tenant not found')
def buildset(self, tenant_name, tenant, auth, uuid):
connection = self._get_connection()
data = connection.getBuildset(tenant_name, uuid)
if not data:
raise cherrypy.HTTPError(404, "Buildset not found")
data = BuildsetConverter.toDict(data, data.builds)
data = BuildsetConverter.toDict(data, data.builds,
data.buildset_events)
return data
@cherrypy.expose