Merge "Add buildset event db table"

This commit is contained in:
Zuul 2024-05-17 16:29:38 +00:00 committed by Gerrit Code Review
commit 01608f1e94
7 changed files with 192 additions and 9 deletions

View File

@ -78,6 +78,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
artifact_table = table_prefix + 'zuul_artifact'
provides_table = table_prefix + 'zuul_provides'
build_event_table = table_prefix + 'zuul_build_event'
buildset_event_table = table_prefix + 'zuul_buildset_event'
self.assertEqual(9, len(insp.get_columns(ref_table)))
self.assertEqual(11, len(insp.get_columns(buildset_table)))
@ -86,6 +87,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
self.assertEqual(5, len(insp.get_columns(artifact_table)))
self.assertEqual(3, len(insp.get_columns(provides_table)))
self.assertEqual(5, len(insp.get_columns(build_event_table)))
self.assertEqual(5, len(insp.get_columns(buildset_event_table)))
def test_sql_tables_created(self):
"Test the tables for storing results are created properly"
@ -105,6 +107,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
artifact_table = table_prefix + 'zuul_artifact'
provides_table = table_prefix + 'zuul_provides'
build_event_table = table_prefix + 'zuul_build_event'
buildset_event_table = table_prefix + 'zuul_buildset_event'
indexes_ref = insp.get_indexes(ref_table)
indexes_buildset = insp.get_indexes(buildset_table)
@ -113,6 +116,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
indexes_artifact = insp.get_indexes(artifact_table)
indexes_provides = insp.get_indexes(provides_table)
indexes_build_event = insp.get_indexes(build_event_table)
indexes_buildset_event = insp.get_indexes(buildset_event_table)
self.assertEqual(8, len(indexes_ref))
self.assertEqual(2, len(indexes_buildset))
@ -121,12 +125,13 @@ class TestSQLConnectionMysql(ZuulTestCase):
self.assertEqual(1, len(indexes_artifact))
self.assertEqual(1, len(indexes_provides))
self.assertEqual(1, len(indexes_build_event))
self.assertEqual(1, len(indexes_buildset_event))
# check if all indexes are prefixed
if table_prefix:
indexes = (indexes_ref + indexes_buildset + indexes_buildset_ref +
indexes_build + indexes_artifact + indexes_provides +
indexes_build_event)
indexes_build_event + indexes_buildset_event)
for index in indexes:
self.assertTrue(index['name'].startswith(table_prefix))
@ -353,6 +358,9 @@ class TestSQLConnectionMysql(ZuulTestCase):
f"delete from {self.expected_table_prefix}zuul_buildset_ref;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}zuul_build;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}"
"zuul_buildset_event;"))
result = conn.execute(sa.text(
f"delete from {self.expected_table_prefix}zuul_buildset;"))
result = conn.execute(sa.text("commit;"))

View File

@ -2061,6 +2061,11 @@ class TestBuildInfo(BaseTestWeb):
"api/tenant/tenant-one/buildset/%s" % project_bs['uuid']).json()
self.assertEqual(3, len(buildset["builds"]))
self.assertEqual(1, len(buildset["events"]))
self.assertEqual('triggered', buildset["events"][0]['event_type'])
self.assertEqual('Triggered by GerritChange org/project 1,1',
buildset["events"][0]['description'])
project_test1_build = [x for x in buildset["builds"]
if x["job_name"] == "project-test1"][0]
self.assertEqual('SUCCESS', project_test1_build['result'])

View File

@ -0,0 +1,57 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""buildset_event_table
Revision ID: 6c1582c1d08c
Revises: ac1dad8c9434
Create Date: 2024-03-25 12:28:58.885794
"""
# revision identifiers, used by Alembic.
revision = '6c1582c1d08c'
down_revision = 'ac1dad8c9434'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
BUILDSET_EVENT_TABLE = "zuul_buildset_event"
BUILDSET_TABLE = "zuul_buildset"
def upgrade(table_prefix=''):
prefixed_buildset_event = table_prefix + BUILDSET_EVENT_TABLE
prefixed_buildset = table_prefix + BUILDSET_TABLE
op.create_table(
prefixed_buildset_event,
sa.Column("id", sa.Integer, primary_key=True),
sa.Column("buildset_id", sa.Integer,
sa.ForeignKey(
f'{prefixed_buildset}.id',
name=f'{prefixed_buildset_event}_buildset_id_fkey',
)),
sa.Column("event_time", sa.DateTime),
sa.Column("event_type", sa.String(255)),
sa.Column("description", sa.TEXT()),
)
op.create_index(
f'{prefixed_buildset_event}_buildset_id_idx',
prefixed_buildset_event,
['buildset_id'])
def downgrade():
raise Exception("Downgrades not supported")

View File

@ -33,8 +33,9 @@ from zuul.zk.locks import CONNECTION_LOCK_ROOT, locked, SessionAwareLock
BUILDSET_TABLE = 'zuul_buildset'
REF_TABLE = 'zuul_ref'
BUILDSET_REF_TABLE = 'zuul_buildset_ref'
BUILDSET_EVENT_TABLE = 'zuul_buildset_event'
BUILD_TABLE = 'zuul_build'
BUILD_EVENTS_TABLE = 'zuul_build_event'
BUILD_EVENT_TABLE = 'zuul_build_event'
ARTIFACT_TABLE = 'zuul_artifact'
PROVIDES_TABLE = 'zuul_provides'
@ -498,6 +499,8 @@ class DatabaseSession(object):
q = self.session().query(self.connection.buildSetModel).\
options(orm.joinedload(self.connection.buildSetModel.refs)).\
options(orm.joinedload(
self.connection.buildSetModel.buildset_events)).\
options(orm.joinedload(self.connection.buildSetModel.builds).
subqueryload(self.connection.buildModel.artifacts)).\
options(orm.joinedload(self.connection.buildSetModel.builds).
@ -528,6 +531,9 @@ class DatabaseSession(object):
orm.selectinload(
self.connection.buildSetModel.refs,
),
orm.selectinload(
self.connection.buildSetModel.buildset_events,
),
orm.selectinload(
self.connection.buildSetModel.builds,
self.connection.buildModel.provides,
@ -724,6 +730,15 @@ class SQLConnection(BaseConnection):
session.flush()
return b
def createBuildSetEvent(self, *args, **kw):
session = orm.session.Session.object_session(self)
e = BuildSetEventModel(*args, **kw)
e.buildset_id = self.id
self.buildset_events.append(e)
session.add(e)
session.flush()
return e
class BuildSetRefModel(Base):
__tablename__ = self.table_prefix + BUILDSET_REF_TABLE
__table_args__ = (
@ -742,6 +757,24 @@ class SQLConnection(BaseConnection):
sa.Index(self.table_prefix + 'zuul_buildset_ref_ref_id_idx',
ref_id)
class BuildSetEventModel(Base):
__tablename__ = self.table_prefix + BUILDSET_EVENT_TABLE
id = sa.Column(sa.Integer, primary_key=True)
buildset_id = sa.Column(sa.Integer, sa.ForeignKey(
self.table_prefix + BUILDSET_TABLE + ".id",
name=(self.table_prefix +
'zuul_buildset_event_buildset_id_fkey'),
))
event_time = sa.Column(sa.DateTime)
event_type = sa.Column(sa.String(255))
description = sa.Column(sa.TEXT())
buildset = orm.relationship(BuildSetModel,
backref=orm.backref(
"buildset_events",
cascade="all, delete-orphan"))
sa.Index(self.table_prefix + 'zuul_buildset_event_buildset_id_idx',
buildset_id)
class BuildModel(Base):
__tablename__ = self.table_prefix + BUILD_TABLE
id = sa.Column(sa.Integer, primary_key=True)
@ -860,7 +893,7 @@ class SQLConnection(BaseConnection):
build_id)
class BuildEventModel(Base):
__tablename__ = self.table_prefix + BUILD_EVENTS_TABLE
__tablename__ = self.table_prefix + BUILD_EVENT_TABLE
id = sa.Column(sa.Integer, primary_key=True)
build_id = sa.Column(sa.Integer, sa.ForeignKey(
self.table_prefix + BUILD_TABLE + ".id",
@ -879,6 +912,9 @@ class SQLConnection(BaseConnection):
self.buildEventModel = BuildEventModel
self.zuul_build_event_table = self.buildEventModel.__table__
self.buildSetEventModel = BuildSetEventModel
self.zuul_buildset_event_table = self.buildSetEventModel.__table__
self.providesModel = ProvidesModel
self.zuul_provides_table = self.providesModel.__table__

View File

@ -75,6 +75,14 @@ class SQLReporter(BaseReporter):
branch=getattr(change, 'branch', ''),
)
db_buildset.refs.append(ref)
event_change = item.getEventChange()
if event_change:
db_buildset.createBuildSetEvent(
event_time=datetime.datetime.fromtimestamp(
item.event.timestamp, tz=datetime.timezone.utc),
event_type='triggered',
description=f'Triggered by {event_change.toString()}',
)
return db_buildset
def reportBuildsetStart(self, buildset):

View File

@ -6046,6 +6046,16 @@ class QueueItem(zkobject.ZKObject):
keys.add(secret['blob'])
return keys
def getEventChange(self):
if not self.event:
return None
if not self.event.ref:
return None
sched = self.pipeline.manager.sched
key = ChangeKey.fromReference(self.event.ref)
source = sched.connections.getSource(key.connection_name)
return source.getChange(key)
# Cache info of a ref
CacheStat = namedtuple("CacheStat",
@ -6116,6 +6126,28 @@ class Ref(object):
self.ref, self.oldrev, self.newrev)
return rep
def toString(self):
# Not using __str__ because of prevalence in log lines and we
# prefer the repr syntax.
rep = None
pname = None
if self.project and self.project.name:
pname = self.project.name
if self.newrev == '0000000000000000000000000000000000000000':
rep = '%s %s deletes %s from %s' % (
type(self).__name__, pname,
self.ref, self.oldrev)
elif self.oldrev == '0000000000000000000000000000000000000000':
rep = '%s %s creates %s on %s' % (
type(self).__name__, pname,
self.ref, self.newrev)
else:
# Catch all
rep = '%s %s %s updated %s..%s' % (
type(self).__name__, pname,
self.ref, self.oldrev, self.newrev)
return rep
def equals(self, other):
if (self.project == other.project
and self.ref == other.ref
@ -6348,6 +6380,12 @@ class Change(Branch):
pname = self.project.name
return '<Change 0x%x %s %s>' % (id(self), pname, self._id())
def toString(self):
pname = None
if self.project and self.project.name:
pname = self.project.name
return '%s %s %s' % (type(self).__name__, pname, self._id())
def equals(self, other):
if (super().equals(other) and
isinstance(other, Change) and

View File

@ -296,10 +296,31 @@ class BuildConverter:
return ret
class BuildsetEventConverter:
# A class to encapsulate the conversion of database BuildsetEvent
# objects to API output.
def toDict(event):
event_time = _datetimeToString(event.event_time)
ret = {
'event_time': event_time,
'event_type': event.event_type,
'description': event.description,
}
return ret
def schema(builds=False):
ret = {
'event_time': str,
'event_type': str,
'description': str,
}
return Prop('The buildset event', ret)
class BuildsetConverter:
# A class to encapsulate the conversion of database Buildset
# objects to API output.
def toDict(buildset, builds=[]):
def toDict(buildset, builds=None, events=None):
event_timestamp = _datetimeToString(buildset.event_timestamp)
start = _datetimeToString(buildset.first_build_start_time)
end = _datetimeToString(buildset.last_build_end_time)
@ -319,12 +340,12 @@ class BuildsetConverter:
],
}
if builds:
ret['builds'] = []
for build in builds:
ret['builds'].append(BuildConverter.toDict(build))
ret['builds'] = [BuildConverter.toDict(b) for b in builds]
if events:
ret['events'] = [BuildsetEventConverter.toDict(e) for e in events]
return ret
def schema(builds=False):
def schema(builds=False, events=False):
ret = {
'_id': str,
'uuid': str,
@ -341,6 +362,8 @@ class BuildsetConverter:
}
if builds:
ret['builds'] = [BuildConverter.schema()]
if events:
ret['events'] = [BuildsetEventConverter.schema()]
return Prop('The buildset', ret)
@ -2007,13 +2030,21 @@ class ZuulWebAPI(object):
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
@cherrypy.tools.handle_options()
@cherrypy.tools.check_tenant_auth()
@openapi_response(
code=200,
content_type='application/json',
description='Returns a buildset',
schema=BuildsetConverter.schema(builds=True, events=True),
)
@openapi_response(404, 'Tenant not found')
def buildset(self, tenant_name, tenant, auth, uuid):
connection = self._get_connection()
data = connection.getBuildset(tenant_name, uuid)
if not data:
raise cherrypy.HTTPError(404, "Buildset not found")
data = BuildsetConverter.toDict(data, data.builds)
data = BuildsetConverter.toDict(data, data.builds,
data.buildset_events)
return data
@cherrypy.expose