Browse Source

Merge "Report retried builds via sql reporter."

changes/83/703983/7
Zuul 2 weeks ago
committed by Gerrit Code Review
parent
commit
35128d24c9
6 changed files with 196 additions and 33 deletions
  1. +9
    -0
      releasenotes/notes/non-final-builds-d9024df868c7e036.yaml
  2. +82
    -1
      tests/unit/test_connection.py
  3. +43
    -0
      zuul/driver/sql/alembic/versions/269691d2220e_add_build_final_column.py
  4. +3
    -1
      zuul/driver/sql/sqlconnection.py
  5. +46
    -28
      zuul/driver/sql/sqlreporter.py
  6. +13
    -3
      zuul/web/__init__.py

+ 9
- 0
releasenotes/notes/non-final-builds-d9024df868c7e036.yaml View File

@@ -0,0 +1,9 @@
---
features:
- |
The builds/ and buildset/ API endpoints now include information about
retried builds. They are called non-final as those are all builds that
were retried at least once and thus weren't visible to the user so far.

The builds/ API can filter those retried builds and you might exclude
them from API result by setting ``final=false`` in the API request.

+ 82
- 1
tests/unit/test_connection.py View File

@@ -75,7 +75,7 @@ class TestSQLConnection(ZuulDBTestCase):
build_table = table_prefix + 'zuul_build'

self.assertEqual(16, len(insp.get_columns(buildset_table)))
self.assertEqual(11, len(insp.get_columns(build_table)))
self.assertEqual(12, len(insp.get_columns(build_table)))

def test_sql_tables_created(self):
"Test the tables for storing results are created properly"
@@ -226,6 +226,87 @@ class TestSQLConnection(ZuulDBTestCase):
check_results('resultsdb_mysql')
check_results('resultsdb_postgresql')

def test_sql_results_retry_builds(self):
"Test that retry results are entered into an sql table correctly"

# Check the results
def check_results(connection_name):
# Grab the sa tables
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
reporter = _get_reporter_from_connection_name(
tenant.layout.pipelines["check"].success_actions,
connection_name
)

with self.connections.connections[
connection_name].engine.connect() as conn:

result = conn.execute(
sa.sql.select([reporter.connection.zuul_buildset_table])
)

buildsets = result.fetchall()
self.assertEqual(1, len(buildsets))
buildset0 = buildsets[0]

self.assertEqual('check', buildset0['pipeline'])
self.assertEqual('org/project', buildset0['project'])
self.assertEqual(1, buildset0['change'])
self.assertEqual('1', buildset0['patchset'])
self.assertEqual('SUCCESS', buildset0['result'])
self.assertEqual('Build succeeded.', buildset0['message'])
self.assertEqual('tenant-one', buildset0['tenant'])
self.assertEqual(
'https://review.example.com/%d' % buildset0['change'],
buildset0['ref_url'])

buildset0_builds = conn.execute(
sa.sql.select(
[reporter.connection.zuul_build_table]
).where(
reporter.connection.zuul_build_table.c.buildset_id ==
buildset0['id']
)
).fetchall()

# Check the retry results
self.assertEqual('project-merge', buildset0_builds[0]['job_name'])
self.assertEqual('SUCCESS', buildset0_builds[0]['result'])
self.assertTrue(buildset0_builds[0]['final'])

self.assertEqual('project-test1', buildset0_builds[1]['job_name'])
self.assertEqual('RETRY', buildset0_builds[1]['result'])
self.assertFalse(buildset0_builds[1]['final'])
self.assertEqual('project-test1', buildset0_builds[2]['job_name'])
self.assertEqual('SUCCESS', buildset0_builds[2]['result'])
self.assertTrue(buildset0_builds[2]['final'])

self.assertEqual('project-test2', buildset0_builds[3]['job_name'])
self.assertEqual('RETRY', buildset0_builds[3]['result'])
self.assertFalse(buildset0_builds[3]['final'])
self.assertEqual('project-test2', buildset0_builds[4]['job_name'])
self.assertEqual('SUCCESS', buildset0_builds[4]['result'])
self.assertTrue(buildset0_builds[4]['final'])

self.executor_server.hold_jobs_in_build = True

# Add a retry result
self.log.debug("Adding retry FakeChange")
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Release the merge job (which is the dependency for the other jobs)
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Let both test jobs fail on the first run, so they are both run again.
self.builds[0].requeue = True
self.builds[1].requeue = True
self.orderedRelease()
self.waitUntilSettled()

check_results('resultsdb_mysql')
check_results('resultsdb_postgresql')

def test_multiple_sql_connections(self):
"Test putting results in different databases"
# Add a successful result


+ 43
- 0
zuul/driver/sql/alembic/versions/269691d2220e_add_build_final_column.py View File

@@ -0,0 +1,43 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""add build final column

Revision ID: 269691d2220e
Revises: e0eda5d09eae
Create Date: 2020-01-03 07:53:15.962739

"""

# revision identifiers, used by Alembic.
revision = '269691d2220e'
down_revision = '16c1dc9054d0'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

BUILD_TABLE = 'zuul_build'


def upgrade(table_prefix=''):
op.add_column(table_prefix + BUILD_TABLE, sa.Column('final', sa.Boolean))

# Set all existing build entries to final (otherwise they will vanish from
# the UI)
new_column = sa.table(table_prefix + BUILD_TABLE, sa.column('final'))
op.execute(new_column.update().values(**{'final': True}))


def downgrade():
raise Exception("Downgrades not supported")

+ 3
- 1
zuul/driver/sql/sqlconnection.py View File

@@ -60,7 +60,7 @@ class DatabaseSession(object):
change=None, branch=None, patchset=None, ref=None,
newrev=None, event_id=None, uuid=None, job_name=None,
voting=None, node_name=None, result=None, provides=None,
limit=50, offset=0):
final=None, limit=50, offset=0):

build_table = self.connection.zuul_build_table
buildset_table = self.connection.zuul_buildset_table
@@ -102,6 +102,7 @@ class DatabaseSession(object):
q = self.listFilter(q, build_table.c.voting, voting)
q = self.listFilter(q, build_table.c.node_name, node_name)
q = self.listFilter(q, build_table.c.result, result)
q = self.listFilter(q, build_table.c.final, final)
q = self.listFilter(q, provides_table.c.name, provides)

q = q.order_by(build_table.c.id.desc()).\
@@ -301,6 +302,7 @@ class SQLConnection(BaseConnection):
log_url = sa.Column(sa.String(255))
node_name = sa.Column(sa.String(255))
error_detail = sa.Column(sa.TEXT())
final = sa.Column(sa.Boolean)
buildset = orm.relationship(BuildSetModel, backref="builds")

def createArtifact(self, *args, **kw):


+ 46
- 28
zuul/driver/sql/sqlreporter.py View File

@@ -29,6 +29,43 @@ class SQLReporter(BaseReporter):
name = 'sql'
log = logging.getLogger("zuul.SQLReporter")

def _getBuildData(self, item, job, build):
(result, url) = item.formatJobResult(job, build)
log_url = build.result_data.get('zuul', {}).get('log_url')
if log_url and log_url[-1] != '/':
log_url = log_url + '/'
start = end = None
if build.start_time:
start = datetime.datetime.fromtimestamp(
build.start_time,
tz=datetime.timezone.utc)
if build.end_time:
end = datetime.datetime.fromtimestamp(
build.end_time,
tz=datetime.timezone.utc)
return result, log_url, start, end

def createBuildEntry(self, item, job, db_buildset, build, final=True):
# Ensure end_time is defined
if not build.end_time:
build.end_time = time.time()

result, log_url, start, end = self._getBuildData(item, job, build)
db_build = db_buildset.createBuild(
uuid=build.uuid,
job_name=build.job.name,
result=result,
start_time=start,
end_time=end,
voting=build.job.voting,
log_url=log_url,
node_name=build.node_name,
error_detail=build.error_detail,
final=final,
)

return db_build

def report(self, item):
"""Create an entry into a database."""
log = get_annotated_logger(self.log, item.event)
@@ -66,35 +103,16 @@ class SQLReporter(BaseReporter):
# stats about builds. It doesn't understand how to store
# information about the change.
continue
# Ensure end_time is defined
if not build.end_time:
build.end_time = time.time()

(result, url) = item.formatJobResult(job)
log_url = build.result_data.get('zuul', {}).get('log_url')
if log_url and log_url[-1] != '/':
log_url = log_url + '/'
start = end = None
if build.start_time:
start = datetime.datetime.fromtimestamp(
build.start_time,
tz=datetime.timezone.utc)
if build.end_time:
end = datetime.datetime.fromtimestamp(
build.end_time,
tz=datetime.timezone.utc)

db_build = db_buildset.createBuild(
uuid=build.uuid,
job_name=build.job.name,
result=result,
start_time=start,
end_time=end,
voting=build.job.voting,
log_url=log_url,
node_name=build.node_name,
error_detail=build.error_detail,

retry_builds = item.current_build_set.getRetryBuildsForJob(
job.name
)
for retry_build in retry_builds:
self.createBuildEntry(
item, job, db_buildset, retry_build, final=False
)

db_build = self.createBuildEntry(item, job, db_buildset, build)

for provides in job.provides:
db_build.createProvides(name=provides)


+ 13
- 3
zuul/web/__init__.py View File

@@ -829,6 +829,7 @@ class ZuulWebAPI(object):
'log_url': build.log_url,
'node_name': build.node_name,
'error_detail': build.error_detail,
'final': build.final,
'artifacts': [],
'provides': [],
}
@@ -880,14 +881,18 @@ class ZuulWebAPI(object):
def builds(self, tenant, project=None, pipeline=None, change=None,
branch=None, patchset=None, ref=None, newrev=None,
uuid=None, job_name=None, voting=None, node_name=None,
result=None, limit=50, skip=0):
result=None, final=None, limit=50, skip=0):
connection = self._get_connection(tenant)

# If final is None, we return all builds, both final and non-final
if final is not None:
final = final.lower() == "true"

builds = connection.getBuilds(
tenant=tenant, project=project, pipeline=pipeline, change=change,
branch=branch, patchset=patchset, ref=ref, newrev=newrev,
uuid=uuid, job_name=job_name, voting=voting, node_name=node_name,
result=result, limit=limit, offset=skip)
result=result, final=final, limit=limit, offset=skip)

resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'
@@ -924,8 +929,13 @@ class ZuulWebAPI(object):
}
if builds:
ret['builds'] = []
ret['retry_builds'] = []
for build in builds:
ret['builds'].append(self.buildToDict(build))
# Put all non-final (retry) builds under a different key
if not build.final:
ret['retry_builds'].append(self.buildToDict(build))
else:
ret['builds'].append(self.buildToDict(build))
return ret

@cherrypy.expose


Loading…
Cancel
Save