# Copyright 2015 Rackspace Australia # Copyright 2024 Acme Gating, LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import datetime import json import logging import time import voluptuous as v import sqlalchemy.exc from zuul.lib.result_data import get_artifacts_from_result_data from zuul.reporter import BaseReporter class SQLReporter(BaseReporter): """Sends off reports to a database.""" name = 'sql' log = logging.getLogger("zuul.SQLReporter") retry_count = 3 retry_delay = 5 def _getBuildData(self, item, job, build): (result, _) = item.formatJobResult(job, build) start = end = None if build.start_time: start = datetime.datetime.fromtimestamp( build.start_time, tz=datetime.timezone.utc) if build.end_time: end = datetime.datetime.fromtimestamp( build.end_time, tz=datetime.timezone.utc) return result, build.log_url, start, end def _createBuildset(self, db, buildset): event_id = None event_timestamp = None item = buildset.item if item.event is not None: event_id = getattr(item.event, "zuul_event_id", None) event_timestamp = datetime.datetime.fromtimestamp( item.event.timestamp, tz=datetime.timezone.utc) db_buildset = db.createBuildSet( uuid=buildset.uuid, tenant=item.pipeline.tenant.name, pipeline=item.pipeline.name, event_id=event_id, event_timestamp=event_timestamp, updated=datetime.datetime.utcnow(), ) for change in item.changes: ref = db.getOrCreateRef( project=change.project.name, change=getattr(change, 'number', None), patchset=getattr(change, 'patchset', None), ref_url=change.url, ref=getattr(change, 'ref', ''), oldrev=getattr(change, 'oldrev', ''), newrev=getattr(change, 'newrev', ''), branch=getattr(change, 'branch', ''), ) db_buildset.refs.append(ref) return db_buildset def reportBuildsetStart(self, buildset): """Create the initial buildset entry in the db""" if not buildset.uuid: return for retry_count in range(self.retry_count): try: with self.connection.getSession() as db: return self._createBuildset(db, buildset) except sqlalchemy.exc.DBAPIError: if retry_count < self.retry_count - 1: self.log.error("Unable to create buildset, will retry") time.sleep(self.retry_delay) else: self.log.exception("Unable to create buildset") def reportBuildsetEnd(self, buildset, action, final, result=None): if not buildset.uuid: return if final: message = self._formatItemReport( buildset.item, with_jobs=False, action=action) else: message = None for retry_count in range(self.retry_count): try: with self.connection.getSession() as db: db_buildset = db.getBuildset( tenant=buildset.item.pipeline.tenant.name, uuid=buildset.uuid) if not db_buildset: db_buildset = self._createBuildset(db, buildset) db_buildset.result = buildset.result or result db_buildset.message = message end_time = db_buildset.first_build_start_time for build in db_buildset.builds: if (build.end_time and end_time and build.end_time > end_time): end_time = build.end_time db_buildset.last_build_end_time = end_time db_buildset.updated = datetime.datetime.utcnow() return except sqlalchemy.exc.DBAPIError: if retry_count < self.retry_count - 1: self.log.error("Unable to update buildset, will retry") time.sleep(self.retry_delay) else: self.log.exception("Unable to update buildset") def reportBuildStart(self, build): for retry_count in range(self.retry_count): try: with self.connection.getSession() as db: db_build = self._createBuild(db, build) return db_build except sqlalchemy.exc.DBAPIError: if retry_count < self.retry_count - 1: self.log.error("Unable to create build, will retry") time.sleep(self.retry_delay) else: self.log.exception("Unable to create build") def reportBuildEnd(self, build, tenant, final): for retry_count in range(self.retry_count): try: with self.connection.getSession() as db: db_build = db.getBuild(tenant=tenant, uuid=build.uuid) if not db_build: db_build = self._createBuild(db, build) end_time = build.end_time or time.time() end = datetime.datetime.fromtimestamp( end_time, tz=datetime.timezone.utc) db_build.result = build.result db_build.end_time = end db_build.log_url = build.log_url db_build.error_detail = build.error_detail db_build.final = final db_build.held = build.held for provides in build.job.provides: db_build.createProvides(name=provides) for artifact in get_artifacts_from_result_data( build.result_data, logger=self.log): if 'metadata' in artifact: artifact['metadata'] = json.dumps( artifact['metadata']) db_build.createArtifact(**artifact) for event in build.events: # Reformat the event_time so it's compatible to SQL. # Don't update the event object in place, but only # the generated dict representation to not alter the # datastructure for other reporters. ev = event.toDict() ev["event_time"] = datetime.datetime.fromtimestamp( event.event_time, tz=datetime.timezone.utc) db_build.createBuildEvent(**ev) return db_build except sqlalchemy.exc.DBAPIError: if retry_count < self.retry_count - 1: self.log.error("Unable to update build, will retry") time.sleep(self.retry_delay) else: self.log.exception("Unable to update build") def _createBuild(self, db, build): start_time = build.start_time or time.time() start = datetime.datetime.fromtimestamp(start_time, tz=datetime.timezone.utc) buildset = build.build_set if not buildset: return db_buildset = db.getBuildset( tenant=buildset.item.pipeline.tenant.name, uuid=buildset.uuid) if not db_buildset: self.log.warning("Creating missing buildset %s", buildset.uuid) db_buildset = self._createBuildset(db, buildset) if db_buildset.first_build_start_time is None: db_buildset.first_build_start_time = start item = buildset.item change = item.getChangeForJob(build.job) ref = db.getOrCreateRef( project=change.project.name, change=getattr(change, 'number', None), patchset=getattr(change, 'patchset', None), ref_url=change.url, ref=getattr(change, 'ref', ''), oldrev=getattr(change, 'oldrev', ''), newrev=getattr(change, 'newrev', ''), branch=getattr(change, 'branch', ''), ) db_build = db_buildset.createBuild( ref=ref, uuid=build.uuid, job_name=build.job.name, start_time=start, voting=build.job.voting, nodeset=build.job.nodeset.name, ) return db_build def getBuilds(self, *args, **kw): """Return a list of Build objects""" return self.connection.getBuilds(*args, **kw) def report(self, item): # We're not a real reporter, but we use _formatItemReport, so # we inherit from the reporters. raise NotImplementedError() def getSchema(): sql_reporter = v.Schema(None) return sql_reporter