Annotate logs around build states
It's useful to have logs around build states annotated with the event ids that caused it. Change-Id: Ib2173210516823cfe16e34710e8d01319aba8cce
This commit is contained in:
parent
6b37468a38
commit
a703f42c24
|
@ -503,12 +503,12 @@ class ExecutorClient(object):
|
||||||
self.onBuildCompleted(job, 'LOST')
|
self.onBuildCompleted(job, 'LOST')
|
||||||
|
|
||||||
def cancelJobInQueue(self, build):
|
def cancelJobInQueue(self, build):
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
job = build.__gearman_job
|
job = build.__gearman_job
|
||||||
|
|
||||||
req = gear.CancelJobAdminRequest(job.handle)
|
req = gear.CancelJobAdminRequest(job.handle)
|
||||||
job.connection.sendAdminRequest(req, timeout=300)
|
job.connection.sendAdminRequest(req, timeout=300)
|
||||||
self.log.debug("Response to cancel build %s request: %s" %
|
log.debug("Response to cancel build request: %s", req.response.strip())
|
||||||
(build, req.response.strip()))
|
|
||||||
if req.response.startswith(b"OK"):
|
if req.response.startswith(b"OK"):
|
||||||
try:
|
try:
|
||||||
del self.builds[job.unique]
|
del self.builds[job.unique]
|
||||||
|
@ -522,29 +522,31 @@ class ExecutorClient(object):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def cancelRunningBuild(self, build):
|
def cancelRunningBuild(self, build):
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
if not build.__gearman_worker:
|
if not build.__gearman_worker:
|
||||||
self.log.error("Build %s has no manager while canceling" %
|
log.error("Build %s has no manager while canceling", build)
|
||||||
(build,))
|
|
||||||
stop_uuid = str(uuid4().hex)
|
stop_uuid = str(uuid4().hex)
|
||||||
data = dict(uuid=build.__gearman_job.unique)
|
data = dict(uuid=build.__gearman_job.unique,
|
||||||
|
zuul_event_id=build.zuul_event_id)
|
||||||
stop_job = gear.TextJob("executor:stop:%s" % build.__gearman_worker,
|
stop_job = gear.TextJob("executor:stop:%s" % build.__gearman_worker,
|
||||||
json_dumps(data), unique=stop_uuid)
|
json_dumps(data), unique=stop_uuid)
|
||||||
self.meta_jobs[stop_uuid] = stop_job
|
self.meta_jobs[stop_uuid] = stop_job
|
||||||
self.log.debug("Submitting stop job: %s", stop_job)
|
log.debug("Submitting stop job: %s", stop_job)
|
||||||
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
|
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
|
||||||
timeout=300)
|
timeout=300)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def resumeBuild(self, build):
|
def resumeBuild(self, build):
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
if not build.__gearman_worker:
|
if not build.__gearman_worker:
|
||||||
self.log.error("Build %s has no manager while resuming" %
|
log.error("Build %s has no manager while resuming", build)
|
||||||
(build,))
|
|
||||||
resume_uuid = str(uuid4().hex)
|
resume_uuid = str(uuid4().hex)
|
||||||
data = dict(uuid=build.__gearman_job.unique)
|
data = dict(uuid=build.__gearman_job.unique,
|
||||||
|
zuul_event_id=build.zuul_event_id)
|
||||||
stop_job = gear.TextJob("executor:resume:%s" % build.__gearman_worker,
|
stop_job = gear.TextJob("executor:resume:%s" % build.__gearman_worker,
|
||||||
json_dumps(data), unique=resume_uuid)
|
json_dumps(data), unique=resume_uuid)
|
||||||
self.meta_jobs[resume_uuid] = stop_job
|
self.meta_jobs[resume_uuid] = stop_job
|
||||||
self.log.debug("Submitting resume job: %s", stop_job)
|
log.debug("Submitting resume job: %s", stop_job)
|
||||||
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
|
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
|
||||||
timeout=300)
|
timeout=300)
|
||||||
|
|
||||||
|
|
|
@ -2618,21 +2618,25 @@ class ExecutorServer(object):
|
||||||
if not self._running:
|
if not self._running:
|
||||||
job.sendWorkFail()
|
job.sendWorkFail()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
args = json.loads(job.arguments)
|
||||||
|
zuul_event_id = args.get('zuul_event_id')
|
||||||
|
log = get_annotated_logger(self.log, zuul_event_id)
|
||||||
|
|
||||||
function_name = 'executor:execute'
|
function_name = 'executor:execute'
|
||||||
if self.zone:
|
if self.zone:
|
||||||
function_name += ':%s' % self.zone
|
function_name += ':%s' % self.zone
|
||||||
if job.name == (function_name):
|
if job.name == (function_name):
|
||||||
self.log.debug("Got %s job: %s" %
|
log.debug("Got %s job: %s", function_name, job.unique)
|
||||||
(function_name, job.unique))
|
|
||||||
self.executeJob(job)
|
self.executeJob(job)
|
||||||
elif job.name.startswith('executor:resume'):
|
elif job.name.startswith('executor:resume'):
|
||||||
self.log.debug("Got resume job: %s" % job.unique)
|
log.debug("Got resume job: %s", job.unique)
|
||||||
self.resumeJob(job)
|
self.resumeJob(job)
|
||||||
elif job.name.startswith('executor:stop'):
|
elif job.name.startswith('executor:stop'):
|
||||||
self.log.debug("Got stop job: %s" % job.unique)
|
log.debug("Got stop job: %s", job.unique)
|
||||||
self.stopJob(job)
|
self.stopJob(job)
|
||||||
else:
|
else:
|
||||||
self.log.error("Unable to handle job %s" % job.name)
|
log.error("Unable to handle job %s", job.name)
|
||||||
job.sendWorkFail()
|
job.sendWorkFail()
|
||||||
|
|
||||||
def executeJob(self, job):
|
def executeJob(self, job):
|
||||||
|
@ -2698,42 +2702,46 @@ class ExecutorServer(object):
|
||||||
def resumeJob(self, job):
|
def resumeJob(self, job):
|
||||||
try:
|
try:
|
||||||
args = json.loads(job.arguments)
|
args = json.loads(job.arguments)
|
||||||
self.log.debug("Resume job with arguments: %s" % (args,))
|
zuul_event_id = args.get('zuul_event_id')
|
||||||
|
log = get_annotated_logger(self.log, zuul_event_id)
|
||||||
|
log.debug("Resume job with arguments: %s", args)
|
||||||
unique = args['uuid']
|
unique = args['uuid']
|
||||||
self.resumeJobByUnique(unique)
|
self.resumeJobByUnique(unique, zuul_event_id=zuul_event_id)
|
||||||
finally:
|
finally:
|
||||||
job.sendWorkComplete()
|
job.sendWorkComplete()
|
||||||
|
|
||||||
def stopJob(self, job):
|
def stopJob(self, job):
|
||||||
try:
|
try:
|
||||||
args = json.loads(job.arguments)
|
args = json.loads(job.arguments)
|
||||||
self.log.debug("Stop job with arguments: %s" % (args,))
|
zuul_event_id = args.get('zuul_event_id')
|
||||||
|
log = get_annotated_logger(self.log, zuul_event_id)
|
||||||
|
log.debug("Stop job with arguments: %s", args)
|
||||||
unique = args['uuid']
|
unique = args['uuid']
|
||||||
self.stopJobByUnique(unique)
|
self.stopJobByUnique(unique, zuul_event_id=zuul_event_id)
|
||||||
finally:
|
finally:
|
||||||
job.sendWorkComplete()
|
job.sendWorkComplete()
|
||||||
|
|
||||||
def resumeJobByUnique(self, unique):
|
def resumeJobByUnique(self, unique, zuul_event_id=None):
|
||||||
|
log = get_annotated_logger(self.log, zuul_event_id)
|
||||||
job_worker = self.job_workers.get(unique)
|
job_worker = self.job_workers.get(unique)
|
||||||
if not job_worker:
|
if not job_worker:
|
||||||
self.log.debug("Unable to find worker for job %s" % (unique,))
|
log.debug("Unable to find worker for job %s", unique)
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
job_worker.resume()
|
job_worker.resume()
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Exception sending resume command "
|
log.exception("Exception sending resume command to worker:")
|
||||||
"to worker:")
|
|
||||||
|
|
||||||
def stopJobByUnique(self, unique, reason=None):
|
def stopJobByUnique(self, unique, reason=None, zuul_event_id=None):
|
||||||
|
log = get_annotated_logger(self.log, zuul_event_id)
|
||||||
job_worker = self.job_workers.get(unique)
|
job_worker = self.job_workers.get(unique)
|
||||||
if not job_worker:
|
if not job_worker:
|
||||||
self.log.debug("Unable to find worker for job %s" % (unique,))
|
log.debug("Unable to find worker for job %s", unique)
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
job_worker.stop(reason)
|
job_worker.stop(reason)
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Exception sending stop command "
|
log.exception("Exception sending stop command to worker:")
|
||||||
"to worker:")
|
|
||||||
|
|
||||||
def cat(self, job):
|
def cat(self, job):
|
||||||
args = json.loads(job.arguments)
|
args = json.loads(job.arguments)
|
||||||
|
|
|
@ -825,12 +825,14 @@ class PipelineManager(object):
|
||||||
return changed
|
return changed
|
||||||
|
|
||||||
def onBuildStarted(self, build):
|
def onBuildStarted(self, build):
|
||||||
self.log.debug("Build %s started" % build)
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
|
log.debug("Build %s started", build)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def onBuildPaused(self, build):
|
def onBuildPaused(self, build):
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
item = build.build_set.item
|
item = build.build_set.item
|
||||||
self.log.debug("Build %s of %s paused", build, item.change)
|
log.debug("Build %s of %s paused", build, item.change)
|
||||||
item.setResult(build)
|
item.setResult(build)
|
||||||
|
|
||||||
# We need to resume builds because we could either have no children
|
# We need to resume builds because we could either have no children
|
||||||
|
|
|
@ -1163,32 +1163,30 @@ class Scheduler(threading.Thread):
|
||||||
|
|
||||||
def _doBuildStartedEvent(self, event):
|
def _doBuildStartedEvent(self, event):
|
||||||
build = event.build
|
build = event.build
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
if build.build_set is not build.build_set.item.current_build_set:
|
if build.build_set is not build.build_set.item.current_build_set:
|
||||||
self.log.warning("Build %s is not in the current build set" %
|
log.warning("Build %s is not in the current build set", build)
|
||||||
(build,))
|
|
||||||
return
|
return
|
||||||
pipeline = build.build_set.item.pipeline
|
pipeline = build.build_set.item.pipeline
|
||||||
if not pipeline:
|
if not pipeline:
|
||||||
self.log.warning("Build %s is not associated with a pipeline" %
|
log.warning("Build %s is not associated with a pipeline", build)
|
||||||
(build,))
|
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
build.estimated_time = float(self.time_database.getEstimatedTime(
|
build.estimated_time = float(self.time_database.getEstimatedTime(
|
||||||
build))
|
build))
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Exception estimating build time:")
|
log.exception("Exception estimating build time:")
|
||||||
pipeline.manager.onBuildStarted(event.build)
|
pipeline.manager.onBuildStarted(event.build)
|
||||||
|
|
||||||
def _doBuildPausedEvent(self, event):
|
def _doBuildPausedEvent(self, event):
|
||||||
build = event.build
|
build = event.build
|
||||||
|
log = get_annotated_logger(self.log, build.zuul_event_id)
|
||||||
if build.build_set is not build.build_set.item.current_build_set:
|
if build.build_set is not build.build_set.item.current_build_set:
|
||||||
self.log.warning("Build %s is not in the current build set" %
|
log.warning("Build %s is not in the current build set", build)
|
||||||
(build,))
|
|
||||||
return
|
return
|
||||||
pipeline = build.build_set.item.pipeline
|
pipeline = build.build_set.item.pipeline
|
||||||
if not pipeline:
|
if not pipeline:
|
||||||
self.log.warning("Build %s is not associated with a pipeline" %
|
log.warning("Build %s is not associated with a pipeline", build)
|
||||||
(build,))
|
|
||||||
return
|
return
|
||||||
pipeline.manager.onBuildPaused(event.build)
|
pipeline.manager.onBuildPaused(event.build)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue