Return executor errors to user

There are some errors that the executor may encounter where it will
be unable to, or refuse to, run a job.  We know that these errors
will not be corrected by retrying the build, so return them as
errors to the user.  The build result will be "ERROR" and the message
which is brief, but hopefully sufficient to illuminate the problem,
will be added to the job report.

Change-Id: Iad486199de19583eb1e9f67c89a8ed8dac75dea1
Story: 2001105
Story: 2001106
changes/53/484953/1
James E. Blair 5 years ago
parent 679b0dc0ec
commit 6f6997350f
  1. 24
      tests/unit/test_v3.py
  2. 1
      zuul/executor/client.py
  3. 38
      zuul/executor/server.py
  4. 2
      zuul/model.py
  5. 8
      zuul/reporter/__init__.py

@ -797,6 +797,30 @@ class TestRoles(ZuulTestCase):
dict(name='project-test', result='SUCCESS', changes='1,1'),
])
def test_role_error(self):
conf = textwrap.dedent(
"""
- job:
name: project-test
roles:
- zuul: common-config
- project:
name: org/project
check:
jobs:
- project-test
""")
file_dict = {'.zuul.yaml': conf}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertIn(
'- project-test project-test : ERROR Unable to find role',
A.messages[-1])
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'

@ -376,6 +376,7 @@ class ExecutorClient(object):
build.node_name = data.get('node_name')
if result is None:
result = data.get('result')
build.error_detail = data.get('error_detail')
if result is None:
if (build.build_set.getTries(build.job.name) >=
build.job.attempts):

@ -41,6 +41,16 @@ COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
DEFAULT_FINGER_PORT = 79
class ExecutorError(Exception):
"""A non-transient run-time executor error
This class represents error conditions detected by the executor
when preparing to run a job which we know are consistently fatal.
Zuul should not reschedule the build in these cases.
"""
pass
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@ -115,8 +125,8 @@ class SshAgent(object):
subprocess.check_output(['ssh-add', key_path], env=env,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
self.log.error('ssh-add failed. stdout: %s, stderr: %s',
e.output, e.stderr)
self.log.exception('ssh-add failed. stdout: %s, stderr: %s',
e.output, e.stderr)
raise
self.log.info('Added SSH Key {}'.format(key_path))
@ -744,6 +754,11 @@ class AnsibleJob(object):
self.executor_server.keep_jobdir,
str(self.job.unique))
self._execute()
except ExecutorError as e:
result_data = json.dumps(dict(result='ERROR',
error_detail=e.args[0]))
self.log.debug("Sending result: %s" % (result_data,))
self.job.sendWorkComplete(result_data)
except Exception:
self.log.exception("Exception while executing job")
self.job.sendWorkException(traceback.format_exc())
@ -913,8 +928,9 @@ class AnsibleJob(object):
project_name, project_default_branch)
repo.checkoutLocalBranch(project_default_branch)
else:
raise Exception("Project %s does not have the default branch %s" %
(project_name, project_default_branch))
raise ExecutorError("Project %s does not have the "
"default branch %s" %
(project_name, project_default_branch))
def runPlaybooks(self, args):
result = None
@ -1005,9 +1021,9 @@ class AnsibleJob(object):
'''
for entry in os.listdir(path):
if os.path.isdir(entry) and entry.endswith('_plugins'):
raise Exception(
"Ansible plugin dir %s found adjacent to playbook %s in"
" non-trusted repo." % (entry, path))
raise ExecutorError(
"Ansible plugin dir %s found adjacent to playbook %s in "
"non-trusted repo." % (entry, path))
def findPlaybook(self, path, required=False, trusted=False):
for ext in ['.yaml', '.yml']:
@ -1018,7 +1034,7 @@ class AnsibleJob(object):
self._blockPluginDirs(playbook_dir)
return fn
if required:
raise Exception("Unable to find playbook %s" % path)
raise ExecutorError("Unable to find playbook %s" % path)
return None
def preparePlaybooks(self, args):
@ -1036,7 +1052,7 @@ class AnsibleJob(object):
break
if self.jobdir.playbook is None:
raise Exception("No valid playbook found")
raise ExecutorError("No valid playbook found")
for playbook in args['post_playbooks']:
jobdir_playbook = self.jobdir.addPostPlaybook()
@ -1124,7 +1140,7 @@ class AnsibleJob(object):
self._blockPluginDirs(os.path.join(d, entry))
return d
# It is neither a bare role, nor a collection of roles
raise Exception("Unable to find role in %s" % (path,))
raise ExecutorError("Unable to find role in %s" % (path,))
def prepareZuulRole(self, jobdir_playbook, role, args, root):
self.log.debug("Prepare zuul role for %s" % (role,))
@ -1162,7 +1178,7 @@ class AnsibleJob(object):
link = os.path.join(root, name)
link = os.path.realpath(link)
if not link.startswith(os.path.realpath(root)):
raise Exception("Invalid role name %s", name)
raise ExecutorError("Invalid role name %s", name)
os.symlink(path, link)
role_path = self.findRole(link, trusted=jobdir_playbook.trusted)

@ -1098,6 +1098,7 @@ class Build(object):
self.url = None
self.result = None
self.result_data = {}
self.error_detail = None
self.build_set = None
self.execute_time = time.time()
self.start_time = None
@ -1118,6 +1119,7 @@ class Build(object):
def getSafeAttributes(self):
return Attributes(uuid=self.uuid,
result=self.result,
error_detail=self.error_detail,
result_data=self.result_data)

@ -138,7 +138,11 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
elapsed = ' in %ds' % (s)
else:
elapsed = ''
if build.error_detail:
error = ' ' + build.error_detail
else:
error = ''
name = job.name + ' '
ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed,
voting)
ret += '- %s%s : %s%s%s%s\n' % (name, url, result, error,
elapsed, voting)
return ret

Loading…
Cancel
Save