Browse Source

Run ansible setup at start of job to verify network

Run the Ansible 'setup' module on all hosts in the inventory
at the start of the job with a 60 second timeout.  If we
aren't able to connect to all the hosts and gather facts
within that timeout, there is likely a network problem
between here and the hosts in the inventory; return them and
reschedule the job.

Change-Id: I121514aa1f3c9ac1a664322c8d7624703146f52f
changes/33/507933/3
James E. Blair 5 years ago
parent
commit
f641f4fd34
  1. 5
      tests/base.py
  2. 43
      zuul/executor/server.py

5
tests/base.py

@ -1485,7 +1485,10 @@ class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
result = super(RecordingAnsibleJob, self).runAnsible(
cmd, timeout, playbook)
else:
result = build.run()
if playbook.path:
result = build.run()
else:
result = (self.RESULT_NORMAL, 0)
return result
def getHostList(self, args):

43
zuul/executor/server.py

@ -346,6 +346,15 @@ class JobDir(object):
self.trusted_projects = []
self.trusted_project_index = {}
# Create a JobDirPlaybook for the Ansible setup run. This
# doesn't use an actual playbook, but it lets us use the same
# methods to write an ansible.cfg as the rest of the Ansible
# runs.
setup_root = os.path.join(self.ansible_root, 'setup_playbook')
os.makedirs(setup_root)
self.setup_playbook = JobDirPlaybook(setup_root)
self.setup_playbook.trusted = True
def addTrustedProject(self, canonical_name, branch):
# Trusted projects are placed in their own directories so that
# we can support using different branches of the same project
@ -1101,6 +1110,17 @@ class AnsibleJob(object):
def runPlaybooks(self, args):
result = None
# Run the Ansible 'setup' module on all hosts in the inventory
# at the start of the job with a 60 second timeout. If we
# aren't able to connect to all the hosts and gather facts
# within that timeout, there is likely a network problem
# between here and the hosts in the inventory; return them and
# reschedule the job.
setup_status, setup_code = self.runAnsibleSetup(
self.jobdir.setup_playbook)
if setup_status != self.RESULT_NORMAL or setup_code != 0:
return result
pre_failed = False
success = False
for index, playbook in enumerate(self.jobdir.pre_playbooks):
@ -1213,6 +1233,8 @@ class AnsibleJob(object):
return None
def preparePlaybooks(self, args):
self.writeAnsibleConfig(self.jobdir.setup_playbook)
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
self.preparePlaybook(jobdir_playbook, playbook,
@ -1289,7 +1311,7 @@ class AnsibleJob(object):
jobdir_playbook.secrets_content = yaml.safe_dump(
secrets, default_flow_style=False)
self.writeAnsibleConfig(jobdir_playbook, playbook)
self.writeAnsibleConfig(jobdir_playbook)
def checkoutTrustedProject(self, project, branch):
root = self.jobdir.getTrustedProject(project.canonical_name,
@ -1420,7 +1442,7 @@ class AnsibleJob(object):
job_output_file=self.jobdir.job_output_file)
logging_config.writeJson(self.jobdir.logging_json)
def writeAnsibleConfig(self, jobdir_playbook, playbook):
def writeAnsibleConfig(self, jobdir_playbook):
trusted = jobdir_playbook.trusted
# TODO(mordred) This should likely be extracted into a more generalized
@ -1467,7 +1489,7 @@ class AnsibleJob(object):
# role. Otherwise, printing the args could be useful for
# debugging.
config.write('display_args_to_stdout = %s\n' %
str(not playbook['secrets']))
str(not jobdir_playbook.secrets_content))
config.write('[ssh_connection]\n')
# NB: when setting pipelining = True, keep_remote_files
@ -1640,6 +1662,21 @@ class AnsibleJob(object):
return (self.RESULT_NORMAL, ret)
def runAnsibleSetup(self, playbook):
if self.executor_server.verbose:
verbose = '-vvv'
else:
verbose = '-v'
cmd = ['ansible', '*', verbose, '-m', 'setup',
'-a', 'gather_subset=!all']
result, code = self.runAnsible(
cmd=cmd, timeout=60, playbook=playbook)
self.log.debug("Ansible complete, result %s code %s" % (
self.RESULT_MAP[result], code))
return result, code
def runAnsiblePlaybook(self, playbook, timeout, success=None,
phase=None, index=None):
if self.executor_server.verbose:

Loading…
Cancel
Save