zuul/zuul
Code Issues Proposed changes

Install ansible during executor startup if needed

Browse Source 
In order to make running zuul easier we want the possiblility that the
executor installs the supported ansible versions during startup. This
adds this functionality as well as a config switch to disable it and a
config option to optionally specify the install location. The default
location is <state_dir>/ansible-bin.

Change-Id: I1858e4fb40190626d001e20b48cf7e69ad35d634
This commit is contained in:
Tobias Henkel 2019-03-03 17:04:03 +01:00
parent 5c2b61e638
commit 3594dc1dfc
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
3 changed files with 91 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/source/admin/components.rst
View File

@ -586,6 +586,32 @@ The following sections of ``zuul.conf`` are used by the executor:
add any site-wide variables. See the :ref:`User's Guide
<user_jobs_sitewide_variables>` for more information.
.. attr:: manage_ansible
:default: True
Specifies wether the zuul-executor should install the supported ansible
versions during startup or not. If this is ``True`` the zuul-executor
will install the ansible versions into :attr:`executor.ansible_root`.
It is recommended to set this to ``False`` and manually install Ansible
after the Zuul installation by running ``zuul-manage-ansible``. This has
the advantage that possible errors during Ansible installation can be
spotted earlier. Further especially containerized deployments of Zuul
will have the advantage of predictable versions.
.. attr:: ansible_root
:default: <state_dir>/ansible-bin
Specifies where the zuul-executor should look for its supported ansible
installations. By default it looks in the following directories and uses
the first which it can find.
* ``<zuul_install_dir>/lib/zuul/ansible``
* ``<ansible_root>``
The ``ansible_root`` setting allows you to override the second location
which is also used for installation if ``manage_ansible`` is ``True``.
.. attr:: ansible_setup_timeout
:default: 60

24
zuul/executor/server.py
View File

@ -1859,6 +1859,9 @@ class AnsibleJob(object):
rw_paths = rw_paths.split(":") if rw_paths else []
ro_paths.append(ansible_dir)
ro_paths.append(
self.executor_server.ansible_manager.getAnsibleInstallDir(
ansible_version))
ro_paths.append(self.jobdir.ansible_root)
ro_paths.append(self.jobdir.trusted_root)
ro_paths.append(self.jobdir.untrusted_root)
@ -2281,14 +2284,23 @@ class ExecutorServer(object):
StartingBuildsSensor(self, cpu_sensor.max_load_avg)
]
manage_ansible = get_default(
self.config, 'executor', 'manage_ansible', True)
ansible_dir = os.path.join(state_dir, 'ansible')
self.ansible_manager = AnsibleManager(ansible_dir)
ansible_install_root = get_default(
self.config, 'executor', 'ansible_root', None)
if not ansible_install_root:
ansible_install_root = os.path.join(state_dir, 'ansible-bin')
self.ansible_manager = AnsibleManager(
ansible_dir, runtime_install_path=ansible_install_root)
if not self.ansible_manager.validate():
# TODO(tobiash): Install ansible here if auto install on startup is
# requested
raise Exception('Error while validating ansible installations. '
'Please run zuul-manage-ansible to install all '
'supported ansible versions.')
if not manage_ansible:
raise Exception('Error while validating ansible '
'installations. Please run '
'zuul-manage-ansible to install all supported '
'ansible versions.')
else:
self.ansible_manager.install()
self.ansible_manager.copyAnsibleFiles()
def _getMerger(self, root, cache_root, logger=None):

64
zuul/lib/ansible.py
View File

@ -28,7 +28,7 @@ from zuul.lib.config import get_default
class ManagedAnsible:
log = logging.getLogger('zuul.managed_ansible')
def __init__(self, config, version):
def __init__(self, config, version, runtime_install_root=None):
self.version = version
requirements = get_default(config, version, 'requirements')
@ -37,8 +37,12 @@ class ManagedAnsible:
self.default = get_default(config, version, 'default', False)
self.deprecated = get_default(config, version, 'deprecated', False)
self._ansible_root = os.path.join(
sys.exec_prefix, 'lib', 'zuul', 'ansible')
self._ansible_roots = [os.path.join(
sys.exec_prefix, 'lib', 'zuul', 'ansible')]
if runtime_install_root:
self._ansible_roots.append(runtime_install_root)
self.install_root = self._ansible_roots[-1]
def ensure_ansible(self, upgrade=False):
self._ensure_venv()
@ -67,12 +71,13 @@ class ManagedAnsible:
self.log.debug('Successfully installed packages %s', requirements)
def _ensure_venv(self):
if os.path.exists(self.python_path):
if self.python_path:
self.log.debug(
'Virtual environment %s already existing', self.venv_path)
return
self.log.info('Creating venv %s', self.venv_path)
venv_path = os.path.join(self.install_root, self.version)
self.log.info('Creating venv %s', venv_path)
python_executable = sys.executable
if hasattr(sys, 'real_prefix'):
@ -83,7 +88,7 @@ class ManagedAnsible:
# We don't use directly the venv module here because its behavior is
# broken if we're already in a virtual environment.
cmd = ['virtualenv', '-p', python_executable, self.venv_path]
cmd = ['virtualenv', '-p', python_executable, venv_path]
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.returncode != 0:
@ -94,11 +99,18 @@ class ManagedAnsible:
@property
def venv_path(self):
return os.path.join(self._ansible_root, self.version)
for root in self._ansible_roots:
venv_path = os.path.join(root, self.version)
if os.path.exists(venv_path):
return venv_path
return None
@property
def python_path(self):
return os.path.join(self.venv_path, 'bin', 'python')
venv_path = self.venv_path
if venv_path:
return os.path.join(self.venv_path, 'bin', 'python')
return None
@property
def extra_packages(self):
@ -123,10 +135,12 @@ class ManagedAnsible:
class AnsibleManager:
log = logging.getLogger('zuul.ansible_manager')
def __init__(self, zuul_ansible_dir=None, default_version=None):
def __init__(self, zuul_ansible_dir=None, default_version=None,
runtime_install_path=None):
self._supported_versions = {}
self.default_version = None
self.zuul_ansible_dir = zuul_ansible_dir
self.runtime_install_root = runtime_install_path
self.load_ansible_config()
@ -142,7 +156,9 @@ class AnsibleManager:
for version in config.sections():
ansible = ManagedAnsible(config, version)
ansible = ManagedAnsible(
config, version,
runtime_install_root=self.runtime_install_root)
if ansible.version in self._supported_versions:
raise RuntimeError(
@ -169,23 +185,25 @@ class AnsibleManager:
def validate(self):
result = True
for version in self._supported_versions:
command = [
self.getAnsibleCommand(version, 'ansible'),
'--version',
]
try:
command = [
self.getAnsibleCommand(version, 'ansible'),
'--version',
]
result = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True)
self.log.info('Ansible version %s information: \n%s',
version, result.stdout.decode())
except FileNotFoundError:
result = False
self.log.exception('Ansible version %s not found' % version)
except subprocess.CalledProcessError:
result = False
self.log.exception("Ansible version %s not working" % version)
except Exception:
result = False
self.log.exception(
'Ansible version %s not installed' % version)
return result
@ -200,8 +218,20 @@ class AnsibleManager:
def getAnsibleCommand(self, version, command='ansible-playbook'):
ansible = self._getAnsible(version)
venv_path = ansible.venv_path
if not venv_path:
raise Exception('Requested ansible version \'%s\' is not '
'installed' % version)
return os.path.join(ansible.venv_path, 'bin', command)
def getAnsibleInstallDir(self, version):
ansible = self._getAnsible(version)
venv_path = ansible.venv_path
if not venv_path:
raise Exception('Requested ansible version \'%s\' is not '
'installed' % version)
return venv_path
def getAnsibleDir(self, version):
ansible = self._getAnsible(version)
return os.path.join(self.zuul_ansible_dir, ansible.version)