From 3594dc1dfc9a802864482d81477278c4a37f1f97 Mon Sep 17 00:00:00 2001 From: Tobias Henkel Date: Sun, 3 Mar 2019 17:04:03 +0100 Subject: [PATCH] Install ansible during executor startup if needed In order to make running zuul easier we want the possiblility that the executor installs the supported ansible versions during startup. This adds this functionality as well as a config switch to disable it and a config option to optionally specify the install location. The default location is /ansible-bin. Change-Id: I1858e4fb40190626d001e20b48cf7e69ad35d634 --- doc/source/admin/components.rst | 26 ++++++++++++++ zuul/executor/server.py | 24 +++++++++---- zuul/lib/ansible.py | 64 ++++++++++++++++++++++++--------- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index f6cd110ed3..2db22d9064 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -586,6 +586,32 @@ The following sections of ``zuul.conf`` are used by the executor: add any site-wide variables. See the :ref:`User's Guide ` for more information. + .. attr:: manage_ansible + :default: True + + Specifies wether the zuul-executor should install the supported ansible + versions during startup or not. If this is ``True`` the zuul-executor + will install the ansible versions into :attr:`executor.ansible_root`. + + It is recommended to set this to ``False`` and manually install Ansible + after the Zuul installation by running ``zuul-manage-ansible``. This has + the advantage that possible errors during Ansible installation can be + spotted earlier. Further especially containerized deployments of Zuul + will have the advantage of predictable versions. + + .. attr:: ansible_root + :default: /ansible-bin + + Specifies where the zuul-executor should look for its supported ansible + installations. By default it looks in the following directories and uses + the first which it can find. + + * ``/lib/zuul/ansible`` + * ```` + + The ``ansible_root`` setting allows you to override the second location + which is also used for installation if ``manage_ansible`` is ``True``. + .. attr:: ansible_setup_timeout :default: 60 diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 1954c19ccf..308495b371 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -1859,6 +1859,9 @@ class AnsibleJob(object): rw_paths = rw_paths.split(":") if rw_paths else [] ro_paths.append(ansible_dir) + ro_paths.append( + self.executor_server.ansible_manager.getAnsibleInstallDir( + ansible_version)) ro_paths.append(self.jobdir.ansible_root) ro_paths.append(self.jobdir.trusted_root) ro_paths.append(self.jobdir.untrusted_root) @@ -2281,14 +2284,23 @@ class ExecutorServer(object): StartingBuildsSensor(self, cpu_sensor.max_load_avg) ] + manage_ansible = get_default( + self.config, 'executor', 'manage_ansible', True) ansible_dir = os.path.join(state_dir, 'ansible') - self.ansible_manager = AnsibleManager(ansible_dir) + ansible_install_root = get_default( + self.config, 'executor', 'ansible_root', None) + if not ansible_install_root: + ansible_install_root = os.path.join(state_dir, 'ansible-bin') + self.ansible_manager = AnsibleManager( + ansible_dir, runtime_install_path=ansible_install_root) if not self.ansible_manager.validate(): - # TODO(tobiash): Install ansible here if auto install on startup is - # requested - raise Exception('Error while validating ansible installations. ' - 'Please run zuul-manage-ansible to install all ' - 'supported ansible versions.') + if not manage_ansible: + raise Exception('Error while validating ansible ' + 'installations. Please run ' + 'zuul-manage-ansible to install all supported ' + 'ansible versions.') + else: + self.ansible_manager.install() self.ansible_manager.copyAnsibleFiles() def _getMerger(self, root, cache_root, logger=None): diff --git a/zuul/lib/ansible.py b/zuul/lib/ansible.py index 7e75f3c6b7..1684159d9b 100644 --- a/zuul/lib/ansible.py +++ b/zuul/lib/ansible.py @@ -28,7 +28,7 @@ from zuul.lib.config import get_default class ManagedAnsible: log = logging.getLogger('zuul.managed_ansible') - def __init__(self, config, version): + def __init__(self, config, version, runtime_install_root=None): self.version = version requirements = get_default(config, version, 'requirements') @@ -37,8 +37,12 @@ class ManagedAnsible: self.default = get_default(config, version, 'default', False) self.deprecated = get_default(config, version, 'deprecated', False) - self._ansible_root = os.path.join( - sys.exec_prefix, 'lib', 'zuul', 'ansible') + self._ansible_roots = [os.path.join( + sys.exec_prefix, 'lib', 'zuul', 'ansible')] + if runtime_install_root: + self._ansible_roots.append(runtime_install_root) + + self.install_root = self._ansible_roots[-1] def ensure_ansible(self, upgrade=False): self._ensure_venv() @@ -67,12 +71,13 @@ class ManagedAnsible: self.log.debug('Successfully installed packages %s', requirements) def _ensure_venv(self): - if os.path.exists(self.python_path): + if self.python_path: self.log.debug( 'Virtual environment %s already existing', self.venv_path) return - self.log.info('Creating venv %s', self.venv_path) + venv_path = os.path.join(self.install_root, self.version) + self.log.info('Creating venv %s', venv_path) python_executable = sys.executable if hasattr(sys, 'real_prefix'): @@ -83,7 +88,7 @@ class ManagedAnsible: # We don't use directly the venv module here because its behavior is # broken if we're already in a virtual environment. - cmd = ['virtualenv', '-p', python_executable, self.venv_path] + cmd = ['virtualenv', '-p', python_executable, venv_path] p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.returncode != 0: @@ -94,11 +99,18 @@ class ManagedAnsible: @property def venv_path(self): - return os.path.join(self._ansible_root, self.version) + for root in self._ansible_roots: + venv_path = os.path.join(root, self.version) + if os.path.exists(venv_path): + return venv_path + return None @property def python_path(self): - return os.path.join(self.venv_path, 'bin', 'python') + venv_path = self.venv_path + if venv_path: + return os.path.join(self.venv_path, 'bin', 'python') + return None @property def extra_packages(self): @@ -123,10 +135,12 @@ class ManagedAnsible: class AnsibleManager: log = logging.getLogger('zuul.ansible_manager') - def __init__(self, zuul_ansible_dir=None, default_version=None): + def __init__(self, zuul_ansible_dir=None, default_version=None, + runtime_install_path=None): self._supported_versions = {} self.default_version = None self.zuul_ansible_dir = zuul_ansible_dir + self.runtime_install_root = runtime_install_path self.load_ansible_config() @@ -142,7 +156,9 @@ class AnsibleManager: for version in config.sections(): - ansible = ManagedAnsible(config, version) + ansible = ManagedAnsible( + config, version, + runtime_install_root=self.runtime_install_root) if ansible.version in self._supported_versions: raise RuntimeError( @@ -169,23 +185,25 @@ class AnsibleManager: def validate(self): result = True for version in self._supported_versions: - command = [ - self.getAnsibleCommand(version, 'ansible'), - '--version', - ] try: + command = [ + self.getAnsibleCommand(version, 'ansible'), + '--version', + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) self.log.info('Ansible version %s information: \n%s', version, result.stdout.decode()) - except FileNotFoundError: - result = False - self.log.exception('Ansible version %s not found' % version) except subprocess.CalledProcessError: result = False self.log.exception("Ansible version %s not working" % version) + except Exception: + result = False + self.log.exception( + 'Ansible version %s not installed' % version) return result @@ -200,8 +218,20 @@ class AnsibleManager: def getAnsibleCommand(self, version, command='ansible-playbook'): ansible = self._getAnsible(version) + venv_path = ansible.venv_path + if not venv_path: + raise Exception('Requested ansible version \'%s\' is not ' + 'installed' % version) return os.path.join(ansible.venv_path, 'bin', command) + def getAnsibleInstallDir(self, version): + ansible = self._getAnsible(version) + venv_path = ansible.venv_path + if not venv_path: + raise Exception('Requested ansible version \'%s\' is not ' + 'installed' % version) + return venv_path + def getAnsibleDir(self, version): ansible = self._getAnsible(version) return os.path.join(self.zuul_ansible_dir, ansible.version)