diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index f6cd110ed3..2db22d9064 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -586,6 +586,32 @@ The following sections of ``zuul.conf`` are used by the executor: add any site-wide variables. See the :ref:`User's Guide ` for more information. + .. attr:: manage_ansible + :default: True + + Specifies wether the zuul-executor should install the supported ansible + versions during startup or not. If this is ``True`` the zuul-executor + will install the ansible versions into :attr:`executor.ansible_root`. + + It is recommended to set this to ``False`` and manually install Ansible + after the Zuul installation by running ``zuul-manage-ansible``. This has + the advantage that possible errors during Ansible installation can be + spotted earlier. Further especially containerized deployments of Zuul + will have the advantage of predictable versions. + + .. attr:: ansible_root + :default: /ansible-bin + + Specifies where the zuul-executor should look for its supported ansible + installations. By default it looks in the following directories and uses + the first which it can find. + + * ``/lib/zuul/ansible`` + * ```` + + The ``ansible_root`` setting allows you to override the second location + which is also used for installation if ``manage_ansible`` is ``True``. + .. attr:: ansible_setup_timeout :default: 60 diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 1954c19ccf..308495b371 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -1859,6 +1859,9 @@ class AnsibleJob(object): rw_paths = rw_paths.split(":") if rw_paths else [] ro_paths.append(ansible_dir) + ro_paths.append( + self.executor_server.ansible_manager.getAnsibleInstallDir( + ansible_version)) ro_paths.append(self.jobdir.ansible_root) ro_paths.append(self.jobdir.trusted_root) ro_paths.append(self.jobdir.untrusted_root) @@ -2281,14 +2284,23 @@ class ExecutorServer(object): StartingBuildsSensor(self, cpu_sensor.max_load_avg) ] + manage_ansible = get_default( + self.config, 'executor', 'manage_ansible', True) ansible_dir = os.path.join(state_dir, 'ansible') - self.ansible_manager = AnsibleManager(ansible_dir) + ansible_install_root = get_default( + self.config, 'executor', 'ansible_root', None) + if not ansible_install_root: + ansible_install_root = os.path.join(state_dir, 'ansible-bin') + self.ansible_manager = AnsibleManager( + ansible_dir, runtime_install_path=ansible_install_root) if not self.ansible_manager.validate(): - # TODO(tobiash): Install ansible here if auto install on startup is - # requested - raise Exception('Error while validating ansible installations. ' - 'Please run zuul-manage-ansible to install all ' - 'supported ansible versions.') + if not manage_ansible: + raise Exception('Error while validating ansible ' + 'installations. Please run ' + 'zuul-manage-ansible to install all supported ' + 'ansible versions.') + else: + self.ansible_manager.install() self.ansible_manager.copyAnsibleFiles() def _getMerger(self, root, cache_root, logger=None): diff --git a/zuul/lib/ansible.py b/zuul/lib/ansible.py index 7e75f3c6b7..1684159d9b 100644 --- a/zuul/lib/ansible.py +++ b/zuul/lib/ansible.py @@ -28,7 +28,7 @@ from zuul.lib.config import get_default class ManagedAnsible: log = logging.getLogger('zuul.managed_ansible') - def __init__(self, config, version): + def __init__(self, config, version, runtime_install_root=None): self.version = version requirements = get_default(config, version, 'requirements') @@ -37,8 +37,12 @@ class ManagedAnsible: self.default = get_default(config, version, 'default', False) self.deprecated = get_default(config, version, 'deprecated', False) - self._ansible_root = os.path.join( - sys.exec_prefix, 'lib', 'zuul', 'ansible') + self._ansible_roots = [os.path.join( + sys.exec_prefix, 'lib', 'zuul', 'ansible')] + if runtime_install_root: + self._ansible_roots.append(runtime_install_root) + + self.install_root = self._ansible_roots[-1] def ensure_ansible(self, upgrade=False): self._ensure_venv() @@ -67,12 +71,13 @@ class ManagedAnsible: self.log.debug('Successfully installed packages %s', requirements) def _ensure_venv(self): - if os.path.exists(self.python_path): + if self.python_path: self.log.debug( 'Virtual environment %s already existing', self.venv_path) return - self.log.info('Creating venv %s', self.venv_path) + venv_path = os.path.join(self.install_root, self.version) + self.log.info('Creating venv %s', venv_path) python_executable = sys.executable if hasattr(sys, 'real_prefix'): @@ -83,7 +88,7 @@ class ManagedAnsible: # We don't use directly the venv module here because its behavior is # broken if we're already in a virtual environment. - cmd = ['virtualenv', '-p', python_executable, self.venv_path] + cmd = ['virtualenv', '-p', python_executable, venv_path] p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.returncode != 0: @@ -94,11 +99,18 @@ class ManagedAnsible: @property def venv_path(self): - return os.path.join(self._ansible_root, self.version) + for root in self._ansible_roots: + venv_path = os.path.join(root, self.version) + if os.path.exists(venv_path): + return venv_path + return None @property def python_path(self): - return os.path.join(self.venv_path, 'bin', 'python') + venv_path = self.venv_path + if venv_path: + return os.path.join(self.venv_path, 'bin', 'python') + return None @property def extra_packages(self): @@ -123,10 +135,12 @@ class ManagedAnsible: class AnsibleManager: log = logging.getLogger('zuul.ansible_manager') - def __init__(self, zuul_ansible_dir=None, default_version=None): + def __init__(self, zuul_ansible_dir=None, default_version=None, + runtime_install_path=None): self._supported_versions = {} self.default_version = None self.zuul_ansible_dir = zuul_ansible_dir + self.runtime_install_root = runtime_install_path self.load_ansible_config() @@ -142,7 +156,9 @@ class AnsibleManager: for version in config.sections(): - ansible = ManagedAnsible(config, version) + ansible = ManagedAnsible( + config, version, + runtime_install_root=self.runtime_install_root) if ansible.version in self._supported_versions: raise RuntimeError( @@ -169,23 +185,25 @@ class AnsibleManager: def validate(self): result = True for version in self._supported_versions: - command = [ - self.getAnsibleCommand(version, 'ansible'), - '--version', - ] try: + command = [ + self.getAnsibleCommand(version, 'ansible'), + '--version', + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) self.log.info('Ansible version %s information: \n%s', version, result.stdout.decode()) - except FileNotFoundError: - result = False - self.log.exception('Ansible version %s not found' % version) except subprocess.CalledProcessError: result = False self.log.exception("Ansible version %s not working" % version) + except Exception: + result = False + self.log.exception( + 'Ansible version %s not installed' % version) return result @@ -200,8 +218,20 @@ class AnsibleManager: def getAnsibleCommand(self, version, command='ansible-playbook'): ansible = self._getAnsible(version) + venv_path = ansible.venv_path + if not venv_path: + raise Exception('Requested ansible version \'%s\' is not ' + 'installed' % version) return os.path.join(ansible.venv_path, 'bin', command) + def getAnsibleInstallDir(self, version): + ansible = self._getAnsible(version) + venv_path = ansible.venv_path + if not venv_path: + raise Exception('Requested ansible version \'%s\' is not ' + 'installed' % version) + return venv_path + def getAnsibleDir(self, version): ansible = self._getAnsible(version) return os.path.join(self.zuul_ansible_dir, ansible.version)