Remove built-in ARA support

This has been pinned to a very old version of ARA for some time, and
newer versions of Ansible are no longer compatible with the old version
of ARA.  Since this isn't receiving maintenance keeping it up to date,
remove it.

Note that if there is desire for support for this or other callback
plugins, it would be quite reasonable and relatively straightforward
to add the ability to generically configure additional callback plugins.
This would have the advantage of not requiring tight internal integration
between Zuul and other callback plugins.  Such a change would likely
be welcome.

Change-Id: I733e48127f2b1cf7d2d52153844098163e48bae8

.zuul.yaml

@@ -25,7 +25,6 @@
     run: playbooks/zuul-stream/functional.yaml
     post-run:
       - playbooks/zuul-stream/post.yaml
-      - playbooks/zuul-stream/post-ara.yaml
     files:
       - zuul/ansible/.*
       - zuul/lib/ansible*

playbooks/zuul-stream/functional.yaml

@@ -14,7 +14,6 @@
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
-        ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         PYTHONPATH: "{{ python_path }}"
 
     - name: Run ansible playbook that should fail
@@ -26,7 +25,6 @@
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
-        ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         PYTHONPATH: "{{ python_path }}"
 
     - name: Validate output - setupvar

playbooks/zuul-stream/post-ara.yaml

@@ -1,11 +0,0 @@
-- hosts: controller
-  tasks:
-
-    - name: Generate ARA html
-      command: /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ara generate html ara-output
-
-    - name: Fetch ARA files
-      synchronize:
-        src: "{{ ansible_user_dir }}/ara-output"
-        dest: "{{ zuul.executor.log_root }}/stream-files"
-        mode: pull

playbooks/zuul-stream/templates/ansible.cfg.j2

@@ -4,7 +4,7 @@ gathering = smart
 gather_subset = !all
 lookup_plugins = {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/lookup
 action_plugins = {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/actiongeneral:{{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/action
-callback_plugins = {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/callback:/usr/lib/zuul/ansible/{{ zuul_ansible_version }}/lib/python3.8/site-packages/ara/plugins/callbacks
+callback_plugins = {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/callback
 stdout_callback = zuul_stream
 library = {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/zuul/ansible/{{ zuul_ansible_version }}/library
 retry_files_enabled = False

releasenotes/notes/remove-ara-f64d6d2408dde1e5.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    The built-in support for the ARA Ansible callback plugin has been
+    removed.

zuul/executor/server.py

@@ -1025,9 +1025,6 @@ class AnsibleJob(object):
 
         plugin_dir = self.executor_server.ansible_manager.getAnsiblePluginDir(
             self.ansible_version)
-        self.ara_callbacks = \
-            self.executor_server.ansible_manager.getAraCallbackPlugin(
-                self.ansible_version)
         self.library_dir = os.path.join(plugin_dir, 'library')
         self.action_dir = os.path.join(plugin_dir, 'action')
         self.callback_dir = os.path.join(plugin_dir, 'callback')
@@ -2547,15 +2544,7 @@ class AnsibleJob(object):
         logging_config.writeJson(self.jobdir.logging_json)
 
     def writeAnsibleConfig(self, jobdir_playbook):
-        # TODO(mordred) This should likely be extracted into a more generalized
-        #               mechanism for deployers being able to add callback
-        #               plugins.
-        if self.ara_callbacks:
-            callback_path = '%s:%s' % (
-                self.callback_dir,
-                os.path.dirname(self.ara_callbacks))
-        else:
-            callback_path = self.callback_dir
+        callback_path = self.callback_dir
         with open(jobdir_playbook.ansible_config, 'w') as config:
             config.write('[defaults]\n')
             config.write('inventory = %s\n' % jobdir_playbook.inventory)
@@ -2661,8 +2650,6 @@ class AnsibleJob(object):
                     for key, value in os.environ.copy().items()
                     if not key.startswith("ZUUL_")}
         env_copy.update(self.ssh_agent.env)
-        if self.ara_callbacks:
-            env_copy['ARA_LOG_CONFIG'] = self.jobdir.logging_json
         env_copy['ZUUL_JOB_LOG_CONFIG'] = self.jobdir.logging_json
         env_copy['ZUUL_JOBDIR'] = self.jobdir.root
         if self.executor_server.log_console_port != DEFAULT_STREAM_PORT:

zuul/lib/ansible-config.conf

@@ -1,12 +1,12 @@
 # This file describes the currently supported ansible versions
 [common]
 default_version = 2.9
-# Jinja2 pinned due to 3.1.0 breaking ara
-requirements = ara>=0.16.5,<1.0.0 Jinja2<3.1.0 openstacksdk openshift jmespath google-cloud-storage pywinrm boto3 azure-storage-blob ibm-cos-sdk
+requirements = openstacksdk openshift jmespath google-cloud-storage pywinrm boto3 azure-storage-blob ibm-cos-sdk
 
 [2.8]
 # Ansible 2.8.16 breaks the k8s connection plugin
-requirements = ansible>=2.8,<2.9,!=2.8.16
+# Jinja 3.1.1 is incompatible with 2.8
+requirements = ansible>=2.8,<2.9,!=2.8.16 Jinja2<3.1.0
 
 [2.9]
 # Ansible 2.9.14 breaks the k8s connection plugin

zuul/lib/ansible.py

@@ -14,7 +14,6 @@
 
 import concurrent.futures
 import configparser
-import functools
 import logging
 import os
 import shutil
@@ -277,23 +276,6 @@ class AnsibleManager:
             raise Exception('Requested ansible version %s not found' % version)
         return ansible
 
-    @functools.lru_cache(maxsize=10)
-    def getAraCallbackPlugin(self, version):
-        result = None
-        try:
-            _python = self.getAnsibleCommand(version, 'python')
-            result = subprocess.run(
-                [_python, '-m', 'ara.setup.callback_plugins'],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                check=True).stdout.decode().strip()
-            self.log.info(
-                'Ansible version %s ARA callback plugin: %s', version, result)
-        except Exception:
-            self.log.exception(
-                'Ansible version %s ARA not installed' % version)
-        return result
-
     def getAnsibleCommand(self, version, command='ansible-playbook'):
         ansible = self._getAnsible(version)
         venv_path = ansible.venv_path