Avoid fstrings in test-prepare-workspace-git

Fstrings are not supported in python3.5 which is in use on Xenial.
We don't claim to support Xenial, but this is an easy regression
to avoid.

Also, add test jobs for this role so that we get feedback before
copying it to the prod roles.

Also, add a xenial test job to exercise it since we still have
Xenial nodes available.

Change-Id: Ifc773aa688adb1a01cfe691b3bdca0b3086658cd

roles/test-prepare-workspace-git/library/repo_prep.py

@@ -33,19 +33,20 @@ except ImportError:
 
 def prep_one_project(args, project, output):
     start = time.monotonic()
-    dest = f"{args['zuul_workspace_root']}/{project['src_dir']}"
+    dest = "%s/%s" % (args['zuul_workspace_root'], project['src_dir'])
     output['dest'] = dest
     if not os.path.isdir(dest):
-        cache = f"{args['cached_repos_root']}/{project['canonical_name']}"
+        cache = "%s/%s" % (args['cached_repos_root'],
+                           project['canonical_name'])
         if os.path.isdir(cache):
             # We do a bare clone here first so that we skip creating a working
             # copy that will be overwritten later anyway.
             output['initial_state'] = 'cloned-from-cache'
-            out = run(f"git clone --bare {cache} {dest}/.git")
+            out = run("git clone --bare %s %s/.git" % (cache, dest))
             output['clone'] = out.stdout.decode('utf8').strip()
         else:
             output['initial_state'] = 'git-init'
-            out = run(f"git init {dest}")
+            out = run("git init %s" % (dest,))
             output['init'] = out.stdout.decode('utf8').strip()
     else:
         output['initial_state'] = 'pre-existing'

roles/test-prepare-workspace-git/library/repo_sync.py

@@ -33,27 +33,28 @@ except ImportError:
 
 def get_ssh_dest(args, dest):
     return (
-        f"git+ssh://{args['ansible_user']}"
-        f"@{args['ansible_host']}"
-        f":{args['ansible_port']}"
-        f"/{dest}"
+        "git+ssh://%s@%s:%s/%s" % (
+            args['ansible_user'],
+            args['ansible_host'],
+            args['ansible_port'],
+            dest)
     )
 
 
 def get_k8s_dest(args, dest):
     resources = args['zuul_resources'][args['inventory_hostname']]
     return (
-        f"\"ext::kubectl "
-        f"--context {resources['context']} "
-        f"-n {resources['namespace']} "
-        f"exec -i {resources['pod']} "
-        f"-- %S {dest}\""
+        "\"ext::kubectl --context %s -n %s exec -i %s -- %%S %s\"" % (
+            resources['context'],
+            resources['namespace'],
+            resources['pod'],
+            dest)
     )
 
 
 def sync_one_project(args, project, output):
-    cwd = f"{args['executor_work_root']}/{project['src_dir']}"
-    dest = f"{args['zuul_workspace_root']}/{project['src_dir']}"
+    cwd = "%s/%s" % (args['executor_work_root'], project['src_dir'])
+    dest = "%s/%s" % (args['zuul_workspace_root'], project['src_dir'])
     output['src'] = cwd
     output['dest'] = dest
     env = os.environ.copy()
@@ -70,7 +71,7 @@ def sync_one_project(args, project, output):
                 git_dest = get_k8s_dest(args, dest)
             else:
                 git_dest = get_ssh_dest(args, dest)
-            out = run(f"git push --quiet --mirror {git_dest}",
+            out = run("git push --quiet --mirror %s" % (git_dest,),
                       cwd=cwd, env=env)
             output['push'] = out.stdout.decode('utf8').strip()
             break

roles/test-prepare-workspace-git/library/repo_update.py

@@ -31,7 +31,7 @@ except ImportError:
 
 
 def update_one_project(args, project, output):
-    cwd = f"{args['zuul_workspace_root']}/{project['src_dir']}"
+    cwd = "%s/%s" % (args['zuul_workspace_root'], project['src_dir'])
     output['dest'] = cwd
 
     start = time.monotonic()
@@ -43,7 +43,7 @@ def update_one_project(args, project, output):
     run("git config --local --unset receive.denyCurrentBranch", cwd=cwd)
     run("git config --local --unset receive.denyDeleteCurrent", cwd=cwd)
     # checkout the branch matching the branch set up by the executor
-    out = run(f"git checkout --quiet {project['checkout']}", cwd=cwd)
+    out = run("git checkout --quiet %s" % (project['checkout'],), cwd=cwd)
     output['checkout'] = out.stdout.decode('utf8').strip()
     # put out a status line with the current HEAD
     out = run("git log --pretty=oneline -1", cwd=cwd)

roles/test-prepare-workspace-git/library/test_prepare_workspace.py

@@ -44,7 +44,7 @@ class TestPrepareWorkspace(testtools.TestCase):
             run("git commit -a -m init", cwd=project_root)
         except Exception as e:
             if hasattr(e, 'output'):
-                msg = f'{str(e)} : {e.output}'
+                msg = '%s : %s' % (str(e), e.output)
             else:
                 msg = str(e)
             print(msg)
@@ -139,7 +139,7 @@ class TestPrepareWorkspace(testtools.TestCase):
         self.assertEqual(dest, project_output['dest'])
         head = run("git rev-parse HEAD", cwd=dest,
                    ).stdout.decode('utf8').strip()
-        self.assertEqual(f'{head} init', project_output['HEAD'])
+        self.assertEqual('%s init' % (head,), project_output['HEAD'])
         self.assertTrue(project_output['elapsed'] > 0)
 
     def test_prepare_workspace_ssh_new(self):

roles/test-prepare-workspace-git/module_utils/zuul_jobs/workspace_utils.py

@@ -45,7 +45,7 @@ def for_each_project(func, args, output):
             except Exception as e:
                 msg = str(e)
                 if hasattr(e, 'output'):
-                    msg = f'{str(e)} : {e.output}'
+                    msg = '%s : %s' % (str(e), e.output)
                 else:
                     msg = str(e)
                 project_out['error'] = msg

test-playbooks/base-test-roles/base.yaml

@@ -0,0 +1,15 @@
+# Roles that are part of the 'base' job
+
+# If you add new tests, also update the files section in job
+# base-integration in zuul.d/jobs.yaml.
+
+# Note: set-zuul-log-path-fact is tested by emit-job-header.yaml
+- import_playbook: emit-job-header.yaml
+- import_playbook: ensure-output-dirs.yaml
+- import_playbook: prepare-workspace-git-required-projects-only.yaml
+- import_playbook: prepare-workspace-git.yaml
+- import_playbook: configure-mirrors.yaml
+- import_playbook: fetch-zuul-cloner.yaml
+- import_playbook: validate-host.yaml
+- import_playbook: fetch-output.yaml
+- import_playbook: fetch-subunit-output.yaml

test-playbooks/base-test-roles/configure-mirrors.yaml

@@ -0,0 +1,57 @@
+- name: Test the configure-mirrors role with http
+  hosts: all
+  roles:
+    - role: configure-mirrors
+      mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+      set_apt_mirrors_trusted: True
+  post_tasks:
+    - name: Set emacs package fact for gentoo
+      set_fact:
+        emacs_package: app-editors/emacs
+      when: ansible_distribution == 'Gentoo'
+    - name: Install a package to sanity check the http mirror configuration
+      package:
+        name: "{{ emacs_package | default('emacs') }}"
+        state: "present"
+      become: yes
+
+- name: Test the configure-mirrors role with https
+  hosts: all
+  roles:
+    - role: configure-mirrors
+      mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+      mirror_use_ssl: True
+      set_apt_mirrors_trusted: True
+  post_tasks:
+    - name: Set emacs package fact for gentoo
+      set_fact:
+        emacs_package: app-editors/emacs
+      when: ansible_distribution == 'Gentoo'
+    - name: Remove existing emacs package install
+      package:
+        name: "{{ emacs_package | default('emacs') }}"
+        state: "absent"
+      become: yes
+    - name: Install a package to sanity check the https mirror configuration
+      package:
+        name: "{{ emacs_package | default('emacs') }}"
+        state: "present"
+      become: yes
+
+- name: Ensure no apt warnings
+  hosts: all
+  tasks:
+    - name: Check apt output
+      when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+      block:
+        - name: Run apt-get update
+          command: 'apt-get update'
+          register: _apt_get_output
+          become: yes
+          tags:
+            - skip_ansible_lint
+
+        - name: Check for warnings in output
+          fail:
+            msg: 'Warnings found in apt output'
+          when: _apt_get_output is regex('^W:.*$')

test-playbooks/base-test-roles/emit-job-header.yaml

@@ -0,0 +1,45 @@
+# Note that the order of these two plays is important. We want the second one
+# to run to be the one the creates the correct log path for the currently
+# running system.
+- name: Test the emit-job-header role without swift
+  hosts: all
+  roles:
+    - role: emit-job-header
+      zuul_log_url: "http://logs.openstack.org"
+  post_tasks:
+    # All emit-job-header does is a debug statement so the worst that would
+    # happen would be that the debug task would fail outright and we'd prevent
+    # something breaking that debug statement from merging just by running it.
+    # However, the emit-job-header role includes the set-zuul-log-path-fact
+    # role. We can only test for zuul_log_path against changes, though.
+    - name: Assert zuul_log_path by set-zuul-log-path-fact for a change
+      assert:
+        that:
+          - zuul_log_path is defined
+          - zuul.change in zuul_log_path
+          - zuul.patchset in zuul_log_path
+          - zuul.pipeline in zuul_log_path
+          - zuul.job in zuul_log_path
+          - zuul.build[:3] != zuul_log_path[:3]
+
+- name: Test the emit-job-header role with swift
+  hosts: all
+  roles:
+    - role: emit-job-header
+      zuul_log_url: "http://logs.openstack.org"
+      zuul_log_path_shard_build: true
+  post_tasks:
+    # All emit-job-header does is a debug statement so the worst that would
+    # happen would be that the debug task would fail outright and we'd prevent
+    # something breaking that debug statement from merging just by running it.
+    # However, the emit-job-header role includes the set-zuul-log-path-fact
+    # role. We can only test for zuul_log_path against changes, though.
+    - name: Assert zuul_log_path by set-zuul-log-path-fact for a change
+      assert:
+        that:
+          - zuul_log_path is defined
+          - zuul.change in zuul_log_path
+          - zuul.patchset in zuul_log_path
+          - zuul.pipeline in zuul_log_path
+          - zuul.job in zuul_log_path
+          - zuul.build[:3] == zuul_log_path[:3]

test-playbooks/base-test-roles/ensure-output-dirs.yaml

@@ -0,0 +1,62 @@
+- name: Test the ensure-output-dirs role
+  hosts: all
+  roles:
+    - role: ensure-output-dirs
+  post_tasks:
+    - name: Check that log dir has been created
+      file:
+        path: "{{ zuul_output_dir }}/logs"
+        state: directory
+      register: log_directory
+
+    - name: Check that artifact dir has been created
+      file:
+        path: "{{ zuul_output_dir }}/artifacts"
+        state: directory
+      register: artifact_directory
+
+    - name: Check that doc dir has been created
+      file:
+        path: "{{ zuul_output_dir }}/docs"
+        state: directory
+      register: doc_directory
+
+    - name: Validate that directories were set correctly
+      assert:
+        that:
+          - log_directory is not changed
+          - log_directory is succeeded
+          - artifact_directory is not changed
+          - artifact_directory is succeeded
+          - doc_directory is not changed
+          - doc_directory is succeeded
+
+    - name: Add a logfile, artifact, and doc item
+      file:
+        path: "{{ zuul_output_dir }}/{{ item }}/file.txt"
+        state: touch
+      loop:
+        - docs
+        - artifacts
+        - logs
+
+    - name: Run ensure-output-dirs
+      include_role:
+        name: ensure-output-dirs
+
+    - name: Make sure output dirs were emptied
+      file:
+        path: "{{ zuul_output_dir }}/{{ item }}/file.txt"
+        state: absent
+      register: output_items
+      loop:
+        - docs
+        - artifacts
+        - logs
+
+    - name: Validate that files were removed in ensure-output-dirs
+      loop: "{{ output_items.results }}"
+      assert:
+        that:
+          - item is not changed
+          - item is succeeded

test-playbooks/base-test-roles/fetch-output.yaml

@@ -0,0 +1,46 @@
+- name: Run the fetch-output role
+  hosts: all
+  pre_tasks:
+    # ensure-output-dirs is run before this
+    - name: Write test log file
+      copy:
+        dest: '{{ zuul_output_dir }}/{{ item }}/{{ inventory_hostname }}'
+        content: '{{ item }}'
+      loop:
+        - logs
+        - docs
+        - artifacts
+  roles:
+    - role: fetch-output
+  post_tasks:
+    - name: Check that logs have been pulled
+      delegate_to: localhost
+      file:
+        # log_path fact is set in fetch-output
+        path: "{{ log_path }}/{{ inventory_hostname }}"
+        state: file
+      register: local_log_content
+
+    - name: Check that artifacts have been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.work_root }}/artifacts/{{ inventory_hostname }}"
+        state: file
+      register: local_artifact_content
+
+    - name: Check that docs have been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.work_root }}/docs/{{ inventory_hostname }}"
+        state: file
+      register: local_doc_content
+
+    - name: Validate that files were pulled correctly
+      assert:
+        that:
+          - local_log_content is not changed
+          - local_log_content is succeeded
+          - local_artifact_content is not changed
+          - local_artifact_content is succeeded
+          - local_doc_content is not changed
+          - local_doc_content is succeeded

test-playbooks/base-test-roles/fetch-subunit-output.yaml

@@ -0,0 +1,131 @@
+- name: Run the fetch-subunit-output role
+  hosts: all
+  vars:
+    tests_data:
+      main:
+        directory: "{{ zuul_work_dir }}"
+        test_pattern: "WorkingTest.test_success"
+      secondary:
+        directory: "/var/tmp/extratests"
+        test_pattern: "FailingTest.test_failure"
+  pre_tasks:
+    - name: Ensure pip
+      include_role:
+        name: ensure-pip
+    # Required packages; install them into a .tox path
+    # to cover the find-*.sh scripts in the role a bit more.
+    - name: Install stestr and subunit-output
+      pip:
+        name:
+          - stestr
+          - python-subunit
+          - 'voluptuous==0.13.1'
+        virtualenv: "{{ zuul_work_dir }}/.tox/utests/"
+        virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+
+    - name: Ensure that the test directories exists
+      file:
+        name: "{{ item.value.directory }}"
+        state: directory
+      loop: "{{ tests_data|dict2items }}"
+
+    - name: Copy the test files on all directories
+      copy:
+        src: "subunit_tests"
+        dest: "{{ item.value.directory }}"
+      loop: "{{ tests_data|dict2items }}"
+
+    - name: Prepare the test results on all directories
+      shell: |
+        . {{ zuul_work_dir }}/.tox/utests/bin/activate
+        stestr init
+        stestr run --test-path subunit_tests {{ item.value.test_pattern }}
+      args:
+        chdir: "{{ item.value.directory }}"
+      ignore_errors: yes
+      loop: "{{ tests_data|dict2items }}"
+  roles:
+    - role: fetch-subunit-output
+  post_tasks:
+    - name: Check that the testrepository file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testrepository.subunit"
+        state: file
+      register: local_subunit_file
+
+    - name: Check that HTML test result file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testr_results.html"
+        state: file
+      register: local_html_test_results
+
+    - name: Validate that files were pulled correctly
+      assert:
+        that:
+          - local_subunit_file is not changed
+          - local_subunit_file is succeeded
+          - local_html_test_results is not changed
+          - local_html_test_results is succeeded
+
+    # only one subunit file; the failed result should be hidden
+    - name: Check the content of the HTML file
+      delegate_to: localhost
+      shell: |
+        GLOBAL_RESULT=1
+        zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
+            {{ zuul.executor.log_root }}/testr_results.html
+        T1=$?
+        zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
+            {{ zuul.executor.log_root }}/testr_results.html
+        T2=$?
+        if [ ${T1} -eq 0 ] && [ ${T2} -ne 0 ]; then
+            GLOBAL_RESULT=0
+        fi
+        exit $GLOBAL_RESULT
+
+# The following test(s) require(s) the previous playbook
+- name: Run the fetch-subunit-output role with multiple subunits
+  hosts: all
+  roles:
+    - role: fetch-subunit-output
+      fetch_subunit_output_additional_dirs:
+        - "/var/tmp/extratests"
+  post_tasks:
+    - name: Check that the testrepository file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testrepository.subunit"
+        state: file
+      register: local_subunit_file
+
+    - name: Check that HTML test result file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testr_results.html"
+        state: file
+      register: local_html_test_results
+
+    - name: Validate that files were pulled correctly
+      assert:
+        that:
+          - local_subunit_file is not changed
+          - local_subunit_file is succeeded
+          - local_html_test_results is not changed
+          - local_html_test_results is succeeded
+
+    - name: Check the content of the HTML file
+      delegate_to: localhost
+      shell: |
+        GLOBAL_RESULT=1
+        zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
+            {{ zuul.executor.log_root }}/testr_results.html
+        T1=$?
+        zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
+            {{ zuul.executor.log_root }}/testr_results.html
+        T2=$?
+        if [ ${T1} -eq 0 ] && [ ${T2} -eq 0 ]; then
+            GLOBAL_RESULT=0
+        fi
+        exit $GLOBAL_RESULT

test-playbooks/base-test-roles/fetch-zuul-cloner.yaml

@@ -0,0 +1,84 @@
+- name: Test the fetch-zuul-cloner role
+  hosts: all
+  vars:
+    destination: "/usr/zuul-env/bin/zuul-cloner"
+    repo_src_dir: "{{ ansible_user_dir }}/src/opendev.org"
+  roles:
+    - role: fetch-zuul-cloner
+  post_tasks:
+    - name: Check that the directory exists
+      file:
+        path: "{{ destination | dirname }}"
+        state: directory
+      register: directory
+
+    - name: Check that the zuul-cloner shim exists
+      stat:
+        path: "{{ destination }}"
+      register: cloner
+
+    - name: Validate that the shim was installed successfully
+      assert:
+        that:
+          - directory is not changed
+          - directory is succeeded
+          - cloner.stat.exists
+          - cloner.stat.mode == "0755"
+
+    - name: Zuul clone something in required-projects
+      shell:
+        executable: /bin/bash
+        cmd: |
+          CLONEMAP=`mktemp`
+          function cleanup {
+              rm -f $CLONEMAP
+          }
+          trap cleanup EXIT
+          cat > $CLONEMAP << EOF
+          clonemap:
+            - name: opendev/base-jobs
+              dest: {{ ansible_user_dir }}
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP \
+              --cache-dir /opt/git https://opendev.org \
+              opendev/base-jobs
+      register: clone_with_required
+
+    - name: Check if repository was cloned
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
+      register: with_required_stat
+
+    - name: Zuul clone something not in required-projects
+      shell:
+        executable: /bin/bash
+        cmd: |
+          CLONEMAP=`mktemp`
+          function cleanup {
+              rm -f $CLONEMAP
+          }
+          trap cleanup EXIT
+          cat > $CLONEMAP << EOF
+          clonemap:
+            - name: jjb/jenkins-job-builder
+              dest: {{ ansible_user_dir }}
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP \
+              --cache-dir /opt/git https://git.openstack.org \
+              jjb/jenkins-job-builder
+      ignore_errors: yes
+      register: clone_without_required
+
+    - name: Check if repository was cloned
+      stat:
+        path: "{{ ansible_user_dir }}/src/git.openstack.org/jjb/jenkins-job-builder"
+      register: without_required_stat
+
+    - name: Validate zuul-cloner shim results
+      assert:
+        that:
+          - clone_with_required is succeeded
+          - clone_with_required is changed
+          - with_required_stat.stat.exists
+          - clone_without_required is failed
+          - not without_required_stat.stat.exists

test-playbooks/base-test-roles/prepare-workspace-git-required-projects-only-inner.yaml

@@ -0,0 +1,6 @@
+- name: Test the test-prepare-workspace-git role
+  hosts: all
+  roles:
+    - role: test-prepare-workspace-git
+      vars:
+        prepare_workspace_sync_required_projects_only: true

test-playbooks/base-test-roles/prepare-workspace-git-required-projects-only.yaml

@@ -0,0 +1,72 @@
+- name: Prepare to test the test-prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
+# We need to override the zuul.projects variable, and that is not
+# possible in a Zuul job.  So we use a nested Ansible to perform this
+# test.
+- name: Test the test-prepare-workspace-git role with sync required only
+  hosts: localhost
+  vars:
+    # Mutate the zuul vars supplied to this test job to simulate a
+    # repo being included as non-required (i.e., a depends-on).
+    zuul_mod:
+      projects:
+        opendev.org/zuul/project-config:
+          required: false
+  tasks:
+    - name: Create nested zuul vars
+      set_fact:
+        nested_zuul:
+          zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}"
+    - name: Write nested zuul vars
+      copy:
+        content: '{{ nested_zuul | to_nice_yaml(indent=2) }}'
+        dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml"
+    - name: Run nested Ansible
+      command: >-
+        {{ ansible_playbook_python | dirname}}/ansible-playbook
+        -vvv
+        -e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml
+        -e zuul_execution_phase=nested
+        -e zuul_execution_phase_index=0
+        -e zuul_execution_canonical_name_and_path=opendev.org/zuul/zuul-jobs/test-playbooks/base-test-roles/prepare-workspace-git-required-projects-only-inner.yaml
+        -e zuul_execution_trusted=False
+        -e zuul_execution_branch={{zuul_execution_branch}}
+        {{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/test-playbooks/base-test-roles/prepare-workspace-git-required-projects-only-inner.yaml
+      environment:
+        ANSIBLE_ROLES_PATH: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles"
+
+- name: Verify the test-prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    # opendev/base-jobs is in 'required-projects'.
+    # Also check that the project being tested is being prepared.
+    # We're checking them explicitly rather than with_items on zuul.projects
+    # in case there is a regression which would take an item out.
+    - name: Check that opendev/base-jobs was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
+      register: base_jobs
+
+    - name: Check that zuul/project-config was not prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
+    - name: Check this project was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
+      register: self_config
+
+    - name: Validate that required projects have been prepared
+      assert:
+        that:
+          - base_jobs.stat.exists
+          - not project_config.stat.exists
+          - self_config.stat.exists

test-playbooks/base-test-roles/prepare-workspace-git.yaml

@@ -0,0 +1,37 @@
+- name: Test the test-prepare-workspace-git role
+  hosts: all
+  pre_tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
+  roles:
+    - role: test-prepare-workspace-git
+  post_tasks:
+    # opendev/base-jobs is in 'required-projects'.
+    # Also check that the project being tested is being prepared.
+    # We're checking them explicitly rather than with_items on zuul.projects
+    # in case there is a regression which would take an item out.
+    - name: Check that opendev/base-jobs was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
+      register: base_jobs
+
+    - name: Check that zuul/project-config was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
+    - name: Check this project was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
+      register: self_config
+
+    - name: Validate that required projects have been prepared
+      assert:
+        that:
+          - base_jobs.stat.exists
+          - project_config.stat.exists
+          - self_config.stat.exists

test-playbooks/base-test-roles/subunit_tests/test_failing.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+
+class FailingTest(unittest.TestCase):
+
+    def test_failure(self):
+        self.assertTrue(False)

test-playbooks/base-test-roles/subunit_tests/test_working.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+
+class WorkingTest(unittest.TestCase):
+
+    def test_success(self):
+        self.assertTrue(True)

test-playbooks/base-test-roles/validate-host.yaml

@@ -0,0 +1,14 @@
+- name: Test the validate-host role
+  pre_tasks:
+    # NOTE(pabelanger): Until we hit the validate-host role, we have a minimal
+    # set of ansible variables collected by zuul-executor. This doesn't include
+    # network variables (ansible_default_ipv4 / ansible_default_ipv6) so gather
+    # these variables as they are important to the configure-unbound role.
+    - name: Gather network facts
+      setup:
+        gather_subset: 'network'
+  hosts: all
+  roles:
+    - role: validate-host
+      zuul_site_traceroute_host: files.openstack.org
+      validate_host_hostname: "abc/123"

zuul-tests.d/general-roles-jobs.yaml

@@ -193,6 +193,112 @@
         - name: ubuntu-noble
           label: ubuntu-noble
 
+- job:
+    name: zuul-jobs-test-base-test-roles
+    description: |
+      Tests roles in the 'base-test' job
+
+      These roles are tested together in this job because testing them
+      relies on using base-minimal, which avoids running these roles.
+      However, for the job to fully function, they all need to have
+      been run (once) by the end.
+
+      Currently only prepare-workspace-git has a test-* variant; this
+      is the only difference from zuul-jobs-test-base-roles
+    parent: base-minimal
+    tags: all-platforms
+    abstract: true
+    run: test-playbooks/base-test-roles/base.yaml
+    # Testing of fetch-zuul-cloner and prepare-workspace-git need
+    # these repos in required-projects
+    required-projects:
+      - opendev/base-jobs
+      - zuul/project-config
+    files:
+      - ^roles/configure-mirrors/.*
+      - ^roles/emit-job-header/.*
+      - ^roles/ensure-output-dirs/.*
+      - ^roles/fetch-output/.*
+      - ^roles/fetch-subunit-output/.*
+      - ^roles/fetch-zuul-cloner/.*
+      - ^roles/set-zuul-log-path-fact/.*
+      - ^roles/test-prepare-workspace-git/.*
+      - ^roles/validate-host/.*
+      - ^test-playbooks/base-roles/.*
+
+- job:
+    name: zuul-jobs-test-base-test-roles-centos-9-stream
+    description: Tests roles in the 'base-test' job on centos-9-stream
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-9-stream
+          label: centos-9-stream
+
+- job:
+    name: zuul-jobs-test-base-test-roles-debian-bookworm
+    description: Tests roles in the 'base-test' job on debian-bookworm
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-bookworm
+          label: debian-bookworm
+
+- job:
+    name: zuul-jobs-test-base-test-roles-debian-bullseye
+    description: Tests roles in the 'base-test' job on debian-bullseye
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-bullseye
+          label: debian-bullseye
+
+- job:
+    name: zuul-jobs-test-base-test-roles-ubuntu-focal
+    description: Tests roles in the 'base-test' job on ubuntu-focal
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-focal
+          label: ubuntu-focal
+
+- job:
+    name: zuul-jobs-test-base-test-roles-ubuntu-jammy
+    description: Tests roles in the 'base-test' job on ubuntu-jammy
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-jammy
+          label: ubuntu-jammy
+
+- job:
+    name: zuul-jobs-test-base-test-roles-ubuntu-noble
+    description: Tests roles in the 'base-test' job on ubuntu-noble
+    parent: zuul-jobs-test-base-test-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-noble
+          label: ubuntu-noble
+
+- job:
+    name: zuul-jobs-test-base-test-roles-ubuntu-xenial
+    description: Tests roles in the 'base-test' job on ubuntu-xenial
+    parent: zuul-jobs-test-base-test-roles
+    # Note this is manually curated since xenial is not really
+    # supported on opendev, but we want the job to run until the last
+    # possible moment.
+    ansible-version: '8'
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial
+          label: ubuntu-xenial
+
 - job:
     name: zuul-jobs-test-bindep
     description: Test the bindep role
@@ -740,6 +846,13 @@
         - zuul-jobs-test-base-roles-ubuntu-focal
         - zuul-jobs-test-base-roles-ubuntu-jammy
         - zuul-jobs-test-base-roles-ubuntu-noble
+        - zuul-jobs-test-base-test-roles-centos-9-stream
+        - zuul-jobs-test-base-test-roles-debian-bookworm
+        - zuul-jobs-test-base-test-roles-debian-bullseye
+        - zuul-jobs-test-base-test-roles-ubuntu-focal
+        - zuul-jobs-test-base-test-roles-ubuntu-jammy
+        - zuul-jobs-test-base-test-roles-ubuntu-noble
+        - zuul-jobs-test-base-test-roles-ubuntu-xenial
         - zuul-jobs-test-bindep-centos-9-stream
         - zuul-jobs-test-bindep-debian-bookworm
         - zuul-jobs-test-bindep-debian-bullseye