prepare-workspace-git: Add ability to define synced pojects

The prepare_workspace_sync_required_projects_only variable allows
users to define which projects to sync to the node. This can prevent
syncing of unnecessary repositories. For some builds e.g. the
depends-on repositories dont need to be synced. The projects are
filtered based on the 'required' flag present in each zuul.project
entry and the required projects list also does not contain projects
which are present due to Depends-On or gate queue sequencing.
Having unnecessary repos in the workspace can for example also break
the analysis phase of bazel.

Change-Id: I3cc36cbfc60c81956caf5137da63973aeade4e21
Co-Authored-By: James E. Blair <jim@acmegating.com>
Co-Authored-By: Bernhard Berg <bernhard.berg@bearingpoint.com>

roles/prepare-workspace-git/README.rst

@@ -16,6 +16,12 @@ The cached repos need to be placed using the canonical name under the
 
    The root of the cached repos.
 
+.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
+   :type: bool
+   :default: False
+
+    A flag which if set to true, filters the to be synchronized project list to only use projects which are required by a job.
+
 .. zuul:rolevar:: mirror_workspace_quiet
    :default: false
 

roles/prepare-workspace-git/defaults/main.yaml

@@ -1,3 +1,4 @@
 cached_repos_root: /opt/git
 mirror_workspace_quiet: false
 zuul_workspace_root: "{{ ansible_user_dir }}"
+prepare_workspace_sync_required_projects_only: false

roles/prepare-workspace-git/tasks/main.yaml

@@ -1,3 +1,30 @@
+- name: debug
+  debug:
+    var: zuul
+
+- name: debug
+  debug:
+    var: zuul.projects
+
+- name: Filter zuul projects if sync-only-required-projects flag is set
+  set_fact:
+    _zuul_projects: >
+      {{ _zuul_projects | default({}) |
+            combine({ zj_project.key : zj_project.value }) }}
+  with_dict: "{{ zuul.projects }}"
+  loop_control:
+    loop_var: zj_project
+  when:
+    - prepare_workspace_sync_required_projects_only
+    - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required
+
+# add comment concerning "project under test" case
+
+- name: Don't filter zuul projects if flag is false
+  set_fact:
+    _zuul_projects: "{{ zuul.projects }}"
+  when: prepare_workspace_sync_required_projects_only
+
 # Do all the steps in a single shell script. This reduces the number of times
 # ansible must loop over the list of projects which reduces the amount of
 # task startup time we incur.
@@ -17,7 +44,7 @@
     git remote add origin file:///dev/null
   args:
     creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
-  with_items: "{{ zuul.projects.values() | list }}"
+  with_items: "{{ _zuul_projects.values() }}"
   loop_control:
     loop_var: zj_project
   # We're using git in a shell script because it is faster and the module
@@ -31,7 +58,7 @@
     value: ignore
     scope: local
     repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
 
@@ -46,7 +73,7 @@
     chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
   environment:
     GIT_ALLOW_PROTOCOL: ext:ssh
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   delegate_to: localhost
@@ -80,7 +107,7 @@
     git log --pretty=oneline  -1
   args:
     chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   # ANSIBLE0006: Skip linting since it triggers on the "git" command,

roles/test-prepare-workspace-git/README.rst

@@ -16,6 +16,14 @@ The cached repos need to be placed using the canonical name under the
 
    The root of the cached repos.
 
+.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
+   :type: bool
+   :default: False
+
+    A flag which if set to true, filters the to be synchronized
+    project list to only use projects which are required by the job,
+    including the project under test (zuul.project).
+
 .. zuul:rolevar:: mirror_workspace_quiet
    :default: false
 

roles/test-prepare-workspace-git/defaults/main.yaml

@@ -1,3 +1,4 @@
 cached_repos_root: /opt/git
 mirror_workspace_quiet: false
 zuul_workspace_root: "{{ ansible_user_dir }}"
+prepare_workspace_sync_required_projects_only: false

roles/test-prepare-workspace-git/tasks/main.yaml

@@ -1,3 +1,20 @@
+- name: Filter zuul projects if sync-only-required-projects flag is set
+  set_fact:
+    _zuul_projects: >
+      {{ _zuul_projects | default({}) |
+            combine({ zj_project.key : zj_project.value }) }}
+  with_dict: "{{ zuul.projects }}"
+  loop_control:
+    loop_var: zj_project
+  when:
+    - prepare_workspace_sync_required_projects_only
+    - zj_project.value.canonical_name == zuul.project.canonical_name or zk_project.value.required
+
+- name: Don't filter zuul projects if flag is false
+  set_fact:
+    _zuul_projects: "{{ zuul.projects }}"
+  when: prepare_workspace_sync_required_projects_only
+
 # Do all the steps in a single shell script. This reduces the number of times
 # ansible must loop over the list of projects which reduces the amount of
 # task startup time we incur.
@@ -17,7 +34,7 @@
     git remote add origin file:///dev/null
   args:
     creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
-  with_items: "{{ zuul.projects.values() | list }}"
+  with_items: "{{ _zuul_projects.values() }}"
   loop_control:
     loop_var: zj_project
   # We're using git in a shell script because it is faster and the module
@@ -31,7 +48,7 @@
     value: ignore
     scope: local
     repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
 
@@ -46,7 +63,7 @@
     chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
   environment:
     GIT_ALLOW_PROTOCOL: ext:ssh
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   delegate_to: localhost
@@ -80,7 +97,7 @@
     git log --pretty=oneline  -1
   args:
     chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   # ANSIBLE0006: Skip linting since it triggers on the "git" command,

test-playbooks/base-roles/base.yaml

@@ -6,6 +6,7 @@
 # Note: set-zuul-log-path-fact is tested by emit-job-header.yaml
 - import_playbook: emit-job-header.yaml
 - import_playbook: ensure-output-dirs.yaml
+- import_playbook: prepare-workspace-git-required-projects-only.yaml
 - import_playbook: prepare-workspace-git.yaml
 - import_playbook: configure-mirrors.yaml
 - import_playbook: fetch-zuul-cloner.yaml

test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml

@@ -0,0 +1,68 @@
+- name: Prepare to test the prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
+# We need to override the zuul.projects variable, and that is not
+# possible in a Zuul job.  So we use a nested Ansible to perform this
+# test.
+- name: Test the prepare-workspace-git role with sync required only
+  hosts: localhost
+  vars:
+    # Mutate the zuul vars supplied to this test job to simulate a
+    # repo being included as non-required (i.e., a depends-on).
+    zuul_mod:
+      projects:
+        opendev.org/zuul/project-config:
+          required: false
+  tasks:
+    - name: Create nested zuul vars
+      set_fact:
+        nested_zuul:
+          zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}"
+    - name: Write nested zuul vars
+      copy:
+        content: '{{ nested_zuul | to_nice_yaml(indent=2) }}'
+        dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml"
+    - name: Write inventory
+      include_role:
+        name: write-inventory
+      vars:
+        write_inventory_dest: "{{ zuul.executor.work_root }}/nested-inventory.yaml"
+    - name: Run nested Ansible
+      command: "{{ ansible_playbook_python | dirname}}/ansible-playbook -i {{ zuul.executor.work_root }}/nested-inventory.yaml -e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml"
+      environment:
+        ANSIBLE_ROLES_PATH: "{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles"
+
+- name: Verify the prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    # opendev/base-jobs is in 'required-projects'.
+    # Also check that the project being tested is being prepared.
+    # We're checking them explicitly rather than with_items on zuul.projects
+    # in case there is a regression which would take an item out.
+    - name: Check that opendev/base-jobs was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
+      register: base_jobs
+
+    - name: Check that zuul/project-config was not prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
+    - name: Check this project was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
+      register: self_config
+
+    - name: Validate that required projects have been prepared
+      assert:
+        that:
+          - base_jobs.stat.exists
+          - not project_config.stat.exists
+          - self_config.stat.exists

test-playbooks/base-roles/prepare-workspace-git.yaml

@@ -1,5 +1,12 @@
 - name: Test the prepare-workspace-git role
   hosts: all
+  pre_tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
   roles:
     - role: prepare-workspace-git
   post_tasks:
@@ -12,6 +19,11 @@
         path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
       register: base_jobs
 
+    - name: Check that zuul/project-config was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
     - name: Check this project was prepared
       stat:
         path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
@@ -21,4 +33,5 @@
       assert:
         that:
           - base_jobs.stat.exists
+          - project_config.stat.exists
           - self_config.stat.exists

zuul-tests.d/general-roles-jobs.yaml

@@ -116,10 +116,11 @@
     tags: all-platforms
     abstract: true
     run: test-playbooks/base-roles/base.yaml
-    # Testing of fetch-zuul-cloner and use-cached-repos need this repo
-    # in required-projects
+    # Testing of fetch-zuul-cloner and prepare-workspace-git need
+    # these repos in required-projects
     required-projects:
       - opendev/base-jobs
+      - zuul/project-config
     files:
       - ^roles/configure-mirrors/.*
       - ^roles/emit-job-header/.*