Merge "prepare-workspace-git: Add ability to define synced pojects"

roles/prepare-workspace-git/README.rst

@@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the
 
    The root of the cached repos.
 
+.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
+   :type: bool
+   :default: False
+
+   A flag which if set to true, filters the to be synchronized project
+   list to only use projects which are required by the job.
+
 .. zuul:rolevar:: mirror_workspace_quiet
    :default: false
 

roles/prepare-workspace-git/defaults/main.yaml

@@ -1,3 +1,4 @@
 cached_repos_root: /opt/git
 mirror_workspace_quiet: false
 zuul_workspace_root: "{{ ansible_user_dir }}"
+prepare_workspace_sync_required_projects_only: false

roles/prepare-workspace-git/tasks/main.yaml

@@ -1,3 +1,20 @@
+- name: Filter zuul projects if sync-only-required-projects flag is set
+  set_fact:
+    _zuul_projects: >
+      {{ _zuul_projects | default({}) |
+            combine({ zj_project.key : zj_project.value }) }}
+  with_dict: "{{ zuul.projects }}"
+  loop_control:
+    loop_var: zj_project
+  when:
+    - prepare_workspace_sync_required_projects_only
+    - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required
+
+- name: Don't filter zuul projects if flag is false
+  set_fact:
+    _zuul_projects: "{{ zuul.projects }}"
+  when: not prepare_workspace_sync_required_projects_only
+
 # Do all the steps in a single shell script. This reduces the number of times
 # ansible must loop over the list of projects which reduces the amount of
 # task startup time we incur.
@@ -17,7 +34,7 @@
     git remote add origin file:///dev/null
   args:
     creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
-  with_items: "{{ zuul.projects.values() | list }}"
+  with_items: "{{ _zuul_projects.values() }}"
   loop_control:
     loop_var: zj_project
   # We're using git in a shell script because it is faster and the module
@@ -31,7 +48,7 @@
     value: ignore
     scope: local
     repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
 
@@ -46,7 +63,7 @@
     chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
   environment:
     GIT_ALLOW_PROTOCOL: ext:ssh
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   delegate_to: localhost
@@ -80,7 +97,7 @@
     git log --pretty=oneline  -1
   args:
     chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   # ANSIBLE0006: Skip linting since it triggers on the "git" command,

roles/test-prepare-workspace-git/README.rst

@@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the
 
    The root of the cached repos.
 
+.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
+   :type: bool
+   :default: False
+
+   A flag which if set to true, filters the to be synchronized project
+   list to only use projects which are required by the job.
+
 .. zuul:rolevar:: mirror_workspace_quiet
    :default: false
 

roles/test-prepare-workspace-git/defaults/main.yaml

@@ -1,3 +1,4 @@
 cached_repos_root: /opt/git
 mirror_workspace_quiet: false
 zuul_workspace_root: "{{ ansible_user_dir }}"
+prepare_workspace_sync_required_projects_only: false

roles/test-prepare-workspace-git/tasks/main.yaml

@@ -1,3 +1,20 @@
+- name: Filter zuul projects if sync-only-required-projects flag is set
+  set_fact:
+    _zuul_projects: >
+      {{ _zuul_projects | default({}) |
+            combine({ zj_project.key : zj_project.value }) }}
+  with_dict: "{{ zuul.projects }}"
+  loop_control:
+    loop_var: zj_project
+  when:
+    - prepare_workspace_sync_required_projects_only
+    - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required
+
+- name: Don't filter zuul projects if flag is false
+  set_fact:
+    _zuul_projects: "{{ zuul.projects }}"
+  when: not prepare_workspace_sync_required_projects_only
+
 # Do all the steps in a single shell script. This reduces the number of times
 # ansible must loop over the list of projects which reduces the amount of
 # task startup time we incur.
@@ -17,7 +34,7 @@
     git remote add origin file:///dev/null
   args:
     creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
-  with_items: "{{ zuul.projects.values() | list }}"
+  with_items: "{{ _zuul_projects.values() }}"
   loop_control:
     loop_var: zj_project
   # We're using git in a shell script because it is faster and the module
@@ -31,7 +48,7 @@
     value: ignore
     scope: local
     repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
 
@@ -46,7 +63,7 @@
     chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
   environment:
     GIT_ALLOW_PROTOCOL: ext:ssh
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   delegate_to: localhost
@@ -80,7 +97,7 @@
     git log --pretty=oneline  -1
   args:
     chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
-  with_dict: "{{ zuul.projects }}"
+  with_dict: "{{ _zuul_projects }}"
   loop_control:
     loop_var: zj_project
   # ANSIBLE0006: Skip linting since it triggers on the "git" command,

test-playbooks/base-roles/base.yaml

@@ -6,6 +6,7 @@
 # Note: set-zuul-log-path-fact is tested by emit-job-header.yaml
 - import_playbook: emit-job-header.yaml
 - import_playbook: ensure-output-dirs.yaml
+- import_playbook: prepare-workspace-git-required-projects-only.yaml
 - import_playbook: prepare-workspace-git.yaml
 - import_playbook: configure-mirrors.yaml
 - import_playbook: fetch-zuul-cloner.yaml

test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml

@@ -0,0 +1,6 @@
+- name: Test the prepare-workspace-git role
+  hosts: all
+  roles:
+    - role: prepare-workspace-git
+      vars:
+        prepare_workspace_sync_required_projects_only: true

test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml

@@ -0,0 +1,72 @@
+- name: Prepare to test the prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
+# We need to override the zuul.projects variable, and that is not
+# possible in a Zuul job.  So we use a nested Ansible to perform this
+# test.
+- name: Test the prepare-workspace-git role with sync required only
+  hosts: localhost
+  vars:
+    # Mutate the zuul vars supplied to this test job to simulate a
+    # repo being included as non-required (i.e., a depends-on).
+    zuul_mod:
+      projects:
+        opendev.org/zuul/project-config:
+          required: false
+  tasks:
+    - name: Create nested zuul vars
+      set_fact:
+        nested_zuul:
+          zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}"
+    - name: Write nested zuul vars
+      copy:
+        content: '{{ nested_zuul | to_nice_yaml(indent=2) }}'
+        dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml"
+    - name: Run nested Ansible
+      command: >-
+        {{ ansible_playbook_python | dirname}}/ansible-playbook
+        -vvv
+        -e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml
+        -e zuul_execution_phase=nested
+        -e zuul_execution_phase_index=0
+        -e zuul_execution_canonical_name_and_path=opendev.org/zuul/zuul-jobs/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml
+        -e zuul_execution_trusted=False
+        -e zuul_execution_branch={{zuul_execution_branch}}
+        {{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml
+      environment:
+        ANSIBLE_ROLES_PATH: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles"
+
+- name: Verify the prepare-workspace-git role with sync required only
+  hosts: all
+  tasks:
+    # opendev/base-jobs is in 'required-projects'.
+    # Also check that the project being tested is being prepared.
+    # We're checking them explicitly rather than with_items on zuul.projects
+    # in case there is a regression which would take an item out.
+    - name: Check that opendev/base-jobs was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
+      register: base_jobs
+
+    - name: Check that zuul/project-config was not prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
+    - name: Check this project was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
+      register: self_config
+
+    - name: Validate that required projects have been prepared
+      assert:
+        that:
+          - base_jobs.stat.exists
+          - not project_config.stat.exists
+          - self_config.stat.exists

test-playbooks/base-roles/prepare-workspace-git.yaml

@@ -1,5 +1,12 @@
 - name: Test the prepare-workspace-git role
   hosts: all
+  pre_tasks:
+    - name: Delete remote source directory to start with clean state
+      file:
+        state: absent
+        path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
+      with_dict: "{{ zuul.projects }}"
+
   roles:
     - role: prepare-workspace-git
   post_tasks:
@@ -12,6 +19,11 @@
         path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
       register: base_jobs
 
+    - name: Check that zuul/project-config was prepared
+      stat:
+        path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
+      register: project_config
+
     - name: Check this project was prepared
       stat:
         path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
@@ -21,4 +33,5 @@
       assert:
         that:
           - base_jobs.stat.exists
+          - project_config.stat.exists
           - self_config.stat.exists

zuul-tests.d/general-roles-jobs.yaml

@@ -116,10 +116,11 @@
     tags: all-platforms
     abstract: true
     run: test-playbooks/base-roles/base.yaml
-    # Testing of fetch-zuul-cloner and use-cached-repos need this repo
+    # Testing of fetch-zuul-cloner and prepare-workspace-git need
-    # in required-projects
+    # these repos in required-projects
     required-projects:
       - opendev/base-jobs
+      - zuul/project-config
     files:
       - ^roles/configure-mirrors/.*
       - ^roles/emit-job-header/.*