From 5eca8feda9a2aea39bcd58dd8c7eefb4bca00a01 Mon Sep 17 00:00:00 2001 From: Lukas Kranz Date: Fri, 7 Jul 2023 10:00:12 +0200 Subject: [PATCH] prepare-workspace-git: Add ability to define synced pojects The prepare_workspace_sync_required_projects_only variable allows users to define which projects to sync to the node. This can prevent syncing of unnecessary repositories. For some builds e.g. the depends-on repositories dont need to be synced. The projects are filtered based on the 'required' flag present in each zuul.project entry and the required projects list also does not contain projects which are present due to Depends-On or gate queue sequencing. Having unnecessary repos in the workspace can for example also break the analysis phase of bazel. Change-Id: I3cc36cbfc60c81956caf5137da63973aeade4e21 Co-Authored-By: James E. Blair Co-Authored-By: Bernhard Berg --- roles/prepare-workspace-git/README.rst | 7 ++ .../prepare-workspace-git/defaults/main.yaml | 1 + roles/prepare-workspace-git/tasks/main.yaml | 25 +++++-- roles/test-prepare-workspace-git/README.rst | 7 ++ .../defaults/main.yaml | 1 + .../tasks/main.yaml | 25 +++++-- test-playbooks/base-roles/base.yaml | 1 + ...pace-git-required-projects-only-inner.yaml | 6 ++ ...-workspace-git-required-projects-only.yaml | 72 +++++++++++++++++++ .../base-roles/prepare-workspace-git.yaml | 13 ++++ zuul-tests.d/general-roles-jobs.yaml | 5 +- 11 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml create mode 100644 test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml diff --git a/roles/prepare-workspace-git/README.rst b/roles/prepare-workspace-git/README.rst index 1cc5ce270..4faf51919 100644 --- a/roles/prepare-workspace-git/README.rst +++ b/roles/prepare-workspace-git/README.rst @@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: prepare_workspace_sync_required_projects_only + :type: bool + :default: False + + A flag which if set to true, filters the to be synchronized project + list to only use projects which are required by the job. + .. zuul:rolevar:: mirror_workspace_quiet :default: false diff --git a/roles/prepare-workspace-git/defaults/main.yaml b/roles/prepare-workspace-git/defaults/main.yaml index f6e5da9bb..fbd1b27ef 100644 --- a/roles/prepare-workspace-git/defaults/main.yaml +++ b/roles/prepare-workspace-git/defaults/main.yaml @@ -1,3 +1,4 @@ cached_repos_root: /opt/git mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" +prepare_workspace_sync_required_projects_only: false diff --git a/roles/prepare-workspace-git/tasks/main.yaml b/roles/prepare-workspace-git/tasks/main.yaml index d57ef11f1..5e68daf99 100644 --- a/roles/prepare-workspace-git/tasks/main.yaml +++ b/roles/prepare-workspace-git/tasks/main.yaml @@ -1,3 +1,20 @@ +- name: Filter zuul projects if sync-only-required-projects flag is set + set_fact: + _zuul_projects: > + {{ _zuul_projects | default({}) | + combine({ zj_project.key : zj_project.value }) }} + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + when: + - prepare_workspace_sync_required_projects_only + - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required + +- name: Don't filter zuul projects if flag is false + set_fact: + _zuul_projects: "{{ zuul.projects }}" + when: not prepare_workspace_sync_required_projects_only + # Do all the steps in a single shell script. This reduces the number of times # ansible must loop over the list of projects which reduces the amount of # task startup time we incur. @@ -17,7 +34,7 @@ git remote add origin file:///dev/null args: creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" - with_items: "{{ zuul.projects.values() | list }}" + with_items: "{{ _zuul_projects.values() }}" loop_control: loop_var: zj_project # We're using git in a shell script because it is faster and the module @@ -31,7 +48,7 @@ value: ignore scope: local repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project @@ -46,7 +63,7 @@ chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" environment: GIT_ALLOW_PROTOCOL: ext:ssh - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project delegate_to: localhost @@ -80,7 +97,7 @@ git log --pretty=oneline -1 args: chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, diff --git a/roles/test-prepare-workspace-git/README.rst b/roles/test-prepare-workspace-git/README.rst index 1cc5ce270..4faf51919 100644 --- a/roles/test-prepare-workspace-git/README.rst +++ b/roles/test-prepare-workspace-git/README.rst @@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: prepare_workspace_sync_required_projects_only + :type: bool + :default: False + + A flag which if set to true, filters the to be synchronized project + list to only use projects which are required by the job. + .. zuul:rolevar:: mirror_workspace_quiet :default: false diff --git a/roles/test-prepare-workspace-git/defaults/main.yaml b/roles/test-prepare-workspace-git/defaults/main.yaml index f6e5da9bb..fbd1b27ef 100644 --- a/roles/test-prepare-workspace-git/defaults/main.yaml +++ b/roles/test-prepare-workspace-git/defaults/main.yaml @@ -1,3 +1,4 @@ cached_repos_root: /opt/git mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" +prepare_workspace_sync_required_projects_only: false diff --git a/roles/test-prepare-workspace-git/tasks/main.yaml b/roles/test-prepare-workspace-git/tasks/main.yaml index d57ef11f1..5e68daf99 100644 --- a/roles/test-prepare-workspace-git/tasks/main.yaml +++ b/roles/test-prepare-workspace-git/tasks/main.yaml @@ -1,3 +1,20 @@ +- name: Filter zuul projects if sync-only-required-projects flag is set + set_fact: + _zuul_projects: > + {{ _zuul_projects | default({}) | + combine({ zj_project.key : zj_project.value }) }} + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + when: + - prepare_workspace_sync_required_projects_only + - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required + +- name: Don't filter zuul projects if flag is false + set_fact: + _zuul_projects: "{{ zuul.projects }}" + when: not prepare_workspace_sync_required_projects_only + # Do all the steps in a single shell script. This reduces the number of times # ansible must loop over the list of projects which reduces the amount of # task startup time we incur. @@ -17,7 +34,7 @@ git remote add origin file:///dev/null args: creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" - with_items: "{{ zuul.projects.values() | list }}" + with_items: "{{ _zuul_projects.values() }}" loop_control: loop_var: zj_project # We're using git in a shell script because it is faster and the module @@ -31,7 +48,7 @@ value: ignore scope: local repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project @@ -46,7 +63,7 @@ chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" environment: GIT_ALLOW_PROTOCOL: ext:ssh - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project delegate_to: localhost @@ -80,7 +97,7 @@ git log --pretty=oneline -1 args: chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, diff --git a/test-playbooks/base-roles/base.yaml b/test-playbooks/base-roles/base.yaml index d760f4af6..1c5e93521 100644 --- a/test-playbooks/base-roles/base.yaml +++ b/test-playbooks/base-roles/base.yaml @@ -6,6 +6,7 @@ # Note: set-zuul-log-path-fact is tested by emit-job-header.yaml - import_playbook: emit-job-header.yaml - import_playbook: ensure-output-dirs.yaml +- import_playbook: prepare-workspace-git-required-projects-only.yaml - import_playbook: prepare-workspace-git.yaml - import_playbook: configure-mirrors.yaml - import_playbook: fetch-zuul-cloner.yaml diff --git a/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml new file mode 100644 index 000000000..196364c9d --- /dev/null +++ b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml @@ -0,0 +1,6 @@ +- name: Test the prepare-workspace-git role + hosts: all + roles: + - role: prepare-workspace-git + vars: + prepare_workspace_sync_required_projects_only: true diff --git a/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml new file mode 100644 index 000000000..8967ba2f7 --- /dev/null +++ b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml @@ -0,0 +1,72 @@ +- name: Prepare to test the prepare-workspace-git role with sync required only + hosts: all + tasks: + - name: Delete remote source directory to start with clean state + file: + state: absent + path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + +# We need to override the zuul.projects variable, and that is not +# possible in a Zuul job. So we use a nested Ansible to perform this +# test. +- name: Test the prepare-workspace-git role with sync required only + hosts: localhost + vars: + # Mutate the zuul vars supplied to this test job to simulate a + # repo being included as non-required (i.e., a depends-on). + zuul_mod: + projects: + opendev.org/zuul/project-config: + required: false + tasks: + - name: Create nested zuul vars + set_fact: + nested_zuul: + zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}" + - name: Write nested zuul vars + copy: + content: '{{ nested_zuul | to_nice_yaml(indent=2) }}' + dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml" + - name: Run nested Ansible + command: >- + {{ ansible_playbook_python | dirname}}/ansible-playbook + -vvv + -e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml + -e zuul_execution_phase=nested + -e zuul_execution_phase_index=0 + -e zuul_execution_canonical_name_and_path=opendev.org/zuul/zuul-jobs/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml + -e zuul_execution_trusted=False + -e zuul_execution_branch={{zuul_execution_branch}} + {{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml + environment: + ANSIBLE_ROLES_PATH: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles" + +- name: Verify the prepare-workspace-git role with sync required only + hosts: all + tasks: + # opendev/base-jobs is in 'required-projects'. + # Also check that the project being tested is being prepared. + # We're checking them explicitly rather than with_items on zuul.projects + # in case there is a regression which would take an item out. + - name: Check that opendev/base-jobs was prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs" + register: base_jobs + + - name: Check that zuul/project-config was not prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config" + register: project_config + + - name: Check this project was prepared + stat: + path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" + register: self_config + + - name: Validate that required projects have been prepared + assert: + that: + - base_jobs.stat.exists + - not project_config.stat.exists + - self_config.stat.exists diff --git a/test-playbooks/base-roles/prepare-workspace-git.yaml b/test-playbooks/base-roles/prepare-workspace-git.yaml index 6a83cfbdc..24a8a16d6 100644 --- a/test-playbooks/base-roles/prepare-workspace-git.yaml +++ b/test-playbooks/base-roles/prepare-workspace-git.yaml @@ -1,5 +1,12 @@ - name: Test the prepare-workspace-git role hosts: all + pre_tasks: + - name: Delete remote source directory to start with clean state + file: + state: absent + path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + roles: - role: prepare-workspace-git post_tasks: @@ -12,6 +19,11 @@ path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs" register: base_jobs + - name: Check that zuul/project-config was prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config" + register: project_config + - name: Check this project was prepared stat: path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" @@ -21,4 +33,5 @@ assert: that: - base_jobs.stat.exists + - project_config.stat.exists - self_config.stat.exists diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index b8857a9f7..7e9821bfd 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -116,10 +116,11 @@ tags: all-platforms abstract: true run: test-playbooks/base-roles/base.yaml - # Testing of fetch-zuul-cloner and use-cached-repos need this repo - # in required-projects + # Testing of fetch-zuul-cloner and prepare-workspace-git need + # these repos in required-projects required-projects: - opendev/base-jobs + - zuul/project-config files: - ^roles/configure-mirrors/.* - ^roles/emit-job-header/.*