diff --git a/doc/source/deprecated-roles.rst b/doc/source/deprecated-roles.rst index 593ff252e..fa629ce69 100644 --- a/doc/source/deprecated-roles.rst +++ b/doc/source/deprecated-roles.rst @@ -2,6 +2,7 @@ Deprecated and Test Roles ========================= .. zuul:autorole:: fetch-zuul-cloner +.. zuul:autorole:: mirror-workspace-git-repos .. zuul:autorole:: test-mirror-workspace-git-repos .. zuul:autorole:: test-upload-logs-swift .. zuul:autorole:: test-prepare-workspace-git diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst index cbc9d9255..cb278a343 100644 --- a/doc/source/general-roles.rst +++ b/doc/source/general-roles.rst @@ -28,7 +28,6 @@ General Purpose Roles .. zuul:autorole:: intercept-job .. zuul:autorole:: log-inventory .. zuul:autorole:: markdownlint -.. zuul:autorole:: mirror-workspace-git-repos .. zuul:autorole:: multi-node-bridge .. zuul:autorole:: multi-node-firewall .. zuul:autorole:: multi-node-hosts-file diff --git a/roles/mirror-workspace-git-repos/README.rst b/roles/mirror-workspace-git-repos/README.rst index 118856bc0..452fdb6f8 100644 --- a/roles/mirror-workspace-git-repos/README.rst +++ b/roles/mirror-workspace-git-repos/README.rst @@ -1,5 +1,8 @@ Mirror the local git repos to remote nodes +.. warning:: This role is deprecated. Use + :zuul:role:`prepare-workspace-git` instead. + This role uses git operations (unlike :zuul:role:`prepare-workspace` which uses rsync) to mirror the local prepared git repos to the remote nodes. This may be useful if the remote node already has a copy of diff --git a/roles/prepare-workspace-git/README.rst b/roles/prepare-workspace-git/README.rst index 4e23c5761..1cc5ce270 100644 --- a/roles/prepare-workspace-git/README.rst +++ b/roles/prepare-workspace-git/README.rst @@ -16,6 +16,12 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: mirror_workspace_quiet + :default: false + + If `true` git operations will be silenced and won't print every + changed reference. + .. zuul:rolevar:: zuul_workspace_root :default: "{{ ansible_user_dir }}" diff --git a/roles/prepare-workspace-git/defaults/main.yaml b/roles/prepare-workspace-git/defaults/main.yaml index 032db3c2a..f6e5da9bb 100644 --- a/roles/prepare-workspace-git/defaults/main.yaml +++ b/roles/prepare-workspace-git/defaults/main.yaml @@ -1,2 +1,3 @@ cached_repos_root: /opt/git +mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" diff --git a/roles/prepare-workspace-git/tasks/main.yaml b/roles/prepare-workspace-git/tasks/main.yaml index d1ff3717d..d57ef11f1 100644 --- a/roles/prepare-workspace-git/tasks/main.yaml +++ b/roles/prepare-workspace-git/tasks/main.yaml @@ -25,8 +25,65 @@ tags: - skip_ansible_lint -# TODO(tobiash): we might want to deprecate the role mirror-workspace-git-repos -# and move it here. -- name: Synchronize repos - import_role: - name: mirror-workspace-git-repos +- name: Allow pushing to non-bare repo + git_config: + name: receive.denyCurrentBranch + value: ignore + scope: local + repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + +- name: Synchronize src repos to workspace directory + command: |- + {% if ansible_connection == "kubectl" %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + {% else %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }} + {% endif %} + args: + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" + environment: + GIT_ALLOW_PROTOCOL: ext:ssh + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + delegate_to: localhost + # We occasionally see git pushes in the middle of this loop fail then + # subsequent pushes for other repos succeed. The entire loop ends up + # failing because one of the pushes failed. Mitigate this by retrying + # on failure. + register: git_push + until: git_push is success + retries: 3 + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but push is not supported by ansible git module. + tags: + - skip_ansible_lint + +# Do this as a multi-line shell so that we can do the loop once +- name: Update remote repository state correctly + shell: | + set -eu + + # Reset is needed because we pushed to a non-bare repo + git reset --hard + # Clean is needed because we pushed to a non-bare repo + git clean -xdf + # Undo the config setting we did above + git config --local --unset receive.denyCurrentBranch + # checkout the branch matching the branch set up by the executor + git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ zj_project.value.checkout }} + # put out a status line with the current HEAD + echo "{{ zj_project.value.canonical_name }} checked out to:" + git log --pretty=oneline -1 + args: + chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but we prefer the shell above + tags: + - skip_ansible_lint diff --git a/roles/test-prepare-workspace-git/README.rst b/roles/test-prepare-workspace-git/README.rst index 4e23c5761..1cc5ce270 100644 --- a/roles/test-prepare-workspace-git/README.rst +++ b/roles/test-prepare-workspace-git/README.rst @@ -16,6 +16,12 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: mirror_workspace_quiet + :default: false + + If `true` git operations will be silenced and won't print every + changed reference. + .. zuul:rolevar:: zuul_workspace_root :default: "{{ ansible_user_dir }}" diff --git a/roles/test-prepare-workspace-git/defaults/main.yaml b/roles/test-prepare-workspace-git/defaults/main.yaml index 032db3c2a..f6e5da9bb 100644 --- a/roles/test-prepare-workspace-git/defaults/main.yaml +++ b/roles/test-prepare-workspace-git/defaults/main.yaml @@ -1,2 +1,3 @@ cached_repos_root: /opt/git +mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" diff --git a/roles/test-prepare-workspace-git/tasks/main.yaml b/roles/test-prepare-workspace-git/tasks/main.yaml index 839dbdb04..d57ef11f1 100644 --- a/roles/test-prepare-workspace-git/tasks/main.yaml +++ b/roles/test-prepare-workspace-git/tasks/main.yaml @@ -25,8 +25,65 @@ tags: - skip_ansible_lint -# TODO(tobiash): we might want to deprecate the role mirror-workspace-git-repos -# and move it here. -- name: Synchronize repos - import_role: - name: test-mirror-workspace-git-repos +- name: Allow pushing to non-bare repo + git_config: + name: receive.denyCurrentBranch + value: ignore + scope: local + repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + +- name: Synchronize src repos to workspace directory + command: |- + {% if ansible_connection == "kubectl" %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + {% else %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }} + {% endif %} + args: + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" + environment: + GIT_ALLOW_PROTOCOL: ext:ssh + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + delegate_to: localhost + # We occasionally see git pushes in the middle of this loop fail then + # subsequent pushes for other repos succeed. The entire loop ends up + # failing because one of the pushes failed. Mitigate this by retrying + # on failure. + register: git_push + until: git_push is success + retries: 3 + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but push is not supported by ansible git module. + tags: + - skip_ansible_lint + +# Do this as a multi-line shell so that we can do the loop once +- name: Update remote repository state correctly + shell: | + set -eu + + # Reset is needed because we pushed to a non-bare repo + git reset --hard + # Clean is needed because we pushed to a non-bare repo + git clean -xdf + # Undo the config setting we did above + git config --local --unset receive.denyCurrentBranch + # checkout the branch matching the branch set up by the executor + git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ zj_project.value.checkout }} + # put out a status line with the current HEAD + echo "{{ zj_project.value.canonical_name }} checked out to:" + git log --pretty=oneline -1 + args: + chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but we prefer the shell above + tags: + - skip_ansible_lint