Make prepare-workspace-git fail faster.

In case of aws spot instances, the node can drop the connection anytime.
Ansible loops do not fail when the first item fails, instead they execute all items before reporting the failure.
This leads to a big overhead, if we sync repositories, when the node has dropped the connection beforehand.
Therefore i used include_tasks, which fails for the first unsuccessful item.

Change-Id: Id6079a2cda10a825384d52d47750d9c05d323e00
This commit is contained in:
Lukas Kranz 2024-02-29 09:44:28 +01:00
parent b81ef9e329
commit 5118c28aac
2 changed files with 25 additions and 23 deletions

View File

@ -35,32 +35,11 @@
loop_control:
loop_var: zj_project
- name: Synchronize src repos to workspace directory
command: |-
{% if ansible_connection == "kubectl" %}
git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
{% else %}
git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}
{% endif %}
args:
chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
environment:
GIT_ALLOW_PROTOCOL: ext:ssh
- name: Include tasks to synchronize src repos to workspace directory
include_tasks: sync-project.yaml
with_dict: "{{ zuul.projects }}"
loop_control:
loop_var: zj_project
delegate_to: localhost
# We occasionally see git pushes in the middle of this loop fail then
# subsequent pushes for other repos succeed. The entire loop ends up
# failing because one of the pushes failed. Mitigate this by retrying
# on failure.
register: git_push
until: git_push is success
retries: 3
# ANSIBLE0006: Skip linting since it triggers on the "git" command,
# but push is not supported by ansible git module.
tags:
- skip_ansible_lint
# Do this as a multi-line shell so that we can do the loop once
- name: Update remote repository state correctly

View File

@ -0,0 +1,23 @@
- name: "Synchronize {{ zj_project.value.canonical_name }}"
command: |-
{% if ansible_connection == "kubectl" %}
git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
{% else %}
git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}
{% endif %}
args:
chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
environment:
GIT_ALLOW_PROTOCOL: ext:ssh
delegate_to: localhost
# We occasionally see git pushes in the middle of this loop fail then
# subsequent pushes for other repos succeed. The entire loop ends up
# failing because one of the pushes failed. Mitigate this by retrying
# on failure.
register: git_push
until: git_push is success
retries: 3
# ANSIBLE0006: Skip linting since it triggers on the "git" command,
# but push is not supported by ansible git module.
tags:
- skip_ansible_lint