From bdae8c94333cb8964a79a0c3a4966ca59bc74fca Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Mon, 30 Sep 2024 11:34:48 -0400 Subject: [PATCH] Add other nodes to buildx builder If you need to run native arm64 builds, you can take advantage of this change which will rely on the remote builders in order to build things natively giving a significant speed up in container build time. Change-Id: I962bb2357a2c458d5e72b334b4fe36b55b034864 --- roles/build-container-image/tasks/main.yaml | 1 + .../tasks/setup-buildx.yaml | 27 ++++++++++++----- roles/build-docker-image/tasks/main.yaml | 1 + .../tasks/setup-buildx.yaml | 29 ++++++++++++++----- roles/upload-container-image/tasks/main.yaml | 4 ++- zuul-tests.d/container-roles-jobs.yaml | 17 +++++++++++ 6 files changed, 64 insertions(+), 15 deletions(-) diff --git a/roles/build-container-image/tasks/main.yaml b/roles/build-container-image/tasks/main.yaml index 84bb82c79..8c821d6ea 100644 --- a/roles/build-container-image/tasks/main.yaml +++ b/roles/build-container-image/tasks/main.yaml @@ -87,6 +87,7 @@ loop: "{{ container_images }}" loop_control: loop_var: zj_image + when: inventory_hostname == ansible_play_hosts[0] - name: Multiarch podman block when: diff --git a/roles/build-container-image/tasks/setup-buildx.yaml b/roles/build-container-image/tasks/setup-buildx.yaml index 81f100c48..546a9acf7 100644 --- a/roles/build-container-image/tasks/setup-buildx.yaml +++ b/roles/build-container-image/tasks/setup-buildx.yaml @@ -3,13 +3,26 @@ when: ansible_architecture == 'x86_64' - name: Create builder - command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}" + command: "docker buildx create --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}" + when: inventory_hostname == ansible_play_hosts[0] + +- name: Add host key to known_hosts + shell: "ssh-keyscan -H {{ ansible_host }} >> ~/.ssh/known_hosts" + when: inventory_hostname != ansible_play_hosts[0] + delegate_to: "{{ ansible_play_hosts[0] }}" + +- name: Append builders from other nodes + command: "docker buildx create --append --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %} ssh://{{ ansible_user }}@{{ ansible_host }}" + when: inventory_hostname != ansible_play_hosts[0] + delegate_to: "{{ ansible_play_hosts[0] }}" - name: Use builder command: docker buildx use mybuilder + when: inventory_hostname == ansible_play_hosts[0] - name: Bootstrap builder command: docker buildx inspect --bootstrap + when: inventory_hostname == ansible_play_hosts[0] - name: Make tempfile for registry TLS certificate tempfile: @@ -25,11 +38,11 @@ when: buildset_registry is defined and buildset_registry.cert - name: Copy buildset registry TLS cert into worker container - command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates" + command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/usr/local/share/ca-certificates" when: buildset_registry is defined and buildset_registry.cert - name: Update CA certs in worker container - command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} update-ca-certificates" when: buildset_registry is defined and buildset_registry.cert - name: Remove TLS cert tempfile @@ -44,7 +57,7 @@ register: etc_hosts_tmp - name: Copy /etc/hosts for editing - command: 'docker cp buildx_buildkit_mybuilder0:/etc/hosts {{ etc_hosts_tmp.path }}' + command: "docker cp buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/hosts {{ etc_hosts_tmp.path }}" # Docker buildx has its own /etc/hosts in the builder image. - name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses @@ -58,16 +71,16 @@ when: buildset_registry is defined and buildset_registry.host | ipaddr - name: Unmount the /etc/hosts mount - command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} umount /etc/hosts" # NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts # in the previous step, when we try to copy the file back directly, we get: # unlinkat /etc/hosts: device or resource busy - name: Copy modified hosts file back in - command: 'docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_mybuilder0:/etc/new-hosts' + command: "docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/new-hosts" - name: Copy modified hosts file into place - command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} cp /etc/new-hosts /etc/hosts" - name: Remove tempfile for /etc/hosts file: diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml index b51801cc0..684d46cbc 100644 --- a/roles/build-docker-image/tasks/main.yaml +++ b/roles/build-docker-image/tasks/main.yaml @@ -85,6 +85,7 @@ loop: "{{ docker_images }}" loop_control: loop_var: zj_image + when: inventory_hostname == ansible_play_hosts[0] - name: Cleanup sibling source directory file: diff --git a/roles/build-docker-image/tasks/setup-buildx.yaml b/roles/build-docker-image/tasks/setup-buildx.yaml index f42f93097..f2fefbe14 100644 --- a/roles/build-docker-image/tasks/setup-buildx.yaml +++ b/roles/build-docker-image/tasks/setup-buildx.yaml @@ -5,19 +5,34 @@ when: ansible_architecture == 'x86_64' - name: Create builder - command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}" + command: "docker buildx create --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}" environment: DOCKER_CLI_EXPERIMENTAL: enabled + when: inventory_hostname == ansible_play_hosts[0] + +- name: Add host key to known_hosts + shell: "ssh-keyscan -H {{ ansible_host }} >> ~/.ssh/known_hosts" + when: inventory_hostname != ansible_play_hosts[0] + delegate_to: "{{ ansible_play_hosts[0] }}" + +- name: Append builders from other nodes + command: "docker buildx create --append --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %} ssh://{{ ansible_user }}@{{ ansible_host }}" + environment: + DOCKER_CLI_EXPERIMENTAL: enabled + when: inventory_hostname != ansible_play_hosts[0] + delegate_to: "{{ ansible_play_hosts[0] }}" - name: Use builder command: docker buildx use mybuilder environment: DOCKER_CLI_EXPERIMENTAL: enabled + when: inventory_hostname == ansible_play_hosts[0] - name: Bootstrap builder command: docker buildx inspect --bootstrap environment: DOCKER_CLI_EXPERIMENTAL: enabled + when: inventory_hostname == ansible_play_hosts[0] - name: Make tempfile for registry TLS certificate tempfile: @@ -33,11 +48,11 @@ when: buildset_registry is defined and buildset_registry.cert - name: Copy buildset registry TLS cert into worker container - command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates" + command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/usr/local/share/ca-certificates" when: buildset_registry is defined and buildset_registry.cert - name: Update CA certs in worker container - command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} update-ca-certificates" when: buildset_registry is defined and buildset_registry.cert - name: Remove TLS cert tempfile @@ -52,7 +67,7 @@ register: etc_hosts_tmp - name: Copy /etc/hosts for editing - command: 'docker cp buildx_buildkit_mybuilder0:/etc/hosts {{ etc_hosts_tmp.path }}' + command: "docker cp buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/hosts {{ etc_hosts_tmp.path }}" # Docker buildx has its own /etc/hosts in the builder image. - name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses @@ -66,16 +81,16 @@ when: buildset_registry is defined and buildset_registry.host | ipaddr - name: Unmount the /etc/hosts mount - command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} umount /etc/hosts" # NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts # in the previous step, when we try to copy the file back directly, we get: # unlinkat /etc/hosts: device or resource busy - name: Copy modified hosts file back in - command: 'docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_mybuilder0:/etc/new-hosts' + command: "docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/new-hosts" - name: Copy modified hosts file into place - command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts + command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} cp /etc/new-hosts /etc/hosts" - name: Remove tempfile for /etc/hosts file: diff --git a/roles/upload-container-image/tasks/main.yaml b/roles/upload-container-image/tasks/main.yaml index 9239a436f..ec86ee992 100644 --- a/roles/upload-container-image/tasks/main.yaml +++ b/roles/upload-container-image/tasks/main.yaml @@ -27,4 +27,6 @@ loop_control: loop_var: zj_image include_tasks: push.yaml - when: not upload_container_image_promote|default(true) or promote_container_image_method|default('tag') == 'tag' + when: + - inventory_hostname == ansible_play_hosts[0] + - not upload_container_image_promote|default(true) or promote_container_image_method|default('tag') == 'tag' diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index bc3dffffc..7cb718286 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -124,6 +124,22 @@ container_command: docker multiarch: true +- job: + name: zuul-jobs-test-build-container-image-docker-release-multiarch-multinode + parent: zuul-jobs-test-build-container-image-docker-release-multiarch + description: | + Test building a multi-arch container image with docker in a release pipeline + across two nodes (native multiple architecture system). + + NOTE(mnaser): Since OpenDev doesn't natively support a provider that has + both x86_64 and arm64 nodes, we're using the same architecture for both. + nodeset: + nodes: + - name: amd64 + label: ubuntu-jammy + - name: arm64 + label: ubuntu-jammy + - job: name: zuul-jobs-test-build-container-image-podman-release parent: zuul-jobs-test-build-container-image-base @@ -580,6 +596,7 @@ - zuul-jobs-test-ensure-docker-ubuntu-noble - zuul-jobs-test-build-container-image-docker-release - zuul-jobs-test-build-container-image-docker-release-multiarch + - zuul-jobs-test-build-container-image-docker-release-multiarch-multinode - zuul-jobs-test-build-container-image-podman-release - zuul-jobs-test-build-container-image-docker-promote - zuul-jobs-test-build-container-image-docker-promote-multiarch