Merge "Add other nodes to buildx builder"

roles/build-container-image/tasks/main.yaml

@@ -87,6 +87,7 @@
       loop: "{{ container_images }}"
       loop_control:
         loop_var: zj_image
+      when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Multiarch podman block
   when:

roles/build-container-image/tasks/setup-buildx.yaml

@@ -3,13 +3,26 @@
   when: ansible_architecture == 'x86_64'
 
 - name: Create builder
-  command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
+  command: "docker buildx create --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
+  when: inventory_hostname == ansible_play_hosts[0]
+
+- name: Add host key to known_hosts
+  shell: "ssh-keyscan -H {{ ansible_host }} >> ~/.ssh/known_hosts"
+  when: inventory_hostname != ansible_play_hosts[0]
+  delegate_to: "{{ ansible_play_hosts[0] }}"
+
+- name: Append builders from other nodes
+  command: "docker buildx create --append --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %} ssh://{{ ansible_user }}@{{ ansible_host }}"
+  when: inventory_hostname != ansible_play_hosts[0]
+  delegate_to: "{{ ansible_play_hosts[0] }}"
 
 - name: Use builder
   command: docker buildx use mybuilder
+  when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Bootstrap builder
   command: docker buildx inspect --bootstrap
+  when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Make tempfile for registry TLS certificate
   tempfile:
@@ -25,11 +38,11 @@
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Copy buildset registry TLS cert into worker container
-  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/usr/local/share/ca-certificates"
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Update CA certs in worker container
-  command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} update-ca-certificates"
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Remove TLS cert tempfile
@@ -44,7 +57,7 @@
   register: etc_hosts_tmp
 
 - name: Copy /etc/hosts for editing
-  command: 'docker cp buildx_buildkit_mybuilder0:/etc/hosts {{ etc_hosts_tmp.path }}'
+  command: "docker cp buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/hosts {{ etc_hosts_tmp.path }}"
 
 # Docker buildx has its own /etc/hosts in the builder image.
 - name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
@@ -58,16 +71,16 @@
   when: buildset_registry is defined and buildset_registry.host | ipaddr
 
 - name: Unmount the /etc/hosts mount
-  command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} umount /etc/hosts"
 
 # NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts
 # in the previous step, when we try to copy the file back directly, we get:
 #   unlinkat /etc/hosts: device or resource busy
 - name: Copy modified hosts file back in
-  command: 'docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_mybuilder0:/etc/new-hosts'
+  command: "docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/new-hosts"
 
 - name: Copy modified hosts file into place
-  command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} cp /etc/new-hosts /etc/hosts"
 
 - name: Remove tempfile for /etc/hosts
   file:

roles/build-docker-image/tasks/main.yaml

@@ -85,6 +85,7 @@
       loop: "{{ docker_images }}"
       loop_control:
         loop_var: zj_image
+      when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Cleanup sibling source directory
   file:

roles/build-docker-image/tasks/setup-buildx.yaml

@@ -5,19 +5,34 @@
   when: ansible_architecture == 'x86_64'
 
 - name: Create builder
-  command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
+  command: "docker buildx create --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
+  when: inventory_hostname == ansible_play_hosts[0]
+
+- name: Add host key to known_hosts
+  shell: "ssh-keyscan -H {{ ansible_host }} >> ~/.ssh/known_hosts"
+  when: inventory_hostname != ansible_play_hosts[0]
+  delegate_to: "{{ ansible_play_hosts[0] }}"
+
+- name: Append builders from other nodes
+  command: "docker buildx create --append --name mybuilder --node {{ inventory_hostname | replace('-', '_') }} --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %} ssh://{{ ansible_user }}@{{ ansible_host }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+  when: inventory_hostname != ansible_play_hosts[0]
+  delegate_to: "{{ ansible_play_hosts[0] }}"
 
 - name: Use builder
   command: docker buildx use mybuilder
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
+  when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Bootstrap builder
   command: docker buildx inspect --bootstrap
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
+  when: inventory_hostname == ansible_play_hosts[0]
 
 - name: Make tempfile for registry TLS certificate
   tempfile:
@@ -33,11 +48,11 @@
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Copy buildset registry TLS cert into worker container
-  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/usr/local/share/ca-certificates"
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Update CA certs in worker container
-  command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} update-ca-certificates"
   when: buildset_registry is defined and buildset_registry.cert
 
 - name: Remove TLS cert tempfile
@@ -52,7 +67,7 @@
   register: etc_hosts_tmp
 
 - name: Copy /etc/hosts for editing
-  command: 'docker cp buildx_buildkit_mybuilder0:/etc/hosts {{ etc_hosts_tmp.path }}'
+  command: "docker cp buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/hosts {{ etc_hosts_tmp.path }}"
 
 # Docker buildx has its own /etc/hosts in the builder image.
 - name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
@@ -66,16 +81,16 @@
   when: buildset_registry is defined and buildset_registry.host | ipaddr
 
 - name: Unmount the /etc/hosts mount
-  command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} umount /etc/hosts"
 
 # NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts
 # in the previous step, when we try to copy the file back directly, we get:
 #   unlinkat /etc/hosts: device or resource busy
 - name: Copy modified hosts file back in
-  command: 'docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_mybuilder0:/etc/new-hosts'
+  command: "docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_{{ inventory_hostname | replace('-', '_') }}:/etc/new-hosts"
 
 - name: Copy modified hosts file into place
-  command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts
+  command: "docker exec buildx_buildkit_{{ inventory_hostname | replace('-', '_') }} cp /etc/new-hosts /etc/hosts"
 
 - name: Remove tempfile for /etc/hosts
   file:

roles/upload-container-image/tasks/main.yaml

@@ -27,4 +27,6 @@
       loop_control:
         loop_var: zj_image
       include_tasks: push.yaml
-  when: not upload_container_image_promote|default(true) or promote_container_image_method|default('tag') == 'tag'
+  when:
+    - inventory_hostname == ansible_play_hosts[0]
+    - not upload_container_image_promote|default(true) or promote_container_image_method|default('tag') == 'tag'

zuul-tests.d/container-roles-jobs.yaml

@@ -124,6 +124,22 @@
       container_command: docker
       multiarch: true
 
+- job:
+    name: zuul-jobs-test-build-container-image-docker-release-multiarch-multinode
+    parent: zuul-jobs-test-build-container-image-docker-release-multiarch
+    description: |
+      Test building a multi-arch container image with docker in a release pipeline
+      across two nodes (native multiple architecture system).
+
+      NOTE(mnaser): Since OpenDev doesn't natively support a provider that has
+      both x86_64 and arm64 nodes, we're using the same architecture for both.
+    nodeset:
+      nodes:
+        - name: amd64
+          label: ubuntu-jammy
+        - name: arm64
+          label: ubuntu-jammy
+
 - job:
     name: zuul-jobs-test-build-container-image-podman-release
     parent: zuul-jobs-test-build-container-image-base
@@ -580,6 +596,7 @@
         - zuul-jobs-test-ensure-docker-ubuntu-noble
         - zuul-jobs-test-build-container-image-docker-release
         - zuul-jobs-test-build-container-image-docker-release-multiarch
+        - zuul-jobs-test-build-container-image-docker-release-multiarch-multinode
         - zuul-jobs-test-build-container-image-podman-release
         - zuul-jobs-test-build-container-image-docker-promote
         - zuul-jobs-test-build-container-image-docker-promote-multiarch