Add docker buildx multiarch support to container roleset

This adds support for multiarch container image builds when using docker
as the container command to the container roleset.

Change-Id: I48bf2e34c258e54baf013d3c04c6d4baaacde04b

roles/build-container-image/tasks/build.yaml

@@ -1,33 +1,5 @@
-- name: Check sibling directory
-  stat:
-    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
-  register: _dot_zuul_siblings
-
-# This should have been cleaned up; multiple builds may specify
-# different siblings to include so we need to start fresh.
-- name: Check for clean build
-  assert:
-    that: not _dot_zuul_siblings.stat.exists
-
-- name: Create sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
-    state: directory
-    mode: 0755
-  when: zj_image.siblings is defined
-
-- name: Copy sibling source directories
-  command:
-    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
-    chdir: '~/src'
-  loop: '{{ zj_image.siblings }}'
-  loop_control:
-    loop_var: zj_sibling
-  when: zj_image.siblings is defined
-
-- name: Set container filename arg
-  set_fact:
-    containerfile: "{{ zj_image.container_filename | default(container_filename) | default('') }}"
+- name: Set up siblings
+  include_tasks: siblings.yaml
 
 - name: Build a container image
   vars:
@@ -55,6 +27,4 @@
   environment: "{{ container_build_extra_env }}"
 
 - name: Cleanup sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
-    state: absent
+  include_tasks: clean-siblings.yaml

roles/build-container-image/tasks/buildx.yaml

@@ -0,0 +1,96 @@
+- name: Validate zj_image.repository is full "url"
+  when:
+    - "'/' not in zj_image.repository"
+  fail:
+    msg: "{{ zj_image.repository }} must be a full container image url including registry location"
+
+- name: Parse out repo path from full "url"
+  set_fact:
+    _repopath: "{{ (zj_image.repository | split('/', 1)).1 }}"
+
+- name: Set up siblings
+  include_tasks: siblings.yaml
+
+# The command below always tags the images for the temp_registry (so
+# they can be pulled back onto the host image cache), and also tags
+# them for the buildset registry if one is present.
+- name: Set base docker build command
+  set_fact:
+    docker_buildx_command: >-
+      docker buildx build {{ zj_image.path | default('.') }}
+      {% if containerfile %}-f {{ containerfile }}{% endif %}
+      {% if zj_image.target | default(false) -%}
+        --target {{ zj_image.target }}
+      {% endif -%}
+      {% for build_arg in zj_image.build_args | default([]) -%}
+        --build-arg {{ build_arg }}
+      {% endfor -%}
+      {% if zj_image.siblings | default(false) -%}
+        --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
+      {% endif -%}
+      {% for tag in zj_image.tags | default(['latest']) -%}
+        --tag {{ temp_registry.host }}:{{ temp_registry.port }}/{{ _repopath }}:{{ tag }}
+        {% if buildset_registry | default(false) -%}
+          --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ _repopath }}:{{ tag }}
+        {% endif -%}
+      {% endfor -%}
+      {% for label in zj_image.labels | default([]) -%}
+        --label "{{ label }}"
+      {% endfor %}
+      {% if zuul.change | default(false) -%}
+        --label "org.zuul-ci.change={{ zuul.change }}"
+      {% endif -%}
+      --label "org.zuul-ci.change_url={{ zuul.change_url }}"
+
+- name: Build images for all arches
+  command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }}"
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Push arch-specific layers one at a time
+  command: "{{ docker_buildx_command }} --platform={{ zj_arch }} --push"
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+  loop: '{{ zj_image.arch }}'
+  loop_control:
+    loop_var: zj_arch
+
+- name: Push final image manifest
+  command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }} --push"
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Pull images from temporary registry
+  command: >-
+    docker pull {{ temp_registry.host }}:{{ temp_registry.port }}/{{ _repopath }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+
+- name: Locally tag for changes so push works later
+  command: >-
+    docker tag
+    {{ temp_registry.host }}:{{ temp_registry.port }}/{{ _repopath }}:{{ zj_image_tag }}
+    {{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+  when: zuul.change | default(false)
+
+- name: Locally tag for changes so push works later
+  command: >-
+    docker tag
+    {{ temp_registry.host }}:{{ temp_registry.port }}/{{ _repopath }}:{{ zj_image_tag }}
+    {{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+
+- name: Cleanup sibling source directory
+  include_tasks: clean-siblings.yaml

roles/build-container-image/tasks/clean-siblings.yaml

@@ -0,0 +1,4 @@
+- name: Cleanup sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    state: absent

roles/build-container-image/tasks/main.yaml

@@ -15,12 +15,6 @@
     - "'buildset_registry' in (lookup('file', zuul.executor.result_data_file) | from_json).get('secret_data')"
   no_log: true
 
-- name: Build container images
-  include_tasks: build.yaml
-  loop: "{{ container_images }}"
-  loop_control:
-    loop_var: zj_image
-
 # Docker, and therefore skopeo and podman, don't understand docker
 # push [1234:5678::]:5000/image/path:tag so we set up /etc/hosts with
 # a registry alias name to support ipv6 and 4.
@@ -44,10 +38,65 @@
     buildset_registry_alias: "{{ buildset_registry.host }}"
   when: buildset_registry is defined and not ( buildset_registry.host | ipaddr )
 
-# Push each image.
-- name: Push image to buildset registry
-  when: buildset_registry is defined
-  include_tasks: push.yaml
-  loop: "{{ container_images }}"
-  loop_control:
-    loop_var: zj_image
+- name: Set container filename arg
+  set_fact:
+    containerfile: "{{ zj_image.container_filename | default(container_filename) | default('') }}"
+
+- name: Determine if we are building multiarch or not
+  set_fact:
+    _multiarch: "{{ container_images | selectattr('arch', 'defined') | list }}"
+
+- name: Normal build block
+  when: not _multiarch
+  block:
+    - name: Build container images
+      include_tasks: build.yaml
+      loop: "{{ container_images }}"
+      loop_control:
+        loop_var: zj_image
+
+    # Push each image.
+    - name: Push image to buildset registry
+      when: buildset_registry is defined
+      include_tasks: push.yaml
+      loop: "{{ container_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Multiarch docker block
+  when:
+    - _multiarch
+    - container_command == 'docker'
+  vars:
+    temp_registry:
+      host: "127.0.0.1"
+      port: 5100
+      username: zuul
+      password: tempregistry
+  block:
+    - name: Set up a temporary registry for holding buildx-built images
+      import_tasks: ../../../util-tasks/run-docker-registry.yaml
+      vars:
+        registry: "{{ temp_registry }}"
+
+    - name: Log in to temporary registry
+      command: "docker login -u {{ temp_registry.username }} -p {{ temp_registry.password }} {{ temp_registry.host }}:{{ temp_registry.port }}"
+
+    - name: Set up buildx builders
+      include_tasks: setup-buildx.yaml
+
+    # TODO is push here wrong?
+    - name: Build and push each image using buildx.
+      include_tasks: buildx.yaml
+      loop: "{{ container_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Multiarch podman block
+  when:
+    - _multiarch
+    - container_command == 'podman'
+  block:
+    - name: Unimplemented podman multiarch block
+      fail:
+        msg: "Multiarch podman is not yet implemented"

roles/build-container-image/tasks/setup-buildx.yaml

@@ -0,0 +1,82 @@
+- name: Update qemu-static container settings
+  command: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Create builder
+  command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Use builder
+  command: docker buildx use mybuilder
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Bootstrap builder
+  command: docker buildx inspect --bootstrap
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Make tempfile for registry TLS certificate
+  tempfile:
+    state: file
+  register: buildkit_cert_tmp
+
+- name: Write buildset registry TLS certificate
+  become: true
+  copy:
+    content: "{{ buildset_registry.cert }}"
+    dest: "{{ buildkit_cert_tmp.path }}"
+    mode: preserve
+  when: buildset_registry is defined and buildset_registry.cert
+
+- name: Copy buildset registry TLS cert into worker container
+  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+  when: buildset_registry is defined and buildset_registry.cert
+
+- name: Update CA certs in worker container
+  command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+  when: buildset_registry is defined and buildset_registry.cert
+
+- name: Remove TLS cert tempfile
+  file:
+    state: absent
+    path: '{{ buildkit_cert_tmp.path }}'
+  when: buildset_registry is defined and buildset_registry.cert
+
+- name: Make tempfile for /etc/hosts
+  tempfile:
+    state: file
+  register: etc_hosts_tmp
+
+- name: Copy /etc/hosts for editing
+  command: 'docker cp buildx_buildkit_mybuilder0:/etc/hosts {{ etc_hosts_tmp.path }}'
+
+# Docker buildx has its own /etc/hosts in the builder image.
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  become: yes
+  lineinfile:
+    path: '{{ etc_hosts_tmp.path }}'
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
+    insertafter: EOF
+  when: buildset_registry is defined and buildset_registry.host | ipaddr
+
+- name: Unmount the /etc/hosts mount
+  command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts
+
+# NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts
+# in the previous step, when we try to copy the file back directly, we get:
+#   unlinkat /etc/hosts: device or resource busy
+- name: Copy modified hosts file back in
+  command: 'docker cp {{ etc_hosts_tmp.path }} buildx_buildkit_mybuilder0:/etc/new-hosts'
+
+- name: Copy modified hosts file into place
+  command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts
+
+- name: Remove tempfile for /etc/hosts
+  file:
+    state: absent
+    path: '{{ etc_hosts_tmp.path }}'

roles/build-container-image/tasks/siblings.yaml

@@ -0,0 +1,26 @@
+- name: Check sibling directory
+  stat:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+  register: _dot_zuul_siblings
+
+# This should have been cleaned up; multiple builds may specify
+# different siblings to include so we need to start fresh.
+- name: Check for clean build
+  assert:
+    that: not _dot_zuul_siblings.stat.exists
+
+- name: Create sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    state: directory
+    mode: 0755
+  when: zj_image.siblings is defined
+
+- name: Copy sibling source directories
+  command:
+    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    chdir: '~/src'
+  loop: '{{ zj_image.siblings }}'
+  loop_control:
+    loop_var: zj_sibling
+  when: zj_image.siblings is defined

zuul-tests.d/container-roles-jobs.yaml

@@ -145,6 +145,15 @@
     vars:
       container_command: docker
 
+- job:
+    name: zuul-jobs-test-build-container-image-docker-release-multiarch
+    parent: zuul-jobs-test-build-container-image-base
+    description: |
+      Test building a multi-arch container image with docker in a release pipeline.
+    vars:
+      container_command: docker
+      multiarch: true
+
 - job:
     name: zuul-jobs-test-build-container-image-podman-release
     parent: zuul-jobs-test-build-container-image-base
@@ -170,6 +179,15 @@
     vars:
       container_command: docker
 
+- job:
+    name: zuul-jobs-test-build-container-image-docker-promote-multiarch
+    parent: zuul-jobs-test-build-container-image-promote-base
+    description: |
+      Test building a multi-arch container image with docker in a promote pipeline.
+    vars:
+      container_command: docker
+      multiarch: true
+
 - job:
     name: zuul-jobs-test-build-container-image-podman-promote
     parent: zuul-jobs-test-build-container-image-promote-base
@@ -619,8 +637,10 @@
         - zuul-jobs-test-ensure-docker-ubuntu-focal
         - zuul-jobs-test-ensure-docker-ubuntu-jammy
         - zuul-jobs-test-build-container-image-docker-release
+        - zuul-jobs-test-build-container-image-docker-release-multiarch
         - zuul-jobs-test-build-container-image-podman-release
         - zuul-jobs-test-build-container-image-docker-promote
+        - zuul-jobs-test-build-container-image-docker-promote-multiarch
         - zuul-jobs-test-build-container-image-podman-promote
         - zuul-jobs-test-build-docker-image-release
         - zuul-jobs-test-build-docker-image-release-multiarch
@@ -655,8 +675,10 @@
         - zuul-jobs-test-ensure-docker-ubuntu-focal
         - zuul-jobs-test-ensure-docker-ubuntu-jammy
         - zuul-jobs-test-build-container-image-docker-release
+        - zuul-jobs-test-build-container-image-docker-release-multiarch
         - zuul-jobs-test-build-container-image-podman-release
         - zuul-jobs-test-build-container-image-docker-promote
+        - zuul-jobs-test-build-container-image-docker-promote-multiarch
         - zuul-jobs-test-build-container-image-podman-promote
         - zuul-jobs-test-build-docker-image-release
         - zuul-jobs-test-build-docker-image-release-multiarch