From a7be05b1583329d9461ff3cad46467315c9961ea Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 28 Aug 2025 09:24:41 -0700 Subject: [PATCH] Always build docker images with custom buildx builder Since docker version 23 buildx is the default builder. We had assumed that configuring /etc/buildkitd/buildkitd.toml would configure this default builder. When the buildset registry is in use we added it as a mirror for common registries under the theory that this would enable image builds using speculative image builds housed within the buildset registry. Unfortunately, this doesn't seem to be the case and we have to configure a custom buildx builder and supply the config to that builder to get the behavior we want. The good news is that this is already how multiarch image builds work. Update the two build-(container|docker)-image roles to always use the buildx path for docker based image builds. This ensures we configure a custom builder with a working config, ssl certificates, and /etc/hosts values enabling the build process to lookup images within buildset registries. This does add some complication to the build process but it enables some fundamental zuul functionality so seems worthwhile. In the process I made a slight simplification refactor to the build-docker-image role to make its buildx.yaml task list follow the tasks from build-container-image. This isn't strictly necessary but simplifies the multiarch vs single arch build paths. The parent change updated testing to cover this case then marked the scenarios that did not work (single arch docker build + image hosted outside of docker.io) as nonvoting. We update those cases to be voting now that we've fixed these scenarios. Change-Id: I81c93dd22331b9793f87221eb5b46c9140042be5 --- roles/build-container-image/tasks/buildx.yaml | 2 +- roles/build-container-image/tasks/main.yaml | 31 +++++------ .../tasks/setup-buildx.yaml | 6 ++- roles/build-docker-image/common.rst | 3 +- roles/build-docker-image/tasks/build.yaml | 35 ------------- roles/build-docker-image/tasks/buildx.yaml | 21 +------- roles/build-docker-image/tasks/main.yaml | 25 ++------- roles/build-docker-image/tasks/push.yaml | 12 ----- .../tasks/setup-buildx.yaml | 6 ++- zuul-tests.d/container-roles-jobs.yaml | 52 +------------------ 10 files changed, 33 insertions(+), 160 deletions(-) delete mode 100644 roles/build-docker-image/tasks/build.yaml delete mode 100644 roles/build-docker-image/tasks/push.yaml diff --git a/roles/build-container-image/tasks/buildx.yaml b/roles/build-container-image/tasks/buildx.yaml index 17769e52b..93344c919 100644 --- a/roles/build-container-image/tasks/buildx.yaml +++ b/roles/build-container-image/tasks/buildx.yaml @@ -43,7 +43,7 @@ --label "org.zuul-ci.change_url={{ zuul.change_url }}" - name: Build and push images for all arches - command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }} --push" + command: "{{ docker_buildx_command }} {% if _multiarch %}--platform={{ zj_image.arch | join(',') }}{% endif %} --push" args: chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" diff --git a/roles/build-container-image/tasks/main.yaml b/roles/build-container-image/tasks/main.yaml index 8c821d6ea..f7d1458a2 100644 --- a/roles/build-container-image/tasks/main.yaml +++ b/roles/build-container-image/tasks/main.yaml @@ -42,8 +42,17 @@ set_fact: _multiarch: "{{ container_images | selectattr('arch', 'defined') | list }}" -- name: Normal build block - when: not _multiarch +- name: Multiarch podman block + when: + - _multiarch + - container_command == 'podman' + block: + - name: Unimplemented podman multiarch block + fail: + msg: "Multiarch podman is not yet implemented" + +- name: Normal podman build block + when: container_command == 'podman' block: - name: Build container images include_tasks: build.yaml @@ -59,10 +68,11 @@ loop_control: loop_var: zj_image -- name: Multiarch docker block - when: - - _multiarch - - container_command == 'docker' +# Docker multiarch and native arch all use buildx. This ensures that we are +# able to take advantage of speculative container states through the custom +# buildx builder. +- name: Docker build block + when: container_command == 'docker' vars: temp_registry: host: "127.0.0.1" @@ -88,12 +98,3 @@ loop_control: loop_var: zj_image when: inventory_hostname == ansible_play_hosts[0] - -- name: Multiarch podman block - when: - - _multiarch - - container_command == 'podman' - block: - - name: Unimplemented podman multiarch block - fail: - msg: "Multiarch podman is not yet implemented" diff --git a/roles/build-container-image/tasks/setup-buildx.yaml b/roles/build-container-image/tasks/setup-buildx.yaml index 40fd39717..3431fd1ac 100644 --- a/roles/build-container-image/tasks/setup-buildx.yaml +++ b/roles/build-container-image/tasks/setup-buildx.yaml @@ -1,8 +1,10 @@ - name: Update qemu-static container settings command: docker run --rm --privileged quay.io/opendevmirror/qemu-user-static --reset -p yes - when: ansible_architecture == 'x86_64' + when: + - _multiarch + - ansible_architecture == 'x86_64' -# TODO(clarkb) Use buildkitd.toml configuration to add certifications. That +# TODO(clarkb) Use buildkitd.toml configuration to add certificates. That # may allow us to drop the ca-certificates install and update-ca-certificates # step below. More info here: # https://docs.docker.com/reference/cli/docker/buildx/create/#buildkitd-config diff --git a/roles/build-docker-image/common.rst b/roles/build-docker-image/common.rst index 1d31f5c35..93a547bb7 100644 --- a/roles/build-docker-image/common.rst +++ b/roles/build-docker-image/common.rst @@ -157,8 +157,7 @@ using this role. :type: list :default: [] - A list of architectures to build on. When enabling this on any - image, all of them will be built with ``docker buildx``. + A list of architectures to build on. Valid values are ``linux/amd64``, ``linux/arm64``, ``linux/riscv64``, ``linux/ppc64le``, ``linux/s390x``, ``linux/386``, diff --git a/roles/build-docker-image/tasks/build.yaml b/roles/build-docker-image/tasks/build.yaml deleted file mode 100644 index e726be9b0..000000000 --- a/roles/build-docker-image/tasks/build.yaml +++ /dev/null @@ -1,35 +0,0 @@ -- name: Set up siblings - include_tasks: siblings.yaml - -- name: Build a docker image - vars: - tag_prefix: "{{ ('change_' + zuul.change) if (zuul.change is defined) else zuul.pipeline }}_" - command: >- - docker build {{ zj_image.path | default('.') }} -f {{ zj_image.dockerfile | default(docker_dockerfile) }} - {% if zj_image.target | default(false) -%} - --target {{ zj_image.target }} - {% endif -%} - {% for build_arg in zj_image.build_args | default([]) -%} - --build-arg {{ build_arg }} - {% endfor -%} - {% if zj_image.siblings | default(false) -%} - --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}" - {% endif -%} - {% for tag in zj_image.tags | default(['latest']) -%} - --tag {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ tag_prefix }}{{ tag }} - --tag {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ tag }} - {% endfor -%} - {% for label in zj_image.labels | default([]) -%} - --label "{{ label }}" - {% endfor %} - {% if zuul.change | default(false) -%} - --label "org.zuul-ci.change={{ zuul.change }}" - {% endif -%} - --label "org.zuul-ci.change_url={{ zuul.change_url }}" - args: - chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" - environment: - DOCKER_BUILDKIT: '{{ "1" if docker_use_buildkit else "0" }}' - -- name: Cleanup sibling source directory - include_tasks: clean-siblings.yaml diff --git a/roles/build-docker-image/tasks/buildx.yaml b/roles/build-docker-image/tasks/buildx.yaml index a160f71f9..77678f7a5 100644 --- a/roles/build-docker-image/tasks/buildx.yaml +++ b/roles/build-docker-image/tasks/buildx.yaml @@ -31,25 +31,8 @@ {% endif -%} --label "org.zuul-ci.change_url={{ zuul.change_url }}" -- name: Build images for all arches - command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }}" - args: - chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" - environment: - DOCKER_CLI_EXPERIMENTAL: enabled - -- name: Push arch-specific layers one at a time - command: "{{ docker_buildx_command }} --platform={{ zj_arch }} --push" - args: - chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" - environment: - DOCKER_CLI_EXPERIMENTAL: enabled - loop: '{{ zj_image.arch }}' - loop_control: - loop_var: zj_arch - -- name: Push final image manifest - command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }} --push" +- name: Build and push images for all arches + command: "{{ docker_buildx_command }} {% if _multiarch %}--platform={{ zj_image.arch | join(',') }}{% endif %} --push" args: chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" environment: diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml index 684d46cbc..1eff5dc47 100644 --- a/roles/build-docker-image/tasks/main.yaml +++ b/roles/build-docker-image/tasks/main.yaml @@ -37,30 +37,11 @@ buildset_registry_alias: "{{ buildset_registry.host }}" when: buildset_registry is defined and not ( buildset_registry.host | ipaddr ) -- name: Determine if we need to use buildx or normal build +- name: Determine if we are building multiarch or not set_fact: - use_buildx: "{{ docker_images | selectattr('arch', 'defined') | list }}" + _multiarch: "{{ docker_images | selectattr('arch', 'defined') | list }}" -- name: Normal docker block - when: not use_buildx - block: - - - name: Build docker images - include_tasks: build.yaml - loop: "{{ docker_images }}" - loop_control: - loop_var: zj_image - - # Push each image. - - name: Push image to buildset registry - when: buildset_registry is defined - include_tasks: push.yaml - loop: "{{ docker_images }}" - loop_control: - loop_var: zj_image - -- name: Buildx block - when: use_buildx +- name: Build image with custom buildx builder vars: temp_registry: host: "127.0.0.1" diff --git a/roles/build-docker-image/tasks/push.yaml b/roles/build-docker-image/tasks/push.yaml deleted file mode 100644 index c3cfc2f1c..000000000 --- a/roles/build-docker-image/tasks/push.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: Tag image for buildset registry - command: >- - docker tag {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} - loop: "{{ zj_image.tags | default(['latest']) }}" - loop_control: - loop_var: zj_image_tag -- name: Push tag to buildset registry - command: >- - docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} - loop: "{{ zj_image.tags | default(['latest']) }}" - loop_control: - loop_var: zj_image_tag diff --git a/roles/build-docker-image/tasks/setup-buildx.yaml b/roles/build-docker-image/tasks/setup-buildx.yaml index 5382e7fa9..1db87f17e 100644 --- a/roles/build-docker-image/tasks/setup-buildx.yaml +++ b/roles/build-docker-image/tasks/setup-buildx.yaml @@ -2,9 +2,11 @@ command: docker run --rm --privileged quay.io/opendevmirror/qemu-user-static --reset -p yes environment: DOCKER_CLI_EXPERIMENTAL: enabled - when: ansible_architecture == 'x86_64' + when: + - _multiarch + - ansible_architecture == 'x86_64' -# TODO(clarkb) Use buildkitd.toml configuration to add certifications. That +# TODO(clarkb) Use buildkitd.toml configuration to add certificates. That # may allow us to drop the ca-certificates install and update-ca-certificates # step below. More info here: # https://docs.docker.com/reference/cli/docker/buildx/create/#buildkitd-config diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index 355843b60..0063a2167 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -251,10 +251,6 @@ label: debian-bookworm - name: builder label: ubuntu-jammy - # Docker can't currently build with speculative states hosted anywhere - # but docker.io. This is nonvoting as we're using quay.io as the - # registry. A followup will fix this issue and make the test voting. - voting: false - job: name: zuul-jobs-test-registry-docker-container-role @@ -267,10 +263,6 @@ compatibility with that role using docker as the command. vars: container_role: container - # Docker can't currently build with speculative states hosted anywhere - # but docker.io. This is nonvoting as we're using quay.io as the - # registry. A followup will fix this issue and make the test voting. - voting: false - job: name: zuul-jobs-test-registry-docker-multiarch @@ -748,7 +740,7 @@ - project: check: - jobs: + jobs: &id001 - zuul-jobs-test-ensure-docker-centos-9-stream - zuul-jobs-test-ensure-docker-debian-bookworm - zuul-jobs-test-ensure-docker-debian-bullseye @@ -792,46 +784,6 @@ - zuul-jobs-test-ensure-podman-debian-bullseye - zuul-jobs-test-mirror-container-images gate: - jobs: &id001 - - zuul-jobs-test-ensure-docker-centos-9-stream - - zuul-jobs-test-ensure-docker-debian-bookworm - - zuul-jobs-test-ensure-docker-debian-bullseye - - zuul-jobs-test-ensure-docker-ubuntu-focal - - zuul-jobs-test-ensure-docker-ubuntu-jammy - - zuul-jobs-test-ensure-docker-ubuntu-noble - - zuul-jobs-test-build-container-image-docker-release - - zuul-jobs-test-build-container-image-docker-release-multiarch - - zuul-jobs-test-build-container-image-docker-release-multiarch-multinode - - zuul-jobs-test-build-container-image-podman-release - - zuul-jobs-test-build-container-image-docker-promote - - zuul-jobs-test-build-container-image-docker-promote-multiarch - - zuul-jobs-test-build-container-image-podman-promote - - zuul-jobs-test-build-docker-image-release - - zuul-jobs-test-build-docker-image-release-multiarch - - zuul-jobs-test-registry-docker-multiarch - - zuul-jobs-test-registry-docker-container-role-multiarch - - zuul-jobs-test-registry-podman - - zuul-jobs-test-registry-buildset-registry - - zuul-jobs-test-registry-buildset-registry-k8s-microk8s - - zuul-jobs-test-registry-buildset-registry-k8s-crio - - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal - - zuul-jobs-test-ensure-kubernetes-podman-ubuntu-noble - - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy - - zuul-jobs-test-ensure-kubernetes-microk8s-debian-bookworm - - zuul-jobs-test-ensure-podman-debian-bookworm - - zuul-jobs-test-ensure-podman-debian-bullseye - - zuul-jobs-test-ensure-podman-ubuntu-jammy - - zuul-jobs-test-ensure-podman-ubuntu-noble - - zuul-jobs-test-ensure-podman-socket-debian-bookworm - - zuul-jobs-test-ensure-podman-socket-debian-bullseye - - zuul-jobs-test-ensure-podman-socket-ubuntu-jammy - - zuul-jobs-test-ensure-podman-socket-ubuntu-noble - - zuul-jobs-test-ensure-skopeo-debian-bookworm - - zuul-jobs-test-ensure-skopeo-debian-bullseye - - zuul-jobs-test-ensure-skopeo-ubuntu-focal - - zuul-jobs-test-ensure-skopeo-ubuntu-jammy - - zuul-jobs-test-ensure-skopeo-ubuntu-noble - - zuul-jobs-test-ensure-podman-debian-bullseye - - zuul-jobs-test-mirror-container-images + jobs: *id001 periodic-weekly: jobs: *id001