From 06e35b21259bca7df7e0a8d628e47551af49f955 Mon Sep 17 00:00:00 2001 From: Albin Vass Date: Mon, 22 Jun 2020 18:39:32 +0200 Subject: [PATCH] Test multiarch release builds and use temp registry with buildx When using docker buildx to build a container image, use a temporary registry to receive the built image instead of requiring a buildset registry. A multi-arch test is also added with a publication registry using the same task list to reduce duplication. Change-Id: Ib20d1c97f6cb63e0ff9d8888ea792d1941cd8690 Co-Authored-By: James E. Blair --- roles/build-docker-image/tasks/buildx.yaml | 32 ++++++-- roles/build-docker-image/tasks/main.yaml | 18 +++-- .../tasks/setup-buildx.yaml | 2 +- roles/upload-docker-image/tasks/buildx.yaml | 8 +- ...est-build-container-image-release-pre.yaml | 21 ++++- .../test-build-container-image-release.yaml | 81 +++---------------- util-tasks/README | 3 + util-tasks/run-docker-registry.yaml | 50 ++++++++++++ 8 files changed, 126 insertions(+), 89 deletions(-) create mode 100644 util-tasks/README create mode 100644 util-tasks/run-docker-registry.yaml diff --git a/roles/build-docker-image/tasks/buildx.yaml b/roles/build-docker-image/tasks/buildx.yaml index 2fc2e695a..a4ce98bc7 100644 --- a/roles/build-docker-image/tasks/buildx.yaml +++ b/roles/build-docker-image/tasks/buildx.yaml @@ -1,6 +1,9 @@ - name: Set up siblings include_tasks: siblings.yaml +# The command below always tags the images for the temp_registry (so +# they can be pulled back onto the host image cache), and also tags +# them for the buildset registry if one is present. - name: Set base docker build command set_fact: docker_buildx_command: >- @@ -16,9 +19,15 @@ {% endif -%} {% for tag in zj_image.tags | default(['latest']) -%} {% if zuul.change | default(false) -%} - --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }} + --tag {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }} + {% if buildset_registry | default(false) -%} + --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }} + {% endif -%} + {% endif -%} + --tag {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ tag }} + {% if buildset_registry | default(false) -%} + --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }} {% endif -%} - --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }} {% endfor -%} {% for label in zj_image.labels | default([]) -%} --label "{{ label }}" @@ -45,27 +54,38 @@ loop_control: loop_var: zj_arch -- name: Push final to buildset registry +- name: Push final image manifest command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }} --push" args: chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}" environment: DOCKER_CLI_EXPERIMENTAL: enabled -- name: Pull images from buildset registry +- name: Pull images from temporary registry command: >- - docker pull {{ zj_image.repository }}:{{ zj_image_tag }} + docker pull {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} loop: "{{ zj_image.tags | default(['latest']) }}" loop_control: loop_var: zj_image_tag - name: Locally tag for changes so push works later command: >- - docker tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }} + docker tag + {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} + {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }} loop: "{{ zj_image.tags | default(['latest']) }}" loop_control: loop_var: zj_image_tag when: zuul.change | default(false) +- name: Locally tag for changes so push works later + command: >- + docker tag + {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} + {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ zj_image_tag }} + loop: "{{ zj_image.tags | default(['latest']) }}" + loop_control: + loop_var: zj_image_tag + - name: Cleanup sibling source directory include_tasks: clean-siblings.yaml diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml index d64a949af..2ac708e45 100644 --- a/roles/build-docker-image/tasks/main.yaml +++ b/roles/build-docker-image/tasks/main.yaml @@ -60,13 +60,21 @@ - name: Buildx block when: use_buildx + vars: + temp_registry: + host: "127.0.0.1" + port: 5100 + username: zuul + password: tempregistry block: + - name: Set up a temporary registry for holding buildx-built images + import_tasks: ../../../util-tasks/run-docker-registry.yaml + vars: + registry: "{{ temp_registry }}" + container_command: docker - - name: Assert buildset registry is defined for buildx - assert: - that: - - buildset_registry is defined - fail_msg: "Building multi-arch images requires a buildset registry" + - name: Log in to temporary registry + command: "docker login -u {{ temp_registry.username }} -p {{ temp_registry.password }} {{ temp_registry.host }}:{{ temp_registry.port }}" - name: Set up buildx builders include_tasks: setup-buildx.yaml diff --git a/roles/build-docker-image/tasks/setup-buildx.yaml b/roles/build-docker-image/tasks/setup-buildx.yaml index 818e89bdb..51720a4c9 100644 --- a/roles/build-docker-image/tasks/setup-buildx.yaml +++ b/roles/build-docker-image/tasks/setup-buildx.yaml @@ -4,7 +4,7 @@ DOCKER_CLI_EXPERIMENTAL: enabled - name: Create builder - command: docker buildx create --name mybuilder --driver-opt network=host --config /etc/buildkit/buildkitd.toml + command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}" environment: DOCKER_CLI_EXPERIMENTAL: enabled diff --git a/roles/upload-docker-image/tasks/buildx.yaml b/roles/upload-docker-image/tasks/buildx.yaml index ef8ce42a2..5141f8edf 100644 --- a/roles/upload-docker-image/tasks/buildx.yaml +++ b/roles/upload-docker-image/tasks/buildx.yaml @@ -18,14 +18,14 @@ --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}" {% endif -%} {% for tag in zj_image.tags | default(['latest']) -%} - {% if zuul.change | default(false) -%} - --tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }} - {% endif -%} + --tag {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ upload_docker_image_promote | ternary('change_' + zuul.get('change', '') + '_', '') }}{{ tag }} {% endfor -%} {% for label in zj_image.labels | default([]) -%} --label "{{ label }}" {% endfor %} - --label "org.zuul-ci.change={{ zuul.change }}" + {% if zuul.change | default(false) %} + --label "org.zuul-ci.change={{ zuul.change }}" + {% endif %} --label "org.zuul-ci.change_url={{ zuul.change_url }}" --platform={{ zj_image.arch | join(',') }} --push diff --git a/test-playbooks/container/test-build-container-image-release-pre.yaml b/test-playbooks/container/test-build-container-image-release-pre.yaml index e576fb482..4e1cb54c4 100644 --- a/test-playbooks/container/test-build-container-image-release-pre.yaml +++ b/test-playbooks/container/test-build-container-image-release-pre.yaml @@ -1,5 +1,20 @@ - hosts: all tasks: - - name: Install container system - include_role: - name: "ensure-{{ container_command }}" + - name: Set docker_registry cacheable facts + set_fact: + cacheable: true + docker_registry: "127.0.0.1:5200" + docker_credentials: + username: zuul + password: testpassword + repository: testrepo + + - name: Set up a publication registry to stand in for Docker Hub + import_tasks: ../../util-tasks/run-docker-registry.yaml + vars: + registry: + host: "127.0.0.1" + port: 5200 + username: zuul + password: testpassword + container_command: docker diff --git a/test-playbooks/container/test-build-container-image-release.yaml b/test-playbooks/container/test-build-container-image-release.yaml index d749d2d66..eb896eef1 100644 --- a/test-playbooks/container/test-build-container-image-release.yaml +++ b/test-playbooks/container/test-build-container-image-release.yaml @@ -1,18 +1,19 @@ - hosts: all vars: - docker_registry: localhost:5000 upload_docker_image_promote: false - docker_credentials: - username: zuul - password: testpassword - repository: testrepo - docker_images: + _arch_docker_images: + - context: test-playbooks/container/docker + repository: "testrepo" + # This is what the Zuul repo uses to tag its releases: + tags: "{{ zuul.tag is defined | ternary([zuul.get('tag', '').split('.')[0], '.'.join(zuul.get('tag', '').split('.')[:2]), zuul.get('tag', '')], ['latest']) }}" + arch: ['linux/amd64', 'linux/arm64'] + _normal_docker_images: - context: test-playbooks/container/docker repository: "testrepo" # This is what the Zuul repo uses to tag its releases: tags: "{{ zuul.tag is defined | ternary([zuul.get('tag', '').split('.')[0], '.'.join(zuul.get('tag', '').split('.')[:2]), zuul.get('tag', '')], ['latest']) }}" container_images: "{{ docker_images }}" - pre_tasks: + tasks: - name: Save zuul variables set_fact: old_zuul: "{{ zuul }}" @@ -25,76 +26,16 @@ project: "{{ old_zuul.project }}" ref: refs/tags/3.19.0 tag: 3.19.0 + - name: "Build a container image" include_role: name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image" vars: zuul: "{{ new_zuul }}" - - - name: Create temporary registry working directory - tempfile: - state: directory - register: registry_tempdir - - - name: Create auth directory - file: - path: "{{ registry_tempdir.path }}/auth" - state: directory - - name: Install passlib for htpasswd - become: true - package: - name: - - python3-passlib - - python3-bcrypt - state: present - - name: Write htpasswd file - htpasswd: - create: true - crypt_scheme: bcrypt - path: "{{ registry_tempdir.path }}/auth/htpasswd" - name: "{{ docker_credentials.username }}" - password: "{{ docker_credentials.password }}" - - - name: Create certs directory - file: - state: directory - path: "{{ registry_tempdir.path }}/certs" - - name: Create self signed certificates - command: > - openssl req - -newkey rsa:4096 -nodes -sha256 -keyout certs/localhost.key - -x509 -days 365 -out certs/localhost.crt - -subj '/CN=localhost' - args: - chdir: "{{ registry_tempdir.path }}" - - name: Create docker certs dir - file: - state: directory - path: /etc/docker/certs.d/localhost:5000/ - become: true - - name: Configure docker to trust certificate - copy: - src: "{{ registry_tempdir.path }}/certs/localhost.crt" - dest: /etc/docker/certs.d/localhost:5000/ca.crt - remote_src: true - become: true - - - name: Start registry with basic auth - command: >- - {{ container_command }} run -d \ - -p 5000:5000 \ - -v {{ registry_tempdir.path }}/auth:/auth \ - -e "REGISTRY_AUTH=htpasswd" \ - -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ - -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ - -v {{ registry_tempdir.path }}/certs:/certs \ - -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/localhost.crt \ - -e REGISTRY_HTTP_TLS_KEY=/certs/localhost.key \ - registry:2 - args: - chdir: "{{ registry_tempdir.path }}" + docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}" - include_role: name: "upload-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image" vars: zuul: "{{ new_zuul }}" + docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}" diff --git a/util-tasks/README b/util-tasks/README new file mode 100644 index 000000000..be34d737f --- /dev/null +++ b/util-tasks/README @@ -0,0 +1,3 @@ +This directory holds task lists for use internally by multiple roles. +They are not meant to be consumed outside of zuul-jobs, and do not +provide a stable API. diff --git a/util-tasks/run-docker-registry.yaml b/util-tasks/run-docker-registry.yaml new file mode 100644 index 000000000..92c2441c1 --- /dev/null +++ b/util-tasks/run-docker-registry.yaml @@ -0,0 +1,50 @@ +# Required vars: +# registry: +# host: "{{ ansible_host }}" +# port: 5000 +# username: zuul +# password: testpassword +# container_command: docker + +- name: Install container system + include_role: + name: "ensure-{{ container_command }}" + +- name: Create temporary registry working directory + tempfile: + state: directory + register: registry_tempdir + +- name: Create auth directory + file: + path: "{{ registry_tempdir.path }}/auth" + state: directory + +- name: Install passlib for htpasswd + become: true + package: + name: + - python3-passlib + - python3-bcrypt + state: present + +- name: Write htpasswd file + htpasswd: + create: true + crypt_scheme: bcrypt + path: "{{ registry_tempdir.path }}/auth/htpasswd" + name: "{{ registry.username }}" + password: "{{ registry.password }}" + +- name: Start registry with basic auth + command: >- + {{ container_command }} run -d \ + -p {{ registry.port }}:5000 + --restart=always + -v {{ registry_tempdir.path }}/auth:/auth \ + -e "REGISTRY_AUTH=htpasswd" \ + -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + registry:2 + args: + chdir: "{{ registry_tempdir.path }}"