Test multiarch release builds and use temp registry with buildx

When using docker buildx to build a container image, use a temporary
registry to receive the built image instead of requiring a buildset
registry.

A multi-arch test is also added with a publication registry
using the same task list to reduce duplication.

Change-Id: Ib20d1c97f6cb63e0ff9d8888ea792d1941cd8690
Co-Authored-By: James E. Blair <jeblair@redhat.com>

roles/build-docker-image/tasks/buildx.yaml

@@ -1,6 +1,9 @@
 - name: Set up siblings
   include_tasks: siblings.yaml
 
+# The command below always tags the images for the temp_registry (so
+# they can be pulled back onto the host image cache), and also tags
+# them for the buildset registry if one is present.
 - name: Set base docker build command
   set_fact:
     docker_buildx_command: >-
@@ -16,9 +19,15 @@
       {% endif -%}
       {% for tag in zj_image.tags | default(['latest']) -%}
         {% if zuul.change | default(false) -%}
-          --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
+          --tag {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
+          {% if buildset_registry | default(false) -%}
+            --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
+          {% endif -%}
+        {% endif -%}
+        --tag {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ tag }}
+        {% if buildset_registry | default(false) -%}
+          --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }}
         {% endif -%}
-        --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }}
       {% endfor -%}
       {% for label in zj_image.labels | default([]) -%}
         --label "{{ label }}"
@@ -45,27 +54,38 @@
   loop_control:
     loop_var: zj_arch
 
-- name: Push final to buildset registry
+- name: Push final image manifest
   command: "{{ docker_buildx_command }} --platform={{ zj_image.arch | join(',') }} --push"
   args:
     chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
 
-- name: Pull images from buildset registry
+- name: Pull images from temporary registry
   command: >-
-    docker pull {{ zj_image.repository }}:{{ zj_image_tag }}
+    docker pull {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
   loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
 
 - name: Locally tag for changes so push works later
   command: >-
-    docker tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}
+    docker tag
+    {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+    {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}
   loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
   when: zuul.change | default(false)
 
+- name: Locally tag for changes so push works later
+  command: >-
+    docker tag
+    {{ temp_registry.host }}:{{ temp_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+    {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+
 - name: Cleanup sibling source directory
   include_tasks: clean-siblings.yaml

roles/build-docker-image/tasks/main.yaml

@@ -60,13 +60,21 @@
 
 - name: Buildx block
   when: use_buildx
+  vars:
+    temp_registry:
+      host: "127.0.0.1"
+      port: 5100
+      username: zuul
+      password: tempregistry
   block:
+    - name: Set up a temporary registry for holding buildx-built images
+      import_tasks: ../../../util-tasks/run-docker-registry.yaml
+      vars:
+        registry: "{{ temp_registry }}"
+        container_command: docker
 
-    - name: Assert buildset registry is defined for buildx
-      assert:
-        that:
-          - buildset_registry is defined
-        fail_msg: "Building multi-arch images requires a buildset registry"
+    - name: Log in to temporary registry
+      command: "docker login -u {{ temp_registry.username }} -p {{ temp_registry.password }} {{ temp_registry.host }}:{{ temp_registry.port }}"
 
     - name: Set up buildx builders
       include_tasks: setup-buildx.yaml

roles/build-docker-image/tasks/setup-buildx.yaml

@@ -4,7 +4,7 @@
     DOCKER_CLI_EXPERIMENTAL: enabled
 
 - name: Create builder
-  command: docker buildx create --name mybuilder --driver-opt network=host --config /etc/buildkit/buildkitd.toml
+  command: "docker buildx create --name mybuilder --driver-opt network=host{% if buildset_registry is defined %} --config /etc/buildkit/buildkitd.toml {% endif %}"
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
 

roles/upload-docker-image/tasks/buildx.yaml

@@ -18,14 +18,14 @@
       --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
     {% endif -%}
     {% for tag in zj_image.tags | default(['latest']) -%}
-      {% if zuul.change | default(false) -%}
-        --tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
-      {% endif -%}
+      --tag {{ docker_registry | ternary(docker_registry + '/', '') }}{{ zj_image.repository }}:{{ upload_docker_image_promote | ternary('change_' + zuul.get('change', '') + '_', '') }}{{ tag }}
     {% endfor -%}
     {% for label in zj_image.labels | default([]) -%}
       --label "{{ label }}"
     {% endfor %}
-    --label "org.zuul-ci.change={{ zuul.change }}"
+    {% if zuul.change | default(false) %}
+      --label "org.zuul-ci.change={{ zuul.change }}"
+    {% endif %}
     --label "org.zuul-ci.change_url={{ zuul.change_url }}"
     --platform={{ zj_image.arch | join(',') }}
     --push

test-playbooks/container/test-build-container-image-release-pre.yaml

@@ -1,5 +1,20 @@
 - hosts: all
   tasks:
-    - name: Install container system
-      include_role:
-        name: "ensure-{{ container_command }}"
+    - name: Set docker_registry cacheable facts
+      set_fact:
+        cacheable: true
+        docker_registry: "127.0.0.1:5200"
+        docker_credentials:
+          username: zuul
+          password: testpassword
+          repository: testrepo
+
+    - name: Set up a publication registry to stand in for Docker Hub
+      import_tasks: ../../util-tasks/run-docker-registry.yaml
+      vars:
+        registry:
+          host: "127.0.0.1"
+          port: 5200
+          username: zuul
+          password: testpassword
+        container_command: docker

test-playbooks/container/test-build-container-image-release.yaml

@@ -1,18 +1,19 @@
 - hosts: all
   vars:
-    docker_registry: localhost:5000
     upload_docker_image_promote: false
-    docker_credentials:
-      username: zuul
-      password: testpassword
-      repository: testrepo
-    docker_images:
+    _arch_docker_images:
+      - context: test-playbooks/container/docker
+        repository: "testrepo"
+        # This is what the Zuul repo uses to tag its releases:
+        tags: "{{ zuul.tag is defined | ternary([zuul.get('tag', '').split('.')[0], '.'.join(zuul.get('tag', '').split('.')[:2]), zuul.get('tag', '')], ['latest']) }}"
+        arch: ['linux/amd64', 'linux/arm64']
+    _normal_docker_images:
       - context: test-playbooks/container/docker
         repository: "testrepo"
         # This is what the Zuul repo uses to tag its releases:
         tags: "{{ zuul.tag is defined | ternary([zuul.get('tag', '').split('.')[0], '.'.join(zuul.get('tag', '').split('.')[:2]), zuul.get('tag', '')], ['latest']) }}"
     container_images: "{{ docker_images }}"
-  pre_tasks:
+  tasks:
     - name: Save zuul variables
       set_fact:
         old_zuul: "{{ zuul }}"
@@ -25,76 +26,16 @@
           project: "{{ old_zuul.project }}"
           ref: refs/tags/3.19.0
           tag: 3.19.0
+
     - name: "Build a container image"
       include_role:
         name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
         zuul: "{{ new_zuul }}"
-
-    - name: Create temporary registry working directory
-      tempfile:
-        state: directory
-      register: registry_tempdir
-
-    - name: Create auth directory
-      file:
-        path: "{{ registry_tempdir.path }}/auth"
-        state: directory
-    - name: Install passlib for htpasswd
-      become: true
-      package:
-        name:
-          - python3-passlib
-          - python3-bcrypt
-        state: present
-    - name: Write htpasswd file
-      htpasswd:
-        create: true
-        crypt_scheme: bcrypt
-        path: "{{ registry_tempdir.path }}/auth/htpasswd"
-        name: "{{ docker_credentials.username }}"
-        password: "{{ docker_credentials.password }}"
-
-    - name: Create certs directory
-      file:
-        state: directory
-        path: "{{ registry_tempdir.path }}/certs"
-    - name: Create self signed certificates
-      command: >
-        openssl req
-        -newkey rsa:4096 -nodes -sha256 -keyout certs/localhost.key
-        -x509 -days 365 -out certs/localhost.crt
-        -subj '/CN=localhost'
-      args:
-        chdir: "{{ registry_tempdir.path }}"
-    - name: Create docker certs dir
-      file:
-        state: directory
-        path: /etc/docker/certs.d/localhost:5000/
-      become: true
-    - name: Configure docker to trust certificate
-      copy:
-        src: "{{ registry_tempdir.path }}/certs/localhost.crt"
-        dest: /etc/docker/certs.d/localhost:5000/ca.crt
-        remote_src: true
-      become: true
-
-    - name: Start registry with basic auth
-      command: >-
-        {{ container_command }} run -d \
-        -p 5000:5000 \
-        -v {{ registry_tempdir.path }}/auth:/auth \
-        -e "REGISTRY_AUTH=htpasswd" \
-        -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-        -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-        -v {{ registry_tempdir.path }}/certs:/certs \
-        -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/localhost.crt \
-        -e REGISTRY_HTTP_TLS_KEY=/certs/localhost.key \
-        registry:2
-      args:
-        chdir: "{{ registry_tempdir.path }}"
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"
 
     - include_role:
         name: "upload-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
         zuul: "{{ new_zuul }}"
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"

util-tasks/README

@@ -0,0 +1,3 @@
+This directory holds task lists for use internally by multiple roles.
+They are not meant to be consumed outside of zuul-jobs, and do not
+provide a stable API.

util-tasks/run-docker-registry.yaml

@@ -0,0 +1,50 @@
+# Required vars:
+#  registry:
+#      host: "{{ ansible_host }}"
+#      port: 5000
+#      username: zuul
+#      password: testpassword
+#  container_command: docker
+
+- name: Install container system
+  include_role:
+    name: "ensure-{{ container_command }}"
+
+- name: Create temporary registry working directory
+  tempfile:
+    state: directory
+  register: registry_tempdir
+
+- name: Create auth directory
+  file:
+    path: "{{ registry_tempdir.path }}/auth"
+    state: directory
+
+- name: Install passlib for htpasswd
+  become: true
+  package:
+    name:
+      - python3-passlib
+      - python3-bcrypt
+    state: present
+
+- name: Write htpasswd file
+  htpasswd:
+    create: true
+    crypt_scheme: bcrypt
+    path: "{{ registry_tempdir.path }}/auth/htpasswd"
+    name: "{{ registry.username }}"
+    password: "{{ registry.password }}"
+
+- name: Start registry with basic auth
+  command: >-
+    {{ container_command }} run -d \
+    -p {{ registry.port }}:5000
+    --restart=always
+    -v {{ registry_tempdir.path }}/auth:/auth \
+    -e "REGISTRY_AUTH=htpasswd" \
+    -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
+    -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
+    registry:2
+  args:
+    chdir: "{{ registry_tempdir.path }}"