WIP: use-buildset-registry: Add podman support

Change-Id: I41718073962c8e7eb3d8810276e550fb84bd6e99

roles/pull-from-intermediate-registry/tasks/main.yaml

@@ -74,13 +74,18 @@
     mode: 0600
 
 # Pull the images
+
+# To support usage with both docker and podman, the buildset registry
+# keeps "docker.io" entries un-namespaced, and any other namespaces
+# are namespaced.  Therefore, if we see docker.io in the repository
+# name, we strip it here.
 - name: Pull artifacts from intermediate registry
   block:
     - name: Pull artifacts from intermediate registry
       command: >-
         skopeo --insecure-policy copy
         {{ item.url }}
-        docker://127.0.0.1:{{ socat_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
+        docker://127.0.0.1:{{ socat_port }}/{{ item.metadata.repository | regex_replace('^docker\.io/(.*)', '\1') }}:{{ item.metadata.tag }}
       retries: 3
       register: result
       until: result is success

roles/push-to-intermediate-registry/tasks/push-image.yaml

@@ -1,7 +1,11 @@
+# To support usage with both docker and podman, the buildset registry
+# keeps "docker.io" entries un-namespaced, and any other namespaces
+# are namespaced.  Therefore, if we see docker.io in the repository
+# name, we strip it here.
 - name: Push tag to intermediate registry
   command: >-
     skopeo --insecure-policy copy
-    docker://127.0.0.1:{{ socat_port }}/{{ image.repository }}:{{ image_tag }}
+    docker://127.0.0.1:{{ socat_port }}/{{ image.repository | regex_replace('^docker\.io/(.*)', '\1') }}:{{ image_tag }}
     docker://{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }}
   retries: 3
   register: result

roles/run-buildset-registry/README.rst

@@ -1,4 +1,4 @@
-Runs a docker registry for the use of this buildset.
+Runs a container registry for the use of this buildset.
 
 This may be used for a single job running on a single node, or it may
 be used at the root of a job graph so that multiple jobs running for a
@@ -16,6 +16,12 @@ single change can share the registry.
 
    The port on which the registry should listen.
 
+.. zuul:rolevar:: container_command
+   :default: docker
+
+   The command to use to run the registry container (E.g., ``podman``).
+
+
 **Return Values**
 
 .. zuul:rolevar:: buildset_registry

roles/run-buildset-registry/defaults/main.yaml

@@ -1,2 +1,3 @@
 buildset_registry_root: "{{ ansible_user_dir }}/buildset_registry"
 buildset_registry_port: 5000
+container_command: docker

roles/run-buildset-registry/tasks/main.yaml

@@ -2,18 +2,18 @@
   become: yes
   package:
     name:
-      - python-docker
       - openssl
       - python-passlib
+      - socat
     state: present
   when: ansible_python_version is version('3', '<')
 - name: Install packages
   become: yes
   package:
     name:
-      - python3-docker
       - openssl
       - python3-passlib
+      - socat
     state: present
   when: ansible_python_version is version('3', '>=')
 - name: Ensure registry volume directories exists
@@ -41,16 +41,21 @@
   set_fact:
     certificate: "{{ certificate.content | b64decode }}"
 - name: Start the buildset registry
-  docker_container:
-    name: "{{ (buildset_registry_port == 5000) | ternary('buildset_registry', 'buildset_registry_' + buildset_registry_port|string) }}"
-    image: zuul/zuul-registry:latest
-    state: started
-    restart_policy: always
-    ports:
-     - "{{ buildset_registry_port }}:5000"
-    volumes:
-      - "{{ buildset_registry_root }}/tls:/tls"
-      - "{{ buildset_registry_root }}/conf:/conf"
+  command: >-
+    {{ container_command }} run -d
+    --name="{{ (buildset_registry_port == 5000) | ternary('buildset_registry', 'buildset_registry_' + buildset_registry_port|string) }}"
+    --restart=always
+    --publish="1{{ buildset_registry_port }}:5000"
+    --volume="{{ buildset_registry_root }}/tls:/tls"
+    --volume="{{ buildset_registry_root }}/conf:/conf"
+    docker.io/zuul/zuul-registry:latest
+
+# Start a socat tunnel to the buildset registry to work around
+# https://github.com/containers/libpod/issues/4311
+# in case we're using podman.
+- name: Start socat to work around https://github.com/containers/libpod/issues/4311
+  shell: "socat -d -d TCP6-LISTEN:{{ buildset_registry_port }},fork TCP:127.0.0.1:1{{ buildset_registry_port }} 2> {{ buildset_registry_root }}/socat_port &"
+
 - name: Set registry information fact
   set_fact:
     buildset_registry:

roles/use-buildset-registry/README.rst

@@ -35,3 +35,15 @@ Use this role on any host which should use the buildset registry.
    The system user to configure to use the docker registry.  The
    docker configuration file for this user will be updated.  By
    default, the user Ansible is running as.
+
+.. zuul:rolevar:: buildset_registry_namespaces
+   :default: ['docker.io', 'quay.io', 'gcr.io']
+
+   The namespaces that the buildset registry supports.  The buildset
+   registry will be consulted first for images in these namespaces.
+   Any others will be fetched only from their upstream sources.
+
+   Add any local or third-party registries necessary here.
+
+   The default may change in the future as more general-purpose public
+   registries become known.

roles/use-buildset-registry/defaults/main.yaml

@@ -0,0 +1,4 @@
+buildset_registry_namespaces:
+  - docker.io
+  - quay.io
+  - gcr.io

roles/use-buildset-registry/library/modify_registries_conf.py

@@ -0,0 +1,76 @@
+# Copyright 2019 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import remarshal
+
+
+def get_location(prefix, location):
+    # To support usage with both docker and podman, the buildset
+    # registry keeps "docker.io" entries un-namespaced.
+    if prefix == 'docker.io':
+        return location
+    else:
+        return location + '/' + prefix
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            path=dict(required=True, type='path'),
+            buildset_registry=dict(type='raw'),
+            namespaces=dict(type='raw'),
+        )
+    )
+    p = module.params
+    location = '%s:%s' % (p['buildset_registry']['host'],
+                          p['buildset_registry']['port'])
+
+    if os.path.exists(p['path']):
+        with open(p['path'], 'rb') as f:
+            input_data = f.read()
+        data = remarshal.decode('toml', input_data, True)
+    else:
+        data = {}
+
+    unseen = set(p['namespaces'])
+    if 'registry' not in data:
+        data['registry'] = []
+    for reg in data['registry']:
+        if reg['prefix'] in unseen:
+            unseen.remove(reg['prefix'])
+        else:
+            continue
+        mirrors = reg.setdefault('mirror', [])
+        mirrors.insert(0, {
+            'location': get_location(reg['prefix'], location)})
+    for prefix in unseen:
+        mirrors = [{'location': get_location(prefix, location)},
+                   {'location': prefix}]
+        reg = {'prefix': prefix,
+               'location': prefix,
+               'mirror': mirrors}
+        data['registry'].append(reg)
+
+    output_data = remarshal.encode_toml(data, True)
+    with open(p['path'], 'wb') as f:
+        f.write(output_data.encode('utf8'))
+
+    module.exit_json(changed=True, data=data)
+
+
+if __name__ == '__main__':
+    ansible_main()

roles/use-buildset-registry/tasks/main.yaml

@@ -23,16 +23,14 @@
   file:
     state: directory
     path: /etc/docker
-- name: Ensure buildset registry cert directory exists
-  become: true
-  file:
-    path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/"
-    state: directory
 - name: Write buildset registry TLS certificate
   become: true
   copy:
     content: "{{ buildset_registry.cert }}"
-    dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/ca.crt"
+    dest: "/usr/local/share/ca-certificates/buildset-registry.crt"
+- name: Update CA certs
+  command: update-ca-certificates
+  become: true
 
 # Update daemon config
 - name: Check if docker daemon configuration exists
@@ -73,6 +71,18 @@
   register: docker_restart
   failed_when: docker_restart is failed and not 'Could not find the requested service' in docker_restart.msg
 
+- name: Ensure containers directory exists
+  become: yes
+  file:
+    state: directory
+    path: /etc/containers
+- name: Modify registries.conf
+  become: yes
+  modify_registries_conf:
+    path: /etc/containers/registries.conf
+    buildset_registry: "{{ buildset_registry }}"
+    namespaces: "{{ buildset_registry_namespaces }}"
+
 # We use 'block' here to cause the become to apply to all the tasks
 # (which does not automatically happen with include_tasks).
 - name: Update docker user config to use buildset registry

roles/use-buildset-registry/tasks/user-config.yaml

@@ -37,6 +37,10 @@
     content: "{{ docker_config | to_nice_json }}"
     dest: "~/.docker/config.json"
     mode: 0600
+- name: Write containers auth configuration
+  copy:
+    content: "{{ docker_config | to_nice_json }}"
+    dest: "/run/user/{{ ansible_user_uid }}/auth.json"
 - name: Check if /var/lib/kubelet exists
   stat:
     path: /var/lib/kubelet

test-playbooks/registry/docker/Dockerfile

@@ -1,2 +1,2 @@
-FROM debian:testing
+FROM docker.io/library/debian:testing
 CMD echo "Zuul container test"; sleep infinity

test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml

@@ -1,4 +1,5 @@
 - name: Ensure registry volume directories exists
+  become: true
   file:
     state: directory
     path: "/var/registry/{{ item }}"
@@ -6,6 +7,7 @@
     - certs
     - auth
 - name: Install python packages
+  become: true
   package:
     name:
       - python3-docker
@@ -13,6 +15,7 @@
       - python3-bcrypt
     state: present
 - name: Write htpassword file
+  become: true
   htpasswd:
     create: true
     crypt_scheme: bcrypt
@@ -20,27 +23,26 @@
     name: "{{ intermediate_registry.username }}"
     password: "{{ intermediate_registry.password }}"
 - name: Write TLS private key
+  become: true
   copy:
     content: "{{ intermediate_registry_tls_key }}"
     dest: /var/registry/certs/domain.key
 - name: Write TLS certificate
+  become: true
   copy:
     content: "{{ intermediate_registry_tls_cert }}{{ intermediate_registry_tls_chain | default('') }}"
     dest: /var/registry/certs/domain.crt
 - name: Start intermediate docker registry
-  docker_container:
-    name: intermediate_registry
-    image: registry:2
-    state: started
-    restart_policy: always
-    ports:
-     - "5000:5000"
-    env:
-      REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
-      REGISTRY_HTTP_TLS_KEY: /certs/domain.key
-      REGISTRY_AUTH: htpasswd
-      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
-      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
-    volumes:
-      - "/var/registry/certs:/certs"
-      - "/var/registry/auth:/auth"
+  command: >-
+    {{ container_command }} run -d
+    --name="intermediate_registry"
+    --restart=always
+    --network=host
+    --env REGISTRY_HTTP_TLS_CERTIFICATE="/certs/domain.crt"
+    --env REGISTRY_HTTP_TLS_KEY="/certs/domain.key"
+    --env REGISTRY_AUTH="htpasswd"
+    --env REGISTRY_AUTH_HTPASSWD_PATH="/auth/htpasswd"
+    --env REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm"
+    --volume="/var/registry/certs:/certs"
+    --volume="/var/registry/auth:/auth"
+    docker.io/library/registry:2

test-playbooks/registry/test-registry-post.yaml

@@ -1,24 +1,24 @@
 - hosts: all
   tasks:
     - name: List containers
-      command: "docker ps -a --format '{{ '{{ .Names }}' }}'"
+      command: "{{ container_command }} ps -a --format '{{ '{{ .Names }}' }}'"
       register: docker_containers
       ignore_errors: true
 
     - name: Create container log dir
       file:
-        path: "{{ ansible_user_dir }}/zuul-output/logs/docker"
+        path: "{{ ansible_user_dir }}/zuul-output/logs/{{ container_command }}"
         state: directory
 
     - name: Save container logs
       loop: "{{ docker_containers.stdout_lines | default([]) }}"
-      shell: "docker logs {{ item }} &> {{ ansible_user_dir }}/zuul-output/logs/docker/{{ item }}.txt"
+      shell: "{{ container_command }} logs {{ item }} &> {{ ansible_user_dir }}/zuul-output/logs/{{ container_command }}/{{ item }}.txt"
       args:
         executable: /bin/bash
       ignore_errors: true
 
     - name: Open container logs permissions
       file:
-        dest: "{{ ansible_user_dir }}/zuul-output/logs/docker"
+        dest: "{{ ansible_user_dir }}/zuul-output/logs/{{ container_command }}"
         mode: u=rwX,g=rX,o=rX
         recurse: yes

test-playbooks/registry/test-registry-pre.yaml

@@ -4,10 +4,11 @@
 # though that obviously happens in configuration management rather
 # than a job).
 - hosts: builder:intermediate-registry
-  name: Set up docker and iptables configuration for registry hosts
-  roles:
-    - install-docker
+  name: "Set up container system and iptables configuration for registry hosts"
   tasks:
+    - name: Install container system
+      include_role:
+        name: "install-{{ container_command }}"
     - name: Open the IPv4 port for the buildset registry
       become: true
       iptables:

test-playbooks/registry/test-registry.yaml

@@ -13,8 +13,6 @@
     - name: Run the intermediate registry
       include_role:
         name: run-test-intermediate-registry
-        apply:
-          become: true
     - name: Install the intermediate registry cert
       include_role:
         name: install-registry-cert
@@ -25,17 +23,18 @@
     - name: Set up user credentials for the intermediate registry
       include_role:
         name: intermediate-registry-user-config
-    - name: Build a docker image for the previous build
+    - name: "Build a container image for the previous build"
       include_role:
-        name: build-docker-image
+        name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
         docker_images:
           - context: test-playbooks/registry/docker
             repository: "{{ previous_build_repository }}"
+        container_images: "{{ docker_images }}"
     - name: Tag the previous build
-      command: "docker tag {{ previous_build_repository }}:latest localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
+      command: "{{ container_command }} tag {{ previous_build_repository }}:latest localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
     - name: Push the previous build to the intermediate registry
-      command: "docker push localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
+      command: "{{ container_command }} push localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
 
 # This is also essentially pre-configuration for the real test of the
 # roles.  This sets up a fake executor (since we can't run the
@@ -112,16 +111,16 @@
     - name: Include previous build vars
       include_vars: vars/previous-build.yaml
     - name: Pull the previous build from buildset registry to the builder host
-      command: "docker pull {{ previous_build_repository }}:latest"
-    - name: Show local docker images for debugging
-      command: "docker image ls"
+      command: "{{ container_command }} pull {{ previous_build_repository }}:latest"
+    - name: "Show local container images for debugging"
+      command: "{{ container_command }} image ls"
     - name: Verify previously built image is in buildset registry
-      command: "docker image inspect {{ previous_build_repository }}:latest"
+      command: "{{ container_command }} image inspect {{ previous_build_repository }}:latest"
 
 # Back to straightforward use of the roles under test.
 
 - hosts: builder
-  name: Test building a docker image
+  name: Test building a container image
   tasks:
 
     - name: Create fake sibling projects
@@ -133,7 +132,7 @@
 
     - name: Build docker image
       include_role:
-        name: build-docker-image
+        name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
         docker_images:
           - context: test-playbooks/registry/docker
@@ -141,6 +140,7 @@
             siblings:
               - opendev.org/fake-sibling-1
               - opendev.org/fake-sibling-2
+        container_images: "{{ docker_images }}"
 
 - hosts: executor
   name: Test pushing to the intermediate registry
@@ -154,6 +154,7 @@
         docker_images:
           - context: playbooks/registry/docker
             repository: downstream/image
+        container_images: "{{ docker_images }}"
 
 # And finally an external verification step.
 

test-playbooks/registry/vars/intermediate-registry-auth.yaml

@@ -3,56 +3,59 @@ intermediate_registry:
   port: 5000
   username: "zuul"
   password: dQI83awO8Akuw0WU
+# openssl req -x509 -newkey rsa:2048 -keyout cert.key -out cert.pem -days 365 -nodes -subj '/C=US/ST=California/L=Oakland/O=Company Name/OU=Org/CN=zuul-jobs.intermediate-registry' -addext 'subjectAltName = DNS:zuul-jobs.intermediate-registry,DNS:localhost,IP:127.0.0.1'
 intermediate_registry_tls_key: |
   -----BEGIN PRIVATE KEY-----
-  MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDYkpjfIz7bziCa
-  mFrWqQ84ldeAs2jvSKs2JG0RhYNNLokr2AU/5TUvqtAisyyd5AX5dBHQ7u/7Vgmj
-  towt7loFfAG/2/rpdSGi2Njx11roBUoDsjwdE9w3aNnrDvOCyJcepx5TWYS86+vZ
-  IqodvdnuoWTk9VuolWfHsCgPRQV4uwMbIC5kbv2o4FORsOEzbuRfCEX9UTcAMEGg
-  K/m/kM/valkrYeBbLILsOcivg4Jh0m+PFC7NTcQFo+uwpZzZvlNtVbmQ3LqkHDAE
-  KDK94uBcQtdYjvvl6UZ+pNo+puD9iakYtcpQFuU8rpavMLE87+SuPVgi2Rk6QtTz
-  OAP2mDMJAgMBAAECggEBANM9MfS7WQ1mIXEI19l2roz/wmIbHGgAllbJ8sRbWLWI
-  hW0JWB15gIYM8tRVtVgP2C/3IYWL+PFKez5+yH3odU/SI5ayhyr8/6DqJ7jD2Dxl
-  JEs0puOpwmsdTyixvZy78IKKeM7NiuYGq1VwNUOrMQ1LyLB2DUAC8mXYkUpLhUm6
-  O4wVaGie7XwMOJazRs66ceU9k7Nuv3b57yc3PN2bzTqYUVjmJ1XeuAiBJaAeHts6
-  NfG1+vO9xLXIRTRWvDGKByNsYJJLLPOXZkQZZFYYe8TTduxyCmZgShY6sZmmnWua
-  cAdBL6b/5B3PZ2SkhdLHklaZmH8PTeAoqI2RDz/8eIECgYEA8gofU8LrK1Xjgrig
-  ItQxYxqZCrggm9lMMcaADc7u3nff68NyImZ5bSXhvZCu74cAIMx12HbU1UvSCsQ4
-  /cncHrlBOzG529878+iWgiUrJ29GsQiHGj+qHA4qGBSP0Qan7ISunskj4GezTeHd
-  /A3oTn5rLuld9V++647O35lXArkCgYEA5RBwV5nle49UT38hNqL/K+TUX5oZJXB8
-  Xl9FT1L799toHUPEWEkSpf7Suf1hDwv6+tsIPO6tN7YirxK390JRxPaT948J8n1d
-  TkurGDs1uwLQdUWgXIwvQ8ms+8rYvTU7vg2hI7/BZhH09LmGCiYSwnem0QYXjGnc
-  kk56VeExytECgYBmBDw2Ctcied4eEAF3DKcQVXqiGP+tkMZbyIXazBjEbhRUhBmM
-  RFLz3V6rjtsdHHLCYEtfhJ6qlH2gihpXZgjAbmb/MzNaaFoVsTgW/OGWioFqRuTi
-  /GiP0KyPX8NKYBrRRw9u3+qeQDdEIWp2Pcpno0M8D6LJtKR9FsE9X51cCQKBgQCs
-  8u5/ldjoo91acHhZUlQrhgi7bhQSao3ciz4/mD5ac7R2dBYpOnL0FiRw/VhtDfSf
-  twTPTL5IVCJ34UA5Vj964VnzDnLKPdFXLlauYvY8jvFpufpMJiQBoKIVMqDWqvzC
-  kHPcFAon0OMMa49C1mBPqBuxslHRWJSLeulvMipwIQKBgDFzDTH49cmKP8YQmCuT
-  vC5PJJ+hutbf/dOVJuOZ5KlKwnRkbMwoamYKrkjgmWMBgtzyz12/a46lZ58ul4xW
-  1fKw/nx8uQcbnKnigyjsAUzI9FgBR4d10cYdxPlfYVmj4TAUA3os5Gu6VKySy6SV
-  xuHEIA6nFsXLXGBu25vI5tEv
+  MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC46dQ/20Zsjel3
+  7D2F+9+9WqslsIWfiP+zsqsz+/K5ngkdYyBEjYCBAUoLmGY/6/HkvZRBYE79R2FK
+  gjKIIo/bElKI4H6jq2nke5No+nroPXRlFh7wu0yP+U3P6pSaaDKJuJ5mMXxcboZE
+  z8TyjRs1+RaFo+walNNfcA/ZOg8JRWV5Fe4JBw7GjgR6GH265h6zppakg801rFXV
+  zbf9sCRz7ic3vpNywGgz8klwqQVR3H1GlZ5zvlDr6/lPin+YwlXRd1wgRfFIima9
+  K+IU+nymnBExInO5AyomolpN+bn4bnrx6q2l/FKWvEssVKZPPjT8v7lbzBiXxvb2
+  AKYmNjL5AgMBAAECggEARnqBNpGKBwgT62x0iqPUxGRRhT0BwSvDYieAT4EBI7RT
+  fwrwGpDgYMswALlmh4iTmv6TClP951WUhISZY//gWrxiDt+aBSHpa3eaWNHXlLsP
+  qRPEWTbaWKnJ+axMVYnPcWSXoxXLc6OAs6uJQnV74Jd++RLgg8Ujx2V79OzHHF3c
+  AwwH1NHHWXPaxrItB+nLiV0Q9eQh5nibW12IFmyknYaAeYmanzVwDplBubtsS1T3
+  X1kjUUaG58qCT/XyyM4YvagaDyy75T6J4XYnRsV7b/FKwc8FuF0vgbI+yY3B8nrz
+  h4z14QLNvNmUiGbkJRpDzKQb/BeWvT/GPXzvD2ObHQKBgQDfrxIiO4bWa+IjJNSn
+  FySIBBoKzh70LC8ElQ8AkrzjeucgtPQIY0zJcdT9nhAcS4mYqw4tp1snhm9mbyuB
+  huF83MwNFJ/O46IrWWpji0fXKQGgmPNex7yDGHYaVAE/nbzajGXXlGB1+w3tHCvM
+  1fxKxtLURHNtjfDBZUqDcz+PhwKBgQDToNXiMo5fKc3PninxaHrjnDQmDxk5t1y6
+  hEOTJf12BJDw4syh3YzxFcAH0CA47OTy7o7dMZVt5RZ53XKR5fbYfKD7KGX/claw
+  sfutpskuVgFj4pnwShylFB6dQueFiHcWHf/DjMJmKyNM1dxzZcfqnH6P7FZaiYoW
+  eoLdy1vJfwKBgQCtTL845HOgNq9aWROkbQqxkrP2gSF8Pasj2rRn1kgf2j2tmmSj
+  BwQb4mSJJegHdAKj1ItEla/K3J38d872KGEU0yAIVl1F9hjTixAhFWzQZwXKvhV3
+  7jnAO7hsx368IeKKVFInBt8BKUPt23CX34X7DTWUnX/sdhb8TxS+6RBqiwKBgQCF
+  3GAtuejQTPL/9n11U68XtcBOqpI8Lb2bxPmxZABU3EKJ/AuP/0GdZTKYPo+DMmUH
+  PNplE23/mz6CSw6jNqDTAtIYy87oq3wmPA6EItFyW7h5Y+YXVemUiYtr0dv8XPtm
+  pAcZvDliwrqLaWMOIz03K1Hq24Urs4ADA+8vN+iRJQKBgQCtNXP4sTXjRhO+leiM
+  3YXc/qBof9TNlMcKS0g6C0s/+KFZ1CG3DbN0CizDCxqYWedOB7CKWklmfg7ENEhL
+  NI6NTo10Q376UZE1+TmaWjGdIdvaxDnUeabSeqUXQxinWOS1pGzMgwXULw1BMLCq
+  Zy9ZnBgOFe2NJl4U7EN8Xdmfrw==
   -----END PRIVATE KEY-----
 intermediate_registry_tls_cert: |
   -----BEGIN CERTIFICATE-----
-  MIIDtDCCApygAwIBAgIJANpxowfzYw4vMA0GCSqGSIb3DQEBCwUAMG8xCzAJBgNV
-  BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-  aWRnaXRzIFB0eSBMdGQxKDAmBgNVBAMMH3p1dWwtam9icy5pbnRlcm1lZGlhdGUt
-  cmVnaXN0cnkwHhcNMTkwNTMwMjAwOTQxWhcNMzkwNTI1MjAwOTQxWjBvMQswCQYD
-  VQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQg
-  V2lkZ2l0cyBQdHkgTHRkMSgwJgYDVQQDDB96dXVsLWpvYnMuaW50ZXJtZWRpYXRl
-  LXJlZ2lzdHJ5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2JKY3yM+
-  284gmpha1qkPOJXXgLNo70irNiRtEYWDTS6JK9gFP+U1L6rQIrMsneQF+XQR0O7v
-  +1YJo7aMLe5aBXwBv9v66XUhotjY8dda6AVKA7I8HRPcN2jZ6w7zgsiXHqceU1mE
-  vOvr2SKqHb3Z7qFk5PVbqJVnx7AoD0UFeLsDGyAuZG79qOBTkbDhM27kXwhF/VE3
-  ADBBoCv5v5DP72pZK2HgWyyC7DnIr4OCYdJvjxQuzU3EBaPrsKWc2b5TbVW5kNy6
-  pBwwBCgyveLgXELXWI775elGfqTaPqbg/YmpGLXKUBblPK6WrzCxPO/krj1YItkZ
-  OkLU8zgD9pgzCQIDAQABo1MwUTAdBgNVHQ4EFgQU00qH9bMUPRacZwgvBgczgR8Z
-  424wHwYDVR0jBBgwFoAU00qH9bMUPRacZwgvBgczgR8Z424wDwYDVR0TAQH/BAUw
-  AwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAHEX2Tw19w5okaJ+6gHMFjA338ffwU9n5
-  2piBMypbYr50yyPyUaTmz4SIBsTLkIWu00a0pdo9pqZDnv1KwxtJtP4o4qQXhMd4
-  Ve3FFF+6AMaOy5y5+hRkE8iHOOik/rNPFqkVDatNGuOMSNYO/jUFXc+C6Ol7gM/J
-  edyWaafjQbvdKapKPbdP4Y69R8OlRTNK1lJMIGJrsCdaeaK4EpLpbJPHnagIMdmQ
-  HDsTf978weRrjJ4JEODTabsKVHKyx0GBwe8CmR0NzpfO2ORCyNUO1rLK2rzh5YTQ
-  qKGyfY0DAyiSHxKaUeGiskc4/WMxaYv2FzD63Xvzmot9atSwCMjN1A==
+  MIIEKDCCAxCgAwIBAgIUWVQQugUNh53VhvVfb3S49zw3GvgwDQYJKoZIhvcNAQEL
+  BQAwgYMxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRAwDgYDVQQH
+  DAdPYWtsYW5kMRUwEwYDVQQKDAxDb21wYW55IE5hbWUxDDAKBgNVBAsMA09yZzEo
+  MCYGA1UEAwwfenV1bC1qb2JzLmludGVybWVkaWF0ZS1yZWdpc3RyeTAeFw0xOTEx
+  MjExODQ5MjhaFw0yMDExMjAxODQ5MjhaMIGDMQswCQYDVQQGEwJVUzETMBEGA1UE
+  CAwKQ2FsaWZvcm5pYTEQMA4GA1UEBwwHT2FrbGFuZDEVMBMGA1UECgwMQ29tcGFu
+  eSBOYW1lMQwwCgYDVQQLDANPcmcxKDAmBgNVBAMMH3p1dWwtam9icy5pbnRlcm1l
+  ZGlhdGUtcmVnaXN0cnkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4
+  6dQ/20Zsjel37D2F+9+9WqslsIWfiP+zsqsz+/K5ngkdYyBEjYCBAUoLmGY/6/Hk
+  vZRBYE79R2FKgjKIIo/bElKI4H6jq2nke5No+nroPXRlFh7wu0yP+U3P6pSaaDKJ
+  uJ5mMXxcboZEz8TyjRs1+RaFo+walNNfcA/ZOg8JRWV5Fe4JBw7GjgR6GH265h6z
+  ppakg801rFXVzbf9sCRz7ic3vpNywGgz8klwqQVR3H1GlZ5zvlDr6/lPin+YwlXR
+  d1wgRfFIima9K+IU+nymnBExInO5AyomolpN+bn4bnrx6q2l/FKWvEssVKZPPjT8
+  v7lbzBiXxvb2AKYmNjL5AgMBAAGjgZEwgY4wHQYDVR0OBBYEFCXcx6YJW0L1JMSA
+  rQDbbc9LyQN3MB8GA1UdIwQYMBaAFCXcx6YJW0L1JMSArQDbbc9LyQN3MA8GA1Ud
+  EwEB/wQFMAMBAf8wOwYDVR0RBDQwMoIfenV1bC1qb2JzLmludGVybWVkaWF0ZS1y
+  ZWdpc3RyeYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBMQR4u
+  bcdeS6ML/X/BLh3HBjWf0DYobU5GVBoMC9c+L9Fxh82ck/CAK3Oeozr9iHFu5YLj
+  OsJWlAbRl0Getz7HOnVH9rMyL/ac9c99CKixjY1Vsf49x1itpOQULoZ+zJixFROk
+  07KhnkaqsYs4SIfDSoa18UmBROEVT2y7yT0uYAwyxwMtZVJWUg7L9OuxPE/tMB0/
+  NyNwMzhdKBL0V54rXH0dxOQ0yE5mGkaOOgKS5x43r78xRRNZ3JM5iRj3S0P75Nbg
+  YDvkkOd0Pf+5UPBJyc4wh5TA+vOrU63lKa6RwIWIbA+xXJn5WJQFoQOjO2dCcEka
+  8p2tutWB2+G+3F12
   -----END CERTIFICATE-----
-#intermediate_registry_tls_chain

test-playbooks/registry/vars/previous-build.yaml

@@ -2,7 +2,7 @@
 # buildset via provides/requires.  This build should be copied from
 # the intermediate registry to the buildset registry.
 
-previous_build_repository: upstream/image
+previous_build_repository: docker.io/upstream/image
 previous_build_uuid: 48a84fe22a744cb5b0310f396358d912
 previous_build_zuul:
   artifacts:

zuul-tests.d/container-roles-jobs.yaml

@@ -1,5 +1,5 @@
 - job:
-    name: zuul-jobs-test-registry
+    name: zuul-jobs-test-registry-docker
     description: |
       Test the intermediate registry roles.
 
@@ -17,6 +17,8 @@
     pre-run: test-playbooks/registry/test-registry-pre.yaml
     run: test-playbooks/registry/test-registry.yaml
     post-run: test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: docker
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -26,6 +28,36 @@
         - name: builder
           label: ubuntu-bionic
 
+- job:
+    name: zuul-jobs-test-registry-podman
+    description: |
+      Test the intermediate registry roles.
+
+      This job tests changes to the intermediate registry roles using
+      podman rather than docker.  It is not meant to be used directly
+      but rather run on changes to roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/install-podman/.*
+      - roles/build-container-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    pre-run: test-playbooks/registry/test-registry-pre.yaml
+    run: test-playbooks/registry/test-registry.yaml
+    post-run: test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: podman
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-bionic-limestone-debug
+        - name: executor
+          label: ubuntu-bionic-limestone-debug
+        - name: builder
+          label: ubuntu-bionic-limestone-debug
+
 - job:
     name: zuul-jobs-test-registry-buildset-registry
     parent: opendev-buildset-registry
@@ -52,6 +84,8 @@
     pre-run: test-playbooks/registry/buildset-registry-pre.yaml
     run: test-playbooks/registry/buildset-registry.yaml
     post-run: test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: docker
 
 - job:
     name: zuul-jobs-test-registry-buildset-registry-k8s-docker
@@ -74,6 +108,8 @@
     post-run:
       - test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
       - test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: docker
 
 - job:
     name: zuul-jobs-test-install-kubernetes-docker
@@ -126,7 +162,8 @@
 - project:
     check:
       jobs: &id001
-        - zuul-jobs-test-registry
+        - zuul-jobs-test-registry-docker
+        - zuul-jobs-test-registry-podman
         - zuul-jobs-test-registry-buildset-registry
         - zuul-jobs-test-registry-buildset-registry-k8s-docker
         - zuul-jobs-test-install-kubernetes-docker