Add a registry test job

This exercises the intermediate and buildset registry roles.

Change-Id: Ic0da2d6b48a7b9f9b616033c70db471ba98181b4

doc/source/index.rst

@@ -8,6 +8,7 @@
    jobs
    roles
    docker-image
+   test-jobs
 
 Indices and tables
 ==================

doc/source/test-jobs.rst

@@ -0,0 +1,9 @@
+Zuul-Jobs Tests
+===============
+
+These are jobs which are used for testing changes to roles in this
+repository.  They are not meant for general consumption and are only
+expected to be run by the OpenDev Zuul.  Generally you can ignore
+these unless you are making changes to the roles in this repository.
+
+.. zuul:autojob:: zuul-jobs-test-registry

roles/pull-from-intermediate-registry/tasks/main.yaml

@@ -9,9 +9,16 @@
 # process will terminate when the playbook ends.
 - name: Start socat to work around https://github.com/moby/moby/issues/39033
   shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
-- name: Find socat port
+
+# Use slurp instead of file lookup to make this testable on a fake
+# executor node.
+- name: Read socat port
+  slurp:
+    src: "{{ zuul.executor.work_root }}/socat_port"
+  register: read_socat_port
+- name: Set socat port
   set_fact:
-    socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
+    socat_port: "{{ read_socat_port['content'] | b64decode | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') | regex_replace('\n', '') }}"
 
 # Set up cert files for the buildset registry
 - name: Ensure registry cert directory exists

roles/push-to-intermediate-registry/tasks/push.yaml

@@ -9,9 +9,16 @@
 # process will terminate when the playbook ends.
 - name: Start socat to work around https://github.com/moby/moby/issues/39033
   shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
-- name: Find socat port
+
+# Use slurp instead of file lookup to make this testable on a fake
+# executor node.
+- name: Read socat port
+  slurp:
+    src: "{{ zuul.executor.work_root }}/socat_port"
+  register: read_socat_port
+- name: Set socat port
   set_fact:
-    socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
+    socat_port: "{{ read_socat_port['content'] | b64decode | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') | regex_replace('\n', '') }}"
 
 # Set up cert files for the buildset registry
 - name: Ensure registry cert directory exists

test-playbooks/registry/docker/Dockerfile

@@ -0,0 +1,2 @@
+FROM debian:testing
+RUN touch /tmp/foo

test-playbooks/registry/host_vars/intermediate-registry.yaml

@@ -0,0 +1 @@
+ansible_python_interpreter: python3

test-playbooks/registry/roles/install-registry-cert/tasks/main.yaml

@@ -0,0 +1,10 @@
+- name: Ensure registry cert directory exists
+  become: true
+  file:
+    path: "/etc/docker/certs.d/{{ registry_host }}:{{ registry_port }}/"
+    state: directory
+- name: Write registry TLS certificate
+  become: true
+  copy:
+    content: "{{ registry_cert }}"
+    dest: "/etc/docker/certs.d/{{ registry_host }}:{{ registry_port }}/ca.crt"

test-playbooks/registry/roles/intermediate-registry-user-config/tasks/main.yaml

@@ -0,0 +1,39 @@
+# Update user config
+- name: Ensure docker user directory exists
+  file:
+    state: directory
+    path: "~/.docker"
+    mode: 0700
+- name: Check if docker user configuration exists
+  stat:
+    path: "~/.docker/config.json"
+  register: docker_config_stat
+- name: Load docker user configuration
+  when: docker_config_stat.stat.exists
+  slurp:
+    path: "~/.docker/config.json"
+  register: docker_config
+- name: Parse docker user configuration
+  when: docker_config_stat.stat.exists
+  set_fact:
+    docker_config: "{{ docker_config.content | b64decode | from_json }}"
+- name: Set default docker user configuration
+  when: not docker_config_stat.stat.exists
+  set_fact:
+    docker_config:
+      auths: {}
+- name: Add registry to docker user configuration
+  vars:
+    new_config:
+      auths: |
+        {
+          "localhost:5000":
+            {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
+        }
+  set_fact:
+    docker_config: "{{ docker_config | combine(new_config, recursive=True) }}"
+- name: Save docker user configuration
+  copy:
+    content: "{{ docker_config | to_nice_json }}"
+    dest: "~/.docker/config.json"
+    mode: 0600

test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml

@@ -0,0 +1,46 @@
+- name: Ensure registry volume directories exists
+  file:
+    state: directory
+    path: "/var/registry/{{ item }}"
+  loop:
+    - certs
+    - auth
+- name: Install python packages
+  package:
+    name:
+      - python3-docker
+      - python3-passlib
+      - python3-bcrypt
+    state: present
+- name: Write htpassword file
+  htpasswd:
+    create: true
+    crypt_scheme: bcrypt
+    path: /var/registry/auth/htpasswd
+    name: "{{ intermediate_registry.username }}"
+    password: "{{ intermediate_registry.password }}"
+- name: Write TLS private key
+  copy:
+    content: "{{ intermediate_registry_tls_key }}"
+    dest: /var/registry/certs/domain.key
+- name: Write TLS certificate
+  copy:
+    content: "{{ intermediate_registry_tls_cert }}{{ intermediate_registry_tls_chain | default('') }}"
+    dest: /var/registry/certs/domain.crt
+- name: Start intermediate docker registry
+  docker_container:
+    name: intermediate_registry
+    image: registry:2
+    state: started
+    restart_policy: always
+    ports:
+     - "5000:5000"
+    env:
+      REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
+      REGISTRY_HTTP_TLS_KEY: /certs/domain.key
+      REGISTRY_AUTH: htpasswd
+      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
+      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
+    volumes:
+      - "/var/registry/certs:/certs"
+      - "/var/registry/auth:/auth"

test-playbooks/registry/test-registry.yaml

@@ -0,0 +1,177 @@
+- hosts: intermediate-registry:builder
+  name: Install docker on registry and builder hosts
+  roles:
+    - install-docker
+
+# Run the intermediate registry on this host, and also build an image
+# and place it in the registry to simulate an artifact from a previous
+# build which has been passed to this one (so that we can test pulling
+# from the intermediate registry in the correct order).
+
+- hosts: intermediate-registry
+  name: Set up the intermediate registry and add a build
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Include previous build vars
+      include_vars: vars/previous-build.yaml
+    - name: Run the intermediate registry
+      include_role:
+        name: run-test-intermediate-registry
+        apply:
+          become: true
+    - name: Install the intermediate registry cert
+      include_role:
+        name: install-registry-cert
+      vars:
+        registry_host: localhost
+        registry_port: 5000
+        registry_cert: "{{ intermediate_registry_tls_cert }}"
+    - name: Set up user credentials for the intermediate registry
+      include_role:
+        name: intermediate-registry-user-config
+    - name: Build a docker image for the previous build
+      include_role:
+        name: build-docker-image
+      vars:
+        docker_images:
+          - context: test-playbooks/registry/docker
+            repository: "{{ previous_build_repository }}"
+    - name: Tag the previous build
+      command: "docker tag {{ previous_build_repository }}:latest localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
+    - name: Push the previous build to the intermediate registry
+      command: "docker push localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
+
+# This is also essentially pre-configuration for the real test of the
+# roles.  This sets up a fake executor (since we can't run the
+# necessary commands untrusted on the real one).
+
+- hosts: executor
+  name: Set up a simulated executor
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Create simulated zuul work directory
+      become: true
+      file:
+        state: directory
+        path: "{{ zuul.executor.work_root }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+    - name: Add project atomic PPA
+      become: true
+      apt_repository:
+        repo: ppa:projectatomic/ppa
+    - name: Install packages
+      become: true
+      package:
+        name:
+          - socat
+          - skopeo
+        state: present
+    - name: Install the intermediate registry cert
+      include_role:
+        name: install-registry-cert
+      vars:
+        registry_host: "{{ intermediate_registry.host }}"
+        registry_port: "{{ intermediate_registry.port }}"
+        registry_cert: "{{ intermediate_registry_tls_cert }}"
+    - name: Make /etc/docker directory zuul-owned
+      become: true
+      file:
+        state: directory
+        path: "/etc/docker"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        recurse: true
+    - name: Configure /etc/hosts for intermediate registry
+      become: true
+      lineinfile:
+        path: /etc/hosts
+        state: present
+        regex: "^{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}$"
+        line: "{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}"
+        insertafter: EOF
+
+# This begins the simulation of what we would expect to happen in a
+# normal job.
+
+- hosts: builder
+  name: Test the buildset registry roles
+  roles:
+    - run-buildset-registry
+    - use-buildset-registry
+
+- hosts: executor
+  name: Test pulling from the intermediate registry
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Include previous build vars
+      include_vars: vars/previous-build.yaml
+    - name: Prepare a replacement zuul variable
+      set_fact:
+        test_zuul: "{{ previous_build_zuul }}"
+    - name: Run pull-from-intermediate-registry role
+      include_role:
+        name: pull-from-intermediate-registry
+      vars:
+        zuul: "{{ test_zuul }}"
+
+# This simulates a build actually using the previous build.
+
+- hosts: builder
+  name: Test that the previous build is available
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Include previous build vars
+      include_vars: vars/previous-build.yaml
+    - name: Pull the previous build from buildset registry to the builder host
+      command: "docker pull {{ previous_build_repository }}:latest"
+    - name: Show local docker images for debugging
+      command: "docker image ls"
+    - name: Verify previously built image is in buildset registry
+      command: "docker image inspect {{ previous_build_repository }}:latest"
+
+# Back to straightforward use of the roles under test.
+
+- hosts: builder
+  name: Test building a docker image
+  roles:
+    - role: build-docker-image
+      vars:
+        docker_images:
+          - context: test-playbooks/registry/docker
+            repository: downstream/image
+
+- hosts: executor
+  name: Test pushing to the intermediate registry
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Run push-to-intermediate-registry role
+      include_role:
+        name: push-to-intermediate-registry
+      vars:
+        docker_images:
+          - context: playbooks/registry/docker
+            repository: downstream/image
+
+# And finally an external verification step.
+
+- hosts: executor
+  name: Test that the newly built image was pushed to the intermediate registry
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Fetch intermediate registry catalog
+      uri:
+        url: "https://{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/v2/_catalog"
+        validate_certs: false
+        user: "{{ intermediate_registry.username }}"
+        password: "{{ intermediate_registry.password }}"
+      register: catalog
+    - name: Verify newly built image is in intermediate registry catalog
+      assert:
+        that: "'downstream/image' in catalog.json.repositories"

test-playbooks/registry/vars/intermediate-registry-auth.yaml

@@ -0,0 +1,58 @@
+intermediate_registry:
+  host: zuul-jobs.intermediate-registry
+  port: 5000
+  username: "zuul"
+  password: dQI83awO8Akuw0WU
+intermediate_registry_tls_key: |
+  -----BEGIN PRIVATE KEY-----
+  MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDYkpjfIz7bziCa
+  mFrWqQ84ldeAs2jvSKs2JG0RhYNNLokr2AU/5TUvqtAisyyd5AX5dBHQ7u/7Vgmj
+  towt7loFfAG/2/rpdSGi2Njx11roBUoDsjwdE9w3aNnrDvOCyJcepx5TWYS86+vZ
+  IqodvdnuoWTk9VuolWfHsCgPRQV4uwMbIC5kbv2o4FORsOEzbuRfCEX9UTcAMEGg
+  K/m/kM/valkrYeBbLILsOcivg4Jh0m+PFC7NTcQFo+uwpZzZvlNtVbmQ3LqkHDAE
+  KDK94uBcQtdYjvvl6UZ+pNo+puD9iakYtcpQFuU8rpavMLE87+SuPVgi2Rk6QtTz
+  OAP2mDMJAgMBAAECggEBANM9MfS7WQ1mIXEI19l2roz/wmIbHGgAllbJ8sRbWLWI
+  hW0JWB15gIYM8tRVtVgP2C/3IYWL+PFKez5+yH3odU/SI5ayhyr8/6DqJ7jD2Dxl
+  JEs0puOpwmsdTyixvZy78IKKeM7NiuYGq1VwNUOrMQ1LyLB2DUAC8mXYkUpLhUm6
+  O4wVaGie7XwMOJazRs66ceU9k7Nuv3b57yc3PN2bzTqYUVjmJ1XeuAiBJaAeHts6
+  NfG1+vO9xLXIRTRWvDGKByNsYJJLLPOXZkQZZFYYe8TTduxyCmZgShY6sZmmnWua
+  cAdBL6b/5B3PZ2SkhdLHklaZmH8PTeAoqI2RDz/8eIECgYEA8gofU8LrK1Xjgrig
+  ItQxYxqZCrggm9lMMcaADc7u3nff68NyImZ5bSXhvZCu74cAIMx12HbU1UvSCsQ4
+  /cncHrlBOzG529878+iWgiUrJ29GsQiHGj+qHA4qGBSP0Qan7ISunskj4GezTeHd
+  /A3oTn5rLuld9V++647O35lXArkCgYEA5RBwV5nle49UT38hNqL/K+TUX5oZJXB8
+  Xl9FT1L799toHUPEWEkSpf7Suf1hDwv6+tsIPO6tN7YirxK390JRxPaT948J8n1d
+  TkurGDs1uwLQdUWgXIwvQ8ms+8rYvTU7vg2hI7/BZhH09LmGCiYSwnem0QYXjGnc
+  kk56VeExytECgYBmBDw2Ctcied4eEAF3DKcQVXqiGP+tkMZbyIXazBjEbhRUhBmM
+  RFLz3V6rjtsdHHLCYEtfhJ6qlH2gihpXZgjAbmb/MzNaaFoVsTgW/OGWioFqRuTi
+  /GiP0KyPX8NKYBrRRw9u3+qeQDdEIWp2Pcpno0M8D6LJtKR9FsE9X51cCQKBgQCs
+  8u5/ldjoo91acHhZUlQrhgi7bhQSao3ciz4/mD5ac7R2dBYpOnL0FiRw/VhtDfSf
+  twTPTL5IVCJ34UA5Vj964VnzDnLKPdFXLlauYvY8jvFpufpMJiQBoKIVMqDWqvzC
+  kHPcFAon0OMMa49C1mBPqBuxslHRWJSLeulvMipwIQKBgDFzDTH49cmKP8YQmCuT
+  vC5PJJ+hutbf/dOVJuOZ5KlKwnRkbMwoamYKrkjgmWMBgtzyz12/a46lZ58ul4xW
+  1fKw/nx8uQcbnKnigyjsAUzI9FgBR4d10cYdxPlfYVmj4TAUA3os5Gu6VKySy6SV
+  xuHEIA6nFsXLXGBu25vI5tEv
+  -----END PRIVATE KEY-----
+intermediate_registry_tls_cert: |
+  -----BEGIN CERTIFICATE-----
+  MIIDtDCCApygAwIBAgIJANpxowfzYw4vMA0GCSqGSIb3DQEBCwUAMG8xCzAJBgNV
+  BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+  aWRnaXRzIFB0eSBMdGQxKDAmBgNVBAMMH3p1dWwtam9icy5pbnRlcm1lZGlhdGUt
+  cmVnaXN0cnkwHhcNMTkwNTMwMjAwOTQxWhcNMzkwNTI1MjAwOTQxWjBvMQswCQYD
+  VQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQg
+  V2lkZ2l0cyBQdHkgTHRkMSgwJgYDVQQDDB96dXVsLWpvYnMuaW50ZXJtZWRpYXRl
+  LXJlZ2lzdHJ5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2JKY3yM+
+  284gmpha1qkPOJXXgLNo70irNiRtEYWDTS6JK9gFP+U1L6rQIrMsneQF+XQR0O7v
+  +1YJo7aMLe5aBXwBv9v66XUhotjY8dda6AVKA7I8HRPcN2jZ6w7zgsiXHqceU1mE
+  vOvr2SKqHb3Z7qFk5PVbqJVnx7AoD0UFeLsDGyAuZG79qOBTkbDhM27kXwhF/VE3
+  ADBBoCv5v5DP72pZK2HgWyyC7DnIr4OCYdJvjxQuzU3EBaPrsKWc2b5TbVW5kNy6
+  pBwwBCgyveLgXELXWI775elGfqTaPqbg/YmpGLXKUBblPK6WrzCxPO/krj1YItkZ
+  OkLU8zgD9pgzCQIDAQABo1MwUTAdBgNVHQ4EFgQU00qH9bMUPRacZwgvBgczgR8Z
+  424wHwYDVR0jBBgwFoAU00qH9bMUPRacZwgvBgczgR8Z424wDwYDVR0TAQH/BAUw
+  AwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAHEX2Tw19w5okaJ+6gHMFjA338ffwU9n5
+  2piBMypbYr50yyPyUaTmz4SIBsTLkIWu00a0pdo9pqZDnv1KwxtJtP4o4qQXhMd4
+  Ve3FFF+6AMaOy5y5+hRkE8iHOOik/rNPFqkVDatNGuOMSNYO/jUFXc+C6Ol7gM/J
+  edyWaafjQbvdKapKPbdP4Y69R8OlRTNK1lJMIGJrsCdaeaK4EpLpbJPHnagIMdmQ
+  HDsTf978weRrjJ4JEODTabsKVHKyx0GBwe8CmR0NzpfO2ORCyNUO1rLK2rzh5YTQ
+  qKGyfY0DAyiSHxKaUeGiskc4/WMxaYv2FzD63Xvzmot9atSwCMjN1A==
+  -----END CERTIFICATE-----
+#intermediate_registry_tls_chain

test-playbooks/registry/vars/previous-build.yaml

@@ -0,0 +1,14 @@
+# This simulates a build from a previous change which appears in this
+# buildset via provides/requires.  This build should be copied from
+# the intermediate registry to the buildset registry.
+
+previous_build_repository: upstream/image
+previous_build_uuid: 48a84fe22a744cb5b0310f396358d912
+previous_build_zuul:
+  artifacts:
+    - url: "docker://{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
+      metadata:
+        repository: "{{ previous_build_repository }}"
+        tag: latest
+        type: container_image
+  executor: "{{ zuul.executor }}"

zuul.yaml

@@ -596,3 +596,32 @@
          Override the default searching above with explicit
          domain/path references (see validate-zone-db role)
     run: playbooks/validate-zone-db/run.yaml
+
+- job:
+    name: zuul-jobs-test-registry
+    description: |
+      Test the intermediate registry roles.
+
+      This job tests changes to the intermediate registry roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    allowed-projects:
+      - zuul/zuul-jobs
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/install-docker/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    run:
+      test-playbooks/registry/test-registry.yaml
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-bionic
+        - name: executor
+          label: ubuntu-bionic
+        - name: builder
+          label: ubuntu-bionic