Only alias docker registries on test nodes

Skopeo has problems with ipv6 address literals just like docker as they
use the same underlying checks for url validity. However, we think we
can fix that by using a port forward from the executor to the buildset
registry so that skopeo can connect via ipv4

Go back to aliases the registries on test nodes via /etc/hosts.

Change-Id: I5f9316ffe84de06cb2fb2b65a7e1c31d9f8b0e35
Co-Authored-By: James E. Blair <jeblair@redhat.com>
This commit is contained in:
Clark Boylan 2019-04-22 09:04:10 -07:00 committed by James E. Blair
parent 20c7a54390
commit 23826e05a6
9 changed files with 75 additions and 18 deletions

View File

@ -1,12 +1,32 @@
# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
# so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
become: yes
lineinfile:
path: /etc/hosts
state: present
regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$"
line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry"
insertafter: EOF
when: buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using ip
set_fact:
buildset_registry_alias: zuul-jobs.buildset_registry
when: buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using name
set_fact:
buildset_registry_alias: "{{ buildset_registry.host }}"
when: not ( buildset_registry.host | ipaddr )
- name: Tag image for buildset registry - name: Tag image for buildset registry
command: >- command: >-
docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
loop: "{{ image.tags | default(['latest']) }}" loop: "{{ image.tags | default(['latest']) }}"
loop_control: loop_control:
loop_var: image_tag loop_var: image_tag
- name: Push tag to buildset registry - name: Push tag to buildset registry
command: >- command: >-
docker push {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
loop: "{{ image.tags | default(['latest']) }}" loop: "{{ image.tags | default(['latest']) }}"
loop_control: loop_control:
loop_var: image_tag loop_var: image_tag

View File

@ -10,8 +10,8 @@ root of the job dependency graph).
This requires the :zuul:role:`run-buildset-registry` role already This requires the :zuul:role:`run-buildset-registry` role already
applied. It also requires an externally managed "intermediate" applied. It also requires an externally managed "intermediate"
registry operating for the use of Zuul, and it requires "skopeo" to be registry operating for the use of Zuul, and it requires "skopeo" and
installed on the Zuul executors. "socat" to be installed on the Zuul executors.
**Role Variables** **Role Variables**

View File

@ -12,6 +12,14 @@
content: "{{ buildset_registry.cert }}" content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt"
# Start a socat tunnel to the buildset registry to work around the
# fact that docker does not correctly parse ipv6 addresses. The socat
# process will terminate when the playbook ends.
- name: Start socat to work around https://github.com/moby/moby/issues/39033
shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
- name: Find socat port
set_fact:
socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
# Update user config for intermediate and buildset registries # Update user config for intermediate and buildset registries
- name: Ensure docker user directory exists - name: Ensure docker user directory exists
@ -44,7 +52,7 @@
{ {
"{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}": "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}":
{"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"}, {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
"{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": "127.0.0.1:{{ socat_port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
} }
set_fact: set_fact:
@ -62,7 +70,7 @@
command: >- command: >-
skopeo --insecure-policy copy skopeo --insecure-policy copy
{{ item.url }} {{ item.url }}
docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }} docker://127.0.0.1:{{ socat_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
when: "'metadata' in item and item.metadata.type | default('') == 'container_image'" when: "'metadata' in item and item.metadata.type | default('') == 'container_image'"
loop: "{{ zuul.artifacts | default([]) }}" loop: "{{ zuul.artifacts | default([]) }}"
always: always:

View File

@ -8,8 +8,8 @@ image build.
This requires the :zuul:role:`run-buildset-registry` role already This requires the :zuul:role:`run-buildset-registry` role already
applied. It also requires an externally managed "intermediate" applied. It also requires an externally managed "intermediate"
registry operating for the use of Zuul, and it requires "skopeo" to be registry operating for the use of Zuul, and it requires "skopeo" and
installed on the Zuul executors. "socat" to be installed on the Zuul executors.
**Role Variables** **Role Variables**

View File

@ -1,7 +1,7 @@
- name: Push tag to intermediate registry - name: Push tag to intermediate registry
command: >- command: >-
skopeo --insecure-policy copy skopeo --insecure-policy copy
docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} docker://127.0.0.1:{{ socat_port }}/{{ image.repository }}:{{ image_tag }}
docker://{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }} docker://{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }}
loop: "{{ image.tags | default(['latest']) }}" loop: "{{ image.tags | default(['latest']) }}"
loop_control: loop_control:

View File

@ -12,6 +12,15 @@
content: "{{ buildset_registry.cert }}" content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt"
# Start a socat tunnel to the buildset registry to work around the
# fact that docker does not correctly parse ipv6 addresses. The socat
# process will terminate when the playbook ends.
- name: Start socat to work around https://github.com/moby/moby/issues/39033
shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
- name: Find socat port
set_fact:
socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
# Update user config for intermediate and buildset registries # Update user config for intermediate and buildset registries
- name: Ensure docker user directory exists - name: Ensure docker user directory exists
file: file:
@ -43,7 +52,7 @@
{ {
"{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}": "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}":
{"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"}, {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
"{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": "localhost:{{ socat_port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
} }
set_fact: set_fact:

View File

@ -44,7 +44,7 @@
path: "{{ buildset_registry_root}}/certs/domain.csr" path: "{{ buildset_registry_root}}/certs/domain.csr"
privatekey_path: "{{ buildset_registry_root}}/certs/domain.key" privatekey_path: "{{ buildset_registry_root}}/certs/domain.key"
common_name: "{{ ansible_host }}" common_name: "{{ ansible_host }}"
subject_alt_name: "DNS:{{ ansible_host }},IP:{{ ansible_host }}" subject_alt_name: "DNS:zuul-jobs.buildset-registry,DNS:{{ ansible_host }},IP:{{ ansible_host }},IP:127.0.0.1"
- name: Generate a TLS cert for the Docker registry - name: Generate a TLS cert for the Docker registry
openssl_certificate: openssl_certificate:
path: "{{ buildset_registry_root}}/certs/domain.crt" path: "{{ buildset_registry_root}}/certs/domain.crt"

View File

@ -1,3 +1,23 @@
# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
# so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
become: yes
lineinfile:
path: /etc/hosts
state: present
regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$"
line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry"
insertafter: EOF
when: buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using ip
set_fact:
buildset_registry_alias: zuul-jobs.buildset_registry
when: buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using name
set_fact:
buildset_registry_alias: "{{ buildset_registry.host }}"
when: not ( buildset_registry.host | ipaddr )
- name: Ensure docker directory exists - name: Ensure docker directory exists
become: yes become: yes
file: file:
@ -6,23 +26,23 @@
- name: Ensure buildset registry cert directory exists - name: Ensure buildset registry cert directory exists
become: true become: true
file: file:
path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/" path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/"
state: directory state: directory
- name: Ensure proxy registry cert directory exists - name: Ensure proxy registry cert directory exists
become: true become: true
file: file:
path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/" path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/"
state: directory state: directory
- name: Write buildset registry TLS certificate - name: Write buildset registry TLS certificate
become: true become: true
copy: copy:
content: "{{ buildset_registry.cert }}" content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/ca.crt"
- name: Write proxy registry TLS certificate - name: Write proxy registry TLS certificate
become: true become: true
copy: copy:
content: "{{ buildset_registry.cert }}" content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/ca.crt" dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/ca.crt"
# Update daemon config # Update daemon config
- name: Check if docker daemon configuration exists - name: Check if docker daemon configuration exists
@ -46,7 +66,7 @@
- name: Add registry to docker daemon configuration - name: Add registry to docker daemon configuration
vars: vars:
new_config: new_config:
registry-mirrors: "['https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port}}/']" registry-mirrors: "['https://{{ buildset_registry_alias }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port}}/']"
set_fact: set_fact:
docker_config: "{{ docker_config | combine(new_config) }}" docker_config: "{{ docker_config | combine(new_config) }}"
- name: Save docker daemon configuration - name: Save docker daemon configuration

View File

@ -29,9 +29,9 @@
{ {
"https://index.docker.io/v1/": "https://index.docker.io/v1/":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
"{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": "{{ buildset_registry_alias }}:{{ buildset_registry.port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
"{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}": "{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"} {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
} }
set_fact: set_fact: