From 23826e05a65c9f11406279fde4dfbea6782da13c Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 22 Apr 2019 09:04:10 -0700 Subject: [PATCH] Only alias docker registries on test nodes Skopeo has problems with ipv6 address literals just like docker as they use the same underlying checks for url validity. However, we think we can fix that by using a port forward from the executor to the buildset registry so that skopeo can connect via ipv4 Go back to aliases the registries on test nodes via /etc/hosts. Change-Id: I5f9316ffe84de06cb2fb2b65a7e1c31d9f8b0e35 Co-Authored-By: James E. Blair --- roles/build-docker-image/tasks/push.yaml | 24 +++++++++++++-- .../README.rst | 4 +-- .../tasks/main.yaml | 12 ++++++-- .../push-to-intermediate-registry/README.rst | 4 +-- .../tasks/push-image.yaml | 2 +- .../tasks/push.yaml | 11 ++++++- roles/run-buildset-registry/tasks/main.yaml | 2 +- roles/use-buildset-registry/tasks/main.yaml | 30 +++++++++++++++---- .../tasks/user-config.yaml | 4 +-- 9 files changed, 75 insertions(+), 18 deletions(-) diff --git a/roles/build-docker-image/tasks/push.yaml b/roles/build-docker-image/tasks/push.yaml index 582849329..e37d9ea80 100644 --- a/roles/build-docker-image/tasks/push.yaml +++ b/roles/build-docker-image/tasks/push.yaml @@ -1,12 +1,32 @@ +# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag +# so we set up /etc/hosts with a registry alias name to support ipv6 and 4. +- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses + become: yes + lineinfile: + path: /etc/hosts + state: present + regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$" + line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry" + insertafter: EOF + when: buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using ip + set_fact: + buildset_registry_alias: zuul-jobs.buildset_registry + when: buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using name + set_fact: + buildset_registry_alias: "{{ buildset_registry.host }}" + when: not ( buildset_registry.host | ipaddr ) + - name: Tag image for buildset registry command: >- - docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} + docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} loop: "{{ image.tags | default(['latest']) }}" loop_control: loop_var: image_tag - name: Push tag to buildset registry command: >- - docker push {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} + docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} loop: "{{ image.tags | default(['latest']) }}" loop_control: loop_var: image_tag diff --git a/roles/pull-from-intermediate-registry/README.rst b/roles/pull-from-intermediate-registry/README.rst index 435d3453f..a0be3ab2d 100644 --- a/roles/pull-from-intermediate-registry/README.rst +++ b/roles/pull-from-intermediate-registry/README.rst @@ -10,8 +10,8 @@ root of the job dependency graph). This requires the :zuul:role:`run-buildset-registry` role already applied. It also requires an externally managed "intermediate" -registry operating for the use of Zuul, and it requires "skopeo" to be -installed on the Zuul executors. +registry operating for the use of Zuul, and it requires "skopeo" and +"socat" to be installed on the Zuul executors. **Role Variables** diff --git a/roles/pull-from-intermediate-registry/tasks/main.yaml b/roles/pull-from-intermediate-registry/tasks/main.yaml index 448f01cf7..6fd26d892 100644 --- a/roles/pull-from-intermediate-registry/tasks/main.yaml +++ b/roles/pull-from-intermediate-registry/tasks/main.yaml @@ -12,6 +12,14 @@ content: "{{ buildset_registry.cert }}" dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" +# Start a socat tunnel to the buildset registry to work around the +# fact that docker does not correctly parse ipv6 addresses. The socat +# process will terminate when the playbook ends. +- name: Start socat to work around https://github.com/moby/moby/issues/39033 + shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &" +- name: Find socat port + set_fact: + socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}" # Update user config for intermediate and buildset registries - name: Ensure docker user directory exists @@ -44,7 +52,7 @@ { "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}": {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"}, - "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": + "127.0.0.1:{{ socat_port }}": {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, } set_fact: @@ -62,7 +70,7 @@ command: >- skopeo --insecure-policy copy {{ item.url }} - docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }} + docker://127.0.0.1:{{ socat_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }} when: "'metadata' in item and item.metadata.type | default('') == 'container_image'" loop: "{{ zuul.artifacts | default([]) }}" always: diff --git a/roles/push-to-intermediate-registry/README.rst b/roles/push-to-intermediate-registry/README.rst index ee3939f93..93a2a6ee4 100644 --- a/roles/push-to-intermediate-registry/README.rst +++ b/roles/push-to-intermediate-registry/README.rst @@ -8,8 +8,8 @@ image build. This requires the :zuul:role:`run-buildset-registry` role already applied. It also requires an externally managed "intermediate" -registry operating for the use of Zuul, and it requires "skopeo" to be -installed on the Zuul executors. +registry operating for the use of Zuul, and it requires "skopeo" and +"socat" to be installed on the Zuul executors. **Role Variables** diff --git a/roles/push-to-intermediate-registry/tasks/push-image.yaml b/roles/push-to-intermediate-registry/tasks/push-image.yaml index 6e1e72d59..a1a2e28c8 100644 --- a/roles/push-to-intermediate-registry/tasks/push-image.yaml +++ b/roles/push-to-intermediate-registry/tasks/push-image.yaml @@ -1,7 +1,7 @@ - name: Push tag to intermediate registry command: >- skopeo --insecure-policy copy - docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }} + docker://127.0.0.1:{{ socat_port }}/{{ image.repository }}:{{ image_tag }} docker://{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }} loop: "{{ image.tags | default(['latest']) }}" loop_control: diff --git a/roles/push-to-intermediate-registry/tasks/push.yaml b/roles/push-to-intermediate-registry/tasks/push.yaml index 3c71df47a..42ae563a1 100644 --- a/roles/push-to-intermediate-registry/tasks/push.yaml +++ b/roles/push-to-intermediate-registry/tasks/push.yaml @@ -12,6 +12,15 @@ content: "{{ buildset_registry.cert }}" dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" +# Start a socat tunnel to the buildset registry to work around the +# fact that docker does not correctly parse ipv6 addresses. The socat +# process will terminate when the playbook ends. +- name: Start socat to work around https://github.com/moby/moby/issues/39033 + shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &" +- name: Find socat port + set_fact: + socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}" + # Update user config for intermediate and buildset registries - name: Ensure docker user directory exists file: @@ -43,7 +52,7 @@ { "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}": {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"}, - "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": + "localhost:{{ socat_port }}": {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, } set_fact: diff --git a/roles/run-buildset-registry/tasks/main.yaml b/roles/run-buildset-registry/tasks/main.yaml index 3f7c858d9..226e5f764 100644 --- a/roles/run-buildset-registry/tasks/main.yaml +++ b/roles/run-buildset-registry/tasks/main.yaml @@ -44,7 +44,7 @@ path: "{{ buildset_registry_root}}/certs/domain.csr" privatekey_path: "{{ buildset_registry_root}}/certs/domain.key" common_name: "{{ ansible_host }}" - subject_alt_name: "DNS:{{ ansible_host }},IP:{{ ansible_host }}" + subject_alt_name: "DNS:zuul-jobs.buildset-registry,DNS:{{ ansible_host }},IP:{{ ansible_host }},IP:127.0.0.1" - name: Generate a TLS cert for the Docker registry openssl_certificate: path: "{{ buildset_registry_root}}/certs/domain.crt" diff --git a/roles/use-buildset-registry/tasks/main.yaml b/roles/use-buildset-registry/tasks/main.yaml index f98820c91..97c5bb9ce 100644 --- a/roles/use-buildset-registry/tasks/main.yaml +++ b/roles/use-buildset-registry/tasks/main.yaml @@ -1,3 +1,23 @@ +# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag +# so we set up /etc/hosts with a registry alias name to support ipv6 and 4. +- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses + become: yes + lineinfile: + path: /etc/hosts + state: present + regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$" + line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry" + insertafter: EOF + when: buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using ip + set_fact: + buildset_registry_alias: zuul-jobs.buildset_registry + when: buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using name + set_fact: + buildset_registry_alias: "{{ buildset_registry.host }}" + when: not ( buildset_registry.host | ipaddr ) + - name: Ensure docker directory exists become: yes file: @@ -6,23 +26,23 @@ - name: Ensure buildset registry cert directory exists become: true file: - path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/" + path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/" state: directory - name: Ensure proxy registry cert directory exists become: true file: - path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/" + path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/" state: directory - name: Write buildset registry TLS certificate become: true copy: content: "{{ buildset_registry.cert }}" - dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt" + dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/ca.crt" - name: Write proxy registry TLS certificate become: true copy: content: "{{ buildset_registry.cert }}" - dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/ca.crt" + dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/ca.crt" # Update daemon config - name: Check if docker daemon configuration exists @@ -46,7 +66,7 @@ - name: Add registry to docker daemon configuration vars: new_config: - registry-mirrors: "['https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port}}/']" + registry-mirrors: "['https://{{ buildset_registry_alias }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port}}/']" set_fact: docker_config: "{{ docker_config | combine(new_config) }}" - name: Save docker daemon configuration diff --git a/roles/use-buildset-registry/tasks/user-config.yaml b/roles/use-buildset-registry/tasks/user-config.yaml index 56b2d2124..09b0602df 100644 --- a/roles/use-buildset-registry/tasks/user-config.yaml +++ b/roles/use-buildset-registry/tasks/user-config.yaml @@ -29,9 +29,9 @@ { "https://index.docker.io/v1/": {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, - "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}": + "{{ buildset_registry_alias }}:{{ buildset_registry.port }}": {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, - "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}": + "{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}": {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"} } set_fact: