Browse Source

Fix buildset registry

The approach of having the proxy serve the local data as well as
the remote wasn't working -- it seems that the proxy would always
check upstream and prefer that data even if it had been pushed
locally.

To correct this, separate the data stores of the two registries,
and add both of them to the registry_mirror setting for the
docker daemon.  Now we will pull from our buildset registry first,
and fall back on the proxy to talk to upstream if an image is not
found locally.

The proxy is still required in order to mask out the username and
password which dockerd will otherwise use when talking to upstream.

Change-Id: Iab11954a4b5431d3b1a4d4753f519b6b71f64094
changes/57/640557/1
James E. Blair 2 months ago
parent
commit
9c0d25f349

+ 2
- 2
roles/build-docker-image/tasks/push.yaml View File

@@ -1,12 +1,12 @@
1 1
 - name: Tag image for buildset registry
2 2
   command: >-
3
-    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
3
+    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
4 4
   loop: "{{ image.tags | default(['latest']) }}"
5 5
   loop_control:
6 6
     loop_var: image_tag
7 7
 - name: Push tag to buildset registry
8 8
   command: >-
9
-    docker push {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
9
+    docker push {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
10 10
   loop: "{{ image.tags | default(['latest']) }}"
11 11
   loop_control:
12 12
     loop_var: image_tag

+ 3
- 3
roles/pull-from-intermediate-registry/tasks/main.yaml View File

@@ -5,19 +5,19 @@
5 5
     buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
6 6
 - name: Ensure registry cert directory exists
7 7
   file:
8
-    path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
8
+    path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
9 9
     state: directory
10 10
 - name: Write registry TLS certificate
11 11
   copy:
12 12
     content: "{{ buildset_registry.cert }}"
13
-    dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
13
+    dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
14 14
 - name: Pull artifact from intermediate registry
15 15
   command: >-
16 16
     skopeo --insecure-policy copy
17 17
     --src-creds={{ intermediate_registry.username }}:{{ intermediate_registry.password }}
18 18
     --dest-creds={{ buildset_registry.username }}:{{ buildset_registry.password }}
19 19
     {{ item.url }}
20
-    docker://{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
20
+    docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
21 21
   when: "item.metadata.type | default('') == 'container_image'"
22 22
   loop: "{{ zuul.artifacts | default([]) }}"
23 23
   # no_log: true  TODO(corvus): replace

+ 3
- 9
roles/run-buildset-registry/README.rst View File

@@ -3,9 +3,7 @@ Runs a docker registry for the use of this buildset.
3 3
 This may be used for a single job running on a single node, or it may
4 4
 be used at the root of a job graph so that multiple jobs running for a
5 5
 single change can share the registry.  Two registry endpoints are
6
-provided -- one is a read-only endpoint which acts as a pull-through
7
-proxy and serves upstream images as well as those which are pushed to
8
-the registry.  The second is intended only for pushing images.
6
+provided -- one is a local registry, the second is an upstream proxy.
9 7
 
10 8
 **Role Variables**
11 9
 
@@ -28,13 +26,9 @@ the registry.  The second is intended only for pushing images.
28 26
 
29 27
       The port on which the registry is listening.
30 28
 
31
-   .. zuul:rolevar:: push_host
29
+   .. zuul:rolevar:: proxy_port
32 30
 
33
-      The host (IP address) to use when pushing images to the registry.
34
-
35
-   .. zuul:rolevar:: push_port
36
-
37
-      The port to use when pushing images to the registry.
31
+      The port on which the proxy is listening.
38 32
 
39 33
    .. zuul:rolevar:: username
40 34
 

+ 8
- 11
roles/run-buildset-registry/tasks/main.yaml View File

@@ -59,9 +59,9 @@
59 59
 - name: Decode TLS certificate
60 60
   set_fact:
61 61
     certificate: "{{ certificate.content | b64decode }}"
62
-- name: Start a docker proxy
62
+- name: Start a docker registry
63 63
   docker_container:
64
-    name: buildset_proxy
64
+    name: buildset_registry
65 65
     image: registry:2
66 66
     state: started
67 67
     restart_policy: always
@@ -73,16 +73,12 @@
73 73
       REGISTRY_AUTH: htpasswd
74 74
       REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
75 75
       REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
76
-      REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
77
-      REGISTRY_PROXY_USERNAME: ''
78
-      REGISTRY_PROXY_PASSWORD: ''
79 76
     volumes:
80
-      - "{{ buildset_registry_root}}/data:/var/lib/registry"
81 77
       - "{{ buildset_registry_root}}/certs:/certs"
82 78
       - "{{ buildset_registry_root}}/auth:/auth"
83
-- name: Start a docker registry
79
+- name: Start a docker proxy
84 80
   docker_container:
85
-    name: buildset_registry
81
+    name: buildset_proxy
86 82
     image: registry:2
87 83
     state: started
88 84
     restart_policy: always
@@ -94,8 +90,10 @@
94 90
       REGISTRY_AUTH: htpasswd
95 91
       REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
96 92
       REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
93
+      REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
94
+      REGISTRY_PROXY_USERNAME: ''
95
+      REGISTRY_PROXY_PASSWORD: ''
97 96
     volumes:
98
-      - "{{ buildset_registry_root}}/data:/var/lib/registry"
99 97
       - "{{ buildset_registry_root}}/certs:/certs"
100 98
       - "{{ buildset_registry_root}}/auth:/auth"
101 99
 - name: Set registry information fact
@@ -103,8 +101,7 @@
103 101
     buildset_registry:
104 102
       host: "{{ ansible_host }}"
105 103
       port: 5000
106
-      push_host: "{{ ansible_host }}"
107
-      push_port: 5001
104
+      proxy_port: 5001
108 105
       username: zuul
109 106
       password: "{{ registry_password }}"
110 107
       cert: "{{ certificate }}"

+ 2
- 6
roles/use-buildset-registry/README.rst View File

@@ -17,13 +17,9 @@ Use this role on any host which should use the buildset registry.
17 17
 
18 18
       The port on which the registry is listening.
19 19
 
20
-   .. zuul:rolevar:: push_host
20
+   .. zuul:rolevar:: proxy_port
21 21
 
22
-      The host (IP address) to use when pushing images to the registry.
23
-
24
-   .. zuul:rolevar:: push_port
25
-
26
-      The port to use when pushing images to the registry.
22
+      The port on which the registry proxy is listening.
27 23
 
28 24
    .. zuul:rolevar:: username
29 25
 

+ 7
- 7
roles/use-buildset-registry/tasks/main.yaml View File

@@ -3,26 +3,26 @@
3 3
   file:
4 4
     state: directory
5 5
     path: /etc/docker
6
-- name: Ensure registry cert directory exists
6
+- name: Ensure buildset registry cert directory exists
7 7
   become: true
8 8
   file:
9 9
     path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
10 10
     state: directory
11
-- name: Ensure push registry cert directory exists
11
+- name: Ensure proxy registry cert directory exists
12 12
   become: true
13 13
   file:
14
-    path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
14
+    path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/"
15 15
     state: directory
16
-- name: Write registry TLS certificate
16
+- name: Write buildset registry TLS certificate
17 17
   become: true
18 18
   copy:
19 19
     content: "{{ buildset_registry.cert }}"
20 20
     dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
21
-- name: Write push registry TLS certificate
21
+- name: Write proxy registry TLS certificate
22 22
   become: true
23 23
   copy:
24 24
     content: "{{ buildset_registry.cert }}"
25
-    dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
25
+    dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/ca.crt"
26 26
 
27 27
 # Update daemon config
28 28
 - name: Check if docker daemon configuration exists
@@ -46,7 +46,7 @@
46 46
 - name: Add registry to docker daemon configuration
47 47
   vars:
48 48
     new_config:
49
-      registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/']"
49
+      registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host }}:{{ buildset_registry.proxy_port}}/']"
50 50
   set_fact:
51 51
     docker_config: "{{ docker_config | combine(new_config) }}"
52 52
 - name: Save docker daemon configuration

+ 1
- 1
roles/use-buildset-registry/tasks/user-config.yaml View File

@@ -31,7 +31,7 @@
31 31
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
32 32
           "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
33 33
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
34
-          "{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}":
34
+          "{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}":
35 35
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
36 36
         }
37 37
   set_fact:

Loading…
Cancel
Save