Write buildkitd.toml in use-buildset-registry

It's a generally useful file for people using buildkitd. It's also
more appropriate to write it in use-buildset-registry and then
just have build-docker-image copy it.

Do the same thing with writing the cert - we don't need to know
which path on the host use-buildset-registry wrote the file to,
we can just write the content from the dir into the container,
and then it's just a consistent command to apply it.

Change-Id: Iaa485c2e8628900dccbed1f4b0773b6d1b5f7983

roles/build-docker-image/tasks/setup-buildx.yaml

@@ -1,31 +1,10 @@
-- name: Include OS-specific variables
-  include_vars: "{{ zj_distro_os }}"
-  with_first_found:
-    - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
-    - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml"
-    - "{{ ansible_distribution }}.yaml"
-    - "{{ ansible_os_family }}.yaml"
-    - "default.yaml"
-  loop_control:
-    loop_var: zj_distro_os
-
-- name: Make tempfile for buildkit.toml
-  tempfile:
-    state: file
-  register: buildkit_toml_tmp
-
-- name: Write buildkit.toml file
-  template:
-    dest: '{{ buildkit_toml_tmp.path }}'
-    src: buildkitd.toml.j2
-
 - name: Run binfmt container
   command: docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
 
 - name: Create builder
-  command: 'docker buildx create --name mybuilder --driver-opt network=host --config {{ buildkit_toml_tmp.path }}'
+  command: docker buildx create --name mybuilder --driver-opt network=host --config /etc/buildkit/buildkitd.toml
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
 
@@ -39,11 +18,31 @@
   environment:
     DOCKER_CLI_EXPERIMENTAL: enabled
 
+- name: Make tempfile for registry TLS certificate
+  tempfile:
+    state: file
+  register: buildkit_cert_tmp
+
+- name: Write buildset registry TLS certificate
+  become: true
+  copy:
+    content: "{{ buildset_registry.cert }}"
+    dest: "{{ buildkit_cert_tmp.path }}"
+  when: buildset_registry is defined and buildset_registry.cert
+
 - name: Copy buildset registry TLS cert into worker container
-  command: "docker cp {{ ca_dir }}/buildset-registry.crt buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+  command: "docker cp {{ buildkit_cert_tmp.path }} buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+  when: buildset_registry is defined and buildset_registry.cert
 
 - name: Update CA certs in worker container
   command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+  when: buildset_registry is defined and buildset_registry.cert
+
+- name: Remove TLS cert tempfile
+  file:
+    state: absent
+    path: '{{ buildkit_cert_tmp.path }}'
+  when: buildset_registry is defined and buildset_registry.cert
 
 - name: Make tempfile for /etc/hosts
   tempfile:

roles/build-docker-image/templates/buildkitd.toml.j2

@@ -1,8 +0,0 @@
-[registry."docker.io"]
-  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}"]
-
-[registry."quay.io"]
-  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/quay.io"]
-
-[registry."gcr.io"]
-  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/gcr.io"]

roles/build-docker-image/vars/CentOS.yaml

@@ -1,2 +0,0 @@
-ca_dir: /etc/pki/ca-trust/source/anchors
-ca_command: update-ca-trust

roles/build-docker-image/vars/default.yaml

@@ -1,2 +0,0 @@
-ca_dir: /usr/local/share/ca-certificates
-ca_command: update-ca-certificates

roles/use-buildset-registry/library/modify_buildkitd_toml.py

@@ -0,0 +1,70 @@
+# Copyright 2019 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import remarshal
+
+
+def get_location(prefix, location):
+    # To support usage with both docker and podman, the buildset
+    # registry keeps "docker.io" entries un-namespaced.
+    if prefix == 'docker.io':
+        return location
+    else:
+        return location + '/' + prefix
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            path=dict(required=True, type='path'),
+            buildset_registry=dict(type='raw'),
+            buildset_registry_alias=dict(type='str'),
+            namespaces=dict(type='raw'),
+        )
+    )
+    p = module.params
+    location = '%s:%s' % (p['buildset_registry_alias'],
+                          p['buildset_registry']['port'])
+
+    if os.path.exists(p['path']):
+        with open(p['path'], 'rb') as f:
+            input_data = f.read()
+        data = remarshal.decode('toml', input_data, True)
+    else:
+        input_data = None
+        data = {}
+
+    if 'registry' not in data:
+        data['registry'] = {}
+    for namespace in set(p['namespaces']):
+        n_config = data['registry'].setdefault(namespace, {})
+        mirrors = n_config.setdefault('mirrors', [])
+        new_loc = get_location(namespace, location)
+        if not mirrors or new_loc != mirrors[0]:
+            mirrors.insert(0, new_loc)
+
+    output_data = remarshal.encode_toml(data, True)
+    changed = input_data is None or input_data != output_data
+    if changed:
+        with open(p['path'], 'wb') as f:
+            f.write(output_data.encode('utf8'))
+
+    module.exit_json(changed=changed, data=data)
+
+
+if __name__ == '__main__':
+    ansible_main()

roles/use-buildset-registry/library/modify_registries_conf.py

@@ -58,7 +58,7 @@ def ansible_main():
             continue
         mirrors = reg.setdefault('mirror', [])
         new_loc = dict(location=get_location(reg['prefix'], location))
-        if mirrors and new_loc != mirrors[0]:
+        if not mirrors or new_loc != mirrors[0]:
             mirrors.insert(0, new_loc)
     for prefix in unseen:
         mirrors = [{'location': get_location(prefix, location)},

roles/use-buildset-registry/tasks/main.yaml

@@ -95,6 +95,19 @@
     buildset_registry_alias: "{{ buildset_registry_alias }}"
     namespaces: "{{ buildset_registry_namespaces }}"
 
+- name: Ensure buildkit directory exists
+  become: yes
+  file:
+    state: directory
+    path: /etc/buildkit/
+- name: Modify buildkitd.toml
+  become: yes
+  modify_buildkitd_toml:
+    path: /etc/buildkit/buildkitd.toml
+    buildset_registry: "{{ buildset_registry }}"
+    buildset_registry_alias: "{{ buildset_registry_alias }}"
+    namespaces: "{{ buildset_registry_namespaces }}"
+
 # We use 'block' here to cause the become to apply to all the tasks
 # (which does not automatically happen with include_tasks).
 - name: Update docker user config to use buildset registry