From 2cbb80a9237d3124b3ebaeff10ad98ab35fa2246 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 3 Dec 2019 10:53:10 +1100 Subject: [PATCH] build-docker-image: add option to install siblings When you build from a Dockerfile, it runs in a given "context"; that is the directory the Dockerfile is in and the directories below it. It can not access anything outside that context during the build. When building a container for a project in the gate, you may wish to install sibling projects that Zuul has checked-out into your container (i.e. so that Depends-On works). As mentioned, because /home/zuul/src/ is not in the context of the current project, you will not be able to access this source code during the container build. So to help facilitate dependencies, add a siblings: tag which can copy some or all of the required-projects already specified for the job into a special sub-directory of the current source. Because all the code is now in the same context, this will allow build scripts to be written that look for directories in .zuul-siblings and can install the source code from there. To further help the scripts, the ZUUL_SIBLINGS arg is set for the docker build giving the copied paths. The test is updated with some paths to test the copy. Change-Id: I079d823e7194e15b1b496aea0f53f70f6b563f02 --- roles/build-docker-image/common.rst | 11 +++++ roles/build-docker-image/tasks/build.yaml | 50 ++++++++++++++++++++++ roles/build-docker-image/tasks/main.yaml | 19 ++------ test-playbooks/registry/test-registry.yaml | 17 +++++++- 4 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 roles/build-docker-image/tasks/build.yaml diff --git a/roles/build-docker-image/common.rst b/roles/build-docker-image/common.rst index 95d37adcf..8a5cc977d 100644 --- a/roles/build-docker-image/common.rst +++ b/roles/build-docker-image/common.rst @@ -119,4 +119,15 @@ using this role. A list of tags to be added to the image when promoted. + .. zuul:rolevar:: siblings + :type: list + :default: [] + + A list of sibling projects to be copied into + ``{{zuul_work_dir}}/.zuul-siblings``. This can be useful to + collect multiple projects to be installed within the same Docker + context. A ``-build-arg`` called ``ZUUL_SIBLINGS`` will be + added with each sibling project. Note that projects here must + be listed in ``required-projects``. + .. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor// diff --git a/roles/build-docker-image/tasks/build.yaml b/roles/build-docker-image/tasks/build.yaml new file mode 100644 index 000000000..31863e5c5 --- /dev/null +++ b/roles/build-docker-image/tasks/build.yaml @@ -0,0 +1,50 @@ +- name: Check sibling directory + stat: + path: '{{ zuul_work_dir }}/.zuul-siblings' + register: _dot_zuul_siblings + +# This should have been cleaned up; multiple builds may specify +# different siblings to include so we need to start fresh. +- assert: + that: _dot_zuul_siblings.exists == False + +- name: Create sibling source directory + file: + path: '{{ zuul_work_dir }}/.zuul-siblings' + state: directory + mode: 0755 + when: item.siblings is defined + +# NOTE(ianw): could use recursive copy: with remote_src, but it's +# Ansible 2.8 only. take the simple approach. +- name: Copy sibling source directories + command: 'cp -r ~/src/{{ sibling }} {{ zuul_work_dir }}/.zuul-siblings/' + loop: '{{ item.siblings }}' + loop_control: + loop_var: sibling + when: item.siblings is defined + +- name: Build a docker image + command: >- + docker build {{ item.path | default('.') }} -f {{ item.dockerfile | default(docker_dockerfile) }} + {% if item.target | default(false) -%} + --target {{ item.target }} + {% endif -%} + {% for build_arg in item.build_args | default([]) -%} + --build-arg {{ build_arg }} + {% endfor -%} + {% if item.siblings | default(false) -%} + --build-arg "ZUUL_SIBLINGS={{ item.siblings | join(' ') }}" + {% endif -%} + {% for tag in item.tags | default(['latest']) -%} + --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }} + --tag {{ item.repository }}:{{ tag }} + {% endfor -%} + args: + chdir: "{{ zuul_work_dir }}/{{ item.context }}" + +- name: Cleanup sibling source directory + file: + path: '{{ zuul_work_dir }}/.zuul-siblings' + state: absent + diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml index 4dceac00a..c5d089880 100644 --- a/roles/build-docker-image/tasks/main.yaml +++ b/roles/build-docker-image/tasks/main.yaml @@ -4,22 +4,11 @@ set_fact: buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}" ignore_errors: true -- name: Build a docker image - command: >- - docker build {{ item.path | default('.') }} -f {{ item.dockerfile | default(docker_dockerfile) }} - {% if item.target | default(false) -%} - --target {{ item.target }} - {% endif -%} - {% for build_arg in item.build_args | default([]) -%} - --build-arg {{ build_arg }} - {% endfor -%} - {% for tag in item.tags | default(['latest']) -%} - --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }} - --tag {{ item.repository }}:{{ tag }} - {% endfor -%} - args: - chdir: "{{ zuul_work_dir }}/{{ item.context }}" + +- name: Build docker images + include_tasks: build.yaml loop: "{{ docker_images }}" + # Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag # so we set up /etc/hosts with a registry alias name to support ipv6 and 4. - name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses diff --git a/test-playbooks/registry/test-registry.yaml b/test-playbooks/registry/test-registry.yaml index da1251908..c85ae8ea5 100644 --- a/test-playbooks/registry/test-registry.yaml +++ b/test-playbooks/registry/test-registry.yaml @@ -122,12 +122,25 @@ - hosts: builder name: Test building a docker image - roles: - - role: build-docker-image + tasks: + + - name: Create fake sibling projects + command: >- + mkdir -p src/opendev.org/fake-sibling-1 && + mkdir -p src/opendev.org/fake-sibling-2 && + touch src/opendev.org/fake-sibling-1/file && + touch src/opendev.org/fake-sibling-2/file + + - name: Build docker image + include_role: + name: build-docker-image vars: docker_images: - context: test-playbooks/registry/docker repository: downstream/image + siblings: + - opendev.org/fake-sibling-1 + - opendev.org/fake-sibling-2 - hosts: executor name: Test pushing to the intermediate registry