From c506e626e7443fb95180807013082fb281390376 Mon Sep 17 00:00:00 2001 From: "Michal (inc0) Jastrzebski" Date: Fri, 20 Oct 2017 14:21:49 -0700 Subject: [PATCH] Dockerhub publisher job This patchset includes pushing mechanism for daily periodic jobs. Change-Id: Ic4179ffd3a6e640abfb181dbd000bc75cc64b750 --- .zuul.yaml | 63 +++++++++++++++++++++ tests/playbooks/publish.yml | 8 +++ tests/playbooks/run.yml | 38 ++++++++++++- tests/templates/kolla-build.conf.j2 | 12 ++++ tests/templates/pip.conf.j2 | 4 ++ tests/templates/template_overrides.j2 | 56 +++++++++++++++++++ tools/setup_gate.sh | 79 --------------------------- 7 files changed, 180 insertions(+), 80 deletions(-) create mode 100644 tests/playbooks/publish.yml create mode 100644 tests/templates/kolla-build.conf.j2 create mode 100644 tests/templates/pip.conf.j2 create mode 100644 tests/templates/template_overrides.j2 diff --git a/.zuul.yaml b/.zuul.yaml index 9fc950e287..6df06bc952 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -37,6 +37,37 @@ - kolla-build-centos-source - kolla-build-ubuntu-source - kolla-build-oraclelinux-source + periodic: + jobs: + - kolla-publish-ubuntu-source + - kolla-publish-centos-source + - kolla-publish-oraclelinux-source + +- secret: + name: kolla_dockerhub_creds + data: + password: !encrypted/pkcs1-oaep + - QLe52Ymma5HJg3K2kgeSEMp7TwarkH8AbEiwcnDTqZ276BUF9wrt+5gPJRfVU1BYty2lq + CCzhawJJ09TV0WU2SEUKlicWoXQ/hcbYWNlOHVL6/gm9UxZP/GC8d1eyQfbCS7UUHfiHF + BP5Vbrn5vmK6DHhaVc9cEdXpcrA9cghLINpH6EeLE6ujD9gH5Apol30Qvj0IcpildWCYC + 1rJqc8e/6BwOcLiPhli+A/bbpWxUzJgnFqogEiHLPvUTRyHDA8aCLv9Q6uryzaA0pzt7I + VY4JcNeM3J43QSTa6G7FxCGDiNOhEDvS31EyoxGZCdDAsN3Yh2nWPCfqr2X8vDk1Sbou7 + ouiU+fKV+sv0f2+2fWNh+bLUOcOpp2gKvFKxpIxt7j5Zm6KBpHRxZelp4wVEPyEyecDN1 + /fV7H4Z4QzH+Lah1F3H19IPvz1FPPaEFdrK5HWsbxa1aS9AY2hn+Th9TPJQSn/jxQvn8D + IT1GtO+Lr1SNUzrJJXzAdLmnyD9iVFOujzGsVfEbFpMlSetoBTNJl0MxrsYp6cJOqMRSS + sQZ56jPLxjCBTXoof2E81wbatOXWTLBpBzI5mtTfbcJnWfO93ZB9z2or6PMkWmTtYVji8 + J1xFe2GTCHuPYjrETCPj5k0eCT3DS0PJ71N2mvRnPv5mKJKjhKBokhWscpuq1k= + user: !encrypted/pkcs1-oaep + - oRkTqW+FpsUy89wPVYJYk0ZU92yt5C6BKJr7ixVZo9ybcXfYwKDeZCoOeugYNkewJo1hV + ksCtqbqMc4v2r1cblJK003XCWkXroU5Vx5h/rzFjhxtDK59xUIYS8GzG2OThduqabzyKz + xm883COjULKPMEL8Q8OxmOvOJ4FcdJnToRWJO9lBQJKLaKleKCor6aadslkSACkDXIcC3 + INIn0t02MZCL96VSBI9RtXaQ5VeyZDzv5QBIquouKYP8/j/yNUb4ilNXJKCJa8kYgo8O8 + sEDtzITJuHorIYxTUc1syzAs9v1BlJF7SUqxTvR2YMFMlV6VADa7r8X4iyyWPmFL5O9c7 + aN9JZhLNGgFLLAK4FISyOsNLZQS/JRjjGT5h9rsslZJsbco4q/Wwj51ezb78NJFhNY7lM + IpjKWByqSjg2k+1rEXpo6msgZskwYIHPi3xH3njP76127o3adeZQCT72LSnc4LIV6en/G + tIDqK72tkJpuwiA6u6ti764xXmIhl8Njfhn49cm6Fex9F8YieqPOj3t9mxMzdEZ7gd4Go + Oz6nUZlML6sEyT0vilbDpo1RRweinE5J8mU+8joxeDpCHXBUYekFmgXDlpg58XmSp96fM + C/rWcxTKlGJjjlVFZfhRHHy6gWBYH/SoozkbpkWbk0g43SLSxAuITy7nqLggOU= - nodeset: name: kolla-centos @@ -64,6 +95,8 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.* + vars: + publisher: false - job: name: kolla-build-centos-binary @@ -83,6 +116,16 @@ base_distro: centos install_type: source +- job: + name: kolla-publish-centos-source + parent: kolla-build-centos-source + timeout: 10800 + post-run: tests/playbooks/publish.yml + vars: + publisher: true + secrets: + - kolla_dockerhub_creds + - job: name: kolla-build-ubuntu-binary parent: kolla-base @@ -101,6 +144,16 @@ base_distro: ubuntu install_type: source +- job: + name: kolla-publish-ubuntu-source + parent: kolla-build-ubuntu-source + timeout: 10800 + post-run: tests/playbooks/publish.yml + vars: + publisher: true + secrets: + - kolla_dockerhub_creds + - job: name: kolla-build-oraclelinux-binary parent: kolla-base @@ -118,3 +171,13 @@ action: build base_distro: oraclelinux install_type: source + +- job: + name: kolla-publish-oraclelinux-source + parent: kolla-build-oraclelinux-source + timeout: 10800 + post-run: tests/playbooks/publish.yml + vars: + publisher: true + secrets: + - kolla_dockerhub_creds diff --git a/tests/playbooks/publish.yml b/tests/playbooks/publish.yml new file mode 100644 index 0000000000..0fad441a06 --- /dev/null +++ b/tests/playbooks/publish.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + tasks: + - name: Login to Dockerhub + command: "docker login -u {{ kolla_dockerhub_creds.user }} -p {{ kolla_dockerhub_creds.password }}" + no_log: true + + - shell: "for img in $(docker images --format '{{ .Repository }}:{{ .Tag }}' | grep kolla ); do docker push $img; done" diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index 5d08a21a58..a041446591 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -35,6 +35,43 @@ mode: 666 become: true + - name: Prepare mirror configuration + set_fact: + nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo" + nodepool_pypi_mirror: "{{ zuul_site_mirror_fqdn }}:8080/pypi/simple" + nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}" + + - name: Ensure /etc/kolla exists + file: + path: /etc/kolla + state: directory + mode: 0777 + become: true + + - name: Template kolla-build.conf + template: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2" + dest: /etc/kolla/kolla-build.conf + + - set_fact: + template_override_path: zuul.executor.work_root + '/' + zuul.project.src_dir '/tests/templates/pip.conf.j2' + + - name: Prepare pip.conf + template: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/pip.conf.j2" + dest: /etc/kolla/pip.conf + + - name: Read contents of pip.conf + command: "cat /etc/kolla/pip.conf" + register: pip_conf_file + + - name: Template override + template: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2" + dest: /etc/kolla/template_overrides.j2 + vars: + pip_conf: "{{ pip_conf_file.stdout | b64encode }}" + - shell: cmd: | set -e @@ -44,7 +81,6 @@ export BASE_DISTRO={{ base_distro }} export INSTALL_TYPE={{ install_type }} export IN_PROJECT_JOBS=1 - tools/gate_run.sh executable: /bin/bash chdir: "{{ zuul.project.src_dir }}" diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2 new file mode 100644 index 0000000000..3b1ea10518 --- /dev/null +++ b/tests/templates/kolla-build.conf.j2 @@ -0,0 +1,12 @@ +[DEFAULT] +debug = true +{% if publisher %} +profile = default +profile = infra +profile = main +namespace = kolla +tag = queens +{% endif %} +push = false +logs_dir = /tmp/logs/build +template_override = /etc/kolla/template_overrides.j2 diff --git a/tests/templates/pip.conf.j2 b/tests/templates/pip.conf.j2 new file mode 100644 index 0000000000..8f0ac66ad3 --- /dev/null +++ b/tests/templates/pip.conf.j2 @@ -0,0 +1,4 @@ +[global] +timeout = 60 +index-url = {{ nodepool_pypi_mirror }} +trusted-host = {{ nodepool_mirror_host }} diff --git a/tests/templates/template_overrides.j2 b/tests/templates/template_overrides.j2 new file mode 100644 index 0000000000..fda0420d58 --- /dev/null +++ b/tests/templates/template_overrides.j2 @@ -0,0 +1,56 @@ +{% raw %} +{% extends parent_template %} + +{% block base_header %} +{% endraw %} + +RUN echo $(base64 -w0 "{{ pip_conf }}") | base64 -d > /etc/pip.conf + +{% raw %} +{% if base_distro == 'ubuntu' %} + +RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated + +{% endif %} +{% endblock %} + +{% block base_footer %} +{% if base_distro == "centos" %} +{% endraw %} + +RUN sed -i -e "/^mirrorlist/d" \ + -e "s|^#baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \ + /etc/yum.repos.d/CentOS-Base.repo \ + && sed -i -e "/^mirrorlist/d" \ + -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ + /etc/yum.repos.d/epel.repo \ + && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \ + /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \ + && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=http://{{ nodepool_rdo_proxy }}|" \ + /etc/yum.repos.d/delorean.repo + +{% raw %} +{% elif base_distro == "oraclelinux" %} +{% endraw %} + +RUN sed -i -e "/^mirrorlist/d" \ + -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \ + /etc/yum.repos.d/epel.repo \ + && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \ + /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \ + && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=http://{{ nodepool_rdo_proxy }}|" \ + /etc/yum.repos.d/delorean.repo + +{% raw %} +{% elif base_distro == "ubuntu" %} +{% endraw %} + +RUN sed -i -e "s|http://archive.ubuntu.com|http://{{ nodepool_mirror_host }}|" \ + -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \ + /etc/apt/sources.list \ + && apt-get update + +{% raw %} +{% endif %} +{% endblock %} +{% endraw %} diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index a0951a8d78..e2c617dd0e 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -6,89 +6,10 @@ set -o errexit # Enable unbuffered output for Ansible in Jenkins. export PYTHONUNBUFFERED=1 -. /etc/ci/mirror_info.sh - -function setup_config { - - sudo mkdir -p /etc/kolla - - # Use Infra provided pypi. - # Wheel package mirror may be not compatible. So do not enable it. - PIP_CONF=$(mktemp) - cat > ${PIP_CONF} < /etc/pip.conf - -{% if base_distro == 'ubuntu' %} - -RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated - -{% endif %} -{% endblock %} - -{% block base_footer %} -{% if base_distro == "centos" %} - -RUN sed -i -e "/^mirrorlist/d" \ - -e "s|^#baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \ - /etc/yum.repos.d/CentOS-Base.repo \ - && sed -i -e "/^mirrorlist/d" \ - -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://$NODEPOOL_MIRROR_HOST|" \ - /etc/yum.repos.d/epel.repo \ - && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \ - /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \ - && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=$NODEPOOL_RDO_PROXY|" \ - /etc/yum.repos.d/delorean.repo - -{% elif base_distro == "oraclelinux" %} - -RUN sed -i -e "/^mirrorlist/d" \ - -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://$NODEPOOL_MIRROR_HOST|" \ - /etc/yum.repos.d/epel.repo \ - && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \ - /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \ - && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=$NODEPOOL_RDO_PROXY|" \ - /etc/yum.repos.d/delorean.repo - -{% elif base_distro == "ubuntu" %} - -RUN sed -i -e "s|http://archive.ubuntu.com|http://$NODEPOOL_MIRROR_HOST|" \ - -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive|" \ - /etc/apt/sources.list \ - && apt-get update - -{% endif %} -{% endblock %} -EOF - -cat <