Dockerhub publisher job

This patchset includes pushing mechanism for daily periodic jobs.

Change-Id: Ic4179ffd3a6e640abfb181dbd000bc75cc64b750

.zuul.yaml

@@ -37,6 +37,37 @@
         - kolla-build-centos-source
         - kolla-build-ubuntu-source
         - kolla-build-oraclelinux-source
+    periodic:
+      jobs:
+        - kolla-publish-ubuntu-source
+        - kolla-publish-centos-source
+        - kolla-publish-oraclelinux-source
+
+- secret:
+    name: kolla_dockerhub_creds
+    data:
+      password: !encrypted/pkcs1-oaep
+        - QLe52Ymma5HJg3K2kgeSEMp7TwarkH8AbEiwcnDTqZ276BUF9wrt+5gPJRfVU1BYty2lq
+          CCzhawJJ09TV0WU2SEUKlicWoXQ/hcbYWNlOHVL6/gm9UxZP/GC8d1eyQfbCS7UUHfiHF
+          BP5Vbrn5vmK6DHhaVc9cEdXpcrA9cghLINpH6EeLE6ujD9gH5Apol30Qvj0IcpildWCYC
+          1rJqc8e/6BwOcLiPhli+A/bbpWxUzJgnFqogEiHLPvUTRyHDA8aCLv9Q6uryzaA0pzt7I
+          VY4JcNeM3J43QSTa6G7FxCGDiNOhEDvS31EyoxGZCdDAsN3Yh2nWPCfqr2X8vDk1Sbou7
+          ouiU+fKV+sv0f2+2fWNh+bLUOcOpp2gKvFKxpIxt7j5Zm6KBpHRxZelp4wVEPyEyecDN1
+          /fV7H4Z4QzH+Lah1F3H19IPvz1FPPaEFdrK5HWsbxa1aS9AY2hn+Th9TPJQSn/jxQvn8D
+          IT1GtO+Lr1SNUzrJJXzAdLmnyD9iVFOujzGsVfEbFpMlSetoBTNJl0MxrsYp6cJOqMRSS
+          sQZ56jPLxjCBTXoof2E81wbatOXWTLBpBzI5mtTfbcJnWfO93ZB9z2or6PMkWmTtYVji8
+          J1xFe2GTCHuPYjrETCPj5k0eCT3DS0PJ71N2mvRnPv5mKJKjhKBokhWscpuq1k=
+      user: !encrypted/pkcs1-oaep
+        - oRkTqW+FpsUy89wPVYJYk0ZU92yt5C6BKJr7ixVZo9ybcXfYwKDeZCoOeugYNkewJo1hV
+          ksCtqbqMc4v2r1cblJK003XCWkXroU5Vx5h/rzFjhxtDK59xUIYS8GzG2OThduqabzyKz
+          xm883COjULKPMEL8Q8OxmOvOJ4FcdJnToRWJO9lBQJKLaKleKCor6aadslkSACkDXIcC3
+          INIn0t02MZCL96VSBI9RtXaQ5VeyZDzv5QBIquouKYP8/j/yNUb4ilNXJKCJa8kYgo8O8
+          sEDtzITJuHorIYxTUc1syzAs9v1BlJF7SUqxTvR2YMFMlV6VADa7r8X4iyyWPmFL5O9c7
+          aN9JZhLNGgFLLAK4FISyOsNLZQS/JRjjGT5h9rsslZJsbco4q/Wwj51ezb78NJFhNY7lM
+          IpjKWByqSjg2k+1rEXpo6msgZskwYIHPi3xH3njP76127o3adeZQCT72LSnc4LIV6en/G
+          tIDqK72tkJpuwiA6u6ti764xXmIhl8Njfhn49cm6Fex9F8YieqPOj3t9mxMzdEZ7gd4Go
+          Oz6nUZlML6sEyT0vilbDpo1RRweinE5J8mU+8joxeDpCHXBUYekFmgXDlpg58XmSp96fM
+          C/rWcxTKlGJjjlVFZfhRHHy6gWBYH/SoozkbpkWbk0g43SLSxAuITy7nqLggOU=
 
 - nodeset:
     name: kolla-centos
@@ -64,6 +95,8 @@
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
+    vars:
+      publisher: false
 
 - job:
     name: kolla-build-centos-binary
@@ -83,6 +116,16 @@
       base_distro: centos
       install_type: source
 
+- job:
+    name: kolla-publish-centos-source
+    parent: kolla-build-centos-source
+    timeout: 10800
+    post-run: tests/playbooks/publish.yml
+    vars:
+      publisher: true
+    secrets:
+      - kolla_dockerhub_creds
+
 - job:
     name: kolla-build-ubuntu-binary
     parent: kolla-base
@@ -101,6 +144,16 @@
       base_distro: ubuntu
       install_type: source
 
+- job:
+    name: kolla-publish-ubuntu-source
+    parent: kolla-build-ubuntu-source
+    timeout: 10800
+    post-run: tests/playbooks/publish.yml
+    vars:
+      publisher: true
+    secrets:
+      - kolla_dockerhub_creds
+
 - job:
     name: kolla-build-oraclelinux-binary
     parent: kolla-base
@@ -118,3 +171,13 @@
       action: build
       base_distro: oraclelinux
       install_type: source
+
+- job:
+    name: kolla-publish-oraclelinux-source
+    parent: kolla-build-oraclelinux-source
+    timeout: 10800
+    post-run: tests/playbooks/publish.yml
+    vars:
+      publisher: true
+    secrets:
+      - kolla_dockerhub_creds

tests/playbooks/publish.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Login to Dockerhub
+      command: "docker login -u {{ kolla_dockerhub_creds.user }} -p {{ kolla_dockerhub_creds.password }}"
+      no_log: true
+
+    - shell: "for img in $(docker images  --format '{{ .Repository }}:{{ .Tag }}' | grep kolla ); do docker push $img; done"

tests/playbooks/run.yml

@@ -35,6 +35,43 @@
         mode: 666
       become: true
 
+    - name: Prepare mirror configuration
+      set_fact:
+        nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
+        nodepool_pypi_mirror: "{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
+        nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
+
+    - name: Ensure /etc/kolla exists
+      file:
+        path: /etc/kolla
+        state: directory
+        mode: 0777
+      become: true
+
+    - name: Template kolla-build.conf
+      template:
+        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
+        dest: /etc/kolla/kolla-build.conf
+
+    - set_fact:
+        template_override_path: zuul.executor.work_root + '/' + zuul.project.src_dir '/tests/templates/pip.conf.j2'
+
+    - name: Prepare pip.conf
+      template:
+        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/pip.conf.j2"
+        dest: /etc/kolla/pip.conf
+
+    - name: Read contents of pip.conf
+      command: "cat /etc/kolla/pip.conf"
+      register: pip_conf_file
+
+    - name: Template override
+      template:
+        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
+        dest: /etc/kolla/template_overrides.j2
+      vars:
+        pip_conf: "{{ pip_conf_file.stdout | b64encode }}"
+
     - shell:
         cmd: |
           set -e
@@ -44,7 +81,6 @@
           export BASE_DISTRO={{ base_distro }}
           export INSTALL_TYPE={{ install_type }}
           export IN_PROJECT_JOBS=1
-
           tools/gate_run.sh
         executable: /bin/bash
         chdir: "{{ zuul.project.src_dir }}"

tests/templates/kolla-build.conf.j2

@@ -0,0 +1,12 @@
+[DEFAULT]
+debug = true
+{% if publisher %}
+profile = default
+profile = infra
+profile = main
+namespace = kolla
+tag = queens
+{% endif %}
+push = false
+logs_dir = /tmp/logs/build
+template_override = /etc/kolla/template_overrides.j2

tests/templates/pip.conf.j2

@@ -0,0 +1,4 @@
+[global]
+timeout = 60
+index-url = {{ nodepool_pypi_mirror }}
+trusted-host = {{ nodepool_mirror_host }}

tests/templates/template_overrides.j2

@@ -0,0 +1,56 @@
+{% raw %}
+{% extends parent_template %}
+
+{% block base_header %}
+{% endraw %}
+
+RUN echo $(base64 -w0 "{{ pip_conf }}") | base64 -d > /etc/pip.conf
+
+{% raw %}
+{% if base_distro == 'ubuntu' %}
+
+RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated
+
+{% endif %}
+{% endblock %}
+
+{% block base_footer %}
+{% if base_distro == "centos" %}
+{% endraw %}
+
+RUN sed -i -e "/^mirrorlist/d" \
+        -e "s|^#baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \
+        /etc/yum.repos.d/CentOS-Base.repo \
+    && sed -i -e "/^mirrorlist/d" \
+        -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \
+        /etc/yum.repos.d/epel.repo \
+    && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \
+        /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \
+    && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=http://{{ nodepool_rdo_proxy }}|" \
+        /etc/yum.repos.d/delorean.repo
+
+{% raw %}
+{% elif base_distro == "oraclelinux" %}
+{% endraw %}
+
+RUN sed -i -e "/^mirrorlist/d" \
+        -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://{{ nodepool_mirror_host }}|" \
+        /etc/yum.repos.d/epel.repo \
+    && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://{{ nodepool_mirror_host }}|" \
+        /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \
+    && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=http://{{ nodepool_rdo_proxy }}|" \
+        /etc/yum.repos.d/delorean.repo
+
+{% raw %}
+{% elif base_distro == "ubuntu" %}
+{% endraw %}
+
+RUN sed -i -e "s|http://archive.ubuntu.com|http://{{ nodepool_mirror_host }}|" \
+        -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://{{ nodepool_mirror_host }}/ubuntu-cloud-archive|" \
+        /etc/apt/sources.list \
+    && apt-get update
+
+{% raw %}
+{% endif %}
+{% endblock %}
+{% endraw %}

tools/setup_gate.sh

@@ -6,89 +6,10 @@ set -o errexit
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
-. /etc/ci/mirror_info.sh
-
-function setup_config {
-
-    sudo mkdir -p /etc/kolla
-
-    # Use Infra provided pypi.
-    # Wheel package mirror may be not compatible. So do not enable it.
-    PIP_CONF=$(mktemp)
-    cat > ${PIP_CONF} <<EOF
-[global]
-timeout = 60
-index-url = $NODEPOOL_PYPI_MIRROR
-trusted-host = $NODEPOOL_MIRROR_HOST
-EOF
-    TEMPLATE_OVERRIDES=$(mktemp)
-
-    cat <<EOF | tee "${TEMPLATE_OVERRIDES}"
-{% extends parent_template %}
-
-{% block base_header %}
-
-RUN echo $(base64 -w0 "${PIP_CONF}") | base64 -d > /etc/pip.conf
-
-{% if base_distro == 'ubuntu' %}
-
-RUN echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated
-
-{% endif %}
-{% endblock %}
-
-{% block base_footer %}
-{% if base_distro == "centos" %}
-
-RUN sed -i -e "/^mirrorlist/d" \
-        -e "s|^#baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \
-        /etc/yum.repos.d/CentOS-Base.repo \
-    && sed -i -e "/^mirrorlist/d" \
-        -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://$NODEPOOL_MIRROR_HOST|" \
-        /etc/yum.repos.d/epel.repo \
-    && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \
-        /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \
-    && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=$NODEPOOL_RDO_PROXY|" \
-        /etc/yum.repos.d/delorean.repo
-
-{% elif base_distro == "oraclelinux" %}
-
-RUN sed -i -e "/^mirrorlist/d" \
-        -e "s|^#baseurl=http://download.fedoraproject.org/pub|baseurl=http://$NODEPOOL_MIRROR_HOST|" \
-        /etc/yum.repos.d/epel.repo \
-    && sed -i -e "s|^baseurl=http://mirror.centos.org|baseurl=http://$NODEPOOL_MIRROR_HOST|" \
-        /etc/yum.repos.d/CentOS-Ceph-Jewel.repo \
-    && sed -i -e "s|^baseurl=https://trunk.rdoproject.org|baseurl=$NODEPOOL_RDO_PROXY|" \
-        /etc/yum.repos.d/delorean.repo
-
-{% elif base_distro == "ubuntu" %}
-
-RUN sed -i -e "s|http://archive.ubuntu.com|http://$NODEPOOL_MIRROR_HOST|" \
-        -e "s|http://ubuntu-cloud.archive.canonical.com/ubuntu|http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive|" \
-        /etc/apt/sources.list \
-    && apt-get update
-
-{% endif %}
-{% endblock %}
-EOF
-
-cat <<EOF | sudo tee /etc/kolla/kolla-build.conf
-[DEFAULT]
-# NOTE(Jeffrey4l): use different a docker namespace name in case it pull image from hub.docker.io when deplying
-namespace = lokolla
-template_override = ${TEMPLATE_OVERRIDES}
-registry = 127.0.0.1:4000
-push = true
-logs_dir = /tmp/logs/build
-EOF
-}
-
 function setup_registry {
-
     sudo mkdir /opt/kolla_registry
     sudo chmod -R 644 /opt/kolla_registry
     docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
 }
 
 setup_registry
-setup_config