Add inital zuulv3 gate

Some work to do includes: * Test multiple profiles * Send wheels tarball to tarball.o.o * Use infra mirrors for packages * Add debian support back (issues with the image upstream) Change-Id: I6014aa57a63ed2d62bedaee350180bb776861b67
2017-09-30 22:30:24 -04:00 · 2017-09-30 22:30:24 -04:00 · 8d90703fad
commit 8d90703fad
parent 9200d9060e
10 changed files with 333 additions and 273 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -2,7 +2,173 @@
    name: openstack/loci
    check:
      jobs:
-        - noop
+        - loci-cinder
+        #- loci-designate
+        - loci-glance
+        - loci-heat
+        - loci-horizon
+        - loci-ironic
+        - loci-keystone
+        #- loci-murano
+        - loci-neutron
+        - loci-nova
+        - loci-requirements
+        #- loci-swift
+        #- loci-trove
    gate:
      jobs:
-        - noop
+        - loci-cinder
+        #- loci-designate
+        - loci-glance
+        - loci-heat
+        - loci-horizon
+        - loci-ironic
+        - loci-keystone
+        #- loci-murano
+        - loci-neutron
+        - loci-nova
+        - loci-requirements
+        #- loci-swift
+        #- loci-trove
+
+- job:
+    name: loci-cinder
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: cinder
+    required-projects:
+      - openstack/cinder
+
+- job:
+    name: loci-designate
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: designate
+    required-projects:
+      - openstack/designate
+
+- job:
+    name: loci-glance
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    vars:
+      project: glance
+    required-projects:
+      - openstack/glance
+
+- job:
+    name: loci-heat
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: heat
+    required-projects:
+      - openstack/heat
+
+- job:
+    name: loci-horizon
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: horizon
+    required-projects:
+      - openstack/horizon
+
+- job:
+    name: loci-ironic
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: ironic
+    required-projects:
+      - openstack/ironic
+
+- job:
+    name: loci-keystone
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: keystone
+    required-projects:
+      - openstack/keystone
+
+- job:
+    name: loci-murano
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: murano
+    required-projects:
+      - openstack/murano
+
+- job:
+    name: loci-neutron
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: neutron
+    required-projects:
+      - openstack/neutron
+
+- job:
+    name: loci-nova
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: nova
+    required-projects:
+      - openstack/nova
+
+- job:
+    name: loci-requirements
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 3600
+    vars:
+      project: requirements
+    required-projects:
+      - openstack/requirements
+
+- job:
+    name: loci-swift
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: swift
+    required-projects:
+      - openstack/swift
+
+- job:
+    name: loci-trove
+    pre-run: playbooks/setup-gate
+    run: playbooks/loci-builder
+    post-run: playbooks/post
+    timeout: 1200
+    vars:
+      project: trove
+    required-projects:
+      - openstack/trove
--- a/playbooks/files/apache.conf
+++ b/playbooks/files/apache.conf
@ -1,7 +1,7 @@
-User jenkins
-Group jenkins
-ErrorLog ##WORKSPACE##/logs/git-server/error.log
-TransferLog ##WORKSPACE##/logs/git-server/access.log
+User zuul
+Group zuul
+ErrorLog /logs/error.log
+TransferLog /logs/access.log

 LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
 LoadModule env_module /usr/lib/apache2/modules/mod_env.so
@ -13,7 +13,7 @@ LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
 Listen 80

 <VirtualHost *:80>
-    SetEnv GIT_PROJECT_ROOT ##WORKSPACE##/repos
+    SetEnv GIT_PROJECT_ROOT /home/zuul/src/git.openstack.org/
    SetEnv GIT_HTTP_EXPORT_ALL
    ScriptAlias / /usr/lib/git-core/git-http-backend/
 </VirtualHost>
--- a/playbooks/files/apache2-systemd.conf
+++ b/playbooks/files/apache2-systemd.conf
@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/sbin/apache2 -k start -f /webroot/apache.conf
--- a/playbooks/files/docker-systemd.conf
+++ b/playbooks/files/docker-systemd.conf
@ -0,0 +1,8 @@
+# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses
+# cap_set_file capability and there is no way to pass that in at build as of
+# docker 17.06.
+# TODO(SamYaple): Periodically check to see if this is possible in newer
+# versions of Docker
+[Service]
+ExecStart=
+ExecStart=/usr/bin/dockerd
--- a/playbooks/loci-builder.yaml
+++ b/playbooks/loci-builder.yaml
@ -0,0 +1,46 @@
+- hosts: all
+  tasks:
+    - include_vars: vars.yaml
+
+    # NOTE(SamYaple): Unused currently
+    - name: Gather wheels
+      block:
+        - docker_image:
+            name: openstackloci/requirements:{{ item }}
+          with_items: "{{ distros }}"
+        - command: "docker save -o /tmp/wheels-{{ item }}.img openstackloci/requirements:{{ item }}"
+          with_items: "{{ distros }}"
+        - command: "{{ zuul.project.src_dir }}/scripts/fetch_wheels.py"
+          environment:
+            WHEELS: /tmp/wheels-{{ item }}.img
+            WHEELS_DEST: "/webroot/{{ item }}.tar.gz"
+          with_items: "{{ distros }}"
+      when: False
+
+    - name: Build images
+      block:
+        - docker_image:
+            path: "{{ zuul.project.src_dir }}/dockerfiles/{{ item }}"
+            name: base
+            tag: "{{ item }}"
+          with_items: "{{ distros }}"
+        - docker_image:
+            path: "{{ zuul.project.src_dir }}"
+            name: openstackloci/{{ project }}:master-{{ item }}
+            pull: False
+            buildargs:
+              PROJECT: "{{ project }}"
+              #PROJECT_REPO: http://172.17.0.1/openstack/{{ project }}
+              #WHEELS: http://172.17.0.1/{{ item }}.tar.gz
+              FROM: base:{{ item }}
+          with_items: "{{ distros }}"
+          async: 1000
+          poll: 0
+          register: builder
+        - async_status:
+            jid: "{{ item.ansible_job_id }}"
+          with_items: "{{ builder.results }}"
+          register: builder_result
+          until: builder_result.finished
+          retries: 30
+          delay: 60
--- a/playbooks/post.yaml
+++ b/playbooks/post.yaml
@ -0,0 +1,14 @@
+- hosts: all
+  tasks:
+    - name: Copy logs
+      synchronize:
+        src: '/logs'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
--- a/playbooks/setup-gate.yaml
+++ b/playbooks/setup-gate.yaml
@ -0,0 +1,79 @@
+- hosts: all
+  tasks:
+    - include_vars: vars.yaml
+
+    - name: Setup swap
+      block:
+        - command: fallocate -l20g /swap
+        - file:
+            path: /swap
+            mode: 0600
+        - command: mkswap /swap
+        - command: swapon /swap
+      become: True
+
+    - name: Install Docker
+      block:
+        - file:
+            path: "{{ item }}"
+            state: directory
+          with_items:
+            - /etc/docker/
+            - /etc/systemd/system/docker.service.d/
+            - /var/lib/docker/
+        - mount:
+            path: /var/lib/docker/
+            src: tmpfs
+            fstype: tmpfs
+            opts: size=25g
+            state: mounted
+        - copy: "{{ item }}"
+          with_items:
+            - content: "{{ docker_daemon | to_json }}"
+              dest: /etc/docker/daemon.json
+            - src: files/docker-systemd.conf
+              dest: /etc/systemd/system/docker.service.d/
+        - apt_key:
+            url: https://download.docker.com/linux/ubuntu/gpg
+        - apt_repository:
+            repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
+        - apt:
+            name: "{{ item }}"
+          with_items:
+            - docker-ce=17.06*
+            - python-pip
+        - pip:
+            name: docker
+        # NOTE(SamYaple): Allow all connections from containers to host so the
+        # containers can access the http server for git and wheels
+        - iptables:
+            chain: INPUT
+            in_interface: docker0
+            policy: ACCEPT
+      become: True
+
+    - name: Setup http server for git and wheels
+      block:
+        - file:
+            path: "{{ item.path }}"
+            owner: "{{ item.owner }}"
+            state: directory
+          with_items:
+            - path: /logs/
+              owner: zuul
+            - path: /webroot/
+              owner: zuul
+            - path: /etc/systemd/system/apache2.service.d/
+              owner: root
+        - copy: "{{ item }}"
+          with_items:
+            - src: files/apache2-systemd.conf
+              dest: /etc/systemd/system/apache2.service.d/
+            - src: files/apache.conf
+              dest: /webroot/
+        - apt:
+            name: "{{ item }}"
+          with_items:
+            - apache2
+            - gitweb
+      become: True
--- a/playbooks/vars.yaml
+++ b/playbooks/vars.yaml
@ -0,0 +1,10 @@
+docker_daemon:
+  group: zuul
+  registry-mirrors:
+    - "http://{{ zuul_site_mirror_fqdn }}:8081/registry-1.docker/"
+  storage-driver: overlay2
+  debug: True
+
+distros:
+  - centos
+  - ubuntu
--- a/scripts/gate-setup.sh
+++ b/scripts/gate-setup.sh
@ -1,87 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-function setup_git_server {
-    sudo apt-get install --no-install-recommends -y apache2 gitweb
-    sudo systemctl stop apache2
-
-    mkdir repos logs/git-server
-    pushd repos
-    local repos=(openstack/{loci,${ZUUL_PROJECT#*-}})
-    /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git git://git.openstack.org ${repos[@]}
-    for p in ${repos[@]}; do
-        git --git-dir ${p}/.git checkout -b zuul
-    done
-    popd
-
-    sed -i "s|##WORKSPACE##|${WORKSPACE}|g" openstack/loci/confs/git-server.conf
-    sudo apache2 -f ${WORKSPACE}/openstack/loci/confs/git-server.conf
-}
-
-function debug_info {
-    set +x
-    local PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    sudo parted -l
-    sudo mount
-    df -h
-    uname -a
-    cat /etc/*release*
-    cat /proc/meminfo
-    env
-    set -x
-}
-
-function setup_docker {
-    sudo apt-get update
-    sudo apt-get install --no-install-recommends -y apt-transport-https
-    echo 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' | sudo tee /etc/apt/sources.list.d/docker.list
-    for ks in hkp://pgp.mit.edu hkp://keyserver.ubuntu.com; do
-        sudo apt-key adv --keyserver ${ks} --recv-keys 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 && break || continue
-    done
-
-    sudo apt-get update
-    sudo apt-get install --no-install-recommends -y docker-ce
-
-    sudo systemctl stop docker
-    sudo mount -o size=25g -t tmpfs tmpfs /var/lib/docker
-    source /etc/ci/mirror_info.sh
-
-    # TODO(SamYaple): CentOS cannot be build with userns-remap enabled. httpd
-    # uses cap_set_file capability and there is no way to pass that in at build
-    # time yet.
-    sudo tee /etc/systemd/system/docker.service <<-EOF
-	[Service]
-	ExecStart=
-	ExecStart=/usr/bin/dockerd
-	EOF
-    sudo tee /etc/docker/daemon.json <<-EOF
-	{
-	    "group": "jenkins",
-	    "registry-mirrors": ["${NODEPOOL_DOCKER_REGISTRY_PROXY}"],
-	    "storage-driver": "overlay2"
-	}
-	EOF
-    sudo systemctl daemon-reload
-    sudo systemctl start docker
-
-    # NOTE(SamYaple): Allow all connections from containers to host ports
-    sudo iptables -I INPUT -i docker0 -j ACCEPT
-}
-
-function setup_swap {
-    sudo fallocate -l20G /swap
-    sudo chmod 0600 /swap
-    sudo mkswap /swap
-    sudo swapon /swap
-}
-
-function setup_logs {
-    mkdir logs
-}
-
-setup_logs
-debug_info | tee logs/gate_info.log
-setup_swap
-setup_docker
-setup_git_server
--- a/scripts/run-tests.sh
+++ b/scripts/run-tests.sh
@ -1,179 +0,0 @@
-#!/bin/bash
-
-# NOTE(SamYaple): It is not safe to have multiple instances of this script
-# running at once due to (poor) error handling
-# TODO(SamYaple): Make this script safer if running outside the gate
-
-set -eux
-
-if [[ -e /etc/nodepool/provider ]]; then
-    export RUNNING_IN_GATE=true
-    export LOGS_DIR=${WORKSPACE}/logs
-else
-    export RUNNING_IN_GATE=false
-    export LOGS_DIR=$(mktemp -d)
-fi
-
-function prep_log_dir {
-    rm -rf ${LOGS_DIR}/build_error
-    mkdir -p ${LOGS_DIR}/builds
-}
-
-function dump_error_logs {
-    while read -r line; do
-        cat $line
-    done < ${LOGS_DIR}/build_error
-    exit 1
-}
-
-function debian_override {
-    mkdir -p etc/apt/
-    echo 'APT::Get::AllowUnauthenticated "true";' > etc/apt/apt.conf
-    cat <<-EOF > etc/apt/sources.list
-	deb http://${NODEPOOL_MIRROR_HOST}/debian jessie main
-	deb http://${NODEPOOL_MIRROR_HOST}/debian jessie-updates main
-	deb http://${NODEPOOL_MIRROR_HOST}/debian jessie-security main
-	EOF
-}
-
-function ubuntu_override {
-    mkdir -p etc/apt/
-    echo 'APT::Get::AllowUnauthenticated "true";' > etc/apt/apt.conf
-    cat <<-EOF > etc/apt/sources.list
-	deb http://${NODEPOOL_MIRROR_HOST}/ubuntu xenial main restricted universe
-	deb http://${NODEPOOL_MIRROR_HOST}/ubuntu xenial-updates main restricted universe
-	deb http://${NODEPOOL_MIRROR_HOST}/ubuntu xenial-security main restricted universe
-	EOF
-}
-
-function centos_override {
-    mkdir -p etc/yum.repos.d/
-    cat <<-EOF > etc/yum.repos.d/CentOS-Base.repo
-	[base]
-	name=CentOS-\$releasever - Base
-	baseurl=http://${NODEPOOL_MIRROR_HOST}/centos/\$releasever/os/\$basearch/
-	gpgcheck=0
-
-	[updates]
-	name=CentOS-\$releasever - Updates
-	baseurl=http://${NODEPOOL_MIRROR_HOST}/centos/\$releasever/updates/\$basearch/
-	gpgcheck=0
-
-	[extras]
-	name=CentOS-\$releasever - Extras
-	baseurl=http://${NODEPOOL_MIRROR_HOST}/centos/\$releasever/extras/\$basearch/
-	gpgcheck=0
-	EOF
-}
-
-function debian_backports_override {
-    mkdir -p etc/apt/sources.list.d/
-    cat <<-EOF > etc/apt/sources.list.d/backports.list
-	deb http://${NODEPOOL_MIRROR_HOST}/debian jessie-backports main
-	EOF
-}
-
-function debian_ceph_override {
-    mkdir -p etc/apt/sources.list.d/
-    # NOTE(SamYaple): Update after https://review.openstack.org/#/c/452547/
-    # Currently Jewel repos are not mirrored.
-    cat <<-EOF > etc/apt/sources.list.d/ceph.list
-	deb http://download.ceph.com/debian-jewel/ jessie main
-	EOF
-}
-
-function ubuntu_ceph_override {
-    mkdir -p etc/apt/sources.list.d/
-    cat <<-EOF > etc/apt/sources.list.d/ceph.list
-	deb http://${NODEPOOL_MIRROR_HOST}/ceph-deb-jewel/ xenial main
-	EOF
-}
-
-function centos_ceph_override {
-    # TODO(SamYaple): Add centos mirror to infra
-    cat <<-EOF > etc/yum.repos.d/Ceph.repo
-	[centos-ceph-jewel]
-	name=CentOS-7 - Ceph Jewel
-	baseurl=http://download.ceph.com/rpm-jewel/el7/noarch
-	gpgcheck=0
-	EOF
-}
-
-function centos_openstack_override {
-    cat <<-EOF > etc/yum.repos.d/CentOS-OpenStack.repo
-	[centos-openstack-ocata]
-	includepkgs=liberasurecode*
-	name=CentOS-7 - OpenStack Ocata
-	baseurl=http://${NODEPOOL_MIRROR_HOST}/centos/7/cloud/\$basearch/openstack-ocata/
-	gpgcheck=0
-	EOF
-}
-
-function generate_override {
-    set -eux
-
-    source /etc/nodepool/provider
-
-    if [[ -z "${NODEPOOL_MIRROR_HOST-}" ]]; then
-        local NODEPOOL_MIRROR_HOST=mirror.${NODEPOOL_REGION,,}.${NODEPOOL_CLOUD}.openstack.org
-    fi
-
-    local TARBALL=${PWD}/override.tar.gz
-    cd $(mktemp -d)
-
-    ${DISTRO}_override
-    if [[ -n ${PLUGIN-} ]] && type -t ${DISTRO}_${PLUGIN}_override; then
-        ${DISTRO}_${PLUGIN}_override
-    fi
-    if [[ -n ${EXTRA-} ]] && type -t ${DISTRO}_${EXTRA}_override; then
-        ${DISTRO}_${EXTRA}_override
-    fi
-
-    tar cfz ${TARBALL} .
-}
-
-function builder {
-    set -eux
-
-    local directory=$1
-    cd ${directory}
-    source testvars
-    if [[ ! -n "${PLUGIN-}" ]]; then
-        local log=${LOGS_DIR}/builds/${DISTRO}.log
-    else
-        local log=${LOGS_DIR}/builds/${DISTRO}-${PLUGIN}.log
-    fi
-
-    local build_args=""
-
-    if $RUNNING_IN_GATE; then
-        build_args+="--build-arg OVERRIDE=override.tar.gz"
-        build_args+=" --build-arg PROJECT_REPO=http://172.17.0.1/openstack/${ZUUL_PROJECT#*-} --build-arg PROJECT_REF=zuul"
-        build_args+=" --build-arg SCRIPTS_REPO=http://172.17.0.1/openstack/loci --build-arg SCRIPTS_REF=zuul"
-        old_path=${PWD}
-        generate_override
-        cd ${old_path}
-    fi
-    if [[ ! -n "${PLUGIN-}" ]]; then
-        docker build --tag openstackloci/${PROJECT}:${DISTRO} --no-cache ${build_args} . 2>&1 > ${log} || echo ${log} >> ${LOGS_DIR}/build_error
-    else
-        docker build --tag openstackloci/${PROJECT}:${DISTRO}-${PLUGIN} --no-cache ${build_args} . 2>&1 > ${log} || echo ${log} >> ${LOGS_DIR}/build_error
-    fi
-}
-
-# NOTE(SamYaple): We must export the functions for use with subshells (xargs)
-export -f $(compgen -A function)
-
-prep_log_dir
-
-echo "Building images"
-find . -mindepth 2 -maxdepth 2 -type f -name Dockerfile -printf '%h\0' | xargs -r -0 -P10 -n1 bash -c 'builder $1' _
-echo "Building plugins"
-find . -mindepth 3 -maxdepth 3 -type f -name Dockerfile -printf '%h\0' | xargs -r -0 -P10 -n1 bash -c 'builder $1' _
-
-if [[ -f ${LOGS_DIR}/build_error ]]; then
-    echo "Building images failure; Dumping failed logs to stdout"
-    dump_error_logs
-else
-    echo "Building images successful"
-fi