Add Zuul V3 support for Puppet Jobs

This patch moves in the Puppet OpenStack integration
jobs as well as the module tests in-repo.

Depends-On: Ib96ee354cbf9c88f9265f5743a97b2d013addc70
Change-Id: Idd846c569a6e067125e130f189be9ae8ffba1646
This commit is contained in:
Mohammed Naser 2017-09-28 16:37:47 -04:00
parent 8334c21dcc
commit bde052c8e4
No known key found for this signature in database
GPG Key ID: 481CBC90384AEC42
11 changed files with 586 additions and 9 deletions

356
.zuul.yaml Normal file
View File

@ -0,0 +1,356 @@
- job:
name: puppet-openstack-base
# TODO: REMOVE base-test BEFORE MERGE (https://review.openstack.org/#/c/508337/)
parent: base-test
pre-run: playbooks/prepare-node-common
- job:
name: puppet-openstack-integration-base
parent: puppet-openstack-base
pre-run: playbooks/prepare-node-integration
run: playbooks/run-integration-tests
post-run: playbooks/upload-logs
timeout: 5400
required-projects:
- openstack/openstack
- openstack/puppet-aodh
- openstack/puppet-barbican
- openstack/puppet-ceilometer
- openstack/puppet-ceph
- openstack/puppet-cinder
- openstack/puppet-cloudkitty
- openstack/puppet-congress
- openstack/puppet-designate
- openstack/puppet-ec2api
- openstack/puppet-glance
- openstack/puppet-gnocchi
- openstack/puppet-heat
- openstack/puppet-horizon
- openstack/puppet-ironic
- openstack/puppet-keystone
- openstack/puppet-manila
- openstack/puppet-mistral
- openstack/puppet-monasca
- openstack/puppet-murano
- openstack/puppet-neutron
- openstack/puppet-nova
- openstack/puppet-octavia
- openstack/puppet-openstack-integration
- openstack/puppet-openstack_extras
- openstack/puppet-openstacklib
- openstack/puppet-oslo
- openstack/puppet-ovn
- openstack/puppet-panko
- openstack/puppet-qdr
- openstack/puppet-sahara
- openstack/puppet-swift
- openstack/puppet-tacker
- openstack/puppet-tempest
- openstack/puppet-trove
- openstack/puppet-vswitch
- openstack/puppet-vitrage
- openstack/puppet-watcher
- openstack/puppet-zaqar
- openstack/tempest
- openstack/tempest-horizon
irrelevant-files:
- ^.*\.md$
- ^metadata.json$
- ^releasenotes/.*$
- ^spec/.*$
- ^test-requirements.txt$
roles:
- zuul: openstack-infra/zuul-jobs
- job:
name: puppet-openstack-integration-4
parent: puppet-openstack-integration-base
vars:
puppet: 4
- job:
name: puppet-openstack-integration-4-scenario001
parent: puppet-openstack-integration-4
vars:
scenario: scenario001
- job:
name: puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
parent: puppet-openstack-integration-4-scenario001
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-4-scenario001-tempest-centos-7
parent: puppet-openstack-integration-4-scenario001
nodeset: centos-7
- job:
name: puppet-openstack-integration-4-scenario002
parent: puppet-openstack-integration-4
vars:
scenario: scenario002
- job:
name: puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
parent: puppet-openstack-integration-4-scenario002
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-4-scenario002-tempest-centos-7
parent: puppet-openstack-integration-4-scenario002
nodeset: centos-7
- job:
name: puppet-openstack-integration-4-scenario003
parent: puppet-openstack-integration-4
vars:
scenario: scenario003
- job:
name: puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
parent: puppet-openstack-integration-4-scenario003
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-4-scenario003-tempest-centos-7
parent: puppet-openstack-integration-4-scenario003
nodeset: centos-7
- job:
name: puppet-openstack-integration-4-scenario004
parent: puppet-openstack-integration-4
vars:
scenario: scenario004
- job:
name: puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
parent: puppet-openstack-integration-4-scenario004
nodeset: ubuntu-xenial
branches: ^(?!stable/newton).*$
- job:
name: puppet-openstack-integration-4-scenario004-tempest-centos-7
parent: puppet-openstack-integration-4-scenario004
nodeset: centos-7
branches: ^(?!stable/newton).*$
- job:
name: puppet-openstack-integration-5
parent: puppet-openstack-integration-base
voting: false
vars:
puppet: 5
- job:
name: puppet-openstack-integration-5-scenario001
parent: puppet-openstack-integration-5
vars:
scenario: scenario001
- job:
name: puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
parent: puppet-openstack-integration-5-scenario001
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-5-scenario001-tempest-centos-7
parent: puppet-openstack-integration-5-scenario001
nodeset: centos-7
- job:
name: puppet-openstack-integration-5-scenario002
parent: puppet-openstack-integration-5
vars:
scenario: scenario002
- job:
name: puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial
parent: puppet-openstack-integration-5-scenario002
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-5-scenario002-tempest-centos-7
parent: puppet-openstack-integration-5-scenario002
nodeset: centos-7
- job:
name: puppet-openstack-integration-5-scenario003
parent: puppet-openstack-integration-5
vars:
scenario: scenario003
- job:
name: puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial
parent: puppet-openstack-integration-5-scenario003
nodeset: ubuntu-xenial
- job:
name: puppet-openstack-integration-5-scenario003-tempest-centos-7
parent: puppet-openstack-integration-5-scenario003
nodeset: centos-7
- job:
name: puppet-openstack-integration-5-scenario004
parent: puppet-openstack-integration-5
vars:
scenario: scenario004
- job:
name: puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
parent: puppet-openstack-integration-5-scenario004
nodeset: ubuntu-xenial
branches: ^(?!stable/newton).*$
- job:
name: puppet-openstack-integration-5-scenario004-tempest-centos-7
parent: puppet-openstack-integration-5-scenario004
nodeset: centos-7
branches: ^(?!stable/newton).*$
- project-template:
name: puppet-openstack-integration-jobs-all
check:
jobs:
- puppet-openstack-integration-4-scenario001-tempest-centos-7
- puppet-openstack-integration-4-scenario002-tempest-centos-7
- puppet-openstack-integration-4-scenario003-tempest-centos-7
- puppet-openstack-integration-4-scenario004-tempest-centos-7
- puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
gate:
jobs:
- puppet-openstack-integration-4-scenario001-tempest-centos-7
- puppet-openstack-integration-4-scenario002-tempest-centos-7
- puppet-openstack-integration-4-scenario003-tempest-centos-7
- puppet-openstack-integration-4-scenario004-tempest-centos-7
- puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
experimental:
jobs:
- puppet-openstack-integration-5-scenario001-tempest-centos-7
- puppet-openstack-integration-5-scenario002-tempest-centos-7
- puppet-openstack-integration-5-scenario003-tempest-centos-7
- puppet-openstack-integration-5-scenario004-tempest-centos-7
- puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
- project-template:
name: puppet-openstack-integration-jobs-scenario001
check:
jobs:
- puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario001-tempest-centos-7
gate:
jobs:
- puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario001-tempest-centos-7
experimental:
jobs:
- puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario001-tempest-centos-7
- project-template:
name: puppet-openstack-integration-jobs-scenario002
check:
jobs:
- puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario002-tempest-centos-7
gate:
jobs:
- puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario002-tempest-centos-7
experimental:
jobs:
- puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario002-tempest-centos-7
- project-template:
name: puppet-openstack-integration-jobs-scenario003
check:
jobs:
- puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario003-tempest-centos-7
gate:
jobs:
- puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario003-tempest-centos-7
experimental:
jobs:
- puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario003-tempest-centos-7
- project-template:
name: puppet-openstack-integration-jobs-scenario004
check:
jobs:
- puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario004-tempest-centos-7
gate:
jobs:
- puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
- puppet-openstack-integration-4-scenario004-tempest-centos-7
experimental:
jobs:
- puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
- puppet-openstack-integration-5-scenario004-tempest-centos-7
- job:
name: puppet-openstack-module-base
parent: puppet-openstack-base
pre-run: playbooks/prepare-node-unit
- job:
name: puppet-openstack-lint
parent: puppet-openstack-module-base
run: playbooks/run-lint-tests
- job:
name: puppet-openstack-syntax
parent: puppet-openstack-module-base
run: playbooks/run-syntax-tests
- job:
name: puppet-openstack-syntax-3-legacy
parent: puppet-openstack-syntax
branches: ^(stable/(newton|ocata)).*$
vars:
puppet: 3
- job:
name: puppet-openstack-syntax-4
parent: puppet-openstack-syntax
vars:
puppet: 4
- job:
name: puppet-openstack-module-build
parent: puppet-openstack-module-base
run: playbooks/run-build-tests
files: ^metadata.json$
- project-template:
name: puppet-openstack-check-jobs
check:
jobs:
- puppet-openstack-lint
- puppet-openstack-syntax-3-legacy
- puppet-openstack-syntax-4
- puppet-openstack-module-build
gate:
jobs:
- puppet-openstack-lint
- puppet-openstack-syntax-3-legacy
- puppet-openstack-syntax-4
- puppet-openstack-module-build
- project:
name: openstack/puppet-openstack-integration
templates:
- puppet-openstack-check-jobs
- puppet-openstack-integration-jobs-all

View File

@ -17,9 +17,7 @@ install_external() {
#
# - ``PUPPETFILE_DIR`` must be set to Puppet modules directory
# - ``SCRIPT_DIR`` must be set to script path
# - ``ZUUL_REF`` must be set to Zuul ref. Fallback to 'None'.
# - ``ZUUL_BRANCH`` must be set to Zuul branch. Fallback to 'master'.
# - ``ZUUL_URL`` must be set to Zuul URL
install_openstack() {
cat > clonemap.yaml <<EOF
clonemap:
@ -28,7 +26,6 @@ clonemap:
EOF
# Periodic jobs run without ref on master
ZUUL_REF=${ZUUL_REF:-None}
ZUUL_BRANCH=${ZUUL_BRANCH:-master}
local project_names=$(awk '{ if ($1 == ":git") print $3 }' \
@ -37,9 +34,7 @@ EOF
project_names="${project_names} openstack/puppet-openstack-integration"
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml \
--cache-dir /opt/git \
--zuul-ref $ZUUL_REF \
--zuul-branch $ZUUL_BRANCH \
--zuul-url $ZUUL_URL \
git://git.openstack.org $project_names
# Because openstack-integration can't be a class name.
@ -64,9 +59,7 @@ install_all() {
#
# - ``PUPPETFILE_DIR`` must be set to Puppet modules directory
# - ``SCRIPT_DIR`` must be set to script path
# - ``ZUUL_REF`` must be set to Zuul ref
# - ``ZUUL_BRANCH`` must be set to Zuul branch
# - ``ZUUL_URL`` must be set to Zuul URL
install_modules() {
# If zuul-cloner is there, have it install modules using zuul refs
if [ -e /usr/zuul-env/bin/zuul-cloner ] ; then

View File

@ -0,0 +1,40 @@
- hosts: all
tasks:
- name: Ensure legacy workspace directory
file:
path: "{{ ansible_user_dir }}/workspace"
state: directory
- name: Clean-up system state (RedHat)
yum:
name: "{{ item }}"
state: absent
become: true
when: ansible_os_family == 'RedHat'
with_items:
- rdo-release
- centos-release-openstack-*
- centos-release-ceph-*
- name: Install Ruby dependencies (RedHat)
yum:
name: "{{ item }}"
become: true
when: ansible_os_family == 'RedHat'
with_items:
- "@Development tools"
- libxml2-devel
- libxslt-devel
- ruby-devel
- zlib-devel
- name: Install Ruby dependencies (Debian)
apt:
name: "{{ item }}"
become: true
when: ansible_os_family == 'Debian'
with_items:
- libxml2-dev
- libxslt-dev
- ruby-dev
- zlib1g-dev

View File

@ -0,0 +1,48 @@
- hosts: all
tasks:
- shell:
cmd: |
set -e
set -x
ZUUL_BRANCH_REAL=${ZUUL_BRANCH:-master}
# Workaround for puppet-ceph, where we need to checkout
# puppet-openstack-integration from stable/pike when working on
# stable/jewel.
# Ceph Jewel works with Newton to Pike
if [[ "$ZUUL_BRANCH" == "stable/jewel" ]]; then
ZUUL_BRANCH_REAL='stable/pike'
fi
CLONEMAP=`mktemp`
function cleanup {
rm -f $CLONEMAP
}
trap cleanup EXIT
cat > $CLONEMAP << EOF
clonemap:
- name: openstack/puppet-openstack-integration
dest: puppet-openstack-integration
EOF
/usr/zuul-env/bin/zuul-cloner -m $CLONEMAP \
--cache-dir /opt/git \
--zuul-branch $ZUUL_BRANCH_REAL \
git://git.openstack.org openstack/puppet-openstack-integration
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- include_role:
name: bindep
vars:
bindep_dir: "src/git.openstack.org/openstack/puppet-openstack-integration"
- name: Create folder for gems
file:
path: "{{ ansible_user_dir }}/workspace/puppet-openstack-integration/.bundled_gems"
state: directory
- name: Install bundler
gem:
name: bundler
user_install: false
environment:
GEM_HOME: "{{ ansible_user_dir }}/workspace/puppet-openstack-integration/.bundled_gems"

View File

@ -0,0 +1,36 @@
- hosts: all
tasks:
- shell:
cmd: |
set -e
set -x
CLONEMAP=`mktemp`
function cleanup {
# In cases where zuul-cloner is aborted during a git
# clone operation, git will remove the git work tree in
# its cleanup. The work tree in these jobs is the
# workspace directory, which means that subsequent
# jenkins post-build actions can not run because the
# workspace has been removed.
# To reduce the likelihood of this having an impact,
# recreate the workspace directory if needed
mkdir -p $WORKSPACE
rm -f $CLONEMAP
}
trap cleanup EXIT
cat > $CLONEMAP << EOF
clonemap:
- name: $ZUUL_PROJECT
dest: .
EOF
/usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \
git://git.openstack.org $ZUUL_PROJECT
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- include_role:
name: bindep
- include_role:
name: revoke-sudo

View File

@ -0,0 +1,29 @@
- hosts: all
tasks:
- shell:
cmd: |
set -e
set -x
if [ -f /usr/bin/yum ]; then
sudo yum install -y https://yum.puppetlabs.com/puppetlabs-release-pc1-el-7.noarch.rpm
sudo yum install -y puppet-agent
elif [ -f /usr/bin/apt-get ]; then
wget https://apt.puppetlabs.com/puppetlabs-release-pc1-xenial.deb -O /tmp/puppet.deb
sudo dpkg -i /tmp/puppet.deb
sudo apt-get update
sudo apt-get install puppet-agent
rm -rf /tmp/puppet.deb
fi
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
cd $ZUUL_PROJECT
puppet module build .
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -0,0 +1,13 @@
- hosts: all
tasks:
- shell:
cmd: |
set -ex
trap "./copy_logs.sh" EXIT
export PUPPET_MAJ_VERSION={{ puppet }}
export SCENARIO={{ scenario }}
export GEM_HOME=`pwd`/.bundled_gems
./run_tests.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace/puppet-openstack-integration'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -0,0 +1,32 @@
- hosts: all
tasks:
- shell:
cmd: |
if [ -f Modulefile -o -f metadata.json ]; then
if [ -f Modulefile ]; then
MODULE=$(awk '/^name/ {print $NF}' Modulefile |tr -d \"\')
elif [ -f metadata.json ]; then
MODULE=$(python -c 'import json;print json.load(open("metadata.json"))["name"]')
fi
if [ -z "$MODULE" ]; then
echo "Module name not defined in Modulefile or metadata.json"
else
mkdir -p "$MODULE"
rsync -a --exclude="$MODULE" --exclude ".*" . "$MODULE"
cd "$MODULE"
fi
fi
mkdir .bundled_gems
export GEM_HOME=`pwd`/.bundled_gems
if [ -f Gemfile ]; then
gem install bundler --no-rdoc --no-ri --verbose
$GEM_HOME/bin/bundle install --without system_tests
$GEM_HOME/bin/bundle exec rake lint 2>&1
else
gem install rake -n ./.bundled_gems/
gem install puppet-lint
gem install puppetlabs_spec_helper
./.bundled_gems/rake lint 2>&1
fi
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -0,0 +1,16 @@
- hosts: all
tasks:
- shell:
cmd: |
export PUPPET_GEM_VERSION='~> {{ puppet }}'
mkdir .bundled_gems
export GEM_HOME=`pwd`/.bundled_gems
gem install bundler --no-rdoc --no-ri --verbose
$GEM_HOME/bin/bundle install --retry 3
# FUTURE_PARSER=yes is only supported by Puppet 3.x
if [ "{{ puppet }}" -lt "4" ]; then
export FUTURE_PARSER=yes
fi
$GEM_HOME/bin/bundle exec rake syntax
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -0,0 +1,14 @@
- hosts: all
tasks:
- name: Upload log files
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

View File

@ -31,7 +31,7 @@ export DISTRO=$(lsb_release -c -s)
# If openstack/tempest is broken on master, we can pin the repository to a specific commit
# by using the following line:
# export TEMPEST_VERSION=${TEMPEST_VERSION:-'382a2065f3364a36c110bfcc6275a0f8f6894773'}
export TEMPEST_VERSION=${TEMPEST_VERSION:-'origin/master'}
export TEMPEST_VERSION=${TEMPEST_VERSION:-'master'}
# For installing Tempest from RPM keep TEMPEST_FROM_SOURCE to false
export TEMPEST_FROM_SOURCE=${TEMPEST_FROM_SOURCE:-true}
# Cirros Image directory
@ -124,7 +124,7 @@ elif [ "${TEMPEST_FROM_SOURCE}" = true ]; then
git clone git://git.openstack.org/openstack/tempest-horizon /tmp/openstack/tempest-horizon
fi
pushd /tmp/openstack/tempest
git reset --hard $TEMPEST_VERSION
git reset --hard origin/$TEMPEST_VERSION
popd
fi