Browse Source

Migrate to Zuul v3

Change-Id: I0d5d6a9038c9a150a74df955bccb3f9ed263b736
changes/76/508676/17
yatin 4 years ago
parent
commit
0b49f38069
10 changed files with 650 additions and 3 deletions
  1. +241
    -0
      .zuul.yaml
  2. +3
    -3
      magnum/tests/contrib/post_test_hook.sh
  3. +102
    -0
      playbooks/magnum-buildimages-base.yaml
  4. +82
    -0
      playbooks/magnum-functional-base.yaml
  5. +85
    -0
      playbooks/magnum-tox-migration.yaml
  6. +21
    -0
      playbooks/post/upload-images.yaml
  7. +67
    -0
      playbooks/post/upload-logs-tox.yaml
  8. +14
    -0
      playbooks/post/upload-logs.yaml
  9. +12
    -0
      playbooks/pre/prepare-workspace-images.yaml
  10. +23
    -0
      playbooks/pre/prepare-workspace.yaml

+ 241
- 0
.zuul.yaml View File

@ -0,0 +1,241 @@
- job:
name: magnum-functional-base
parent: legacy-dsvm-base
timeout: 7800
nodeset: legacy-ubuntu-xenial
pre-run: playbooks/pre/prepare-workspace
run: playbooks/magnum-functional-base
post-run: playbooks/post/upload-logs
required-projects:
- openstack-infra/devstack-gate
- openstack/diskimage-builder
- openstack/ironic
- openstack/ironic-lib
- openstack/ironic-python-agent
- openstack/magnum
- openstack/pyghmi
- openstack/python-ironicclient
- openstack/python-magnumclient
- openstack/virtualbmc
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^specs/.*$
- ^install-guide/.*$
- ^releasenotes/.*$
vars:
ironic: 0
ceilometer: 0
swift: 0
horizon: 0
multinode: 0
neutron: 1
tempest: 0
branch_override: "default"
- job:
name: magnum-functional-multinode-base
parent: legacy-dsvm-base-multinode
timeout: 7800
nodeset: legacy-ubuntu-xenial-2-node
pre-run: playbooks/pre/prepare-workspace
run: playbooks/magnum-functional-base
post-run: playbooks/post/upload-logs
required-projects:
- openstack-infra/devstack-gate
- openstack/diskimage-builder
- openstack/ironic
- openstack/ironic-lib
- openstack/ironic-python-agent
- openstack/magnum
- openstack/pyghmi
- openstack/python-ironicclient
- openstack/python-magnumclient
- openstack/virtualbmc
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^specs/.*$
- ^install-guide/.*$
- ^releasenotes/.*$
vars:
ironic: 0
ceilometer: 0
swift: 0
horizon: 0
multinode: 1
neutron: 1
tempest: 0
branch_override: "default"
- job:
name: magnum-functional-api
parent: magnum-functional-base
vars:
coe: "api"
- job:
name: magnum-functional-k8s
parent: magnum-functional-base
voting: false
vars:
coe: "k8s"
irrelevant-files:
- ^magnum/drivers/swarm.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-swarm-mode
parent: magnum-functional-base
voting: false
branches: ^(?!stable/(newton|ocata)).*$
vars:
coe: "swarm-mode"
irrelevant-files:
- ^magnum/drivers/swarm_fedora_atomic_v1/.*$
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-dcos
parent: magnum-functional-base
voting: false
branches: ^(?!stable/(newton|ocata)).*$
vars:
coe: "dcos"
irrelevant-files:
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/mesos.*$
- ^magnum/drivers/swarm.*$
- job:
name: magnum-functional-mesos
parent: magnum-functional-base
voting: false
vars:
coe: "mesos"
irrelevant-files:
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/swarm.*$
- job:
name: magnum-functional-swarm
parent: magnum-functional-base
voting: false
vars:
coe: "swarm"
irrelevant-files:
- ^magnum/drivers/swarm_fedora_atomic_v2/.*$
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-k8s-ironic
parent: magnum-functional-base
voting: false
vars:
coe: "k8s"
ironic: 1
irrelevant-files:
- ^magnum/drivers/swarm.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-swarm-ironic
parent: magnum-functional-base
voting: false
vars:
coe: "swarm"
ironic: 1
irrelevant-files:
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-k8s-multinode
parent: magnum-functional-multinode-base
voting: false
vars:
coe: "k8s"
irrelevant-files:
- ^magnum/drivers/swarm.*$
- ^magnum/drivers/mesos.*$
- job:
name: magnum-functional-swarm-mode-multinode
parent: magnum-functional-multinode-base
voting: false
branches: ^(?!stable/(newton|ocata)).*$
vars:
coe: "swarm-mode"
irrelevant-files:
- ^magnum/drivers/k8s.*$
- ^magnum/drivers/mesos.*$
- ^magnum/drivers/swarm_fedora_atomic_v1/.*$
- job:
name: magnum-non-functional-tox-migration
nodeset: legacy-ubuntu-xenial
run: playbooks/magnum-tox-migration
post-run: playbooks/post/upload-logs-tox
timeout: 2400
required-projects:
- openstack/requirements
- job:
name: magnum-buildimages-base
parent: legacy-publish-openstack-artifacts
timeout: 3600
nodeset: legacy-ubuntu-xenial
pre-run: playbooks/pre/prepare-workspace-images
run: playbooks/magnum-buildimages-base
post-run: playbooks/post/upload-images
required-projects:
- openstack/dib-utils
- openstack/diskimage-builder
- openstack/magnum
- job:
name: magnum-dib-buildimage-fedora-atomic-25
parent: magnum-buildimages-base
vars:
image_name: "fedora-atomic-25"
- job:
name: magnum-dib-buildimage-ubuntu-mesos
parent: magnum-buildimages-base
vars:
image_name: "ubuntu-mesos"
- job:
name: magnum-dib-buildimage-centos-dcos
parent: magnum-buildimages-base
vars:
image_name: "centos-dcos"
- project:
name: openstack/magnum
check:
jobs:
- magnum-functional-api
- magnum-functional-k8s
- magnum-functional-swarm-mode
gate:
jobs:
- magnum-functional-api
experimental:
jobs:
- magnum-functional-dcos
- magnum-functional-mesos
- magnum-functional-swarm
- magnum-functional-k8s-ironic
- magnum-functional-swarm-ironic
- magnum-functional-k8s-multinode
- magnum-functional-swarm-mode-multinode
- magnum-non-functional-tox-migration
periodic:
jobs:
- magnum-dib-buildimage-fedora-atomic-25
- magnum-dib-buildimage-ubuntu-mesos
- magnum-dib-buildimage-centos-dcos

+ 3
- 3
magnum/tests/contrib/post_test_hook.sh View File

@ -154,7 +154,7 @@ constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt"
sudo -H pip install $constraints -U -r requirements.txt -r test-requirements.txt
export MAGNUM_DIR="$BASE/new/magnum"
sudo chown -R jenkins:stack $MAGNUM_DIR
sudo chown -R $USER:stack $MAGNUM_DIR
# Run functional tests
# Currently we support functional-api, functional-k8s, will support swarm,
@ -180,7 +180,7 @@ create_test_data $coe $special
local _magnum_tests=""
if [[ "api" == "$coe" ]]; then
sudo chown -R jenkins:stack $BASE/new/tempest
sudo chown -R $USER:stack $BASE/new/tempest
export TEMPEST_CONFIG=$BASE/new/tempest/etc/tempest.conf
@ -211,7 +211,7 @@ if [[ "api" == "$coe" ]]; then
fi
target="${coe}${special}"
sudo -E -H -u jenkins tox -e functional-"$target" $_magnum_tests -- --concurrency=1
sudo -E -H -u $USER tox -e functional-"$target" $_magnum_tests -- --concurrency=1
EXIT_CODE=$?


+ 102
- 0
playbooks/magnum-buildimages-base.yaml View File

@ -0,0 +1,102 @@
- hosts: primary
tasks:
- shell:
cmd: |
set -u
set -e
set -x
cd ~
if [[ "{{ image_name }}" =~ ^(ubuntu-mesos|centos-dcos)$ ]]; then
EXTRA_PROJECTS="openstack/tripleo-image-elements openstack/heat-templates"
else
EXTRA_PROJECTS=""
fi
/usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \
git://git.openstack.org \
openstack/diskimage-builder \
openstack/dib-utils \
openstack/magnum $EXTRA_PROJECTS
virtualenv env
./env/bin/pip install $(pwd)/openstack/dib-utils
./env/bin/pip install $(pwd)/openstack/diskimage-builder
# TODO(pabelanger): Remove once we migrated to bindep
./openstack/diskimage-builder/tests/install_test_deps.sh
# activate the virtualenv so that any tools run by dib run
# using the python inside it
set +u
source ./env/bin/activate
set -u
DIB_ELEMENTS=./openstack/diskimage-builder/diskimage_builder/elements
if [ "{{ image_name }}" == "ubuntu-mesos" ]; then
TRIPLEO_ELEMENTS=./openstack/tripleo-image-elements/elements
HEAT_ELEMENTS=./openstack/heat-templates/hot/software-config/elements
MESOS_ELEMENTS=./openstack/magnum/magnum/drivers/mesos_ubuntu_v1/image
export ELEMENTS_PATH=$TRIPLEO_ELEMENTS:$HEAT_ELEMENTS:$MESOS_ELEMENTS
$MESOS_ELEMENTS/install_imagebuild_deps.sh
export DIB_RELEASE=trusty
export DIB_IMAGE_SIZE=2.2
disk-image-create ubuntu vm docker mesos \
os-collect-config os-refresh-config os-apply-config \
heat-config heat-config-script -o $WORKSPACE/{{ image_name }}.qcow2
$MESOS_ELEMENTS/validate_image.sh $WORKSPACE/{{ image_name }}.qcow2
elif [ "{{ image_name }}" == "centos-dcos" ]; then
DCOS_ELEMENTS=./openstack/magnum/contrib/drivers/dcos_centos_v1/image
TRIPLEO_ELEMENTS=./openstack/tripleo-image-elements/elements
HEAT_ELEMENTS=./openstack/heat-templates/hot/software-config/elements
# Order matters, we need the docker elements from DCOS_ELEMENTS to be used first
export ELEMENTS_PATH=$DCOS_ELEMENTS:$DIB_ELEMENTS:$TRIPLEO_ELEMENTS:$HEAT_ELEMENTS
$DCOS_ELEMENTS/install_imagebuild_deps.sh
export DIB_IMAGE_SIZE=3.0
export FS_TYPE=xfs
curl -O https://downloads.dcos.io/dcos/stable/commit/e64024af95b62c632c90b9063ed06296fcf38ea5/dcos_generate_config.sh
export DCOS_GENERATE_CONFIG_SRC=`pwd`/dcos_generate_config.sh
disk-image-create \
centos7 vm docker dcos selinux-permissive \
os-collect-config os-refresh-config os-apply-config \
heat-config heat-config-script \
-o $WORKSPACE/{{ image_name }}.qcow2
#TODO: Add size validation
else
MAGNUM_ELEMENTS=./openstack/magnum/magnum/drivers/common/image
export ELEMENTS_PATH=$DIB_ELEMENTS:$MAGNUM_ELEMENTS
$MAGNUM_ELEMENTS/fedora-atomic/install_imagebuild_deps.sh
export DIB_RELEASE="25"
export DIB_IMAGE_SIZE=2.5
export FEDORA_ATOMIC_TREE_URL="https://kojipkgs.fedoraproject.org/atomic/25/"
export FEDORA_ATOMIC_TREE_REF="8b15e9b988b4b02f4cb8b39bdd63d182ab7004a8926ecdac6314ee5c7ffa646b"
disk-image-create -x -o $WORKSPACE/{{ image_name }}-dib fedora-atomic
# validate image
$MAGNUM_ELEMENTS/fedora-atomic/validate_atomic_image.sh $WORKSPACE/{{ image_name }}-dib.qcow2
fi
set +u
deactivate
set -u
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

+ 82
- 0
playbooks/magnum-functional-base.yaml View File

@ -0,0 +1,82 @@
- hosts: primary
tasks:
- shell:
cmd: |
set -e
set -x
export PYTHONUNBUFFERED=true
if [ "{{ neutron }}" -eq 1 ] ; then
export DEVSTACK_GATE_NEUTRON=1
fi
export DEVSTACK_GATE_TEMPEST=1
if [ "{{ tempest }}" -eq 0 ] ; then
# Do not run any tempest tests
export DEVSTACK_GATE_TEMPEST_NOTESTS=1
fi
if [ "{{ branch_override }}" != "default" ] ; then
export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
fi
export PROJECTS="openstack/magnum $PROJECTS"
export PROJECTS="openstack/python-magnumclient $PROJECTS"
export PROJECTS="openstack/diskimage-builder $PROJECTS"
if [ "{{ multinode }}" -eq 1 ] ; then
export DEVSTACK_GATE_TOPOLOGY="multinode"
export DEVSTACK_SUBNODE_CONFIG+=$'\n'"disable_service tempest"
fi
if [ "{{ ironic }}" -eq 1 ] ; then
export PROJECTS="openstack/ironic $PROJECTS"
export PROJECTS="openstack/ironic-lib $PROJECTS"
export PROJECTS="openstack/ironic-python-agent $PROJECTS"
export PROJECTS="openstack/python-ironicclient $PROJECTS"
export PROJECTS="openstack/pyghmi $PROJECTS"
export PROJECTS="openstack/virtualbmc $PROJECTS"
export MAGNUM_GATE_SPECIAL="-ironic"
fi
if [ "{{ horizon }}" -eq 0 ] ; then
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service horizon"
else
export DEVSTACK_GATE_HORIZON=1
fi
if [ "{{ swift }}" -eq 0 ] ; then
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy"
fi
if [ "{{ ceilometer }}" -eq 0 ] ; then
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api"
export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector"
fi
# Keep localrc to be able to set some vars in post_test_hook
export KEEP_LOCALRC=1
function gate_hook {
cd /opt/stack/new/magnum/
./magnum/tests/contrib/gate_hook.sh {{ coe }} $MAGNUM_GATE_SPECIAL
}
export -f gate_hook
function post_test_hook {
source $BASE/new/devstack/accrc/admin/admin
cd /opt/stack/new/magnum/
./magnum/tests/contrib/post_test_hook.sh {{ coe }} $MAGNUM_GATE_SPECIAL
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

+ 85
- 0
playbooks/magnum-tox-migration.yaml View File

@ -0,0 +1,85 @@
- hosts: primary
tasks:
- name: Ensure workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
CLONEMAP=`mktemp`
REQS_DIR=`mktemp -d`
function cleanup {
mkdir -p $WORKSPACE
rm -rf $CLONEMAP $REQS_DIR
}
trap cleanup EXIT
cat > $CLONEMAP << EOF
clonemap:
- name: $ZUUL_PROJECT
dest: .
EOF
# zuul cloner works poorly if there are 2 names that are the
# same in here.
if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
cat >> $CLONEMAP << EOF
- name: openstack/requirements
dest: $REQS_DIR
EOF
fi
/usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \
git://git.openstack.org $ZUUL_PROJECT openstack/requirements
# REQS_DIR is not set for openstack/requirements and there is also
# no need to copy in this case.
if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
cp $REQS_DIR/upper-constraints.txt ./
fi
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
if [ -x tools/test-setup.sh ] ; then
tools/test-setup.sh
fi
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -x
sudo rm -f /etc/sudoers.d/zuul
# Prove that general sudo access is actually revoked
! sudo -n true
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: /usr/local/jenkins/slave_scripts/run-tox.sh migration
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
OUT=`git ls-files --other --exclude-standard --directory`
if [ -z "$OUT" ]; then
echo "No extra files created during test."
exit 0
else
echo "The following un-ignored files were created during the test:"
echo "$OUT"
exit 0 # TODO: change to 1 to fail tests.
fi
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

+ 21
- 0
playbooks/post/upload-images.yaml View File

@ -0,0 +1,21 @@
- hosts: primary
tasks:
- name: Ensure artifacts directory exists
file:
path: '{{ zuul.executor.work_root }}/artifacts'
state: directory
delegate_to: localhost
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.work_root }}/artifacts/images'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/*.qcow2
- --include=*/
- --exclude=*
- --prune-empty-dirs

+ 67
- 0
playbooks/post/upload-logs-tox.yaml View File

@ -0,0 +1,67 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*nose_results.html
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testr_results.html.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.testrepository/tmp*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testrepository.subunit.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}/tox'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.tox/*/log/*
- --include=*/
- --exclude=*
- --prune-empty-dirs

+ 14
- 0
playbooks/post/upload-logs.yaml View File

@ -0,0 +1,14 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

+ 12
- 0
playbooks/pre/prepare-workspace-images.yaml View File

@ -0,0 +1,12 @@
- hosts: all
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

+ 23
- 0
playbooks/pre/prepare-workspace.yaml View File

@ -0,0 +1,23 @@
- hosts: all
name: magnum-prepare-workspace
tasks:
- name: Ensure workspace directory exists
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack-infra/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
git://git.openstack.org \
openstack-infra/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

Loading…
Cancel
Save