Move to zuul v3 in project jobs

Partial-Bug: #1720601
Change-Id: Ibc20a6ae8c645ff82f3c14a6286073dffd4cfae2
This commit is contained in:
Jeffrey Zhang 2017-10-01 23:21:38 +08:00 committed by Michal (inc0) Jastrzebski
parent 1201ee0572
commit baa9319a75
13 changed files with 229 additions and 273 deletions

57
.zuul.yaml Normal file
View File

@ -0,0 +1,57 @@
- project:
name: openstack/kolla-ansible
check:
jobs:
- kolla-ansible-centos-source
- kolla-ansible-ubuntu-source
- kolla-ansible-oraclelinux-source
- nodeset:
name: kolla-ansible-centos
nodes:
- name: primary
label: centos-7
- nodeset:
name: kolla-ansible-xenial
nodes:
- name: primary
label: ubuntu-xenial
- job:
name: kolla-ansible-base
pre-run: tests/pre
run: tests/run
post-run: tests/post
attempts: 1
timeout: 10800
required-projects:
- openstack/kolla
- openstack/requirements
irrelevant-files:
- ^.*\.rst$
- ^doc/.*
- job:
name: kolla-ansible-centos-source
parent: kolla-ansible-base
nodeset: kolla-ansible-centos
vars:
base_distro: centos
install_type: source
- job:
name: kolla-ansible-ubuntu-source
parent: kolla-ansible-base
nodeset: kolla-ansible-xenial
vars:
base_distro: ubuntu
install_type: source
- job:
name: kolla-ansible-oraclelinux-source
parent: kolla-ansible-base
nodeset: kolla-ansible-centos
vars:
base_distro: oraclelinux
install_type: source

View File

@ -161,3 +161,6 @@ project_name = service
project_domain_name = {{ default_project_domain_name }}
os_region_name = {{ openstack_region_name }}
os_interface = internal
[privsep]
helper_command=sudo neutron-rootwrap /etc/neutron/rootwrap.conf privsep-helper

View File

@ -1,40 +0,0 @@
---
- hosts: localhost
connection: local
become: True
tasks:
- name: Get api_interface name
set_fact: api_interface_address="{{ lookup('file', '/etc/nodepool/primary_node_private') }}"
- shell: "ip a | grep {{ api_interface_address }}"
register: api_interface_name
- set_fact: api_interface_name="{{ api_interface_name.stdout_lines[0].split(" ")[-1] }}"
- set_fact:
is_multinode: "{{ lookup('file', '/etc/nodepool/sub_nodes') }}"
- name: Ensure /etc/kolla dir
file:
path: /etc/kolla
state: "directory"
- name: Setup globals.yml
template:
src: "templates/globals-default.j2"
dest: "/etc/kolla/globals.yml"
- name: Copy passwords.yml
copy:
src: "../etc/kolla/passwords.yml"
dest: "/etc/kolla/passwords.yml"
- name: Ensure /etc/kolla/config directory
file:
path: /etc/kolla/config/nova
state: "directory"
- name: Setup overrides
template:
src: "templates/nova-compute-overrides.j2"
dest: "/etc/kolla/config/nova/nova-compute.conf"

View File

@ -1,26 +0,0 @@
---
- hosts: localhost
connection: local
tasks:
- name: Get node addresses
set_fact:
primary_node_address: "{{ lookup('file', '/etc/nodepool/primary_node_private') }}"
sub_node_addresses: "{{ lookup('file', '/etc/nodepool/sub_nodes_private').split('\n') }}"
- set_fact: node_group="{{ lookup('template', 'templates/inventory.j2')}}"
- name: Ensure /tmp/kolla exists
file:
path: "/tmp/kolla"
state: "directory"
- name: Copy default ansible kolla-ansible inventory
copy:
src: ../ansible/inventory/all-in-one
dest: /tmp/kolla/raw_inventory
- name: Replace localhost with IPs
replace:
path: /tmp/kolla/raw_inventory
regexp: "localhost.*$"
replace: "{{ node_group }}"

View File

@ -1,88 +0,0 @@
---
- hosts: all
tasks:
- name: Check node role
command: "cat /etc/nodepool/role"
register: node_role
- set_fact: node_role="{{ node_role.stdout }}"
- name: Ensure /tmp/logs dir
file:
path: "/tmp/logs"
state: "directory"
when:
- node_role == "sub"
- name: Get /tmp/logs symlink target
command: "readlink -f /tmp/logs"
register: logs_target
- name: Ensure primary node directories
file:
path: "{{ logs_target.stdout }}/{{ item }}"
state: "directory"
mode: 0777
when:
- node_role == "primary"
with_items:
- "subnodes"
- "docker_logs"
- "kolla_configs"
- "system_logs"
- "kolla"
- name: Ensure sub node directories
file:
path: "/tmp/logs/{{ item }}"
state: "directory"
mode: 0777
when:
- node_role == "sub"
with_items:
- "docker_logs"
- "kolla_configs"
- "system_logs"
- "kolla"
- name: Run diagnostics script
script: get_logs.sh
register: get_logs_result
failed_when: false
- name: Print get_logs output
debug:
msg: "{{ get_logs_result.stdout }}"
- name: Run dump_info script
script: ../tools/dump_info.sh
- name: Download logs from all subnodes
synchronize:
src: "/tmp/logs"
dest: "{{ logs_target.stdout }}/subnodes/{{ ansible_hostname }}"
mode: "pull"
when:
- node_role == "sub"
- name: Change permission of all log files
command: "chmod -R 777 {{ logs_target.stdout }}"
- name: Fail if get_logs has failed
fail:
msg: "{{ get_logs_result.stdout }}"
when: get_logs_result.rc != 0
- name: Copy inventory file to logs
copy:
src: "/tmp/kolla/raw_inventory"
dest: "{{ logs_target.stdout }}/ansible/inventory"
when:
- node_role == "primary"
- name: Copy hosts file to logs
copy:
src: "/etc/hosts"
dest: "{{ logs_target.stdout }}/system_logs/hosts"
when:
- node_role == "primary"

View File

@ -1,8 +1,25 @@
#!/bin/bash
copy_logs() {
set +o errexit
LOG_DIR=$(readlink -f /tmp/logs)
check_failure() {
# All docker container's status are created, restarting, running, removing,
# paused, exited and dead. Containers without running status are treated as
# failure. removing is added in docker 1.13, just ignore it now.
failed_containers=$(docker ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=restarting \
--filter status=paused \
--filter status=exited \
--filter status=dead)
if [[ -n "$failed_containers" ]]; then
exit 1;
fi
}
copy_logs() {
LOG_DIR=/tmp/logs
cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/
cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_configs/
@ -16,12 +33,12 @@ copy_logs() {
cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log
fi
sudo cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
sudo cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
df -h > ${LOG_DIR}/system_logs/df.txt
free > ${LOG_DIR}/system_logs/free.txt
sudo parted -l > ${LOG_DIR}/system_logs/parted-l.txt
parted -l > ${LOG_DIR}/system_logs/parted-l.txt
mount > ${LOG_DIR}/system_logs/mount.txt
env > ${LOG_DIR}/system_logs/env.txt
@ -45,9 +62,9 @@ copy_logs() {
docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
fi
# container logs
mkdir ${LOG_DIR}/kolla/fluentd/
docker logs fluentd > ${LOG_DIR}/kolla/fluentd/fluentd.txt
for container in $(docker ps -a --format "{{.Names}}"); do
docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt
done
# Rename files to .txt; this is so that when displayed via
# logs.openstack.org clicking results in the browser shows the
@ -56,40 +73,13 @@ copy_logs() {
# Rename all .log files to .txt files
for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do
sudo mv $f ${f/.log/.txt}
mv $f ${f/.log/.txt}
done
sudo find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
sudo find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+
set +o errexit
chmod -R 777 ${LOG_DIR}
find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+
}
check_failure() {
# Command failures after this point can be expected
set +o errexit
docker images
docker ps -a
# All docker container's status are created, restarting, running, removing,
# paused, exited and dead. Containers without running status are treated as
# failure. removing is added in docker 1.13, just ignore it now.
failed_containers=$(docker ps -a --format "{{.Names}}" \
--filter status=created \
--filter status=restarting \
--filter status=paused \
--filter status=exited \
--filter status=dead)
for failed in ${failed_containers}; do
docker logs --tail all ${failed} > /tmp/logs/docker_logs/${failed}
done
copy_logs
if [[ -n "$failed_containers" ]]; then
exit 1;
fi
}
check_failure

28
tests/post.yml Normal file
View File

@ -0,0 +1,28 @@
---
- hosts: all
vars:
logs_dir: "/tmp/logs"
tasks:
- name: Run diagnostics script
script: get_logs.sh
register: get_logs_result
become: true
failed_when: false
- name: Print get_logs output
debug:
msg: "{{ get_logs_result.stdout }}"
- name: Download logs to executor
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
mode: pull
src: "{{ logs_dir }}/"
ignore_errors: yes
- name: Download /etc/hosts file to executor
synchronize:
src: "/etc/hosts"
dest: "{{ zuul.executor.log_root }}/{{inventory_hostname }}/"
mode: pull
ignore_errors: yes

20
tests/pre.yml Normal file
View File

@ -0,0 +1,20 @@
- hosts: all
vars:
logs_dir: "/tmp/logs"
tasks:
- name: Ensure /tmp/logs/ dir
file:
path: "{{ logs_dir }}"
state: "directory"
- name: Ensure node directories
file:
path: "{{ logs_dir }}/{{ item }}"
state: "directory"
mode: 0777
with_items:
- "docker_logs"
- "kolla_configs"
- "system_logs"
- "kolla"
- "ansible"

83
tests/run.yml Normal file
View File

@ -0,0 +1,83 @@
---
- hosts: all
vars:
kolla_inventory_path: "/etc/kolla/inventory"
logs_dir: "/tmp/logs"
tasks:
- name: ensure /etc/kolla exists
file:
path: "/etc/kolla"
state: "directory"
mode: 0777
become: true
- set_fact:
nodes: |
{% for host in hostvars %}
{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
{% endfor %}
- name: copy default ansible kolla-ansible inventory
copy:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/ansible/inventory/all-in-one"
dest: "{{ kolla_inventory_path }}"
delegate_to: "primary"
- name: replace localhost with IPs
replace:
path: "{{ kolla_inventory_path }}"
regexp: "localhost.*$"
replace: "{{ nodes }}"
delegate_to: "primary"
- name: generate global.yml file
vars:
api_interface_name: "{{ hostvars['primary']['ansible_default_ipv4'].alias }}"
api_interface_address: "{{ hostvars['primary']['nodepool']['private_ipv4'] }}"
template:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/globals-default.j2"
dest: /etc/kolla/globals.yml
delegate_to: "primary"
- name: ensure nova conf overrides dir exists
file:
path: "/etc/kolla/config/nova"
state: "directory"
mode: 0777
become: true
delegate_to: "primary"
- name: generate config overrides
template:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/nova-compute-overrides.j2"
dest: /etc/kolla/config/nova/nova-compute.conf
delegate_to: "primary"
- name: copy passwords.yml file
copy:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/etc/kolla/passwords.yml"
dest: /etc/kolla/passwords.yml
delegate_to: "primary"
- name: install kolla-ansible requirements
pip:
requirements: "{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}/requirements.txt"
become: true
- name: generate passwords
shell: "{{ zuul.project.src_dir }}/tools/generate_passwords.py"
delegate_to: "primary"
- shell:
cmd: |
set -e
set -x
export BASE_DISTRO={{ base_distro }}
export INSTALL_TYPE={{ install_type }}
export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs
tools/setup_gate.sh
executable: /bin/bash
chdir: "{{ zuul.project.src_dir }}"
delegate_to: "primary"
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -1,8 +1,8 @@
---
kolla_base_distro: "{{ base }}"
kolla_install_type: "{{ type }}"
kolla_base_distro: "{{ base_distro }}"
kolla_install_type: "{{ install_type }}"
{% if is_multinode %}
{% if hostvars|length > 2 %}
enable_haproxy: "no"
kolla_internal_vip_address: "{{ api_interface_address }}"
{% else %}
@ -20,7 +20,7 @@ enable_heat: "no"
openstack_logging_debug: "True"
openstack_service_workers: "1"
{% if 'ceph' in action %}
{% if 'ceph' in zuul.job %}
enable_ceph: "yes"
enable_cinder: "yes"
{% endif %}

View File

@ -1,6 +0,0 @@
{{ primary_node_address }} ansible_become=true ansible_connection=local
{% for addr in sub_node_addresses %}
{% if addr %}
{{ addr }} ansible_user=jenkins ansible_become=true ansible_ssh_private_key_file=/etc/nodepool/id_rsa
{% endif %}
{% endfor %}

View File

@ -82,7 +82,7 @@ sudo service docker stop
if [[ ${DISTRIB_CODENAME} == "trusty" ]]; then
sudo apt-get -y install --no-install-recommends btrfs-tools
setup_disk
echo "DOCKER_OPTS=\"-s btrfs --insecure-registry $(cat /etc/nodepool/primary_node_private):4000\"" | sudo tee /etc/default/docker
echo "DOCKER_OPTS=\"-s btrfs --insecure-registry 0.0.0.0/0\"" | sudo tee /etc/default/docker
sudo mount --make-shared /run
sudo service docker start
else
@ -90,7 +90,7 @@ else
sudo tee /etc/systemd/system/docker.service.d/kolla.conf << EOF
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry $(cat /etc/nodepool/primary_node_private):4000
ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry 0.0.0.0/0
MountFlags=shared
EOF
sudo systemctl daemon-reload

View File

@ -6,20 +6,8 @@ set -o errexit
# Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1
. /etc/nodepool/provider
NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org}
NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]')
NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple}
NODEPOOL_TARBALLS_MIRROR=${NODEPOOL_TARBALLS_MIRROR:-http://$NODEPOOL_MIRROR_HOST:8080/tarballs}
GIT_PROJECT_DIR=$(mktemp -d)
# Just for mandre :)
if [[ ! -f /etc/sudoers.d/jenkins ]]; then
echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins
fi
function clone_repos {
cat > /tmp/clonemap <<EOF
clonemap:
@ -34,8 +22,6 @@ EOF
}
function setup_config {
sudo mkdir /etc/kolla
sudo chmod -R 777 /etc/kolla
# Use Infra provided pypi.
# Wheel package mirror may be not compatible. So do not enable it.
PIP_CONF=$(mktemp)
@ -55,6 +41,7 @@ EOF
GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon"
# TODO(jeffrey4l): this doesn't work with zuulv3
if echo $ACTION | grep -q "ceph"; then
GATE_IMAGES+=",ceph,cinder"
fi
@ -97,40 +84,13 @@ function detect_distro {
DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}')
}
# NOTE(sdake): This works around broken nodepool on systems with only one
# private interface
# The big regex checks for IP addresses in the file
function setup_workaround_broken_nodepool {
if [[ `grep -E -o "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" /etc/nodepool/node_private | wc -l` -eq 0 ]]; then
cp /etc/nodepool/node /etc/nodepool/node_private
cp /etc/nodepool/sub_nodes /etc/nodepool/sub_nodes_private
fi
}
function setup_ssh {
sudo chown jenkins /etc/nodepool/id_rsa
sudo chmod 600 /etc/nodepool/id_rsa
}
function setup_inventory {
echo -e "127.0.0.1\tlocalhost" > /tmp/hosts
ansible-playbook tests/ansible_generate_inventory.yml
sudo chown root: /tmp/hosts
sudo chmod 644 /tmp/hosts
sudo mv /tmp/hosts /etc/hosts
}
function setup_ansible {
RAW_INVENTORY=/tmp/kolla/raw_inventory
mkdir /tmp/kolla
RAW_INVENTORY=/etc/kolla/inventory
# TODO(SamYaple): Move to virtualenv
sudo -H pip install -U "ansible>=2,<2.4" "docker>=2.0.0" "python-openstackclient" "python-neutronclient" "ara"
detect_distro
setup_inventory
sudo mkdir /etc/ansible
ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
sudo tee /etc/ansible/ansible.cfg<<EOF
@ -147,16 +107,6 @@ function setup_node {
ansible-playbook -i ${RAW_INVENTORY} tools/playbook-setup-nodes.yml
}
function setup_logging {
# This directory is the directory that is copied with the devstack-logs
# publisher. It must exist at /home/jenkins/workspace/<job-name>/logs
mkdir logs
# For ease of access we symlink that logs directory to a known path
ln -s $(pwd)/logs /tmp/logs
mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs,system_logs}
}
function prepare_images {
sudo docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
@ -179,8 +129,6 @@ function prepare_images {
fi
}
function sanity_check {
# Wait for service ready
sleep 15
@ -203,30 +151,19 @@ function sanity_check {
fi
}
function get_logs {
ansible-playbook -i ${RAW_INVENTORY} tests/ansible_get_logs.yml > /tmp/logs/ansible/get-logs
}
setup_logging
tools/dump_info.sh
clone_repos
setup_workaround_broken_nodepool
setup_ssh
setup_ansible
setup_config
setup_node
ansible-playbook -e type=$INSTALL_TYPE -e base=$BASE_DISTRO -e action=$ACTION tests/ansible_generate_config.yml > /tmp/logs/ansible/generate_config
tools/kolla-ansible -i ${RAW_INVENTORY} bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
sudo tools/generate_passwords.py
prepare_images
if echo $ACTION | grep -q "ceph"; then
ansible-playbook -i ${RAW_INVENTORY} tests/ansible_setup_ceph_disks.yml > /tmp/logs/ansible/setup_ceph_disks
fi
trap get_logs EXIT
# Create dummy interface for neutron
ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all
@ -252,7 +189,5 @@ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
# run prechecks again
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
get_logs
ara generate html /tmp/logs/playbook_reports/
gzip --recursive --best /tmp/logs/playbook_reports/