[CI] Use Tenks in Ironic job
Installs Tenks [1] and uses it to create virtual machines to pose as bare metal compute nodes. The nodes are registered in Ironic, and used to provision instances. [1] https://docs.openstack.org/tenks/latest/ Depends-On: https://review.opendev.org/c/openstack/tenks/+/830182 Depends-On: https://review.opendev.org/c/openstack/tenks/+/830675 Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/831055 Change-Id: Idfb8fbb50dc7442225967b2a2ec38ae7114f3c11 Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
This commit is contained in:
Mark Goddard
Radosław Piliszek
parent
68bc4f8f52
commit
91778aca76
70
tests/deploy-tenks.sh
Executable file
70
tests/deploy-tenks.sh
Executable file
@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
function deploy_tenks_logged {
|
||||
. /etc/kolla/admin-openrc.sh
|
||||
|
||||
echo "Creating IPA images for Ironic"
|
||||
~/openstackclient-venv/bin/openstack image create --disk-format aki --container-format aki --private \
|
||||
--file /etc/kolla/config/ironic/ironic-agent.kernel ipa.vmlinuz
|
||||
~/openstackclient-venv/bin/openstack image create --disk-format ari --container-format ari --private \
|
||||
--file /etc/kolla/config/ironic/ironic-agent.initramfs ipa.initramfs
|
||||
|
||||
# Install a trivial script for ovs-vsctl that talks to containerised Open
|
||||
# vSwitch.
|
||||
sudo tee /usr/bin/ovs-vsctl >/dev/null <<EOF
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Script installed onto the host to fool tenks into using the containerised
|
||||
# Open vSwitch rather than installing its own.
|
||||
|
||||
sudo docker exec openvswitch_vswitchd ovs-vsctl "\$@"
|
||||
EOF
|
||||
sudo chmod 755 /usr/bin/ovs-vsctl
|
||||
|
||||
# Install the Tenks venv.
|
||||
python3 -m venv ${TENKS_VENV_PATH}
|
||||
${TENKS_VENV_PATH}/bin/pip install -U pip
|
||||
${TENKS_VENV_PATH}/bin/pip install ${TENKS_SRC_PATH}
|
||||
|
||||
local attempt
|
||||
attempt=1
|
||||
while ! ${TENKS_VENV_PATH}/bin/ansible-galaxy install \
|
||||
--role-file="${TENKS_SRC_PATH}/requirements.yml" \
|
||||
--roles-path="${TENKS_SRC_PATH}/ansible/roles/"; do
|
||||
echo "ansible-galaxy install failed, attempt $attempt"
|
||||
attempt=$((attempt+1))
|
||||
if [[ $attempt -eq 10 ]]; then
|
||||
echo "enough retrying, farewell"
|
||||
return 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
# Run Tenks.
|
||||
${TENKS_VENV_PATH}/bin/ansible-playbook \
|
||||
-vvv \
|
||||
--inventory "${TENKS_SRC_PATH}/ansible/inventory" \
|
||||
--extra-vars=@"$HOME/tenks.yml" \
|
||||
"${TENKS_SRC_PATH}/ansible/deploy.yml"
|
||||
}
|
||||
|
||||
function deploy_tenks {
|
||||
echo "Configuring virtual bare metal via Tenks"
|
||||
deploy_tenks_logged > /tmp/logs/ansible/deploy-tenks 2>&1
|
||||
result=$?
|
||||
if [[ $result != 0 ]]; then
|
||||
echo "Deploying tenks failed. See ansible/deploy-tenks for details"
|
||||
else
|
||||
echo "Successfully deployed tenks. See ansible/deploy-tenks for details"
|
||||
fi
|
||||
return $result
|
||||
}
|
||||
|
||||
deploy_tenks
|
||||
@ -74,7 +74,7 @@
|
||||
when:
|
||||
# NOTE(yoctozepto): build container images if there is any tested
|
||||
# change that impacts them.
|
||||
- item.project.short_name not in ["ansible-collection-kolla", "kayobe", "kolla-ansible"]
|
||||
- item.project.short_name not in ["ansible-collection-kolla", "kayobe", "kolla-ansible", "tenks"]
|
||||
with_items: "{{ zuul['items'] }}"
|
||||
|
||||
# NOTE(yoctozepto): required to template template_overrides.j2 for Zuul
|
||||
@ -178,7 +178,9 @@
|
||||
- src: "tests/templates/ironic-overrides.j2"
|
||||
dest: /etc/kolla/config/ironic.conf
|
||||
when: "{{ scenario == 'ironic' }}"
|
||||
|
||||
- src: "tests/templates/tenks-deploy-config.yml.j2"
|
||||
dest: "{{ ansible_env.HOME }}/tenks.yml"
|
||||
when: "{{ scenario == 'ironic' }}"
|
||||
when: item.when | default(true)
|
||||
|
||||
- block:
|
||||
@ -449,13 +451,23 @@
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
when: scenario == "scenario_nfv"
|
||||
|
||||
- name: Run test-ironic.sh script
|
||||
script:
|
||||
cmd: test-ironic.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
TLS_ENABLED: "{{ tls_enabled }}"
|
||||
- block:
|
||||
- name: Run deploy-tenks.sh script
|
||||
script:
|
||||
cmd: deploy-tenks.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
TENKS_VENV_PATH: "{{ ansible_env.HOME }}/tenks-venv"
|
||||
TENKS_SRC_PATH: "{{ ansible_env.HOME }}/src/opendev.org/openstack/tenks"
|
||||
|
||||
- name: Run test-ironic.sh script
|
||||
script:
|
||||
cmd: test-ironic.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
TLS_ENABLED: "{{ tls_enabled }}"
|
||||
when: scenario == "ironic"
|
||||
|
||||
- name: Run test-magnum.sh script
|
||||
|
||||
@ -1,19 +1,3 @@
|
||||
[DEFAULT]
|
||||
# Enable all fake hardware types and interfaces.
|
||||
enabled_hardware_types = fake-hardware
|
||||
enabled_boot_interfaces = fake
|
||||
enabled_console_interfaces = ipmitool-socat,no-console
|
||||
enabled_deploy_interfaces = fake
|
||||
enabled_inspect_interfaces = inspector,no-inspect
|
||||
enabled_management_interfaces = fake
|
||||
enabled_network_interfaces = noop,flat,neutron
|
||||
default_network_interface = neutron
|
||||
enabled_power_interfaces = fake
|
||||
enabled_raid_interfaces = agent,no-raid
|
||||
default_raid_interface = no-raid
|
||||
enabled_rescue_interfaces = fake
|
||||
enabled_vendor_interfaces = no-vendor
|
||||
|
||||
[neutron]
|
||||
cleaning_network = public1
|
||||
provisioning_network = public1
|
||||
|
||||
55
tests/templates/tenks-deploy-config.yml.j2
Normal file
55
tests/templates/tenks-deploy-config.yml.j2
Normal file
@ -0,0 +1,55 @@
|
||||
---
|
||||
# This file holds the config given to Tenks when running `deploy-tenks.sh`.
|
||||
|
||||
node_types:
|
||||
type0:
|
||||
memory_mb: 1024
|
||||
vcpus: 1
|
||||
volumes:
|
||||
# There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
|
||||
# https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
|
||||
- capacity: 4GiB
|
||||
physical_networks:
|
||||
- physnet1
|
||||
console_log_enabled: true
|
||||
# We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g:
|
||||
# https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38
|
||||
cpu_mode:
|
||||
specs:
|
||||
- type: type0
|
||||
count: 1
|
||||
ironic_config:
|
||||
resource_class: test-rc
|
||||
network_interface: flat
|
||||
|
||||
nova_flavors:
|
||||
- resource_class: test-rc
|
||||
node_type: type0
|
||||
|
||||
physnet_mappings:
|
||||
physnet1: {{ neutron_external_bridge_name }}
|
||||
|
||||
deploy_kernel: ipa.vmlinuz
|
||||
deploy_ramdisk: ipa.initramfs
|
||||
|
||||
default_boot_mode: "bios"
|
||||
|
||||
# Use the libvirt daemon deployed by Kolla Ansible in the nova_libvirt
|
||||
# container. Tenks will install libvirt client packages.
|
||||
libvirt_host_install_daemon: false
|
||||
|
||||
# Nested virtualisation is not working well in CI currently. Force the use of
|
||||
# QEMU.
|
||||
libvirt_vm_engine: "qemu"
|
||||
|
||||
# QEMU may not be installed on the host, so set the path and avoid
|
||||
# autodetection.
|
||||
libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}"
|
||||
|
||||
# Specify a log path in the kolla_logs Docker volume. It is accessible on the
|
||||
# host at the same path.
|
||||
libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks"
|
||||
|
||||
# Console logs are owned by the ID of the Nova user in the nova_libvirt
|
||||
# container.
|
||||
libvirt_vm_log_owner: 42436
|
||||
@ -7,134 +7,48 @@ set -o pipefail
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
# Adapted from the function of the same name in the ironic devstack plugin.
|
||||
function wait_for_placement_resources {
|
||||
# After nodes have been enrolled, we need to wait for both ironic and
|
||||
# nova's periodic tasks to populate the resource tracker with available
|
||||
# nodes and resources. Wait up to 2 minutes for a given resource before
|
||||
# timing out.
|
||||
local expected_count=1
|
||||
local resource_class="RC0"
|
||||
|
||||
curl --fail -L -o jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
|
||||
chmod +x jq
|
||||
|
||||
# TODO(mgoddard): switch to Placement OSC plugin, once it exists
|
||||
local token
|
||||
token=$(openstack token issue -f value -c id)
|
||||
local endpoint
|
||||
endpoint=$(openstack endpoint list --service placement --interface public -f value -c URL)
|
||||
if [[ -z $endpoint ]]; then
|
||||
echo "Cannot find Placement API endpoint"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local i
|
||||
local count
|
||||
echo "Waiting 2 minutes for Nova resource tracker to pick up $expected_count nodes"
|
||||
for i in $(seq 1 120); do
|
||||
# Fetch provider UUIDs from Placement
|
||||
local providers
|
||||
args=(
|
||||
--fail
|
||||
-sH "X-Auth-Token: $token"
|
||||
)
|
||||
if [[ "$TLS_ENABLED" = "True" ]]; then
|
||||
args+=(--cacert $OS_CACERT)
|
||||
fi
|
||||
providers=$(curl "${args[@]}" $endpoint/resource_providers \
|
||||
| ./jq -r '.resource_providers[].uuid')
|
||||
|
||||
local p
|
||||
# Total count of the resource class, has to be equal to nodes count
|
||||
count=0
|
||||
for p in $providers; do
|
||||
local amount
|
||||
# A resource class inventory record looks something like
|
||||
# {"max_unit": 1, "min_unit": 1, "step_size": 1, "reserved": 0, "total": 1, "allocation_ratio": 1}
|
||||
# Subtract reserved from total (defaulting both to 0)
|
||||
amount=$(curl "${args[@]}" $endpoint/resource_providers/$p/inventories \
|
||||
| ./jq ".inventories.CUSTOM_$resource_class as \$cls
|
||||
| (\$cls.total // 0) - (\$cls.reserved // 0)")
|
||||
if [ $amount -gt 0 ]; then
|
||||
count=$(( count + $amount ))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $count -ge $expected_count ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Timed out waiting for Nova to track $expected_count nodes"
|
||||
return 1
|
||||
}
|
||||
|
||||
function create_resources {
|
||||
# Create a bare metal node and port.
|
||||
openstack baremetal node create \
|
||||
--name node-0 \
|
||||
--driver fake-hardware \
|
||||
--network-interface noop \
|
||||
--property cpu_arch=x86_64 \
|
||||
--resource-class rc0
|
||||
node_uuid=$(openstack baremetal node show node-0 -f value -c uuid)
|
||||
openstack baremetal port create \
|
||||
00:11:22:33:44:55 \
|
||||
--node $node_uuid
|
||||
openstack baremetal node power off node-0
|
||||
openstack baremetal node manage node-0 --wait
|
||||
openstack baremetal node provide node-0 --wait
|
||||
|
||||
# Create a bare metal flavor in nova.
|
||||
openstack flavor create \
|
||||
baremetal \
|
||||
--vcpus 1 \
|
||||
--ram 1024 \
|
||||
--disk 10 \
|
||||
--property resources:CUSTOM_RC0=1 \
|
||||
--property resources:VCPU=0 \
|
||||
--property resources:MEMORY_MB=0 \
|
||||
--property resources:DISK_GB=0 \
|
||||
--public
|
||||
}
|
||||
|
||||
function test_ironic_logged {
|
||||
# Assumes init-runonce has been executed.
|
||||
. /etc/kolla/admin-openrc.sh
|
||||
. ~/openstackclient-venv/bin/activate
|
||||
|
||||
echo "Enabling DHCP on the external (\"public\") subnet"
|
||||
openstack subnet set --dhcp public1-subnet
|
||||
|
||||
# Smoke test ironic API.
|
||||
local baremetal_driver_list
|
||||
baremetal_driver_list=$(openstack baremetal driver list)
|
||||
openstack baremetal driver list
|
||||
openstack baremetal node list
|
||||
openstack baremetal port list
|
||||
# Ironic Inspector API
|
||||
openstack baremetal introspection rule list
|
||||
|
||||
# Sanity check.
|
||||
if ! echo "$baremetal_driver_list" | grep fake-hardware; then
|
||||
echo "No active conductors with fake-hardware driver"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create_resources
|
||||
wait_for_placement_resources
|
||||
openstack baremetal node show tk0
|
||||
openstack baremetal node power off tk0
|
||||
openstack baremetal node show tk0
|
||||
openstack baremetal node manage tk0
|
||||
openstack baremetal node show tk0
|
||||
openstack baremetal node provide tk0
|
||||
openstack baremetal node show tk0
|
||||
openstack baremetal node validate tk0
|
||||
|
||||
echo "TESTING: Server creation"
|
||||
openstack server create --wait --image cirros --flavor baremetal --key-name mykey --network demo-net kolla_boot_test
|
||||
openstack --debug server list
|
||||
# If the status is not ACTIVE, print info and exit 1
|
||||
if [[ $(openstack server show kolla_boot_test -f value -c status) != "ACTIVE" ]]; then
|
||||
echo "FAILED: Instance is not active"
|
||||
openstack --debug server show kolla_boot_test
|
||||
return 1
|
||||
fi
|
||||
openstack server create --image cirros --flavor test-rc --key-name mykey --network public1 kolla_bm_boot_test
|
||||
local attempt
|
||||
attempt=1
|
||||
while [[ $(openstack server show kolla_bm_boot_test -f value -c status) != "ACTIVE" ]]; do
|
||||
echo "Server not yet active, check $attempt - retrying"
|
||||
attempt=$((attempt+1))
|
||||
if [[ $attempt -eq 16 ]]; then
|
||||
echo "FAILED: Server did not become active after $attempt checks"
|
||||
openstack server show kolla_bm_boot_test
|
||||
return 1
|
||||
fi
|
||||
sleep 60
|
||||
done
|
||||
echo "SUCCESS: Server creation"
|
||||
|
||||
echo "TESTING: Server deletion"
|
||||
openstack server delete --wait kolla_boot_test
|
||||
openstack server delete --wait kolla_bm_boot_test
|
||||
echo "SUCCESS: Server deletion"
|
||||
}
|
||||
|
||||
|
||||
@ -83,8 +83,13 @@
|
||||
voting: false
|
||||
files:
|
||||
- ^ansible/roles/(ironic|neutron|nova|nova-cell)/
|
||||
- ^tests/test-ironic.sh
|
||||
- ^tests/test-dashboard.sh
|
||||
- ^tests/deploy-tenks\.sh$
|
||||
- ^tests/templates/ironic-overrides\.j2$
|
||||
- ^tests/templates/tenks-deploy-config\.yml\.j2$
|
||||
- ^tests/test-dashboard\.sh$
|
||||
- ^tests/test-ironic\.sh$
|
||||
required-projects:
|
||||
- openstack/tenks
|
||||
vars:
|
||||
scenario: ironic
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user