Merge "CI: Add overcloud host configure job"

dev/functions

@@ -35,9 +35,19 @@ function config_defaults {
     export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
 
     # Whether to build container images for the overcloud services. If 0, they
-    # will be pulled.
+    # will be pulled if $KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL is 1.
     export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
 
+    # Whether to pull container images for the overcloud services if
+    # $KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD is 0.
+    export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL:-1}
+
+    # Whether to deploy overcloud services.
+    export KAYOBE_OVERCLOUD_SERVICE_DEPLOY=${KAYOBE_OVERCLOUD_SERVICE_DEPLOY:-1}
+
+    # Whether to perform overcloud post configuration.
+    export KAYOBE_OVERCLOUD_POST_CONFIGURE=${KAYOBE_OVERCLOUD_POST_CONFIGURE:-1}
+
     # Additional arguments to pass to kayobe commands.
     export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-}
 
@@ -349,17 +359,21 @@ function overcloud_deploy {
     if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
         echo "Building overcloud container images"
         run_kayobe overcloud container image build
-    else
+    elif [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL} = 1 ]]; then
         echo "Pulling overcloud container images"
         run_kayobe overcloud container image pull
     fi
 
+    if [[ ${KAYOBE_OVERCLOUD_SERVICE_DEPLOY} = 1 ]]; then
         echo "Deploying containerised overcloud services"
         run_kayobe overcloud service deploy
+    fi
 
+    if [[ ${KAYOBE_OVERCLOUD_POST_CONFIGURE} = 1 ]]; then
         echo "Performing post-deployment configuration"
         source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
         run_kayobe overcloud post configure
+    fi
 
     echo "Control plane deployment complete"
 }

playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2

@@ -0,0 +1,128 @@
+---
+# The following configuration aims to test some of the 'host configure'
+# command.
+
+# Additional users.
+controller_users:
+  - username: kayobe-test-user
+    name: Kayobe test user
+    password: kayobe-test-user-password
+    groups:
+      - stack
+
+# Additional network interfaces, testing a variety of interface configurations.
+controller_extra_network_interfaces:
+  - test_net_eth
+  - test_net_eth_vlan
+  - test_net_bridge
+  - test_net_bridge_vlan
+  - test_net_bond
+  - test_net_bond_vlan
+
+# dummy2: Ethernet interface.
+test_net_eth_cidr: 192.168.34.0/24
+test_net_eth_routes:
+  - cidr: 192.168.40.0/24
+    gateway: 192.168.34.254
+test_net_eth_interface: dummy2
+
+# dummy2.42: VLAN subinterface of dummy2.
+test_net_eth_vlan_cidr: 192.168.35.0/24
+test_net_eth_vlan_interface: "{% raw %}{{ test_net_eth_interface }}.{{ test_net_eth_vlan_vlan }}{% endraw %}"
+test_net_eth_vlan_vlan: 42
+
+# br0: bridge with ports dummy3, dummy4.
+test_net_bridge_cidr: 192.168.36.0/24
+test_net_bridge_interface: br0
+test_net_bridge_bridge_ports: [dummy3, dummy4]
+
+# br0.43: VLAN subinterface of br0.
+test_net_bridge_vlan_cidr: 192.168.37.0/24
+test_net_bridge_vlan_interface: "{% raw %}{{ test_net_bridge_interface }}.{{ test_net_bridge_vlan_vlan }}{% endraw %}"
+test_net_bridge_vlan_vlan: 43
+
+# bond0: bond with slaves dummy5, dummy6.
+test_net_bond_cidr: 192.168.38.0/24
+test_net_bond_interface: bond0
+test_net_bond_bond_slaves: [dummy5, dummy6]
+
+# bond0.44: VLAN subinterface of bond0.
+test_net_bond_vlan_cidr: 192.168.39.0/24
+test_net_bond_vlan_interface: "{% raw %}{{ test_net_bond_interface }}.{{ test_net_bond_vlan_vlan }}{% endraw %}"
+test_net_bond_vlan_vlan: 44
+
+# Create an LVM volume group for Docker volumes and devicemapper.
+controller_lvm_groups:
+  - "{% raw %}{{ controller_lvm_group_data }}{% endraw %}"
+
+# Provide a disk for use by LVM. Uses the software RAID device created below.
+controller_lvm_group_data_disks:
+  - /dev/md0
+
+# Define a software RAID device consisting of two loopback devices.
+controller_mdadm_arrays:
+  - name: md0
+    devices:
+      - /dev/loop0
+      - /dev/loop1
+    level: '1'
+    state: present
+
+# Set a sysctl.
+controller_sysctl_parameters:
+  fs.mount-max: 99999
+
+# Disable cloud-init.
+disable_cloud_init: true
+
+# Use devicemapper storage driver.
+docker_storage_driver: devicemapper
+
+# Set Honolulu time.
+timezone: Pacific/Honolulu
+
+{% if ansible_os_family == 'RedHat' %}
+{% if ansible_distribution_major_version | int == 7 %}
+# Use a local Yum mirror.
+yum_use_local_mirror: true
+# Mirror FQDN for Yum repos.
+yum_centos_mirror_host: "{{ zuul_site_mirror_fqdn }}"
+# Mirror directory for Yum CentOS repos.
+yum_centos_mirror_directory: 'centos'
+# Mirror FQDN for Yum EPEL repos.
+yum_epel_mirror_host: "{{ zuul_site_mirror_fqdn }}"
+# Mirror directory for Yum EPEL repos.
+yum_epel_mirror_directory: 'epel'
+# Configure a custom Yum repository.
+yum_custom_repos:
+  td-agent:
+    baseurl: http://packages.treasuredata.com/3/redhat/$releasever/$basearch
+    gpgkey: https://packages.treasuredata.com/GPG-KEY-td-agent
+    gpgcheck: yes
+# Don't install EPEL repositories.
+yum_install_epel: false
+# Enable yum-cron.
+yum_cron_enabled: true
+{% else %}
+# Use a local DNF mirror.
+dnf_use_local_mirror: true
+# Mirror FQDN for DNF repos.
+dnf_centos_mirror_host: "{{ zuul_site_mirror_fqdn }}"
+# Mirror directory for DNF CentOS repos.
+dnf_centos_mirror_directory: 'centos'
+# Mirror FQDN for DNF EPEL repos.
+dnf_epel_mirror_host: "{{ zuul_site_mirror_fqdn }}"
+# Mirror directory for DNF EPEL repos.
+dnf_epel_mirror_directory: 'epel'
+# Configure a custom DNF repository.
+dnf_custom_repos:
+  td-agent:
+    baseurl: http://packages.treasuredata.com/3/redhat/$releasever/$basearch
+    gpgkey: https://packages.treasuredata.com/GPG-KEY-td-agent
+    gpgcheck: yes
+# Don't install EPEL repositories.
+dnf_install_epel: false
+# Enable DNF Automatic.
+dnf_automatic_enabled: true
+{% endif %}
+{% endif %}

playbooks/kayobe-overcloud-host-configure-base/pre.yml

@@ -0,0 +1,42 @@
+---
+- hosts: primary
+  vars:
+    testinfra_venv: ~/testinfra-venv
+  tasks:
+    - name: Ensure python3 is installed
+      package:
+        name: python3
+      become: true
+
+    - name: Ensure testinfra is installed
+      pip:
+        name:
+          - distro
+          - testinfra
+          - pytest-html
+        virtualenv: "{{ testinfra_venv }}"
+        virtualenv_python: python3
+
+    # NOTE(mgoddard): Use the name zzz-overrides.yml to ensure this takes
+    # precedence over the standard config files and zz-overrides.yml from
+    # kayobe-overcloud-base.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zzz-overrides.yml"
+
+    # NOTE(mgoddard): Create two loopback devices backed by files. These will
+    # be added to a software RAID volume, then added to an LVM volume group.
+    - name: Ensure a docker storage backing file exists
+      command: truncate -s 2G /tmp/docker-storage{{ item }}
+      loop: [0, 1]
+
+    - name: Ensure the docker storage loopback device is created
+      command: losetup /dev/loop{{ item }} /tmp/docker-storage{{ item }}
+      become: true
+      loop: [0, 1]
+
+    - name: Ensure dummy network interfaces exist
+      command: ip link add dummy{{ item }} type dummy
+      become: true
+      loop: "{{ range(2, 7) | list }}"

playbooks/kayobe-overcloud-host-configure-base/run.yml

@@ -0,0 +1,20 @@
+---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+    # Don't run container deployment.
+    KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL: 0
+    KAYOBE_OVERCLOUD_SERVICE_DEPLOY: 0
+    KAYOBE_OVERCLOUD_POST_CONFIGURE: 0
+  vars:
+    testinfra_venv: ~/testinfra-venv
+    test_path: "{{ kayobe_src_dir }}/playbooks/kayobe-overcloud-host-configure-base/tests/"
+  tasks:
+    - name: Ensure overcloud is deployed
+      shell:
+        cmd: "{{ kayobe_src_dir }}/dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy"
+
+    - name: Run testinfra tests
+      command: "{{ testinfra_venv }}/bin/py.test {{ test_path }} --html={{ logs_dir }}/test-results.html --self-contained-html"
+      environment:
+        SITE_MIRROR_FQDN: "{{ zuul_site_mirror_fqdn }}"

playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+# Kayobe overcloud host configure tests.
+# Uses py.test and TestInfra.
+
+import ipaddress
+import os
+
+import distro
+import pytest
+
+
+def _is_yum():
+    info = distro.linux_distribution()
+    return info[0] == 'CentOS Linux' and info[1].startswith('7')
+
+
+def _is_dnf():
+    info = distro.linux_distribution()
+    return info[0] == 'CentOS Linux' and info[1].startswith('8')
+
+
+def test_network_ethernet(host):
+    interface = host.interface('dummy2')
+    assert interface.exists
+    assert '192.168.34.1' in interface.addresses
+    routes = host.check_output('/sbin/ip route show dev dummy2')
+    assert '192.168.40.0/24 via 192.168.34.254' in routes
+
+
+def test_network_ethernet_vlan(host):
+    interface = host.interface('dummy2.42')
+    assert interface.exists
+    assert '192.168.35.1' in interface.addresses
+    assert host.file('/sys/class/net/dummy2.42/lower_dummy2').exists
+
+
+def test_network_bridge(host):
+    interface = host.interface('br0')
+    assert interface.exists
+    assert '192.168.36.1' in interface.addresses
+    ports = ['dummy3', 'dummy4']
+    sys_ports = host.check_output('ls -1 /sys/class/net/br0/brif')
+    assert sys_ports == "\n".join(ports)
+    for port in ports:
+       interface = host.interface(port)
+       assert interface.exists
+       v4_addresses = [a for a in interface.addresses
+                       if ipaddress.ip_address(a).version == '4']
+       assert not v4_addresses
+
+
+def test_network_bridge_vlan(host):
+    interface = host.interface('br0.43')
+    assert interface.exists
+    assert '192.168.37.1' in interface.addresses
+    assert host.file('/sys/class/net/br0.43/lower_br0').exists
+
+
+def test_network_bond(host):
+    interface = host.interface('bond0')
+    assert interface.exists
+    assert '192.168.38.1' in interface.addresses
+    sys_slaves = host.check_output('cat /sys/class/net/bond0/bonding/slaves')
+    slaves = ['dummy5', 'dummy6']
+    assert sys_slaves == " ".join(slaves)
+    for slave in slaves:
+       interface = host.interface(slave)
+       assert interface.exists
+       assert not interface.addresses
+
+
+def test_network_bond_vlan(host):
+    interface = host.interface('bond0.44')
+    assert interface.exists
+    assert '192.168.39.1' in interface.addresses
+    assert host.file('/sys/class/net/bond0.44/lower_bond0').exists
+
+
+def test_additional_user_account(host):
+      user = host.user("kayobe-test-user")
+      assert user.name == "kayobe-test-user"
+      assert user.group == "kayobe-test-user"
+      assert set(user.groups) == {"kayobe-test-user", "stack"}
+      assert user.gecos == "Kayobe test user"
+      with host.sudo():
+          assert user.password == 'kayobe-test-user-password'
+
+
+def test_software_RAID(host):
+    slaves = host.check_output("ls -1 /sys/class/block/md0/slaves/")
+    assert slaves == "loop0\nloop1"
+
+
+def test_sysctls(host):
+    assert host.sysctl("fs.mount-max") == 99999
+
+
+def test_cloud_init_is_disabled(host):
+    assert host.file("/etc/cloud/cloud-init.disabled").exists
+
+
+def test_docker_storage_driver_is_devicemapper(host):
+    with host.sudo("stack"):
+        info = host.check_output("docker info")
+    assert "devicemapper" in info
+
+
+@pytest.mark.parametrize('user', ['kolla', 'stack'])
+def test_docker_image_download(host, user):
+    with host.sudo(user):
+        host.check_output("docker pull alpine")
+
+
+@pytest.mark.parametrize('user', ['kolla', 'stack'])
+def test_docker_container_run(host, user):
+    with host.sudo(user):
+        host.check_output("docker run --rm alpine /bin/true")
+
+
+def test_timezone(host):
+    status = host.check_output("timedatectl status")
+    assert "Pacific/Honolulu" in status
+
+
+@pytest.mark.parametrize('repo', ["base", "extras", "updates", "epel"])
+@pytest.mark.skipif(not _is_yum(), reason="Yum only supported on CentOS 7")
+def test_yum_local_package_mirrors(host, repo):
+    assert os.getenv('SITE_MIRROR_FQDN')
+    info = host.check_output("yum repoinfo %s", repo)
+    assert os.getenv('SITE_MIRROR_FQDN') in info
+
+
+@pytest.mark.parametrize('repo', ["AppStream", "BaseOS", "Extras", "epel",
+                                  "epel-modular"])
+@pytest.mark.skipif(not _is_dnf(), reason="DNF only supported on CentOS 8")
+def test_dnf_local_package_mirrors(host, repo):
+    # Depends on SITE_MIRROR_FQDN environment variable.
+    assert os.getenv('SITE_MIRROR_FQDN')
+    # NOTE(mgoddard): Should not require sudo but some files
+    # (/var/cache/dnf/expired_repos.json) can have incorrect permissions.
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1636909
+    with host.sudo():
+        info = host.check_output("dnf repoinfo %s", repo)
+    assert os.getenv('SITE_MIRROR_FQDN') in info
+
+
+@pytest.mark.skipif(not _is_yum(), reason="YUM only supported on CentOS 7")
+def test_yum_custom_package_repository_is_available(host):
+    with host.sudo():
+        host.check_output("yum -y install td-agent")
+    assert host.package("td-agent").is_installed
+
+
+@pytest.mark.skipif(not _is_dnf(), reason="DNF only supported on CentOS 8")
+def test_dnf_custom_package_repository_is_available(host):
+    with host.sudo():
+        host.check_output("dnf -y install td-agent")
+    assert host.package("td-agent").is_installed
+
+
+@pytest.mark.skipif(not _is_yum(), reason="YUM only supported on CentOS 7")
+def test_yum_cron(host):
+    assert host.package("yum-cron").is_installed
+    assert host.service("yum-cron").is_enabled
+    assert host.service("yum-cron").is_running
+
+
+@pytest.mark.skipif(not _is_dnf(), reason="DNF only supported on CentOS 8")
+def test_dnf_automatic(host):
+    assert host.package("dnf-automatic").is_installed
+    assert host.service("dnf-automatic.timer").is_enabled
+    assert host.service("dnf-automatic.timer").is_running

zuul.d/jobs.yaml

@@ -130,6 +130,22 @@
     parent: kayobe-seed-base
     nodeset: kayobe-centos8
 
+- job:
+    name: kayobe-overcloud-host-configure-base
+    parent: kayobe-overcloud-base
+    description: |
+      Base job for testing overcloud host configure.
+
+      Configures the primary VM as an overcloud controller.
+    pre-run: playbooks/kayobe-overcloud-host-configure-base/pre.yml
+    run: playbooks/kayobe-overcloud-host-configure-base/run.yml
+    timeout: 7200
+
+- job:
+    name: kayobe-overcloud-host-configure-centos8
+    parent: kayobe-overcloud-host-configure-base
+    nodeset: kayobe-centos8
+
 - job:
     name: kayobe-seed-upgrade-base
     parent: kayobe-base

zuul.d/project.yaml

@@ -11,6 +11,7 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos8
+        - kayobe-overcloud-host-configure-centos8
         - kayobe-overcloud-upgrade-centos8
         - kayobe-seed-centos8
         - kayobe-seed-upgrade-centos8
@@ -22,6 +23,7 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos8
+        - kayobe-overcloud-host-configure-centos8
         - kayobe-overcloud-upgrade-centos8
         - kayobe-seed-centos8
         - kayobe-seed-upgrade-centos8