Merge "libvirt: deploy libvirt on the host"

ansible/compute-libvirt-host.yml

@@ -0,0 +1,56 @@
+---
+- name: Ensure the libvirt daemon is configured
+  hosts: compute
+  tags:
+    - libvirt-host
+  tasks:
+    - name: Ensure Ceph package repository is available
+      package:
+        name: "centos-release-ceph-{{ compute_libvirt_ceph_repo_release }}"
+        state: present
+      when:
+        - compute_libvirt_enabled | bool
+        - ansible_facts.distribution in ['CentOS', 'Rocky']
+        - compute_libvirt_ceph_repo_install | bool
+      become: true
+
+    - name: Include stackhpc.libvirt-host role
+      include_role:
+        name: stackhpc.libvirt-host
+      vars:
+        libvirt_host_libvirtd_conf: "{{ compute_libvirt_conf }}"
+        libvirt_host_qemu_conf: "{{ compute_qemu_conf }}"
+        libvirt_host_tcp_listen: "{{ not compute_libvirt_enable_tls | bool }}"
+        libvirt_host_tcp_listen_address: "{{ internal_net_name | net_ip }}:16509"
+        libvirt_host_tls_listen: "{{ compute_libvirt_enable_tls | bool }}"
+        libvirt_host_tls_listen_address: "{{ internal_net_name | net_ip }}:16514"
+        # TLS server and client certificates.
+        libvirt_host_tls_server_cert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['servercert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_server_key: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['serverkey.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_client_cert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientcert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_client_key: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientkey.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_cacert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['cacert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        lookup_params:
+          paths: "{{ libvirt_tls_cert_paths }}"
+          skip: true
+        # Support loading libvirt TLS certificates & keys from per-host and
+        # global locations.
+        libvirt_tls_cert_paths: >-
+          {{ (libvirt_tls_cert_dirs | unique | product([inventory_hostname]) | map('path_join') | list +
+              libvirt_tls_cert_dirs | unique | list) | list }}
+        libvirt_tls_cert_dirs:
+          - "{{ kayobe_env_config_path }}/certificates/libvirt"
+          - "{{ kayobe_config_path }}/certificates/libvirt"
+        libvirt_host_enable_efi_support: true
+      when:
+        - compute_libvirt_enabled | bool

ansible/group_vars/all/compute

@@ -161,3 +161,54 @@ compute_firewalld_default_zone:
 # - permanent: true
 # - state: enabled
 compute_firewalld_rules: []
+
+###############################################################################
+# Compute node host libvirt configuration.
+
+# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova
+# is true and kolla_enable_nova_libvirt_container is false.
+compute_libvirt_enabled: "{{ kolla_enable_nova | bool and not kolla_enable_nova_libvirt_container | bool }}"
+
+# A dict of default configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+compute_libvirt_conf_default:
+  auth_tcp: "none"
+  log_level: "{{ compute_libvirtd_log_level }}"
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+compute_libvirt_conf_extra: {}
+
+# A dict of configuration options to write to /etc/libvirt/libvirtd.conf.
+# Default is a combination of compute_libvirt_conf_default and
+# compute_libvirt_conf_extra.
+compute_libvirt_conf: "{{ compute_libvirt_conf_default | combine(compute_libvirt_conf_extra) }}"
+
+# Numerical log level for libvirtd. Default is 3.
+compute_libvirtd_log_level: 3
+
+# A dict of default configuration options to write to
+# /etc/libvirt/qemu.conf.
+compute_qemu_conf_default:
+  max_files: 32768
+  max_processes: 131072
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/qemu.conf.
+compute_qemu_conf_extra: {}
+
+# A dict of configuration options to write to /etc/libvirt/qemu.conf.
+# Default is a combination of compute_qemu_conf_default and
+# compute_qemu_conf_extra.
+compute_qemu_conf: "{{ compute_qemu_conf_default | combine(compute_qemu_conf_extra) }}"
+
+# Whether to enable a libvirt TLS listener. Default is false.
+compute_libvirt_enable_tls: false
+
+# Whether to install a Ceph package repository on CentOS and Rocky hosts.
+# Default is true.
+compute_libvirt_ceph_repo_install: true
+
+# Ceph package repository release to install on CentOS and Rocky hosts when
+# compute_libvirt_ceph_repo_install is true. Default is 'pacific'.
+compute_libvirt_ceph_repo_release: pacific

ansible/group_vars/all/kolla

@@ -553,6 +553,7 @@ kolla_enable_murano: "no"
 kolla_enable_neutron_mlnx: "no"
 kolla_enable_neutron_provider_networks: "no"
 kolla_enable_neutron_sriov: "no"
+kolla_enable_nova_libvirt_container: "yes"
 kolla_enable_octavia: "no"
 kolla_enable_openvswitch: "{{ kolla_enable_neutron | bool }}"
 kolla_enable_ovn: "no"

ansible/kolla-ansible.yml

@@ -103,6 +103,7 @@
         kolla_inspector_netmask: "{{ inspection_net_name | net_mask }}"
         kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}"
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
+        kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
         kolla_enable_host_ntp: false
         docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
         kolla_globals_paths_extra:

ansible/kolla-openstack.yml

@@ -249,3 +249,5 @@
       kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
       kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}"
       kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config"
+      kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
+      kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt"

ansible/roles/kolla-ansible/defaults/main.yml

@@ -236,6 +236,8 @@ kolla_openstack_logging_debug:
 # controllers.
 kolla_nova_compute_ironic_host:
 
+kolla_libvirt_tls:
+
 ###############################################################################
 # Extra free-form configuraton.
 

ansible/roles/kolla-ansible/templates/kolla/globals.yml

@@ -393,6 +393,10 @@ enable_{{ feature_flag }}: {{ hostvars[inventory_hostname]['kolla_enable_' ~ fea
 # Valid options are [ none, novnc, spice, rdp ]
 #nova_console: "novnc"
 
+{% if kolla_libvirt_tls is not none %}
+libvirt_tls: {{ kolla_libvirt_tls | bool }}
+{% endif %}
+
 #################
 # Hyper-V options
 #################

ansible/roles/kolla-ansible/vars/main.yml

@@ -184,6 +184,7 @@ kolla_feature_flags:
   - nova
   - nova_fake
   - nova_horizon_policy_file
+  - nova_libvirt_container
   - nova_serialconsole_proxy
   - nova_ssh
   - octavia

ansible/roles/kolla-openstack/defaults/main.yml

@@ -447,9 +447,19 @@ kolla_extra_neutron_ml2:
 # Whether to enable Nova.
 kolla_enable_nova:
 
+# Whether to enable Nova libvirt container.
+kolla_enable_nova_libvirt_container:
+
 # Free form extra configuration to append to nova.conf.
 kolla_extra_nova:
 
+# Whether libvirt TLS is enabled.
+kolla_libvirt_tls:
+
+# Directory containing libvirt certificates for nova-compute when running
+# libvirt on the host.
+kolla_nova_libvirt_certificates_src:
+
 ###############################################################################
 # Octavia configuration.
 

ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml

@@ -15,7 +15,7 @@ provisioner:
   inventory:
     group_vars:
       all:
-        kolla_extra_config_path:
+        kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/kolla/config
         kolla_enable_aodh: true
         kolla_extra_aodh: |
           [extra-aodh.conf]
@@ -116,9 +116,12 @@ provisioner:
           [extra-ml2_conf.ini]
           foo=bar
         kolla_enable_nova: true
+        kolla_enable_nova_libvirt_container: false
         kolla_extra_nova: |
           [extra-nova.conf]
           foo=bar
+        kolla_libvirt_tls: true
+        kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/nova-libvirt/certificates
         kolla_enable_octavia: true
         kolla_extra_octavia: |
           [extra-octavia.conf]

ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml

@@ -25,3 +25,23 @@
       with_items:
         - "{{ kolla_inspector_ipa_kernel_path }}"
         - "{{ kolla_inspector_ipa_ramdisk_path }}"
+
+    - name: Ensure nova libvirt certificates directory exists
+      local_action:
+        module: file
+        path: "{{ kolla_nova_libvirt_certificates_src }}"
+        state: directory
+
+    # NOTE(mgoddard): Previously we were creating empty files for the kernel
+    # and ramdisk, but this was found to cause ansible to hang on recent
+    # versions of docker. Using non-empty files seems to resolve the issue.
+    # See https://github.com/ansible/ansible/issues/36725.
+    - name: Ensure nova libvirt certificates exist
+      local_action:
+        module: copy
+        content: fake cert
+        dest: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}"
+      with_items:
+        - "cacert.pem"
+        - "clientcert.pem"
+        - "clientkey.pem"

ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py

@@ -50,6 +50,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
      'murano',
      'neutron',
      'nova',
+     'nova/nova-libvirt',
      'octavia',
      'placement',
      'prometheus',
@@ -100,7 +101,10 @@ def test_service_ini_file(host, path):
 @pytest.mark.parametrize(
     'path',
     ['ironic/ironic-agent.initramfs',
-     'ironic/ironic-agent.kernel'])
+     'ironic/ironic-agent.kernel',
+     'nova/nova-libvirt/cacert.pem',
+     'nova/nova-libvirt/clientcert.pem',
+     'nova/nova-libvirt/clientkey.pem'])
 def test_service_non_ini_file(host, path):
     # TODO(mgoddard): Check config file contents.
     path = os.path.join('/etc/kolla/config', path)

ansible/roles/kolla-openstack/tasks/config.yml

@@ -80,6 +80,7 @@
     recurse: true
   with_items: "{{ kolla_openstack_custom_config }}"
   register: find_src_result
+  delegate_to: localhost
 
 - name: Find previously generated extra configuration files
   find:
@@ -91,7 +92,6 @@
 - name: Ensure extra configuration parent directories are present
   file:
     path: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) | dirname }}"
-    recurse: true
     state: directory
     mode: 0750
   with_subelements:

ansible/roles/kolla-openstack/vars/main.yml

@@ -178,6 +178,27 @@ kolla_openstack_custom_config:
     dest: "{{ kolla_node_custom_config_path }}/nova"
     patterns: "*"
     enabled: "{{ kolla_enable_nova }}"
+  # Nova.
+  - src: "{{ kolla_nova_libvirt_certificates_src }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
+    patterns:
+      - clientcert.pem
+      - clientkey.pem
+      - cacert.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}"
+    untemplated:
+      - clientcert.pem
+      - clientkey.pem
+      - cacert.pem
+  - src: "{{ kolla_nova_libvirt_certificates_src }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
+    patterns:
+      - servercert.pem
+      - serverkey.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool }}"
+    untemplated:
+      - servercert.pem
+      - serverkey.pem
   # Octavia.
   - src: "{{ kolla_extra_config_path }}/octavia"
     dest: "{{ kolla_node_custom_config_path }}/octavia"

dev/functions

@@ -10,12 +10,8 @@ set -o pipefail
 function config_defaults {
     # Set default values for kayobe development configuration.
 
-    # Try to detect if we are running in a vagrant VM.
-    if [[ -e /vagrant ]]; then
-        KAYOBE_SOURCE_PATH_DEFAULT=/vagrant
-    else
-        KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)"
-    fi
+    PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    KAYOBE_SOURCE_PATH_DEFAULT="$(dirname ${PARENT})"
 
     # Path to the kayobe source code repository. Typically this will be the
     # Vagrant shared directory.
@@ -392,18 +388,26 @@ function overcloud_deploy {
 
     control_host_bootstrap
 
-    echo "Configuring the controller host"
-    run_kayobe overcloud host configure
-
-    # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname
-    # resolves to IP address of API interface for RabbitMQ. This seems to be
-    # required since https://review.openstack.org/#/c/584427 was merged.
-    echo "Workaround: upgrading the controller host"
-    run_kayobe overcloud host upgrade
-
     if [[ ${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES} = 1 ]]; then
         echo "Generate TLS certificates"
-        run_kayobe kolla ansible run certificates --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates
+        run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/kolla-ansible.yml -t config
+        # NOTE(mgoddard): There is a chicken and egg when generating libvirt
+        # TLS certificates using the kolla-ansible certificates command, and
+        # host libvirt. The certificates command needs to be able to gather
+        # facts for all hosts, but since the host configure step hasn't been
+        # run, we don't have SSH or the kolla user configured yet. However, we
+        # can't run host configure without the libvirt TLS certificates.
+        # Workaround: add the host to SSH known hosts and SSH as $USER.
+        run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/ssh-known-host.yml -l overcloud
+
+        # Avoid populating the fact cache with this weird setup.
+        export ANSIBLE_CACHE_PLUGIN=memory
+        run_kayobe kolla ansible run certificates \
+          --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates \
+          --kolla-extra ansible_user=$USER \
+          --kolla-extra ansible_python_interpreter=/usr/bin/python3
+        unset ANSIBLE_CACHE_PLUGIN
+
         # Add CA cert to trust store.
         ca_cert=${KAYOBE_CONFIG_PATH}/kolla/certificates/ca/root.crt
         if [[ -e /etc/debian_version ]]; then
@@ -417,6 +421,15 @@ function overcloud_deploy {
         fi
     fi
 
+    echo "Configuring the controller host"
+    run_kayobe overcloud host configure
+
+    # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname
+    # resolves to IP address of API interface for RabbitMQ. This seems to be
+    # required since https://review.openstack.org/#/c/584427 was merged.
+    echo "Workaround: upgrading the controller host"
+    run_kayobe overcloud host upgrade
+
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.

dev/tenks-deploy-config-compute-libvirt-on-host.yml

@@ -0,0 +1,56 @@
+---
+# This file holds the config given to Tenks when running `tenks-deploy.sh`. It
+# assumes the existence of the bridge `breth1`.
+
+node_types:
+  type0:
+    memory_mb: 1024
+    vcpus: 1
+    volumes:
+      # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
+      # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
+      - capacity: 4GiB
+    physical_networks:
+      - physnet1
+    console_log_enabled: true
+    # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g:
+    # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38
+    cpu_mode:
+
+specs:
+  - type: type0
+    count: 2
+    ironic_config:
+      resource_class: test-rc
+      network_interface: flat
+
+nova_flavors:
+  - resource_class: test-rc
+    node_type: type0
+
+physnet_mappings:
+  physnet1: breth1
+
+deploy_kernel: ipa.kernel
+deploy_ramdisk: ipa.initramfs
+
+default_boot_mode: "bios"
+
+# Use the libvirt daemon deployed by Kayobe. Tenks will install libvirt client
+# packages.
+libvirt_host_install_daemon: false
+
+# Configure AppArmor for the pool on Ubuntu.
+libvirt_host_configure_apparmor: true
+
+# Nested virtualisation is not working well in CI currently. Force the use of
+# QEMU.
+libvirt_vm_engine: "qemu"
+
+# QEMU may not be installed on the host, so set the path and avoid
+# autodetection.
+libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}"
+
+# Specify a log path in the kolla_logs Docker volume. It is accessible on the
+# host at the same path.
+libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks"

doc/source/configuration/reference/hosts.rst

@@ -1044,3 +1044,154 @@ Ansible's containers do), but may be necessary when building images.
 Docker's live restore feature can be configured via
 ``docker_daemon_live_restore``, although it is disabled by default due to
 issues observed.
+
+Compute libvirt daemon
+======================
+*tags:*
+  | ``libvirt-host``
+
+.. note::
+
+   This section is about the libvirt daemon on compute nodes, as opposed to the
+   seed hypervisor.
+
+Since Yoga, Kayobe provides support for deploying and configuring a libvirt
+host daemon, as an alternative to the ``nova_libvirt`` container support by
+Kolla Ansible. The host daemon is not used by default, but it is possible to
+enable it by setting ``kolla_enable_nova_libvirt_container`` to ``false`` in
+``$KAYOBE_CONFIG_PATH/kolla.yml``.
+
+Migration of hosts from a containerised libvirt to host libvirt is currently
+not supported.
+
+The following options are available in ``$KAYOBE_CONFIG_PATH/compute.yml`` and
+are relevant only when using the libvirt daemon rather than the
+``nova_libvirt`` container:
+
+``compute_libvirt_enabled``
+    Whether to enable a host libvirt daemon. Default is true if
+    ``kolla_enable_nova`` is ``true`` and
+    ``kolla_enable_nova_libvirt_container`` is ``false``.
+``compute_libvirt_conf_default``
+    A dict of default configuration options to write to
+    ``/etc/libvirt/libvirtd.conf``.
+``compute_libvirt_conf_extra``
+    A dict of additional configuration options to write to
+    ``/etc/libvirt/libvirtd.conf``.
+``compute_libvirt_conf``
+    A dict of configuration options to write to ``/etc/libvirt/libvirtd.conf``.
+    Default is a combination of ``compute_libvirt_conf_default`` and
+    ``compute_libvirt_conf_extra``.
+``compute_libvirtd_log_level``
+    Numerical log level for libvirtd. Default is 3.
+``compute_qemu_conf_default``
+    A dict of default configuration options to write to
+    ``/etc/libvirt/qemu.conf``.
+``compute_qemu_conf_extra``
+    A dict of additional configuration options to write to
+    ``/etc/libvirt/qemu.conf``.
+``compute_qemu_conf``
+    A dict of configuration options to write to ``/etc/libvirt/qemu.conf``.
+    Default is a combination of ``compute_qemu_conf_default`` and
+    ``compute_qemu_conf_extra``.
+``compute_libvirt_enable_tls``
+    Whether to enable a libvirt TLS listener. Default is false.
+``compute_libvirt_ceph_repo_install``
+    Whether to install a Ceph package repository on CentOS and Rocky hosts.
+    Default is ``true``.
+``compute_libvirt_ceph_repo_release``
+    Ceph package repository release to install on CentOS and Rocky hosts when
+    ``compute_libvirt_ceph_repo_install`` is ``true``. Default is ``pacific``.
+
+Example: custom libvirtd.conf
+-----------------------------
+
+To customise the libvirt daemon log output to send level 3 to the journal:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_conf_extra:
+     log_outputs: "3:journald"
+
+Example: custom qemu.conf
+-------------------------
+
+To customise QEMU to avoid adding timestamps to logs:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_qemu_conf_extra:
+     log_timestamp: 0
+
+Example: enabling libvirt TLS listener
+--------------------------------------
+
+To enable the libvirt TLS listener:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_enable_tls: true
+
+When the TLS listener is enabled, it is necessary to provide client, server and
+CA certificates. The following files should be provided:
+
+``cacert.pem``
+    CA certificate used to sign client and server certificates.
+``clientcert.pem``
+    Client certificate.
+``clientkey.pem``
+    Client key.
+``servercert.pem``
+    Server certificate.
+``serverkey.pem``
+    Server key.
+
+It is recommended to encrypt the key files using Ansible Vault.
+
+The following paths are searched for these files:
+
+* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/{{ inventory_hostname }}/``
+* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/``
+
+In this way, certificates may be generated for each host, or shared using
+wildcard certificates.
+
+If using Kayobe environments, certificates in the environment take precedence.
+
+Kayobe makes the CA certificate and client certificate and key available to
+Kolla Ansible, for use by the ``nova_compute`` service.
+
+Example: disabling Ceph repository installation
+-----------------------------------------------
+
+On CentOS and Rocky hosts, a CentOS Storage SIG Ceph repository is installed
+that provides more recent Ceph libraries than those available in CentOS/Rocky
+AppStream.  This may be necessary when using Ceph for Cinder volumes or Nova
+ephemeral block devices. In some cases, such as when using local package
+mirrors, the upstream repository may not be appropriate. The installation of
+the repository may be disabled as follows:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_ceph_repo_install: false
+
+Example: installing additional packages
+---------------------------------------
+
+In some cases it may be useful to install additional packages on compute hosts
+for use by libvirt. The `stackhpc.libvirt-host
+<https://galaxy.ansible.com/stackhpc/libvirt-host>`__ Ansible role supports
+this via the ``libvirt_host_extra_daemon_packages`` variable. The variable
+should be defined via group variables in the Ansible inventory, to avoid
+applying the change to the seed hypervisor. For example, to install the
+``trousers`` package used for accessing TPM hardware:
+
+.. code-block:: yaml
+   :caption: ``inventory/group_vars/compute/libvirt``
+
+   libvirt_host_extra_daemon_packages:
+     - trousers

doc/source/contributor/automated.rst

@@ -131,11 +131,6 @@ For a control plane with Ironic enabled, a "bare metal" instance can be
 deployed. We can use the `Tenks <https://tenks.readthedocs.io/en/latest/>`__
 project to create fake bare metal nodes.
 
-On Ubuntu, the ``nova_libvirt`` image does not contain the ``qemu-utils``
-package necessary for image operations used by Tenks. Install it as follows::
-
-    sudo docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils'
-
 Clone the tenks repository::
 
     git clone https://opendev.org/openstack/tenks.git

etc/kayobe/compute.yml

@@ -143,6 +143,53 @@
 # - state: enabled
 #compute_firewalld_rules:
 
+###############################################################################
+# Compute node host libvirt configuration.
+
+# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova
+# is true and kolla_enable_nova_libvirt_container is false.
+#compute_libvirt_enabled:
+
+# A dict of default configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+#compute_libvirt_conf_default:
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+#compute_libvirt_conf_extra:
+
+# A dict of configuration options to write to /etc/libvirt/libvirtd.conf.
+# Default is a combination of compute_libvirt_conf_default and
+# compute_libvirt_conf_extra.
+#compute_libvirt_conf:
+
+# Numerical log level for libvirtd. Default is 3.
+#compute_libvirtd_log_level:
+
+# A dict of default configuration options to write to
+# /etc/libvirt/qemu.conf.
+#compute_qemu_conf_default:
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/qemu.conf.
+#compute_qemu_conf_extra:
+
+# A dict of configuration options to write to /etc/libvirt/qemu.conf.
+# Default is a combination of compute_qemu_conf_default and
+# compute_qemu_conf_extra.
+#compute_qemu_conf:
+
+# Whether to enable a libvirt TLS listener. Default is false.
+#compute_libvirt_enable_tls:
+
+# Whether to install a Ceph package repository on CentOS and Rocky hosts.
+# Default is true.
+#compute_libvirt_ceph_repo_install:
+
+# Ceph package repository release to install on CentOS and Rocky hosts when
+# compute_libvirt_ceph_repo_install is true. Default is 'pacific'.
+#compute_libvirt_ceph_repo_release:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -1125,6 +1125,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Optionally, create a virtualenv for kolla-ansible.
     * Configure a user account for kolla-ansible.
     * Configure Docker engine.
+    * Configure libvirt.
     """
 
     def get_parser(self, prog_name):
@@ -1157,7 +1158,8 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers")
 
         # Further kayobe playbooks.
-        playbooks = _build_playbook_list("docker", "swift-block-devices")
+        playbooks = _build_playbook_list(
+            "docker", "swift-block-devices", "compute-libvirt-host")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
 

kayobe/tests/unit/cli/test_commands.py

@@ -1322,6 +1322,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "docker.yml"),
                     utils.get_data_files_path(
                         "ansible", "swift-block-devices.yml"),
+                    utils.get_data_files_path(
+                        "ansible", "compute-libvirt-host.yml"),
                 ],
                 limit="overcloud",
             ),
@@ -1376,6 +1378,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "docker.yml"),
                     utils.get_data_files_path(
                         "ansible", "swift-block-devices.yml"),
+                    utils.get_data_files_path(
+                        "ansible", "compute-libvirt-host.yml"),
                 ],
                 limit="overcloud",
             ),

playbooks/kayobe-overcloud-base/globals.yml.j2

@@ -22,4 +22,5 @@ kolla_enable_tls_backend: "yes"
 openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
 kolla_admin_openrc_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
 libvirt_tls: "yes"
+certificates_libvirt_output_dir: "{% raw %}{{ kayobe_env_config_path }}{% endraw %}/certificates/libvirt"
 {% endif %}

playbooks/kayobe-overcloud-base/overrides.yml.j2

@@ -42,6 +42,9 @@ kolla_ironic_default_boot_interface: ipxe
 {% endif %}
 
 {% if tls_enabled %}
+kolla_enable_nova_libvirt_container: false
+compute_libvirt_enable_tls: true
+
 kolla_enable_tls_external: "yes"
 kolla_enable_tls_internal: "yes"
 

playbooks/kayobe-overcloud-base/run.yml

@@ -3,6 +3,8 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
     KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES: "{{ tls_enabled | ternary(1, 0) }}"
+    # TODO(mgoddard): Remove this when libvirt on host is used by default.
+    TENKS_CONFIG_PATH: "dev/tenks-deploy-config-compute{% if tls_enabled %}-libvirt-on-host{% endif %}.yml"
   tasks:
     - name: Ensure overcloud is deployed
       shell:
@@ -18,8 +20,6 @@
         executable: /bin/bash
 
     - name: Perform testing of the virtualized machines
-      # We must do this before tenks-deploy as that will stop the nova_libvirt
-      # container
       shell:
         cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm
         chdir: "{{ kayobe_src_dir }}"

releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds support for running a libvirt daemon on the host, rather than in a
+    container. This is done by setting ``kolla_enable_nova_libvirt_container``
+    to ``false``. See `story 2009858
+    <https://storyboard.openstack.org/#!/story/2009858>`__ for details.

requirements.yml

@@ -32,7 +32,7 @@ roles:
   - src: stackhpc.grafana-conf
     version: 1.1.1
   - src: stackhpc.libvirt-host
-    version: v1.8.3
+    version: v1.10.0
   - src: stackhpc.libvirt-vm
     version: v1.14.2
   - src: stackhpc.luks