Merge "Stop using kolla-ansible bootstrap-servers"

.gitignore

@@ -56,8 +56,9 @@ ChangeLog
 ansible/*.retry
 ansible/roles/*/tests/*.retry
 
-# Ansible Galaxy roles
+# Ansible Galaxy roles & collections
 ansible/roles/*\.*/
+ansible/collections/
 
 # Virtualenvs
 ansible/kolla-venv/

ansible/apparmor-libvirt.yml

@@ -0,0 +1,21 @@
+---
+- name: Ensure AppArmor is disabled for containerised libvirt
+  hosts: compute
+  tags:
+    - apparmor-libvirt
+  vars:
+    # kolla_overcloud_inventory_top_level_group_map looks like:
+    # kolla_overcloud_inventory_top_level_group_map:
+    #  control:
+    #    groups:
+    #      - controllers
+    hosts_in_kolla_inventory: >-
+      {{ kolla_overcloud_inventory_top_level_group_map.values() |
+         map(attribute='groups') | flatten | unique | join(':') }}
+  tasks:
+    - name: Include openstack.kolla.apparmor_libvirt role
+      include_role:
+        name: openstack.kolla.apparmor_libvirt
+      when:
+        - inventory_hostname in query('inventory_hostnames', hosts_in_kolla_inventory)
+        - ansible_facts.distribution == "Ubuntu"

ansible/docker.yml

@@ -3,7 +3,12 @@
   hosts: docker
   tags:
     - docker
-  vars:
+  tasks:
-    - docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
+    - import_role:
-  roles:
+        name: docker
-    - role: docker
+      vars:
+        docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
+        docker_configure_for_zun: "{{ kolla_enable_zun | bool }}"
+        docker_http_proxy: "{{ kolla_http_proxy }}"
+        docker_https_proxy: "{{ kolla_https_proxy }}"
+        docker_no_proxy: "{{ kolla_no_proxy | select | join(',') }}"

ansible/etc-hosts.yml

@@ -0,0 +1,15 @@
+---
+- name: Ensure /etc/hosts is configured
+  hosts: overcloud
+  tags:
+    - etc-hosts
+  tasks:
+    # NOTE(mgoddard): Need to ensure that all hosts have facts available.
+    - import_role:
+        name: gather-facts-delegated
+      tags:
+        - gather-facts-delegated
+      when: etc_hosts_gather_facts | default(true)
+
+    - import_role:
+        name: etc-hosts

ansible/firewall.yml

@@ -6,6 +6,5 @@
     - firewall
   tasks:
     - name: Configure firewalld
-      include_role:
+      import_role:
         name: "firewalld"
-

ansible/inventory/group_vars/all/docker

@@ -26,3 +26,9 @@ docker_registry:
 
 # CA of docker registry
 docker_registry_ca:
+
+# List of Docker registry mirrors.
+docker_registry_mirrors: []
+
+# Enable live-restore on docker daemon
+docker_daemon_live_restore: false

ansible/kayobe-ansible-user.yml

@@ -70,9 +70,11 @@
     ansible_python_interpreter: /usr/bin/python3
   roles:
     - role: singleplatform-eng.users
+      groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
       users:
         - username: "{{ kayobe_ansible_user }}"
           name: Kayobe deployment user
+          groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
           append: True
           ssh_key:
             - "{{ lookup('file', ssh_public_key_path) }}"

ansible/kayobe-target-venv.yml

@@ -100,3 +100,14 @@
             state: present
           become: True
       when: virtualenv is not defined
+
+    - name: Ensure kolla-ansible virtualenv has docker SDK for python installed
+      pip:
+        name: docker
+        state: latest
+        virtualenv: "{{ virtualenv | default(omit) }}"
+        extra_args: "{% if docker_upper_constraints_file %}-c {{ docker_upper_constraints_file }}{% endif %}"
+      become: "{{ virtualenv is not defined }}"
+      vars:
+        docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
+      when: "'docker' in group_names"

ansible/kolla-ansible.yml

@@ -107,7 +107,6 @@
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
         kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
         kolla_enable_host_ntp: false
-        docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
         kolla_globals_paths_extra:
           - "{{ kayobe_config_path }}"
           - "{{ kayobe_env_config_path }}"

ansible/kolla-packages.yml

@@ -0,0 +1,22 @@
+---
+- name: Ensure Kolla Ansible packages are installed
+  hosts: overcloud
+  tags:
+    - kolla-packages
+  vars:
+    # kolla_overcloud_inventory_top_level_group_map looks like:
+    # kolla_overcloud_inventory_top_level_group_map:
+    #  control:
+    #    groups:
+    #      - controllers
+    hosts_in_kolla_inventory: >-
+      {{ kolla_overcloud_inventory_top_level_group_map.values() |
+         map(attribute='groups') | flatten | unique | join(':') }}
+  tasks:
+    - name: Include openstack.kolla.packages role
+      include_role:
+        name: openstack.kolla.packages
+      vars:
+        enable_multipathd: "{{ kolla_enable_multipathd | bool }}"
+      when:
+        - inventory_hostname in query('inventory_hostnames', hosts_in_kolla_inventory)

ansible/overcloud-docker-sdk-upgrade.yml

@@ -1,31 +0,0 @@
----
-- name: Ensure docker SDK for python is installed
-  hosts: overcloud
-  tags:
-    - docker-sdk-upgrade
-  tasks:
-    # Docker renamed their python SDK from docker-py to docker in the 2.0.0
-    # release, and also broke backwards compatibility. Kolla-ansible requires
-    # docker, so ensure it is installed.
-    - name: Set a fact about the virtualenv on the remote system
-      set_fact:
-        virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
-      when:
-        - ansible_python_interpreter is defined
-        - not ansible_python_interpreter.startswith('/bin/')
-        - not ansible_python_interpreter.startswith('/usr/bin/')
-
-    - name: Ensure legacy docker-py python package is uninstalled
-      pip:
-        name: docker-py
-        state: absent
-        virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-      become: "{{ virtualenv is not defined }}"
-
-    - name: Ensure docker SDK for python is installed
-      pip:
-        name: docker
-        state: latest
-        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
-        virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-      become: "{{ virtualenv is not defined }}"

ansible/overcloud-host-configure.yml

@@ -12,6 +12,7 @@
 - import_playbook: "selinux.yml"
 - import_playbook: "network.yml"
 - import_playbook: "firewall.yml"
+- import_playbook: "etc-hosts.yml"
 - import_playbook: "tuned.yml"
 - import_playbook: "sysctl.yml"
 - import_playbook: "disable-glean.yml"
@@ -25,3 +26,8 @@
 - import_playbook: "kolla-ansible-user.yml"
 - import_playbook: "kolla-pip.yml"
 - import_playbook: "kolla-target-venv.yml"
+- import_playbook: "kolla-packages.yml"
+- import_playbook: "docker.yml"
+- import_playbook: "apparmor-libvirt.yml"
+- import_playbook: "swift-block-devices.yml"
+- import_playbook: "compute-libvirt-host.yml"

ansible/overcloud-host-upgrade.yml

@@ -1,5 +1,4 @@
 ---
 - import_playbook: "kayobe-target-venv.yml"
 - import_playbook: "kolla-target-venv.yml"
-- import_playbook: "overcloud-docker-sdk-upgrade.yml"
 - import_playbook: "overcloud-etc-hosts-fixup.yml"

ansible/roles/docker/defaults/main.yml

@@ -1,10 +1,7 @@
 ---
-# URL of docker registry
+docker_storage_driver: overlay2
-docker_registry:
+docker_storage_volume_group:
-
+docker_storage_volume_thinpool:
-# CA of docker registry
+docker_registry_mirrors: []
-docker_registry_ca:
+docker_daemon_mtu: 1500
-
+docker_daemon_live_restore: false
-# Upper constraints file which is passed to pip when installing packages
-# into a venv.
-docker_upper_constraints_file:

ansible/roles/docker/handlers/main.yml

@@ -1,6 +0,0 @@
----
-- name: reload docker service
-  service:
-    name: docker
-    state: reloaded
-  become: True

ansible/roles/docker/tasks/main.yml

@@ -1,52 +1,5 @@
 ---
-- name: Set a fact about the virtualenv on the remote system
+- import_role:
-  set_fact:
+    name: openstack.kolla.docker
-    virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
+  vars:
-  when:
+    docker_custom_config: "{{ lookup('template', 'daemon.json.j2') | to_nice_json | indent(2) }}"
-    - ansible_python_interpreter is defined
-    - not ansible_python_interpreter.startswith('/bin/')
-    - not ansible_python_interpreter.startswith('/usr/bin/')
-
-- name: Ensure docker SDK for python is installed
-  pip:
-    name: docker
-    state: latest
-    extra_args: "{% if docker_upper_constraints_file %}-c {{ docker_upper_constraints_file }}{% endif %}"
-    virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-  become: "{{ virtualenv is not defined }}"
-
-- name: Ensure user is in the docker group
-  user:
-    name: "{{ ansible_facts.user_id }}"
-    groups: docker
-    append: yes
-  register: group_result
-  become: True
-
-# After adding the user to the docker group, we need to log out and in again to
-# pick up the group membership. We do this by resetting the SSH connection.
-
-- name: Reset connection to activate new group membership
-  meta: reset_connection
-  when: group_result is changed
-
-- name: Ensure Docker daemon is started
-  service:
-    name: docker
-    state: started
-  become: True
-
-- name: Ensure the path for CA file for private registry exists
-  file:
-    path: "/etc/docker/certs.d/{{ docker_registry }}"
-    state: directory
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-
-- name: Ensure the CA file for private registry exists
-  copy:
-    src: "{{ docker_registry_ca }}"
-    dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt"
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-  notify: reload docker service

ansible/roles/etc-hosts/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Whether to add entries to /etc/hosts.
+customize_etc_hosts: true
+
+# List of hosts to add to /etc/hosts.
+etc_hosts_hosts: "{{ ansible_play_hosts_all }}"

ansible/roles/etc-hosts/tasks/etc-hosts.yml

@@ -0,0 +1,56 @@
+---
+- name: Ensure localhost in /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^127.0.0.1.*"
+    line: "127.0.0.1 localhost"
+    state: present
+  become: True
+
+# NOTE(mgoddard): Ubuntu may include a line in /etc/hosts that makes the local
+# hostname and fqdn point to 127.0.1.1. This can break
+# RabbitMQ, which expects the hostname to resolve to the API network address.
+# Remove the troublesome entry.
+# see https://bugs.launchpad.net/kolla-ansible/+bug/1837699
+# and https://bugs.launchpad.net/kolla-ansible/+bug/1862739
+- name: Ensure hostname does not point to 127.0.1.1 in /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^127.0.1.1\\b.*\\s{{ ansible_facts.hostname }}\\b"
+    state: absent
+  become: True
+
+- name: Generate /etc/hosts for all of the nodes
+  blockinfile:
+    dest: /etc/hosts
+    marker: "# {mark} ANSIBLE GENERATED HOSTS"
+    block: |
+        {% for host in etc_hosts_hosts %}
+        {% if hostvars[host].internal_net_name in hostvars[host].network_interfaces %}
+        {% set hostnames = [hostvars[host].ansible_facts.nodename, hostvars[host].ansible_facts.hostname] %}
+        {{ hostvars[host].internal_net_name | net_ip(inventory_hostname=host) }} {{ hostnames | unique | join(' ') }}
+        {% endif %}
+        {% endfor %}
+  become: True
+  when:
+    # Skip hosts that do not have a valid internal network interface.
+    - internal_net_name in network_interfaces
+
+# NOTE(osmanlicilegi): The distribution might come with cloud-init installed, and manage_etc_hosts
+# configuration enabled. If so, it will override the file /etc/hosts from cloud-init templates at
+# every boot, which will break RabbitMQ. To prevent this happens, first we check whether cloud-init
+# has been installed, and then set manage_etc_hosts to false.
+- name: Check whether cloud-init has been installed, and ensure manage_etc_hosts is disabled
+  block:
+    - name: Ensure /etc/cloud/cloud.cfg exists
+      stat:
+        path: /etc/cloud/cloud.cfg
+      register: cloud_init
+
+    - name: Disable cloud-init manage_etc_hosts
+      copy:
+        content: "manage_etc_hosts: false"
+        dest: /etc/cloud/cloud.cfg.d/99-kolla.cfg
+        mode: "0660"
+      when: cloud_init.stat.exists
+  become: True

ansible/roles/etc-hosts/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- include_tasks: etc-hosts.yml
+  when: customize_etc_hosts | bool

ansible/roles/firewall-debian/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Whether to install and enable ufw.
+ufw_enabled: false

ansible/roles/firewall-debian/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+# TODO(inc0): Gates don't seem to have ufw executable, check for it instead of ignore errors
+- name: Set firewall default policy
+  become: True
+  ufw:
+    state: disabled
+    policy: allow
+  when: not ufw_enabled | bool
+  ignore_errors: yes

ansible/roles/gather-facts-delegated/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+gather_facts_delegated_limit_hosts: "{{ ansible_play_hosts_all }}"
+gather_facts_delegated_batch_index: "{{ gather_facts_delegated_limit_hosts.index(inventory_hostname) }}"
+gather_facts_delegated_batch_size: "{{ gather_facts_delegated_limit_hosts | length }}"
+# Use a python list slice to divide the group up.
+# Syntax: [<start index>:<end index>:<step size>]
+gather_facts_delegated_delegate_hosts: >-
+  {{ gather_facts_delegated_limit_hosts[gather_facts_delegated_batch_index | int::gather_facts_delegated_batch_size | int] }}

ansible/roles/gather-facts-delegated/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Gather facts for all hosts (if using --limit)
+  setup:
+    filter: "{{ kayobe_ansible_setup_filter }}"
+    gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+  delegate_facts: True
+  delegate_to: "{{ item }}"
+  with_items: "{{ gather_facts_delegated_delegate_hosts }}"
+  when:
+    - not hostvars[item].ansible_facts

ansible/roles/kolla-ansible/defaults/main.yml

@@ -299,30 +299,9 @@ kolla_enable_host_ntp:
 ###############################################################################
 # Docker configuration.
 
-# Name of the docker storage driver.
-docker_storage_driver: overlay2
-
-# Name of the docker storage LVM volume group.
-docker_storage_volume_group:
-
-# Name of the docker storage data LVM volume.
-docker_storage_volume_thinpool:
-
 # URL of docker registry
 docker_registry:
 
-# CA of docker registry
-docker_registry_ca:
-
-# List of Docker registry mirrors.
-docker_registry_mirrors: []
-
-# MTU to pass through to containers not using net=host
-docker_daemon_mtu: 1500
-
-# Enable live-restore on docker daemon
-docker_daemon_live_restore: false
-
 ###############################################################################
 # Proxy configuration
 

ansible/roles/kolla-ansible/tasks/config.yml

@@ -52,8 +52,6 @@
     sources: "{{ kolla_globals_paths | product(['/kolla/globals.yml']) | map('join') | unique | list }}"
     dest: "{{ kolla_config_path }}/globals.yml"
     mode: 0640
-  vars:
-    kolla_docker_custom_config: "{{ lookup('template', 'daemon.json.j2') }}"
 
 - name: Ensure the Kolla seed inventory file exists
   copy:

ansible/roles/kolla-ansible/templates/kolla/globals.yml

@@ -73,21 +73,6 @@ docker_namespace: "{{ kolla_docker_namespace }}"
 {% if kolla_docker_registry_username %}
 docker_registry_username: "{{ kolla_docker_registry_username }}"
 {% endif %}
-docker_storage_driver: "{{ docker_storage_driver }}"
-docker_custom_config: {{ kolla_docker_custom_config | to_nice_json | indent(2) }}
-{% if kolla_docker_registry_insecure | bool %}
-docker_registry_insecure: "yes"
-{% endif %}
-
-{% if kolla_http_proxy is not none and kolla_http_proxy | length > 0 %}
-docker_http_proxy: "{{ kolla_http_proxy }}"
-{% endif %}
-{% if kolla_https_proxy is not none and kolla_https_proxy | length > 0 %}
-docker_https_proxy: "{{ kolla_https_proxy }}"
-{% endif %}
-{% if kolla_no_proxy is not none and kolla_no_proxy | length > 0 %}
-docker_no_proxy: "{{ kolla_no_proxy | select | join(',') }}"
-{% endif %}
 
 #docker_configure_for_zun: "no"
 

ansible/seed-host-configure.yml

@@ -25,3 +25,5 @@
 - import_playbook: "kolla-ansible-user.yml"
 - import_playbook: "kolla-pip.yml"
 - import_playbook: "kolla-target-venv.yml"
+- import_playbook: "docker.yml"
+- import_playbook: "docker-registry.yml"

doc/source/configuration/reference/hosts.rst

@@ -561,6 +561,22 @@ In the following example, firewalld is enabled on controllers. ``public`` and
      - service: http
        zone: public
 
+UFW
+===
+*tags:*
+  | ``firewall``
+
+Configuration of Uncomplicated Firewall (UFW) on Ubuntu hosts is currently not
+supported. Instead, UFW is disabled. Since Yoga, this may be avoided as
+follows:
+
+.. code-block:: yaml
+
+   ufw_enabled: true
+
+Note that despite the name, this will not actively enable UFW. It may do so in
+the future.
+
 .. _configuration-hosts-tuned:
 
 Tuned
@@ -991,22 +1007,6 @@ custom one.
            create: true
            mount: false
 
-Kolla-Ansible bootstrap-servers
-===============================
-
-Kolla Ansible provides some host configuration functionality via the
-``bootstrap-servers`` command, which may be leveraged by Kayobe.
-
-See the :kolla-ansible-doc:`Kolla Ansible documentation
-<reference/deployment-and-bootstrapping/bootstrap-servers.html>`
-for more information on the functions performed by this command, and how to
-configure it.
-
-Note that from the Ussuri release, Kayobe creates a user account for Kolla
-Ansible rather than this being done by Kolla Ansible during
-``bootstrap-servers``. See :ref:`configuration-kolla-ansible-user-creation` for
-details.
-
 Kolla-Ansible Remote Virtual Environment
 ========================================
 *tags:*
@@ -1023,9 +1023,6 @@ Docker Engine
 *tags:*
   | ``docker``
 
-Docker engine configuration is applied by both Kayobe and Kolla Ansible (during
-bootstrap-servers).
-
 The ``docker_storage_driver`` variable sets the Docker storage driver, and by
 default the ``overlay2`` driver is used. If using the ``devicemapper`` driver,
 see :ref:`configuration-hosts-lvm` for information about configuring LVM for
@@ -1276,3 +1273,65 @@ The following example defines a 1GiB swap file that will be created at
    compute_swap:
      - path: /swapfile
        size_mb: 1024
+
+AppArmor for the libvirt container
+==================================
+*tags:*
+  | ``apparmor-libvirt``
+
+.. note::
+
+   Prior to the Yoga release, this was handled by the ``kolla-ansible
+   bootstrap-servers`` command.
+
+On Ubuntu systems running the ``nova_libvirt`` Kolla container, AppArmor rules
+for libvirt are disabled.
+
+Adding entries to /etc/hosts
+============================
+*tags:*
+  | ``etc-hosts``
+
+.. note::
+
+   Prior to the Yoga release, this was handled by the ``kolla-ansible
+   bootstrap-servers`` command.
+
+Since Yoga, Kayobe adds entries to ``/etc/hosts`` for all hosts in the
+``overcloud`` group.  The entries map the hostname and FQDN of a host to its IP
+address on the internal API network. This may be avoided as follows:
+
+.. code-block:: yaml
+
+   customize_etc_hosts: false
+
+By default, each host gets an entry for every other host in the ``overcloud``
+group by default. The list of hosts that will be added may be customised:
+
+.. code-block:: yaml
+
+   etc_hosts_hosts: "{{ groups['compute'] }}"
+
+It should be noted that this functionality requires facts to be populated for
+all hosts that will be added to any ``/etc/hosts`` file. When using the
+``--limit`` argument, Kayobe will gather facts for all hosts without facts,
+including those outside of the limit. Enabling fact caching for Kayobe may
+reduce the impact of this. This fact gathering process may be avoided as
+follows:
+
+.. code-block:: yaml
+
+   etc_hosts_gather_facts: false
+
+Installing packages required by Kolla Ansible
+=============================================
+*tags:*
+  | ``kolla-packages``
+
+.. note::
+
+   Prior to the Yoga release, this was handled by the ``kolla-ansible
+   bootstrap-servers`` command.
+
+A small number of packages are required to be installed on the hosts for Kolla
+Ansible and the services that it deploys, while some others must be removed.

kayobe/cli/commands.py

@@ -424,8 +424,7 @@ class PhysicalNetworkConfigure(KayobeAnsibleMixin, VaultMixin, Command):
                                  extra_vars=extra_vars)
 
 
-class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
+class SeedHypervisorHostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
-                                  VaultMixin, Command):
     """Configure the seed hypervisor node host OS and services.
 
     * Allocate IP addresses for all configured networks.
@@ -572,8 +571,7 @@ class SeedVMDeprovision(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
                                  _get_playbook_path("seed-vm-deprovision"))
 
 
-class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+class SeedHostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
-                        Command):
     """Configure the seed node host OS and services.
 
     * Allocate IP addresses for all configured networks.
@@ -619,27 +617,12 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
 
         # Run kayobe playbooks.
-        kwargs = {}
+        extra_vars = {"kayobe_action": "deploy"}
         if parsed_args.wipe_disks:
-            kwargs["extra_vars"] = {"wipe_disks": True}
+            extra_vars["wipe_disks"] = True
         playbooks = _build_playbook_list("seed-host-configure")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed",
-                                  **kwargs)
+                                  extra_vars=extra_vars)
-
-        self.generate_kolla_ansible_config(parsed_args, service_config=False)
-
-        # Run kolla-ansible bootstrap-servers.
-        self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers")
-
-        # Run final kayobe playbooks.
-        playbooks = _build_playbook_list("docker")
-        self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
-
-        # Optionally, deploy a Docker Registry.
-        playbooks = _build_playbook_list("docker-registry")
-        extra_vars = {"kayobe_action": "deploy"}
-        self.run_kayobe_playbooks(parsed_args, playbooks,
-                                  extra_vars=extra_vars, limit="seed")
 
 
 class SeedHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):
@@ -689,8 +672,7 @@ class SeedHostCommandRun(KayobeAnsibleMixin, VaultMixin, Command):
                                   extra_vars=extra_vars)
 
 
-class SeedHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+class SeedHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
-                      Command):
     """Upgrade the seed host services.
 
     Performs the changes necessary to make the host services suitable for the
@@ -879,8 +861,7 @@ class InfraVMDeprovision(KayobeAnsibleMixin, VaultMixin, Command):
                                  ignore_limit=True, extra_vars=extra_vars)
 
 
-class InfraVMHostConfigure(KayobeAnsibleMixin, VaultMixin,
+class InfraVMHostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
-                           Command):
     """Configure the infra VMs host OS and services.
 
     * Allocate IP addresses for all configured networks.
@@ -1126,8 +1107,7 @@ class OvercloudFactsGather(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kolla_ansible_overcloud(parsed_args, "gather-facts")
 
 
-class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+class OvercloudHostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
-                             Command):
     """Configure the overcloud host OS and services.
 
     * Allocate IP addresses for all configured networks.
@@ -1179,16 +1159,6 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud",
                                   **kwargs)
 
-        self.generate_kolla_ansible_config(parsed_args, service_config=False)
-
-        # Kolla-ansible bootstrap-servers.
-        self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers")
-
-        # Further kayobe playbooks.
-        playbooks = _build_playbook_list(
-            "docker", "swift-block-devices", "compute-libvirt-host")
-        self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
-
 
 class OvercloudHostPackageUpdate(KayobeAnsibleMixin, VaultMixin, Command):
     """Update packages on the overcloud hosts."""
@@ -1237,8 +1207,7 @@ class OvercloudHostCommandRun(KayobeAnsibleMixin, VaultMixin, Command):
                                   extra_vars=extra_vars)
 
 
-class OvercloudHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
-                           Command):
     """Upgrade the overcloud host services.
 
     Performs the changes necessary to make the host services suitable for the

kayobe/tests/unit/cli/test_commands.py

@@ -482,9 +482,7 @@ class TestCase(unittest.TestCase):
 
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
-    @mock.patch.object(commands.KollaAnsibleMixin,
+    def test_seed_host_configure(self, mock_run):
-                       "run_kolla_ansible_seed")
-    def test_seed_host_configure(self, mock_kolla_run, mock_run):
         command = commands.SeedHostConfigure(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args([])
@@ -505,45 +503,14 @@ class TestCase(unittest.TestCase):
                         "ansible", "seed-host-configure.yml"),
                 ],
                 limit="seed",
-            ),
+                extra_vars={"kayobe_action": "deploy"},
-            mock.call(
-                mock.ANY,
-                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
-                tags="config",
-                ignore_limit=True,
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible", "docker.yml"),
-                ],
-                limit="seed",
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible",
-                                              "docker-registry.yml"),
-                ],
-                limit="seed",
-                extra_vars={'kayobe_action': 'deploy'},
             ),
         ]
         self.assertListEqual(expected_calls, mock_run.call_args_list)
 
-        expected_calls = [
-            mock.call(
-                mock.ANY,
-                "bootstrap-servers",
-            ),
-        ]
-        self.assertListEqual(expected_calls, mock_kolla_run.call_args_list)
-
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
-    @mock.patch.object(commands.KollaAnsibleMixin,
+    def test_seed_host_configure_wipe_disks(self, mock_run):
-                       "run_kolla_ansible_seed")
-    def test_seed_host_configure_wipe_disks(self, mock_kolla_run, mock_run):
         command = commands.SeedHostConfigure(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args(["--wipe-disks"])
@@ -564,43 +531,13 @@ class TestCase(unittest.TestCase):
                         "ansible", "seed-host-configure.yml"),
                 ],
                 limit="seed",
-                extra_vars={"wipe_disks": True},
+                extra_vars={"kayobe_action": "deploy", "wipe_disks": True},
-            ),
-            mock.call(
-                mock.ANY,
-                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
-                tags="config",
-                ignore_limit=True,
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible", "docker.yml"),
-                ],
-                limit="seed",
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible",
-                                              "docker-registry.yml"),
-                ],
-                limit="seed",
-                extra_vars={'kayobe_action': 'deploy'},
             ),
         ]
         print(expected_calls)
         print(mock_run.call_args_list)
         self.assertListEqual(expected_calls, mock_run.call_args_list)
 
-        expected_calls = [
-            mock.call(
-                mock.ANY,
-                "bootstrap-servers",
-            ),
-        ]
-        self.assertListEqual(expected_calls, mock_kolla_run.call_args_list)
-
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_seed_host_command_run(self, mock_run):
@@ -1293,9 +1230,7 @@ class TestCase(unittest.TestCase):
 
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
-    @mock.patch.object(commands.KollaAnsibleMixin,
+    def test_overcloud_host_configure(self, mock_run):
-                       "run_kolla_ansible_overcloud")
-    def test_overcloud_host_configure(self, mock_kolla_run, mock_run):
         command = commands.OvercloudHostConfigure(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args([])
@@ -1317,40 +1252,12 @@ class TestCase(unittest.TestCase):
                 ],
                 limit="overcloud",
             ),
-            mock.call(
-                mock.ANY,
-                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
-                tags="config",
-                ignore_limit=True,
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible", "docker.yml"),
-                    utils.get_data_files_path(
-                        "ansible", "swift-block-devices.yml"),
-                    utils.get_data_files_path(
-                        "ansible", "compute-libvirt-host.yml"),
-                ],
-                limit="overcloud",
-            ),
         ]
         self.assertListEqual(expected_calls, mock_run.call_args_list)
 
-        expected_calls = [
-            mock.call(
-                mock.ANY,
-                "bootstrap-servers",
-            ),
-        ]
-        self.assertListEqual(expected_calls, mock_kolla_run.call_args_list)
-
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
-    @mock.patch.object(commands.KollaAnsibleMixin,
+    def test_overcloud_host_configure_wipe_disks(self, mock_run):
-                       "run_kolla_ansible_overcloud")
-    def test_overcloud_host_configure_wipe_disks(self, mock_kolla_run,
-                                                 mock_run):
         command = commands.OvercloudHostConfigure(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args(["--wipe-disks"])
@@ -1373,34 +1280,9 @@ class TestCase(unittest.TestCase):
                 limit="overcloud",
                 extra_vars={"wipe_disks": True},
             ),
-            mock.call(
-                mock.ANY,
-                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
-                tags="config",
-                ignore_limit=True,
-            ),
-            mock.call(
-                mock.ANY,
-                [
-                    utils.get_data_files_path("ansible", "docker.yml"),
-                    utils.get_data_files_path(
-                        "ansible", "swift-block-devices.yml"),
-                    utils.get_data_files_path(
-                        "ansible", "compute-libvirt-host.yml"),
-                ],
-                limit="overcloud",
-            ),
         ]
         self.assertListEqual(expected_calls, mock_run.call_args_list)
 
-        expected_calls = [
-            mock.call(
-                mock.ANY,
-                "bootstrap-servers",
-            ),
-        ]
-        self.assertListEqual(expected_calls, mock_kolla_run.call_args_list)
-
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_overcloud_host_command_run(self, mock_run):

playbooks/kayobe-overcloud-base/run.yml

@@ -3,6 +3,7 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
     KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES: "{{ tls_enabled | ternary(1, 0) }}"
+    KAYOBE_VAULT_PASSWORD: 'test-password'
     # TODO(mgoddard): Remove this when libvirt on host is used by default.
     TENKS_CONFIG_PATH: "dev/tenks-deploy-config-compute{% if tls_enabled %}-libvirt-on-host{% endif %}.yml"
   tasks:
@@ -11,6 +12,16 @@
         cmd: "{{ kayobe_src_dir }}/dev/overcloud-deploy.sh &> {{ logs_dir }}/ansible/overcloud-deploy"
         executable: /bin/bash
 
+    # Check that passwords are Vault encrypted.
+    - name: View passwords.yml using Ansible Vault
+      vars:
+        kayobe_venv: "{{ ansible_env.HOME }}/kayobe-venv"
+      command: >-
+        {{ kayobe_venv }}/bin/ansible-vault
+        view
+        --vault-password-file {{ kayobe_venv }}/bin/kayobe-vault-password-helper
+        {{ kayobe_config_src_dir }}/etc/kayobe/kolla/passwords.yml
+
     - name: Ensure test Tenks cluster is deployed
       shell:
         # Pass absolute source directory, since otherwise the `chdir` will

playbooks/kayobe-overcloud-host-configure-base/run.yml

@@ -6,7 +6,6 @@
     KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL: 0
     KAYOBE_OVERCLOUD_SERVICE_DEPLOY: 0
     KAYOBE_OVERCLOUD_POST_CONFIGURE: 0
-    KAYOBE_VAULT_PASSWORD: 'test-password'
   vars:
     testinfra_venv: ~/testinfra-venv
     test_path: "{{ kayobe_src_dir }}/playbooks/kayobe-overcloud-host-configure-base/tests/"
@@ -30,13 +29,3 @@
       command: "{{ testinfra_venv }}/bin/py.test {{ test_path }} --html={{ logs_dir }}/test-results.html --self-contained-html"
       environment:
         SITE_MIRROR_FQDN: "{{ zuul_site_mirror_fqdn }}"
-
-    # Check that passwords are Vault encrypted.
-    - name: Decrypt passwords.yml using Ansible Vault
-      vars:
-        kayobe_venv: "{{ ansible_env.HOME }}/kayobe-venv"
-      command: >-
-        {{ kayobe_venv }}/bin/ansible-vault
-        decrypt
-        --vault-password-file {{ kayobe_venv }}/bin/kayobe-vault-password-helper
-        {{ kayobe_config_src_dir }}/etc/kayobe/kolla/passwords.yml

playbooks/kayobe-tox-ansible-syntax/pre.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Update kayobe requirements.yml
+      include_role:
+        name: kayobe-galaxy-requirements
+      vars:
+        kayobe_galaxy_requirements_src_dir: "{{ kayobe_src_dir }}"

releasenotes/notes/drop-bootstrap-servers-4d75713c7009153f.yaml

@@ -0,0 +1,63 @@
+---
+features:
+  - |
+    Improves failure handling in the ``kayobe * host configure`` commands by
+    avoiding use of the ``kolla-ansible bootstrap-servers`` command, and moving
+    all relevant functionality to Kayobe playbooks. This ensures that
+    if a host fails during a host configuration command, other hosts are able
+    to continue to completion. This is useful at scale, where host failures
+    occur more frequently. See `story 2009854
+    <https://storyboard.openstack.org/#!/story/2009854>`__ for details. Refer
+    to the upgrade notes for information about the implications of this change.
+upgrade:
+  - |
+    The ``kayobe * host configure`` commands no longer use the ``kolla-ansible
+    bootstrap-servers`` command, and associated ``baremetal`` role in Kolla
+    Ansible. The functionality provided by the ``baremetal`` role has been
+    extracted into a new ``openstack.kolla`` Ansible collection, and split into
+    separate roles. This allows Kayobe to use it directly, and only the
+    necessary parts.
+
+    This change improves failure handling in these Kayobe commands, and aims to
+    reduce confusion over which ``--limit`` and ``--tags`` arguments to
+    provide.
+
+    This change has implications for configuration of Kayobe, since some
+    variables that were previously in Kolla Ansible are now in Kayobe. The
+    following is an incomplete list of variables that have changed scoped from
+    Kolla Ansible to Kayobe::
+
+    * ``enable_docker_repo``
+    * ``docker_apt_url``
+    * ``docker_apt_repo``
+    * ``docker_apt_key_file``
+    * ``docker_apt_key_id``
+    * ``docker_apt_package``
+    * ``docker_yum_url``
+    * ``docker_yum_baseurl``
+    * ``docker_yum_gpgkey``
+    * ``docker_yum_gpgcheck``
+    * ``docker_yum_package``
+    * ``customize_etc_hosts``
+    * ``docker_storage_driver``
+    * ``docker_custom_option``
+    * ``docker_custom_config``
+    * ``docker_http_proxy``
+    * ``docker_https_proxy``
+    * ``docker_no_proxy``
+    * ``debian_pkg_install``
+    * ``redhat_pkg_install``
+    * ``ubuntu_pkg_removals``
+    * ``redhat_pkg_removals``
+
+    The following Kolla Ansible variables are no longer relevant::
+
+    * ``create_kolla_user``
+    * ``create_kolla_user_sudoers``
+    * ``kolla_user``
+    * ``kolla_group``
+    * ``change_selinux``
+    * ``selinux_state``
+    * ``host_python_version``
+    * ``virtualenv``
+    * ``virtualenv_site_packages``

tox.ini

@@ -77,9 +77,13 @@ commands = /bin/bash -c "ansible-lint {toxinidir}/ansible/*.yml"
 [testenv:ansible-syntax]
 commands =
     # Install ansible role dependencies from Galaxy.
-    bash {toxinidir}/tools/ansible-galaxy-retried.sh install \
+    bash {toxinidir}/tools/ansible-galaxy-retried.sh role install \
         -r {toxinidir}/requirements.yml \
         -p {toxinidir}/ansible/roles
+    # Install ansible collection dependencies from Galaxy.
+    bash {toxinidir}/tools/ansible-galaxy-retried.sh collection install \
+        -r {toxinidir}/requirements.yml \
+        -p {toxinidir}/ansible/collections
     # Perform an Ansible syntax check. Skip some playbooks which require extra
     # variables to be defined.
     bash -c \

zuul.d/jobs.yaml

@@ -4,8 +4,13 @@
     description: |
       Tox job that checks Ansible playbook syntax.
     parent: openstack-tox
+    pre-run: playbooks/kayobe-tox-ansible-syntax/pre.yml
+    required-projects:
+      - name: openstack/ansible-collection-kolla
     vars:
       tox_envlist: ansible-syntax
+      ansible_collection_kolla_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/ansible-collection-kolla'].src_dir }}"
+      kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*