From 3f9935a680a18ae0c1377c6f3a6a2edc5b0b7734 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 15 Mar 2017 09:38:59 +0000 Subject: [PATCH] Improve network configuration & external network support - Support multiple OVS bridges - one for provisioning and another optional bridge for external network access. - Use a more consistent naming scheme for OVS bridges and veth pairs - Add an inspection network used for inspecting the workload hosts. --- ansible/group_vars/all/network | 21 ++ ansible/group_vars/controllers/network | 9 - ansible/kolla-ansible.yml | 179 +++++++++++++++--- ansible/network.yml | 48 ++++- ansible/roles/kolla-ansible/defaults/main.yml | 9 +- .../kolla-ansible/templates/globals.yml.j2 | 7 +- etc/kayobe/networks.yml | 21 ++ 7 files changed, 254 insertions(+), 40 deletions(-) diff --git a/ansible/group_vars/all/network b/ansible/group_vars/all/network index 7a32657e8..f3beea4dd 100644 --- a/ansible/group_vars/all/network +++ b/ansible/group_vars/all/network @@ -22,3 +22,24 @@ storage_net_name: 'storage_net' # Name of the network used to carry storage management traffic. storage_mgmt_net_name: 'storage_mgmt_net' + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +inspection_net_name: 'inspection_net' + +############################################################################### +# Network interface naming conventions. + +# Suffix for Open vSwitch bridge names. +network_bridge_suffix_ovs: '-ovs' + +# Prefix for virtual patch interface names. +network_patch_prefix: 'p-' + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +network_patch_suffix_phy: '-phy' + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +network_patch_suffix_ovs: '-ovs' diff --git a/ansible/group_vars/controllers/network b/ansible/group_vars/controllers/network index 759490f66..c0e90a46d 100644 --- a/ansible/group_vars/controllers/network +++ b/ansible/group_vars/controllers/network @@ -18,12 +18,3 @@ controller_default_network_interfaces: > # List of extra networks to which controller nodes are attached. controller_extra_network_interfaces: [] - -############################################################################### -# Kolla networking. - -# Name of the Neutron OVS bridge for the provisioning network. -neutron_bridge_name: "br-ex" - -# External network interface for Neutron. -neutron_external_interface: "{{ 'patch-' ~ neutron_bridge_name }}" diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index 3eff66666..b8457efcd 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -2,30 +2,163 @@ - name: Ensure Kolla Ansible is configured hosts: config-mgmt vars: - controller_host: "{{ groups['controllers'][0] | default() }}" - seed_host: "{{ groups['seed'][0] | default() }}" + # We need to reference configuration for the controller and seed nodes. + # We pick the first host from each group for this. It is possible that at + # this point these groups have no hosts in, and we should handle that case + # gracefully. + # NOTE: Per-host configuration is not currently supported. + controller_host: "{{ groups['controllers'][0] }}" + seed_host: "{{ groups['seed'][0] }}" pre_tasks: - - name: Check whether a Kolla extra globals configuration file exists - stat: - path: "{{ kayobe_config_path ~ '/kolla/globals.yml' }}" - register: globals_stat + # Configuration of extra user-provided Kolla globals. + - block: + - name: Check whether a Kolla extra globals configuration file exists + stat: + path: "{{ kayobe_config_path ~ '/kolla/globals.yml' }}" + register: globals_stat + + - name: Read the Kolla extra globals configuration file + set_fact: + kolla_extra_globals: "{{ lookup('template', kayobe_config_path ~ '/kolla/globals.yml') | from_yaml }}" + when: globals_stat.stat.exists + tags: + - config + + # Configuration and validation of controller networking. + - block: + - name: Set facts containing the VIP addresses and FQDNs + set_fact: + kolla_internal_vip_address: "{{ internal_net_name | net_vip_address }}" + kolla_internal_fqdn: "{{ internal_net_name | net_fqdn or internal_net_name | net_vip_address }}" + kolla_external_vip_address: "{{ external_net_name | net_vip_address }}" + kolla_external_fqdn: "{{ external_net_name | net_fqdn or external_net_name | net_vip_address }}" + when: "{{ kolla_enable_haproxy | bool }}" + + - name: Set facts containing the VIP addresses and FQDNs + set_fact: + kolla_internal_vip_address: "{{ internal_net_name | net_ip(controller_host) }}" + kolla_internal_fqdn: "{{ internal_net_name | net_ip(controller_host) }}" + kolla_external_vip_address: "{{ external_net_name | net_ip(controller_host) }}" + kolla_external_fqdn: "{{ external_net_name | net_ip(controller_host) }}" + when: + - "{{ not kolla_enable_haproxy | bool }}" + + - name: Set facts containing the controller network interfaces + set_fact: + kolla_network_interface: "{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_external_vip_interface: "{{ external_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_api_interface: "{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_storage_interface: "{{ storage_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_cluster_interface: "{{ storage_mgmt_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_provision_interface: "{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}" + kolla_inspector_dnsmasq_interface: "{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}" + + - name: Set facts containing the Neutron bridge and interface names for the provisioning network + set_fact: + kolla_neutron_bridge_names: + - "{{ provision_wl_net_name | net_interface(controller_host) ~ network_bridge_suffix_ovs }}" + kolla_neutron_external_interfaces: + - "{{ network_patch_prefix ~ provision_wl_net_name | net_interface(controller_host) ~ network_patch_suffix_ovs }}" + + - name: Update facts containing the Neutron bridge and interface names for the external network + set_fact: + kolla_neutron_bridge_names: > + {{ kolla_neutron_bridge_names + + [external_net_name | net_interface(controller_host) ~ network_bridge_suffix_ovs] }} + kolla_neutron_external_interfaces: > + {{ kolla_neutron_external_interfaces + + [network_patch_prefix ~ external_net_name | net_interface(controller_host) ~ network_patch_suffix_ovs] }} + when: + - "{{ provision_wl_net_name != external_net_name }}" + + - name: Validate controller Kolla Ansible network configuration + fail: + msg: > + The Kolla Ansible variable {{ item.var_name }} + ({{ item.description }}) is invalid. Value: + "{{ hostvars[inventory_hostname][item.var_name] | default('') }}". + when: + - "{{ item.required | bool }}" + - "{{ hostvars[inventory_hostname][item.var_name] is not defined or not hostvars[inventory_hostname][item.var_name] }}" + with_items: + - var_name: "kolla_internal_vip_address" + description: "Internal API VIP address" + required: True + - var_name: "kolla_internal_fqdn" + description: "Internal API Fully Qualified Domain Name (FQDN)" + required: True + - var_name: "kolla_external_vip_address" + description: "external API VIP address" + required: True + - var_name: "kolla_external_fqdn" + description: "External API Fully Qualified Domain Name (FQDN)" + required: True + - var_name: "kolla_api_interface" + description: "API network interface name" + required: True + - var_name: "kolla_external_vip_interface" + description: "External network interface name" + required: True + - var_name: "kolla_provision_interface" + description: "Bare metal provisioning network interface name" + required: "{{ kolla_enable_ironic }}" + - var_name: "kolla_inspector_dnsmasq_interface" + description: "Bare metal introspection network interface name" + required: "{{ kolla_enable_ironic }}" + - var_name: "kolla_neutron_bridge_names" + description: "List of Neutron bridge names" + required: True + - var_name: "kolla_neutron_external_interfaces" + description: "List of Neutron interface names" + required: True + + - name: Validate controller Kolla Ansible Neutron bridge and interface configuration + fail: + msg: > + The Kolla Ansible variable {{ item.0.var_name }} + ({{ item.0.description }}) is invalid. Value: + "{{ item.1 | default('') }}". + when: + - "{{ item.0.required | bool }}" + - "{{ item.1 is not defined or not item.1 }}" + with_subelements: + - - var_name: "kolla_neutron_bridge_names" + value: "{{ kolla_neutron_bridge_names }}" + description: "List of Neutron bridge names" + required: True + - var_name: "kolla_neutron_external_interfaces" + value: "{{ kolla_neutron_external_interfaces }}" + description: "List of Neutron interface names" + required: True + - value + when: "{{ controller_host is defined }}" + tags: + - config + - config-validation + + # Configuration and validation of seed networking. + - block: + - name: Set facts containing the seed network interfaces + set_fact: + kolla_bifrost_network_interface: "{{ provision_oc_net_name | net_interface(seed_host) | replace('-', '_') }}" + + - name: Validate seed Kolla Ansible network configuration + fail: + msg: > + The Kolla Ansible variable {{ item.var_name }} + ({{ item.description }}) is invalid. Value: + "{{ hostvars[inventory_hostname][item.var_name] | default('') }}". + when: + - "{{ item.required | bool }}" + - "{{ hostvars[inventory_hostname][item.var_name] is not defined or not hostvars[inventory_hostname][item.var_name] }}" + with_items: + - var_name: "kolla_bifrost_network_interface" + description: "Bifrost network interface name" + required: True + when: "{{ seed_host is defined }}" + tags: + - config + - config-validation - - name: Read the Kolla extra globals configuration file - set_fact: - kolla_extra_globals: "{{ lookup('template', kayobe_config_path ~ '/kolla/globals.yml') | from_yaml }}" - when: globals_stat.stat.exists roles: - role: kolla-ansible - kolla_internal_vip_address: "{% if kolla_enable_haproxy | bool %}{{ internal_net_name | net_vip_address }}{% elif controller_host %}{{ internal_net_name | net_ip(controller_host) }}{% endif %}" - kolla_internal_fqdn: "{% if kolla_enable_haproxy | bool %}{{ internal_net_name | net_fqdn or kolla_internal_vip_address }}{% elif controller_host %}{{ internal_net_name | net_ip(controller_host) }}{% endif %}" - kolla_external_vip_address: "{% if kolla_enable_haproxy | bool %}{{ external_net_name | net_vip_address }}{% elif controller_host %}{{ external_net_name | net_ip(controller_host) }}{% endif %}" - kolla_external_fqdn: "{% if kolla_enable_haproxy | bool %}{{ external_net_name | net_fqdn or kolla_external_vip_address }}{% elif controller_host %}{{ external_net_name | net_ip(controller_host) }}{% endif %}" - kolla_network_interface: "{% if controller_host %}{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" - kolla_external_vip_interface: "{% if controller_host %}{{ external_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" - kolla_api_interface: "{% if controller_host %}{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" - kolla_storage_interface: "{% if controller_host %}{{ storage_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" - kolla_cluster_interface: "{% if controller_host %}{{ storage_mgmt_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" - kolla_neutron_bridge_name: "{% if controller_host %}{{ hostvars[controller_host]['neutron_bridge_name'] }}{% endif %}" - kolla_neutron_external_interface: "{% if controller_host %}{{ hostvars[controller_host]['neutron_external_interface'] }}{% endif %}" - kolla_bifrost_network_interface: "{% if seed_host %}{{ provision_oc_net_name | net_interface(seed_host) | replace('-', '_') }}{% endif %}" - kolla_provision_interface: "{% if controller_host %}{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}" diff --git a/ansible/network.yml b/ansible/network.yml index 3847e7d50..26e124ed2 100644 --- a/ansible/network.yml +++ b/ansible/network.yml @@ -2,6 +2,26 @@ - name: Ensure networking is configured hosts: seed:controllers pre_tasks: + - block: + - name: Validate network interface configuration + fail: + msg: > + Network interface validation failed - no interface configured for + {{ item }}. This should be configured via '{{ item }}_interface'. + with_items: "{{ network_interfaces | net_select_ethers | list }}" + when: "{{ not item | net_interface }}" + + - name: Validate bridge interface configuration + fail: + msg: > + Bridge interface validation failed - no interface configured for + {{ item }}. This should be configured via '{{ item }}_interface'. + with_items: "{{ network_interfaces | net_select_bridges | list }}" + when: "{{ not item | net_interface }}" + tags: + - config + - config-validation + - name: Ensure NetworkManager is disabled service: name: NetworkManager @@ -17,6 +37,8 @@ roles: - role: ahuffman.resolv become: True + tags: + - config - role: MichaelRigart.interfaces interfaces_ether_interfaces: > @@ -30,18 +52,38 @@ map('net_bridge_obj') | list }} become: True + tags: + - config -- name: Ensure controller workload provisioning network is configured +- name: Ensure controller workload OVS patch links exist hosts: controllers roles: # Configure a virtual ethernet patch link to connect the workload provision # network bridge to the Neutron OVS bridge. - role: veth veth_interfaces: - - device: "{{ 'patch-' ~ provision_wl_net_name | net_interface }}" + - device: "{{ network_patch_prefix ~ provision_wl_net_name | net_interface ~ network_patch_suffix_phy }}" bootproto: "static" bridge: "{{ provision_wl_net_name | net_interface }}" - peer_device: "{{ 'patch-' ~ neutron_bridge_name }}" + peer_device: "{{ network_patch_prefix ~ provision_wl_net_name | net_interface ~ network_patch_suffix_ovs }}" peer_bootproto: "static" onboot: yes when: "{{ provision_wl_net_name in network_interfaces }}" + tags: + - config + + # Configure a virtual ethernet patch link to connect the external network + # bridge to the Neutron OVS bridge. + - role: veth + veth_interfaces: + - device: "{{ network_patch_prefix ~ external_net_name | net_interface ~ network_patch_suffix_phy }}" + bootproto: "static" + bridge: "{{ external_net_name | net_interface }}" + peer_device: "{{ network_patch_prefix ~ external_net_name | net_interface ~ network_patch_suffix_ovs }}" + peer_bootproto: "static" + onboot: yes + when: + - "{{ external_net_name in network_interfaces }}" + - "{{ external_net_name != provision_wl_net_name }}" + tags: + - config diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 08a4b570a..2f29c5385 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -81,10 +81,11 @@ kolla_cluster_interface: # though an IP address can exist on this interface, it will be unusable in most # configurations. It is recommended this interface not be configured with any IP # addresses for that reason. -kolla_neutron_external_interface: +# This should be a list of names of interfaces. +kolla_neutron_external_interfaces: [] -# Name of the Neutron external bridge. -kolla_neutron_bridge_name: +# List of names of Neutron bridges. +kolla_neutron_bridge_names: [] # This is the interface to use for Bifrost bare metal provisioning of the # control plane. @@ -94,6 +95,8 @@ kolla_bifrost_network_interface: # standard kolla variable. kolla_provision_interface: +# Interface on which the inspector dnsmasq service listens. +kolla_inspector_dnsmasq_interface: #################### # TLS options diff --git a/ansible/roles/kolla-ansible/templates/globals.yml.j2 b/ansible/roles/kolla-ansible/templates/globals.yml.j2 index 24cf4891a..8669949de 100644 --- a/ansible/roles/kolla-ansible/templates/globals.yml.j2 +++ b/ansible/roles/kolla-ansible/templates/globals.yml.j2 @@ -83,10 +83,10 @@ cluster_interface: "{{ kolla_cluster_interface }}" # though an IP address can exist on this interface, it will be unusable in most # configurations. It is recommended this interface not be configured with any IP # addresses for that reason. -neutron_external_interface: "{{ kolla_neutron_external_interface }}" +neutron_external_interface: "{{ kolla_neutron_external_interfaces | join(',') }}" # Name of the Neutron external bridge. -neutron_bridge_name: "{{ kolla_neutron_bridge_name }}" +neutron_bridge_name: "{{ kolla_neutron_bridge_names | join(',') }}" # This is the interface to use for Bifrost bare metal provisioning of the # control plane. @@ -96,6 +96,9 @@ bifrost_network_interface: "{{ kolla_bifrost_network_interface }}" # standard kolla variable. provision_interface: "{{ kolla_provision_interface }}" +# Interface on which the inspector dnsmasq service listens. +ironic_dnsmasq_interface: "{{ kolla_inspector_dnsmasq_interface }}" + # Valid options are [ openvswitch, linuxbridge ] neutron_plugin_agent: "openvswitch" diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 08bd0036c..bf8631006 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -25,6 +25,10 @@ # Name of the network used to carry storage management traffic. #storage_mgmt_net_name: +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +#inspection_net_name: + ############################################################################### # Network definitions. @@ -74,6 +78,23 @@ # storage_mgmt_net_gateway: # storage_mgmt_net_vlan: +############################################################################### +# Network virtual patch link configuration. + +# Suffix for Open vSwitch bridge names. +#network_bridge_suffix_ovs: + +# Prefix for virtual patch interface names. +#network_patch_prefix: + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +#network_patch_suffix_phy: + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +#network_patch_suffix_ovs: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes