openstack/tripleo-heat-templates
Code Issues Proposed changes

Move KernelArgs and OvS-DPDK deployment to ansible role

Browse Source 
Ealier, KernelArgs had been configured using ansible
tasks part of THT repo. Thoese ansiblet asks has been
moved to tripleo-kernel role of tripleo-ansible. This
role will be invoked from the boot-params-service.
boot-params-service has been moved from pre network to
the deployment/kernel directory.

OvS-DPDK configuration was done using puppet-vswitch
module by invoking puppet in PreNetworkConfig's
ExtraConfig script. A new ansible role tripleo-ovs-dpdk
has been created to apply the DPDK configurations via
ansible instead of puppet. This role will be common
for both ml2-ovs and ml2-ovn. Common parameter merging
has been enhanced to provide common deploy steps.

ODL is not validated as it has been deprecated and
currently no active usage or development.

Depends-On: https://review.opendev.org/#/c/688864/
Change-Id: I4b6f7d73cf76954e93760c59d522d485187157cf
This commit is contained in:
Saravanan KR 2019-08-21 18:59:39 +05:30
parent 5ffad3a3f0
commit 16679d0ec4
10 changed files with 118 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
common/deploy-steps.j2
View File

@ -1609,4 +1609,3 @@ outputs:
- import_tasks: {{role.name}}/post_update_tasks.yaml
when: tripleo_role_name == '{{role.name}}'
{%- endfor %}
boot_param_tasks: {get_file: ../extraconfig/pre_network/boot_param_tasks.yaml}

71
extraconfig/pre_network/boot-params-service.yaml → deployment/kernel/kernel-boot-params-baremetal-ansible.yaml
View File

@ -58,8 +58,22 @@ parameters:
type: number
description: Timeout in seconds to specify the wait time for ansible node reboot
conditions:
is_host_config_required: {not: {equals: [{get_param: [RoleParameters, KernelArgs]}, ""]}}
resources:
RoleParametersValue:
type: OS::Heat::Value
properties:
type: json
value:
map_replace:
- map_replace:
- tuned_profile: TunedProfileName
isolated_cores: IsolCpusList
kernel_args: KernelArgs
- values: {get_param: [RoleParameters]}
- values:
TunedProfileName: {get_param: TunedProfileName}
IsolCpusList: {get_param: IsolCpusList}
KernelArgs: {get_param: KernelArgs}
outputs:
role_data:
@ -67,39 +81,20 @@ outputs:
value:
service_name: boot_params_service
deploy_steps_tasks:
- if:
- is_host_config_required
- - name: Configure kernel args, tuned and reboot
when: step|int == 0
vars:
_KERNEL_ARGS_: {get_param: [RoleParameters, KernelArgs]}
_TUNED_PROFILE_NAME_: {get_param: [RoleParameters, TunedProfileName]}
_TUNED_CORES_: {get_param: [RoleParameters, IsolCpusList]}
block:
- import_tasks: ../boot_param_tasks.yaml
- name: Update gid for hugetlbfs to kolla's gid
group:
name: hugetlbfs
gid: 42477
when: reboot_required is defined and reboot_required
# Reboot the node
- name: Reboot after kernel args update
shell: "sleep 2 && /sbin/shutdown -r now"
async: 1
poll: 0
ignore_errors: true
become: yes
when: reboot_required is defined and reboot_required
- name: Wait for the connection to be ready on the restarted node
wait_for_connection:
connect_timeout: 20
sleep: 5
delay: 30
timeout:
get_param: NodeRebootWaitTimeout
when: reboot_required is defined and reboot_required
- null
# TODO(skramaja): Tasks of Tuned TripleO service can be used here
# But the configuration in that service has to be moved from host_prep_tasks to step 0 of deploy_step_tasks
- name: Configure tuned before reboot
when: step|int == 0
import_role:
name: tuned
vars:
tuned_profile: {get_attr: [RoleParametersValue, value, tuned_profile]}
tuned_isolated_cores: {get_attr: [RoleParametersValue, value, isolated_cores]}
- name: Configure kernel args and reboot
when: step|int == 0
import_role:
name: tripleo-kernel
tasks_from: kernelargs.yml
vars:
tripleo_kernel_args: {get_attr: [RoleParametersValue, value, kernel_args]}
tripleo_kernel_reboot_timeout: {get_param: NodeRebootWaitTimeout}

26
deployment/neutron/neutron-ovn-dpdk-config-container-puppet.yaml
View File

@ -51,6 +51,16 @@ parameters:
resources:
OpenvswitchDpdk:
type: ./../openvswitch/openvswitch-dpdk-baremetal-ansible.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
# Merging role-specific parameters (RoleParameters) with the default parameters.
# RoleParameters will have the precedence over the default parameters.
RoleParametersValue:
@ -67,15 +77,6 @@ resources:
VhostuserSocketGroup: {get_param: VhostuserSocketGroup}
VhostuserSocketUser: {get_param: VhostuserSocketUser}
Ovs:
type: ../../puppet/services/openvswitch.yaml
properties:
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for Neutron ML2/OVN DPDK config
@ -83,9 +84,8 @@ outputs:
service_name: neutron_ovn_dpdk
config_settings:
map_merge:
- ovn::controller::enable_dpdk: true
ovn::controller::datapath_type: "netdev"
- ovn::controller::datapath_type: "netdev"
nova::compute::libvirt::qemu::group: {get_attr: [RoleParametersValue, value, vhostuser_socket_group]}
- get_attr: [RoleParametersValue, value]
- get_attr: [Ovs, role_data, config_settings]
deploy_steps_tasks:
- get_attr: [OpenvswitchDpdk, role_data, deploy_steps_tasks]

25
deployment/neutron/neutron-ovs-dpdk-agent-container-puppet.yaml
View File

@ -79,6 +79,16 @@ resources:
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
OpenvswitchDpdk:
type: ./../openvswitch/openvswitch-dpdk-baremetal-ansible.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
# Merging role-specific parameters (RoleParameters) with the default parameters.
# RoleParameters will have the precedence over the default parameters.
RoleParametersValue:
@ -99,15 +109,6 @@ resources:
VhostuserSocketGroup: {get_param: VhostuserSocketGroup}
VhostuserSocketUser: {get_param: VhostuserSocketUser}
Ovs:
type: ../openvswitch/openvswitch-baremetal-puppet.yaml
properties:
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for Neutron openvswitch DPDK service
@ -119,8 +120,6 @@ outputs:
- get_attr: [NeutronOvsAgent, role_data, config_settings]
- keys:
tripleo::neutron_ovs_agent::firewall_rules: tripleo::neutron_ovs_dpdk_agent::firewall_rules
- neutron::agents::ml2::ovs::enable_dpdk: true
- get_attr: [Ovs, role_data, config_settings]
- get_attr: [RoleParametersValue, value]
service_config_settings:
map_merge:
@ -152,6 +151,10 @@ outputs:
get_attr: [NeutronOvsAgent, role_data, metadata_settings]
host_prep_tasks:
get_attr: [NeutronOvsAgent, role_data, host_prep_tasks]
deploy_steps_tasks:
list_concat:
- get_attr: [NeutronOvsAgent, role_data, deploy_steps_tasks]
- get_attr: [OpenvswitchDpdk, role_data, deploy_steps_tasks]
upgrade_tasks:
get_attr: [NeutronOvsAgent, role_data, upgrade_tasks]
update_tasks:

65
deployment/openvswitch/openvswitch-baremetal-puppet.yaml → deployment/openvswitch/openvswitch-dpdk-baremetal-ansible.yaml
View File

@ -105,22 +105,25 @@ parameters:
tags:
- role_specific
outputs:
role_data:
description: Role data for the Open vSwitch service.
value:
service_name: openvswitch
config_settings:
conditions:
emc_disable: {equals: [{get_param: OvsDisableEMC}, true]}
resources:
RoleParametersDpdk:
type: OS::Heat::Value
properties:
type: json
value:
map_replace:
- map_replace:
- vswitch::dpdk::driver_type: OvsDpdkDriverType
vswitch::dpdk::host_core_list: OvsDpdkCoreList
vswitch::dpdk::pmd_core_list: OvsPmdCoreList
vswitch::dpdk::memory_channels: OvsDpdkMemoryChannels
vswitch::dpdk::socket_mem: OvsDpdkSocketMemory
vswitch::dpdk::disable_emc: OvsDisableEMC
vswitch::dpdk::revalidator_cores: OvsRevalidatorCores
vswitch::dpdk::handler_cores: OvsHandlerCores
- lcore: OvsDpdkCoreList
pmd: OvsPmdCoreList
memory_channels: OvsDpdkMemoryChannels
socket_mem: OvsDpdkSocketMemory
disable_emc: OvsDisableEMC
revalidator: OvsRevalidatorCores
handler: OvsHandlerCores
- values: {get_param: [RoleParameters]}
- values:
OvsDpdkCoreList: {get_param: OvsDpdkCoreList}
@ -131,3 +134,37 @@ outputs:
OvsDisableEMC: {get_param: OvsDisableEMC}
OvsRevalidatorCores: {get_param: OvsRevalidatorCores}
OvsHandlerCores: {get_param: OvsHandlerCores}
BootParams:
type: ./../kernel/kernel-boot-params-baremetal-ansible.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for the Open vSwitch service.
value:
service_name: openvswitch
deploy_steps_tasks:
- get_attr: [BootParams, role_data, deploy_steps_tasks]
- - name: Run ovs-dpdk role
when: step|int == 0
include_role:
name: tripleo-ovs-dpdk
vars:
tripleo_ovs_dpdk_pmd_core_list: {get_attr: [RoleParametersDpdk, value, pmd]}
tripleo_ovs_dpdk_lcore_list: {get_attr: [RoleParametersDpdk, value, lcore]}
tripleo_ovs_dpdk_memory_channels: {get_attr: [RoleParametersDpdk, value, memory_channels]}
tripleo_ovs_dpdk_socket_memory: {get_attr: [RoleParametersDpdk, value, socket_mem]}
tripleo_ovs_dpdk_revalidator_cores: {get_attr: [RoleParametersDpdk, value, revalidator]}
tripleo_ovs_dpdk_handler_cores: {get_attr: [RoleParametersDpdk, value, handler]}
tripleo_ovs_dpdk_emc_insertion_probablity:
if:
- emc_disable
- 0
- ''

7
environments/host-config-and-reboot.j2.yaml
View File

@ -1,6 +1,11 @@
## DEPRECATED
# KernelArgs and TunedProfileName has been moved from PreNetworkConfig to boot-params-service.yaml (TripleO service)
# OvS-DPDK configurations has been moved from PreNetworkConfig to respective dpdk TripleO service (ml2-ovs, ml2-ovn)
# host_config_and_reboot.yaml does not do anything, and soon it will be removed
resource_registry:
{% for role in roles %}
OS::TripleO::{{role.name}}::PreNetworkConfig: ../extraconfig/pre_network/host_config_and_reboot.yaml
OS::TripleO::{{role.name}}::PreNetworkConfig: OS::Heat::None
{% endfor %}
#parameter_defaults:

101
extraconfig/pre_network/boot_param_tasks.yaml
View File

@ -1,101 +0,0 @@
- name: Get the command line args of the node
command: cat /proc/cmdline
register: cmdline
- name: Check tuned active profile
slurp:
src: "/etc/tuned/active_profile"
register: tuned_active_profile
# Kernel Args Configuration
- block:
- name: Ensure the kernel args ( {{ _KERNEL_ARGS_ }} ) is present as TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS
lineinfile:
dest: /etc/default/grub
regexp: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
insertafter: '^GRUB_CMDLINE_LINUX.*'
line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" {{ _KERNEL_ARGS_ }} "'
- name: Add TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS to the GRUB_CMDLINE_LINUX parameter
lineinfile:
dest: /etc/default/grub
line: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX:+$GRUB_CMDLINE_LINUX }${TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS}"'
insertafter: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
- name: Generate grub config file
command: grub2-mkconfig -o /boot/grub2/grub.cfg
- name: Set reboot required fact
set_fact:
reboot_required: true
become: true
when:
- _KERNEL_ARGS_|default("") != ""
- _KERNEL_ARGS_|default("") not in cmdline.stdout_lines[0]
# Kernel modules loading
- name: Load type1 IOMMU driver for VFIO on boot
import_role:
name: tripleo-module-load
vars:
modules:
- name: vfio_iommu_type1
when: _KERNEL_ARGS_ is search("iommu")
# Tune-d Configuration
- block:
- name: Check Tune-d Configuration file exists
stat:
path: "/etc/tuned/{{ _TUNED_PROFILE_NAME_ }}-variables.conf"
register: tuned_conf_stat_result
- name: Tune-d Configuration
lineinfile:
dest: "/etc/tuned/{{ _TUNED_PROFILE_NAME_ }}-variables.conf"
regexp: '^isolated_cores=.*'
line: 'isolated_cores={{ _TUNED_CORES_ }}'
when: tuned_conf_stat_result.stat.exists and _TUNED_CORES_|default("") != ""
- name: Tune-d profile activation
command: >-
tuned-adm profile {{ _TUNED_PROFILE_NAME_ }}
- name: Set reboot required fact
set_fact:
reboot_required: true
become: true
when:
- (_TUNED_PROFILE_NAME_ | default("")) != ""
- (_TUNED_PROFILE_NAME_ | default("")) != (tuned_active_profile['content'] | b64decode).strip()
# Check if os-net-config has run once, if yes, no need for the below workaround
- find:
paths: /etc/sysconfig/network-scripts/
patterns: ifcfg-*
contains: "# This file is autogenerated by os-net-config"
register: os_net_ifcfg_files
when: reboot_required is defined and reboot_required
# Provisioning Network workaround
# The script will be executed before os-net-config, in which case, only Provisioning network will have IP
# BOOTPROTO of all interface config files (except provisioning), will be set to "none" to avoid reboot failing to acquire IP on other networks
- block:
- find:
paths: /etc/sysconfig/network-scripts/
patterns: ifcfg-*
register: ifcfg_files
- replace:
dest: "{{ item.path }}"
regexp: '^BOOTPROTO=.*'
replace: 'BOOTPROTO=none'
when:
- item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') != "lo"
# Ensure the interface information is available in the facts
- hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ] is defined
# This condition will list all the interfaces except the one with valid IP (which is Provisioning network at this stage)
# Simpler Version - hostvars[inventory_hostname]['ansible_' + iface_name ]['ipv4'] is undefined
- hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ]['ipv4'] is undefined
with_items:
- "{{ ifcfg_files.files }}"
become: true
when:
- reboot_required is defined and reboot_required
- os_net_ifcfg_files.matched == 0

163
extraconfig/pre_network/host_config_and_reboot.yaml
View File

@ -1,163 +0,0 @@
heat_template_version: rocky
description: >
All configurations which require reboot should be initiated via PreNetworkConfig. After
this configuration is completed, the corresponding node will be rebooted.
parameters:
server:
type: string
RoleParameters:
type: json
description: Parameters specific to the role
default: {}
ServiceNames:
type: comma_delimited_list
default: []
OvsEnableDpdk:
default: false
description: Whether or not to configure enable DPDK in OVS
type: boolean
tags:
- role_specific
OvsDpdkCoreList:
description: >
List of cores to be used for DPDK lcore threads. Note, these threads
are used by the OVS control path for validator and handling functions.
type: string
constraints:
- allowed_pattern: "[0-9,-]*"
default: ""
tags:
- role_specific
OvsDpdkMemoryChannels:
description: Number of memory channels per socket to be used for DPDK
type: string
constraints:
- allowed_pattern: "[0-9]*"
default: "4"
tags:
- role_specific
OvsDpdkSocketMemory:
default: ""
description: >
Sets the amount of hugepage memory to assign per NUMA node. It is
recommended to use the socket closest to the PCIe slot used for the
desired DPDK NIC. The format should be in "<socket 0 mem>, <socket 1
mem>, <socket n mem>", where the value is specified in MB. For example:
"1024,0".
type: string
tags:
- role_specific
OvsPmdCoreList:
description: >
A list or range of CPU cores for PMD threads to be pinned to. Note, NIC
location to cores on socket, number of hyper-threaded logical cores, and
desired number of PMD threads can all play a role in configuring this
setting. These cores should be on the same socket where
OvsDpdkSocketMemory is assigned. If using hyperthreading then specify
both logical cores that would equal the physical core. Also, specifying
more than one core will trigger multiple PMD threads to be spawned which
may improve dataplane performance.
constraints:
- allowed_pattern: "[0-9,-]*"
type: string
default: ""
tags:
- role_specific
deployment_actions:
default: ['CREATE', 'UPDATE']
type: comma_delimited_list
description: >
List of stack actions that will trigger any deployments in this
templates. The actions will be an empty list of the server is in the
toplevel DeploymentServerBlacklist parameter's value.
EnableDpdkDeploymentActions:
default: ['CREATE']
type: comma_delimited_list
description: >
Exposing the DPDK deployment action, it may be required to run DPDK
config during an upgrade. By default DPDK will be enabled during the
CREATE action only. But on cases when it requires for certain migration,
it may be required to run it for UPDATE action too.
conditions:
is_dpdk_config_required:
or:
- yaql:
expression: $.data.service_names.contains('neutron_ovs_dpdk_agent') or $.data.service_names.contains('neutron_ovn_dpdk')
data:
service_names: {get_param: ServiceNames}
- {equals: [{get_param: [RoleParameters, OvsEnableDpdk]}, true]}
deployment_actions_empty:
equals:
- {get_param: deployment_actions}
- []
resources:
RoleParametersValue:
type: OS::Heat::Value
properties:
type: json
value:
map_replace:
- map_replace:
- OvsDpdkCoreList: OvsDpdkCoreList
OvsDpdkMemoryChannels: OvsDpdkMemoryChannels
OvsDpdkSocketMemory: OvsDpdkSocketMemory
OvsPmdCoreList: OvsPmdCoreList
- values: {get_param: [RoleParameters]}
- values:
OvsDpdkCoreList: {get_param: OvsDpdkCoreList}
OvsDpdkMemoryChannels: {get_param: OvsDpdkMemoryChannels}
OvsDpdkSocketMemory: {get_param: OvsDpdkSocketMemory}
OvsPmdCoreList: {get_param: OvsPmdCoreList}
# With OvS2.7 (which is default with pike), ovs-vswitchd will start dpdk
# immediately after setting dpdk-init (behaviour change from ovs2.6).
# Starting of DPDK require the huge page configuration to be enabled. So
# reboot will happen before DPDK config and we don't need an explicity
# restart after dpdk-init as true because of the behavior change.
# TODO(skramaja): Dependency is that till the service file workaround, is
# maintained, restart of ovs is required.
EnableDpdkConfig:
type: OS::Heat::SoftwareConfig
condition: is_dpdk_config_required
properties:
group: script
config:
str_replace:
template: |
#!/bin/bash
set -x
# DO NOT use --detailed-exitcodes
puppet apply --logdest console --summarize \
--modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \
-e '
class {"vswitch::dpdk":
host_core_list => "$HOST_CORES",
pmd_core_list => "$PMD_CORES",
memory_channels => "$MEMORY_CHANNELS",
socket_mem => "$SOCKET_MEMORY",
}
'
params:
$HOST_CORES: {get_attr: [RoleParametersValue, value, OvsDpdkCoreList]}
$PMD_CORES: {get_attr: [RoleParametersValue, value, OvsPmdCoreList]}
$MEMORY_CHANNELS: {get_attr: [RoleParametersValue, value, OvsDpdkMemoryChannels]}
$SOCKET_MEMORY: {get_attr: [RoleParametersValue, value, OvsDpdkSocketMemory]}
EnableDpdkDeployment:
type: OS::Heat::SoftwareDeployment
condition: is_dpdk_config_required
properties:
name: EnableDpdkDeployment
server: {get_param: server}
config: {get_resource: EnableDpdkConfig}
actions:
if:
- deployment_actions_empty
- []
- {get_param: EnableDpdkDeploymentActions}

2
overcloud-resource-registry-puppet.j2.yaml
View File

@ -351,7 +351,7 @@ resource_registry:
# Tempest
OS::TripleO::Services::Tempest: OS::Heat::None
OS::TripleO::Services::BootParams: extraconfig/pre_network/boot-params-service.yaml
OS::TripleO::Services::BootParams: deployment/kernel/kernel-boot-params-baremetal-ansible.yaml
parameter_defaults:
NeutronMechanismDrivers: ovn

6
tools/yaml-validate.py
View File

@ -251,10 +251,6 @@ WORKFLOW_TASKS_EXCLUSIONS = [
]
ANSIBLE_TASKS_YAMLS = [
'./extraconfig/pre_network/boot_param_tasks.yaml'
]
HEAT_OUTPUTS_EXCLUSIONS = [
'./puppet/extraconfig/tls/ca-inject.yaml',
'./deployed-server/deployed-server.yaml',
@ -1164,8 +1160,6 @@ def validate(filename, param_map):
retval |= validate_nic_config_file(filename, tpl)
except Exception:
if filename in ANSIBLE_TASKS_YAMLS:
return 0
print(traceback.format_exc())
return 1
# yaml is OK, now walk the parameters and output a warning for unused ones