Browse Source

Add kernelargs and tuned core configuration

Add support to update kernel args and reboot the node.
Enhanced support for configuring isolated cores with
few of the tuned profiles.

Change-Id: Iac72c41d3bfc87c0016da4df2228732a778a810e
changes/09/679809/8
Saravanan KR 3 years ago committed by Kevin Carter (cloudnull)
parent
commit
f94d24bddf
  1. 3
      .gitignore
  2. 4
      tripleo_ansible/roles/tripleo-kernel/defaults/main.yml
  3. 37
      tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/Dockerfile
  4. 53
      tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/molecule.yml
  5. 27
      tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/playbook.yml
  6. 21
      tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/prepare.yml
  7. 30
      tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/verify.yml
  8. 64
      tripleo_ansible/roles/tripleo-kernel/tasks/kernelargs.yml
  9. 62
      tripleo_ansible/roles/tripleo-kernel/tasks/reboot.yaml
  10. 26
      tripleo_ansible/roles/tripleo-ovs-dpdk/tasks/workarounds.yml
  11. 1
      tripleo_ansible/roles/tuned/defaults/main.yml
  12. 36
      tripleo_ansible/roles/tuned/molecule/isolated/Dockerfile
  13. 68
      tripleo_ansible/roles/tuned/molecule/isolated/molecule.yml
  14. 25
      tripleo_ansible/roles/tuned/molecule/isolated/playbook.yml
  15. 21
      tripleo_ansible/roles/tuned/molecule/isolated/prepare.yml
  16. 40
      tripleo_ansible/roles/tuned/molecule/isolated/tests/test_isolated.py
  17. 21
      tripleo_ansible/roles/tuned/tasks/tuned_config.yml

3
.gitignore vendored

@ -28,3 +28,6 @@ doc/build
ansible-errors.json
pytestdebug.log
# doc
doc/build/*

4
tripleo_ansible/roles/tripleo-kernel/defaults/main.yml

@ -19,3 +19,7 @@
tripleo_kernel_extra_modules: {}
tripleo_kernel_extra_packages: {}
tripleo_kernel_sysctl_extra_settings: {}
tripleo_kernel_args: ""
tripleo_kernel_reboot_timeout: 3600
tripleo_kernel_post_reboot_delay: 60
tripleo_kernel_defer_reboot: false

37
tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/Dockerfile

@ -0,0 +1,37 @@
# Molecule managed
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
{% if item.registry is defined %}
FROM {{ item.registry.url }}/{{ item.image }}
{% else %}
FROM {{ item.image }}
{% endif %}
RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
{% for pkg in item.easy_install | default([]) %}
# install pip for centos where there is no python-pip rpm in default repos
RUN easy_install {{ pkg }}
{% endfor %}
CMD ["sh", "-c", "while true; do sleep 10000; done"]

53
tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/molecule.yml

@ -0,0 +1,53 @@
---
driver:
name: delegated
options:
managed: false
login_cmd_template: >-
ssh
-o UserKnownHostsFile=/dev/null
-o StrictHostKeyChecking=no
-o Compression=no
-o TCPKeepAlive=yes
-o VerifyHostKeyDNS=no
-o ForwardX11=no
-o ForwardAgent=no
{instance}
ansible_connection_options:
ansible_connection: ssh
log: true
platforms:
- name: instance
provisioner:
name: ansible
config_options:
defaults:
fact_caching: jsonfile
fact_caching_connection: /tmp/molecule/facts
inventory:
hosts:
all:
hosts:
instance:
ansible_host: localhost
log: true
env:
ANSIBLE_STDOUT_CALLBACK: yaml
ANSIBLE_ROLES_PATH: "${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles"
scenario:
test_sequence:
- prepare
- converge
- verify
lint:
enabled: false
verifier:
name: ansible
lint:
name: ansible-lint

27
tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/playbook.yml

@ -0,0 +1,27 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Converge
hosts: all
become: true
tasks:
- include_role:
name: "tripleo-kernel"
tasks_from: kernelargs.yml
vars:
tripleo_kernel_args: "test=1"
tripleo_kernel_defer_reboot: true

21
tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/prepare.yml

@ -0,0 +1,21 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Prepare
hosts: all
roles:
- role: test_deps

30
tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/verify.yml

@ -0,0 +1,30 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Verify
hosts: all
become: true
gather_facts: false
tasks:
- name: Check if the kernel args is applied to the grub file
lineinfile:
name: /etc/default/grub
line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" test=1 "'
state: present
check_mode: true
register: grub
failed_when: (grub is changed) or (grub is failed)

64
tripleo_ansible/roles/tripleo-kernel/tasks/kernelargs.yml

@ -0,0 +1,64 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Get the command line args of the node
command: cat /proc/cmdline
register: cmdline
# Kernel Args Configuration
- block:
- name: Ensure the kernel args ( {{ tripleo_kernel_args }} ) is present as TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS
lineinfile:
dest: /etc/default/grub
regexp: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
insertafter: '^GRUB_CMDLINE_LINUX.*'
line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" {{ tripleo_kernel_args }} "'
- name: Add TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS to the GRUB_CMDLINE_LINUX parameter
lineinfile:
dest: /etc/default/grub
line: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX:+$GRUB_CMDLINE_LINUX }${TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS}"'
insertafter: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
- name: Generate grub config file
command: grub2-mkconfig -o /boot/grub2/grub.cfg
- name: Set reboot required fact
set_fact:
reboot_required: true
become: true
when:
- tripleo_kernel_args|string
- tripleo_kernel_args not in cmdline.stdout_lines[0]
# Apply DPDK workarounds before reboot
- name: Apply DPDK workarounds
include_role:
name: tripleo-ovs-dpdk
tasks_from: workarounds.yml
when: reboot_required is defined and reboot_required
# Kernel modules loading
- name: Load type1 IOMMU driver for VFIO on boot
import_role:
name: tripleo-module-load
vars:
modules:
- name: vfio_iommu_type1
when: tripleo_kernel_args is search("iommu")
- name: Reboot and workaround block
include_tasks: reboot.yaml
when:
- reboot_required is defined and reboot_required
- not tripleo_kernel_defer_reboot|bool

62
tripleo_ansible/roles/tripleo-kernel/tasks/reboot.yaml

@ -0,0 +1,62 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Check if os-net-config has run once, if yes, no need for the below workaround
- name: Find the ifcfg file generated by os-net-config
find:
paths: /etc/sysconfig/network-scripts/
patterns: ifcfg-*
contains: "# This file is autogenerated by os-net-config"
register: os_net_ifcfg_files
# Provisioning Network workaround
# The script will be executed before os-net-config, in which case, only Provisioning network will have IP
# BOOTPROTO of all interface config files (except provisioning), will be set to "none" to avoid reboot failing to acquire IP on other networks
- name: Apply workaround for node reboot
block:
- name: Find the ifcg files
find:
paths: /etc/sysconfig/network-scripts/
patterns: ifcfg-*
register: ifcfg_files
- name: Replace BOOTPROTO to none for interfaces which does not have IP
replace:
dest: "{{ item.path }}"
regexp: '^BOOTPROTO=.*'
replace: 'BOOTPROTO=none'
when:
- item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') != "lo"
# Ensure the interface information is available in the facts
- hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ] is defined
# This condition will list all the interfaces except the one with valid IP (which is Provisioning network at this stage)
# Simpler Version - hostvars[inventory_hostname]['ansible_' + iface_name ]['ipv4'] is undefined
- hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ]['ipv4'] is undefined
with_items:
- "{{ ifcfg_files.files }}"
become: true
when:
- os_net_ifcfg_files.matched == 0
- name: Reboot debug message
debug:
msg: "Going to reboot the node after applying kernel args..."
# Reboot the node
- name: Reboot after kernel args update
reboot:
post_reboot_delay: "{{ tripleo_kernel_post_reboot_delay }}"
reboot_timeout: "{{ tripleo_kernel_reboot_timeout }}"

26
tripleo_ansible/roles/tripleo-ovs-dpdk/tasks/workarounds.yml

@ -0,0 +1,26 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Workaround to allow access of vhostuser sockets from OvS running host to qemu
# running inside the kolla container. GID of the group 'hugetlbfs' is changed
# to the same value as kolla's gid. Apply this workaround for all DPDK nodes,
# along with reboot, so that OvS threads are restarted with this id.
# NOTE: This will not be included in this roles's main.yml, as main.yml should
# be executed after node reboot.
- name: Update gid for hugetlbfs to kolla's gid
group:
name: hugetlbfs
gid: 42477

1
tripleo_ansible/roles/tuned/defaults/main.yml

@ -19,6 +19,7 @@
tuned_profile: "throughput-performance"
tuned_custom_profile: ""
tuned_isolated_cores: ""
# Packages installed on the local system. Allows user to define this list
# otherwise it will inherit from the OS specific variable file(s).

36
tripleo_ansible/roles/tuned/molecule/isolated/Dockerfile

@ -0,0 +1,36 @@
# Molecule managed
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
{% if item.registry is defined %}
FROM {{ item.registry.url }}/{{ item.image }}
{% else %}
FROM {{ item.image }}
{% endif %}
RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
{% for pkg in item.easy_install | default([]) %}
# install pip for centos where there is no python-pip rpm in default repos
RUN easy_install {{ pkg }}
{% endfor %}
CMD ["sh", "-c", "while true; do sleep 10000; done"]

68
tripleo_ansible/roles/tuned/molecule/isolated/molecule.yml

@ -0,0 +1,68 @@
---
driver:
name: docker
log: true
platforms:
- name: centos7
hostname: centos7
image: centos:7
dockerfile: Dockerfile
pkg_extras: python-setuptools
easy_install:
- pip
environment: &env
http_proxy: "{{ lookup('env', 'http_proxy') }}"
https_proxy: "{{ lookup('env', 'https_proxy') }}"
command: /sbin/init
tmpfs:
- /run
- /tmp
capabilities:
- ALL # CENT7 requires all due to the age of the software
volumes:
- /run/udev:/run/udev:ro
- /sys/fs/cgroup:/sys/fs/cgroup:ro
- name: fedora28
hostname: fedora28
image: fedora:28
dockerfile: Dockerfile
pkg_extras: python*-setuptools
environment:
http_proxy: "{{ lookup('env', 'http_proxy') }}"
https_proxy: "{{ lookup('env', 'https_proxy') }}"
command: /sbin/init
privileged: true
tmpfs:
- /run
- /tmp
capabilities:
- SYS_ADMIN
volumes:
- /run/udev:/run/udev:ro
- /sys/fs/cgroup:/sys/fs/cgroup:ro
provisioner:
name: ansible
log: true
env:
ANSIBLE_STDOUT_CALLBACK: yaml
scenario:
test_sequence:
- destroy
- create
- prepare
- converge
- verify
- destroy
lint:
enabled: false
verifier:
name: testinfra
lint:
name: flake8

25
tripleo_ansible/roles/tuned/molecule/isolated/playbook.yml

@ -0,0 +1,25 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Converge
hosts: all
vars:
tuned_system_packages: "tuned-profiles-cpu-partitioning"
tuned_profile: "cpu-partitioning"
tuned_isolated_cores: "1"
roles:
- role: "tuned"

21
tripleo_ansible/roles/tuned/molecule/isolated/prepare.yml

@ -0,0 +1,21 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Prepare
hosts: all
roles:
- role: test_deps

40
tripleo_ansible/roles/tuned/molecule/isolated/tests/test_isolated.py

@ -0,0 +1,40 @@
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import configparser
import os
import testinfra.utils.ansible_runner
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
def test_isolated_cores(host):
assert host.file('/etc/tuned/cpu-partitioning-variables.conf').contains('^isolated_cores=1$')
def test_cpu_affinity(host):
out = host.check_output('nproc --all')
cpus = ''
for i in range(int(out)):
if i == 1:
continue
if cpus:
cpus += ' '
cpus += str(i)
assert host.file('/etc/systemd/system.conf').contains('^CPUAffinity=' + cpus + '$')

21
tripleo_ansible/roles/tuned/tasks/tuned_config.yml

@ -34,6 +34,27 @@
src: "/etc/tuned/active_profile"
register: tuned_active_profile
- name: Check Tuned Configuration file exists
stat:
path: "/etc/tuned/{{ tuned_profile }}-variables.conf"
register: tuned_conf_stat_result
- name: Fail if tuned profile conf is absent but isolated cores is provided
fail:
msg: "Tuned profile conf file is not available to configure isolated cores"
when:
- tuned_isolated_cores
- not tuned_conf_stat_result.stat.exists
- name: "Configure isolated cores for profile {{ tuned_profile }}"
lineinfile:
dest: "/etc/tuned/{{ tuned_profile }}-variables.conf"
regexp: '^isolated_cores=.*'
line: 'isolated_cores={{ tuned_isolated_cores }}'
when:
- tuned_isolated_cores
- tuned_conf_stat_result.stat.exists
- name: Enable tuned profile
command: >-
tuned-adm profile {{ tuned_profile }}

Loading…
Cancel
Save