Add kernelargs and tuned core configuration

Add support to update kernel args and reboot the node.
Enhanced support for configuring isolated cores with
few of the tuned profiles.

Change-Id: Iac72c41d3bfc87c0016da4df2228732a778a810e

.gitignore

@@ -28,3 +28,6 @@ doc/build
 
 ansible-errors.json
 pytestdebug.log
+
+# doc
+doc/build/*

tripleo_ansible/roles/tripleo-kernel/defaults/main.yml

@@ -19,3 +19,7 @@
 tripleo_kernel_extra_modules: {}
 tripleo_kernel_extra_packages: {}
 tripleo_kernel_sysctl_extra_settings: {}
+tripleo_kernel_args: ""
+tripleo_kernel_reboot_timeout: 3600
+tripleo_kernel_post_reboot_delay: 60
+tripleo_kernel_defer_reboot: false

tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/Dockerfile

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/molecule.yml

@@ -0,0 +1,53 @@
+---
+driver:
+  name: delegated
+  options:
+    managed: false
+    login_cmd_template: >-
+      ssh
+      -o UserKnownHostsFile=/dev/null
+      -o StrictHostKeyChecking=no
+      -o Compression=no
+      -o TCPKeepAlive=yes
+      -o VerifyHostKeyDNS=no
+      -o ForwardX11=no
+      -o ForwardAgent=no
+      {instance}
+    ansible_connection_options:
+      ansible_connection: ssh
+
+log: true
+
+platforms:
+  - name: instance
+
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      fact_caching: jsonfile
+      fact_caching_connection: /tmp/molecule/facts
+  inventory:
+    hosts:
+      all:
+        hosts:
+          instance:
+            ansible_host: localhost
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_ROLES_PATH: "${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles"
+
+scenario:
+  test_sequence:
+    - prepare
+    - converge
+    - verify
+
+lint:
+  enabled: false
+
+verifier:
+  name: ansible
+  lint:
+    name: ansible-lint

tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/playbook.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  become: true
+  tasks:
+    - include_role:
+        name: "tripleo-kernel"
+        tasks_from: kernelargs.yml
+      vars:
+        tripleo_kernel_args: "test=1"
+        tripleo_kernel_defer_reboot: true

tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/prepare.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  roles:
+    - role: test_deps

tripleo_ansible/roles/tripleo-kernel/molecule/kernelargs/verify.yml

@@ -0,0 +1,30 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Verify
+  hosts: all
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Check if the kernel args is applied to the grub file
+      lineinfile:
+        name: /etc/default/grub
+        line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" test=1 "'
+        state: present
+      check_mode: true
+      register: grub
+      failed_when: (grub is changed) or (grub is failed)

tripleo_ansible/roles/tripleo-kernel/tasks/kernelargs.yml

@@ -0,0 +1,64 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Get the command line args of the node
+  command: cat /proc/cmdline
+  register: cmdline
+
+# Kernel Args Configuration
+- block:
+    - name: Ensure the kernel args ( {{ tripleo_kernel_args }} ) is present as TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS
+      lineinfile:
+        dest: /etc/default/grub
+        regexp: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
+        insertafter: '^GRUB_CMDLINE_LINUX.*'
+        line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" {{ tripleo_kernel_args }} "'
+    - name: Add TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS to the GRUB_CMDLINE_LINUX parameter
+      lineinfile:
+        dest: /etc/default/grub
+        line: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX:+$GRUB_CMDLINE_LINUX }${TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS}"'
+        insertafter: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*'
+    - name: Generate grub config file
+      command: grub2-mkconfig -o /boot/grub2/grub.cfg
+    - name: Set reboot required fact
+      set_fact:
+        reboot_required: true
+  become: true
+  when:
+    - tripleo_kernel_args|string
+    - tripleo_kernel_args not in cmdline.stdout_lines[0]
+
+# Apply DPDK workarounds before reboot
+- name: Apply DPDK workarounds
+  include_role:
+    name: tripleo-ovs-dpdk
+    tasks_from: workarounds.yml
+  when: reboot_required is defined and reboot_required
+
+# Kernel modules loading
+- name: Load type1 IOMMU driver for VFIO on boot
+  import_role:
+    name: tripleo-module-load
+  vars:
+    modules:
+      - name: vfio_iommu_type1
+  when: tripleo_kernel_args is search("iommu")
+
+- name: Reboot and workaround block
+  include_tasks: reboot.yaml
+  when:
+    - reboot_required is defined and reboot_required
+    - not tripleo_kernel_defer_reboot|bool

tripleo_ansible/roles/tripleo-kernel/tasks/reboot.yaml

@@ -0,0 +1,62 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Check if os-net-config has run once, if yes, no need for the below workaround
+- name: Find the ifcfg file generated by os-net-config
+  find:
+    paths: /etc/sysconfig/network-scripts/
+    patterns: ifcfg-*
+    contains: "# This file is autogenerated by os-net-config"
+  register: os_net_ifcfg_files
+
+# Provisioning Network workaround
+# The script will be executed before os-net-config, in which case, only Provisioning network will have IP
+# BOOTPROTO of all interface config files (except provisioning), will be set to "none" to avoid reboot failing to acquire IP on other networks
+- name: Apply workaround for node reboot
+  block:
+    - name: Find the ifcg files
+      find:
+        paths: /etc/sysconfig/network-scripts/
+        patterns: ifcfg-*
+      register: ifcfg_files
+
+    - name: Replace BOOTPROTO to none for interfaces which does not have IP
+      replace:
+        dest: "{{ item.path }}"
+        regexp: '^BOOTPROTO=.*'
+        replace: 'BOOTPROTO=none'
+      when:
+        - item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') != "lo"
+        # Ensure the interface information is available in the facts
+        - hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ] is defined
+        # This condition will list all the interfaces except the one with valid IP (which is Provisioning network at this stage)
+        # Simpler Version - hostvars[inventory_hostname]['ansible_' + iface_name ]['ipv4'] is undefined
+        - hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') | replace('-', '_') ]['ipv4'] is undefined
+      with_items:
+        - "{{ ifcfg_files.files }}"
+  become: true
+  when:
+    - os_net_ifcfg_files.matched == 0
+
+- name: Reboot debug message
+  debug:
+    msg: "Going to reboot the node after applying kernel args..."
+
+# Reboot the node
+- name: Reboot after kernel args update
+  reboot:
+    post_reboot_delay: "{{ tripleo_kernel_post_reboot_delay }}"
+    reboot_timeout: "{{ tripleo_kernel_reboot_timeout }}"

tripleo_ansible/roles/tripleo-ovs-dpdk/tasks/workarounds.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Workaround to allow access of vhostuser sockets from OvS running host to qemu
+# running inside the kolla container. GID of the group 'hugetlbfs' is changed
+# to the same value as kolla's gid. Apply this workaround for all DPDK nodes,
+# along with reboot, so that OvS threads are restarted with this id.
+# NOTE: This will not be included in this roles's main.yml, as main.yml should
+# be executed after node reboot.
+- name: Update gid for hugetlbfs to kolla's gid
+  group:
+    name: hugetlbfs
+    gid: 42477

tripleo_ansible/roles/tuned/defaults/main.yml

@@ -19,6 +19,7 @@
 
 tuned_profile: "throughput-performance"
 tuned_custom_profile: ""
+tuned_isolated_cores: ""
 
 # Packages installed on the local system. Allows user to define this list
 # otherwise it will inherit from the OS specific variable file(s).

tripleo_ansible/roles/tuned/molecule/isolated/Dockerfile

@@ -0,0 +1,36 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

tripleo_ansible/roles/tuned/molecule/isolated/molecule.yml

@@ -0,0 +1,68 @@
+---
+driver:
+  name: docker
+
+log: true
+
+platforms:
+  - name: centos7
+    hostname: centos7
+    image: centos:7
+    dockerfile: Dockerfile
+    pkg_extras: python-setuptools
+    easy_install:
+      - pip
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    command: /sbin/init
+    tmpfs:
+      - /run
+      - /tmp
+    capabilities:
+      - ALL  # CENT7 requires all due to the age of the software
+    volumes:
+      - /run/udev:/run/udev:ro
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+
+  - name: fedora28
+    hostname: fedora28
+    image: fedora:28
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    environment:
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    command: /sbin/init
+    privileged: true
+    tmpfs:
+      - /run
+      - /tmp
+    capabilities:
+      - SYS_ADMIN
+    volumes:
+      - /run/udev:/run/udev:ro
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+lint:
+  enabled: false
+
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

tripleo_ansible/roles/tuned/molecule/isolated/playbook.yml

@@ -0,0 +1,25 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  vars:
+    tuned_system_packages: "tuned-profiles-cpu-partitioning"
+    tuned_profile: "cpu-partitioning"
+    tuned_isolated_cores: "1"
+  roles:
+    - role: "tuned"

tripleo_ansible/roles/tuned/molecule/isolated/prepare.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  roles:
+    - role: test_deps

tripleo_ansible/roles/tuned/molecule/isolated/tests/test_isolated.py

@@ -0,0 +1,40 @@
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import configparser
+import os
+
+import testinfra.utils.ansible_runner
+
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_isolated_cores(host):
+    assert host.file('/etc/tuned/cpu-partitioning-variables.conf').contains('^isolated_cores=1$')
+
+
+def test_cpu_affinity(host):
+    out = host.check_output('nproc --all')
+    cpus = ''
+    for i in range(int(out)):
+        if i == 1:
+            continue
+        if cpus:
+            cpus += ' '
+        cpus += str(i)
+    assert host.file('/etc/systemd/system.conf').contains('^CPUAffinity=' + cpus + '$')

tripleo_ansible/roles/tuned/tasks/tuned_config.yml

@@ -34,6 +34,27 @@
     src: "/etc/tuned/active_profile"
   register: tuned_active_profile
 
+- name: Check Tuned Configuration file exists
+  stat:
+    path: "/etc/tuned/{{ tuned_profile }}-variables.conf"
+  register: tuned_conf_stat_result
+
+- name: Fail if tuned profile conf is absent but isolated cores is provided
+  fail:
+    msg: "Tuned profile conf file is not available to configure isolated cores"
+  when:
+    - tuned_isolated_cores
+    - not tuned_conf_stat_result.stat.exists
+
+- name: "Configure isolated cores for profile {{ tuned_profile }}"
+  lineinfile:
+    dest: "/etc/tuned/{{ tuned_profile }}-variables.conf"
+    regexp: '^isolated_cores=.*'
+    line: 'isolated_cores={{ tuned_isolated_cores }}'
+  when:
+    - tuned_isolated_cores
+    - tuned_conf_stat_result.stat.exists
+
 - name: Enable tuned profile
   command: >-
     tuned-adm profile {{ tuned_profile }}