diff --git a/puppet-manifests/src/bin/puppet-manifest-apply.sh b/puppet-manifests/src/bin/puppet-manifest-apply.sh index 3774de15c..c98ac7488 100755 --- a/puppet-manifests/src/bin/puppet-manifest-apply.sh +++ b/puppet-manifests/src/bin/puppet-manifest-apply.sh @@ -52,7 +52,23 @@ rm -rf ${PUPPET_TMP} mkdir -p ${PUPPET_TMP}/hieradata cp /etc/puppet/hieradata/global.yaml ${PUPPET_TMP}/hieradata/global.yaml cp /etc/puppet/hieradata/${PERSONALITY}.yaml ${PUPPET_TMP}/hieradata/personality.yaml -cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml + +# When the compute node is first booted and goes online, sysinv-agent reports +# host CPU inventory which triggers the first runtime manifest apply that updates +# the grub. At this time, copying the host file failed due to a timing issue that +# has not yet been fully understood. Subsequent retries worked. +if [ "${PERSONALITY}" = "compute" ]; then + n=0 + until [ $n -ge 3 ] + do + cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml && break + n=$[$n+1] + logger -t $0 "Failed to copy /etc/puppet/hieradata/${HOST}.yaml" + sleep 15 + done +else + cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml +fi cp -f ${HIERADATA}/system.yaml \ ${HIERADATA}/secure_system.yaml \ ${HIERADATA}/static.yaml \ diff --git a/puppet-manifests/src/manifests/compute.pp b/puppet-manifests/src/manifests/compute.pp index 52f4c2e2a..46226316d 100644 --- a/puppet-manifests/src/manifests/compute.pp +++ b/puppet-manifests/src/manifests/compute.pp @@ -13,6 +13,7 @@ include ::platform::sysctl::compute include ::platform::dhclient include ::platform::partitions include ::platform::lvm::compute +include ::platform::compute include ::platform::vswitch include ::platform::network include ::platform::fstab diff --git a/puppet-manifests/src/modules/platform/lib/facter/get_cmdline.rb b/puppet-manifests/src/modules/platform/lib/facter/get_cmdline.rb new file mode 100644 index 000000000..9074d7fce --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/get_cmdline.rb @@ -0,0 +1,5 @@ +# Returns the current boot parameters +Facter.add(:get_cmdline) do + setcode "cat /proc/cmdline 2>/dev/null" +end + diff --git a/puppet-manifests/src/modules/platform/lib/facter/is_broadwell_processor.rb b/puppet-manifests/src/modules/platform/lib/facter/is_broadwell_processor.rb new file mode 100644 index 000000000..9429a29fb --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/is_broadwell_processor.rb @@ -0,0 +1,8 @@ +# Returns true if it is Broadwell processor +# Broadwell specific flags (model: 79) +Facter.add("is_broadwell_processor") do + setcode do + Facter::Core::Execution.exec('grep -q -E "^model\s+:\s+79$" /proc/cpuinfo') + $?.exitstatus == 0 + end +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/is_gb_page_supported.rb b/puppet-manifests/src/modules/platform/lib/facter/is_gb_page_supported.rb new file mode 100644 index 000000000..122768ce2 --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/is_gb_page_supported.rb @@ -0,0 +1,7 @@ +# Returns true if one GB pages is supported +Facter.add("is_gb_page_supported") do + setcode do + Facter::Core::Execution.exec('grep -q pdpe1gb /proc/cpuinfo') + $?.exitstatus == 0 + end +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/is_hugetlbfs_enabled.rb b/puppet-manifests/src/modules/platform/lib/facter/is_hugetlbfs_enabled.rb new file mode 100644 index 000000000..aadada4f7 --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/is_hugetlbfs_enabled.rb @@ -0,0 +1,7 @@ +# Returns true if hugetlbfs not enabled +Facter.add("is_hugetlbfs_enabled") do + setcode do + Facter::Core::Execution.exec('grep -q hugetlbfs /proc/filesystems') + $?.exitstatus == 0 + end +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/is_per_numa_supported.rb b/puppet-manifests/src/modules/platform/lib/facter/is_per_numa_supported.rb new file mode 100644 index 000000000..70061a0f8 --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/is_per_numa_supported.rb @@ -0,0 +1,6 @@ +# Returns true if Resource Control is supported on this node +Facter.add("is_per_numa_supported") do + setcode do + Dir.exist?('/sys/devices/system/node/node0') + end +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/is_resctrl_supported.rb b/puppet-manifests/src/modules/platform/lib/facter/is_resctrl_supported.rb new file mode 100644 index 000000000..4a25e065a --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/is_resctrl_supported.rb @@ -0,0 +1,6 @@ +# Returns true if Resource Control is supported on this node +Facter.add("is_resctrl_supported") do + setcode do + Dir.exist?('/sys/fs/resctrl') + end +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/number_of_logical_cpus.rb b/puppet-manifests/src/modules/platform/lib/facter/number_of_logical_cpus.rb new file mode 100644 index 000000000..652a3db9b --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/number_of_logical_cpus.rb @@ -0,0 +1,4 @@ +# Returns number of logical cpus +Facter.add(:number_of_logical_cpus) do + setcode "cat /proc/cpuinfo 2>/dev/null | awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}'" +end diff --git a/puppet-manifests/src/modules/platform/lib/facter/number_of_numa_nodes.rb b/puppet-manifests/src/modules/platform/lib/facter/number_of_numa_nodes.rb new file mode 100644 index 000000000..b8962abf3 --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/facter/number_of_numa_nodes.rb @@ -0,0 +1,4 @@ +# Returns number of numa nodes +Facter.add(:number_of_numa_nodes) do + setcode "ls -d /sys/devices/system/node/node* 2>/dev/null | wc -l" +end diff --git a/puppet-manifests/src/modules/platform/lib/puppet/parser/functions/check_grub_config.rb b/puppet-manifests/src/modules/platform/lib/puppet/parser/functions/check_grub_config.rb new file mode 100644 index 000000000..c6840432f --- /dev/null +++ b/puppet-manifests/src/modules/platform/lib/puppet/parser/functions/check_grub_config.rb @@ -0,0 +1,34 @@ +module Puppet::Parser::Functions + newfunction(:check_grub_config, + :type => :rvalue, + :doc => <<-EOD + This internal function checks if a list of arguments are configured + in the current boot args based on the input parameters + + EOD + ) do |args| + + func_name = "check_grub_config()" + + raise(Puppet::ParseError, "#{func_name}: Requires 1 argument" + + "#{args.size} given") if args.size != 1 + + expected = args[0] + raise(Puppet::ParseError, "#{func_name}: first argument must be a string") \ + unless expected.instance_of? String + + # get the current boot args + cmd = Facter.value(:get_cmdline) + cmd_array = cmd.split() + + value = true + expected.split().each do |element| + value = cmd_array.include?(element) + if value == false + Puppet.debug("#{element} is not presented in #{cmd}") + return value + end + end + value + end +end diff --git a/puppet-manifests/src/modules/platform/manifests/compute.pp b/puppet-manifests/src/modules/platform/manifests/compute.pp new file mode 100644 index 000000000..4846432e6 --- /dev/null +++ b/puppet-manifests/src/modules/platform/manifests/compute.pp @@ -0,0 +1,246 @@ +class platform::compute::grub::params ( + $n_cpus = '', + $cpu_options = '', + $m_hugepages = 'hugepagesz=2M hugepages=0', + $default_pgsz = 'default_hugepagesz=2M', + $keys = ['kvm-intel.eptad', 'default_hugepagesz', 'hugepagesz', 'hugepages', 'isolcpus', 'nohz_full', 'rcu_nocbs', 'kthread_cpus', 'irqaffinity'], +) { + + if $::is_broadwell_processor { + $eptad = 'kvm-intel.eptad=0' + } else { + $eptad = '' + } + + if $::is_gb_page_supported { + $gb_hugepages = "hugepagesz=1G hugepages=$::number_of_numa_nodes" + } else { + $gb_hugepages = '' + } + + $grub_updates = strip("${eptad} ${$gb_hugepages} ${m_hugepages} ${default_pgsz} ${cpu_options}") +} + +class platform::compute::grub::update + inherits ::platform::compute::grub::params { + + notice("Updating grub configuration") + + $to_be_removed = join($keys, " ") + exec { "Remove the cpu arguments": + command => "grubby --update-kernel=ALL --remove-args='$to_be_removed'", + } -> + exec { "Add the cpu arguments": + command => "grubby --update-kernel=ALL --args='$grub_updates'", + } +} + +class platform::compute::grub::recovery { + + notice("Update Grub and Reboot") + + class {'platform::compute::grub::update': } -> Exec['reboot-recovery'] + + exec { "reboot-recovery": + command => "reboot", + } +} + +class platform::compute::grub::audit + inherits ::platform::compute::grub::params { + + if ! str2bool($::is_initial_config_primary) { + + notice("Audit CPU and Grub Configuration") + + $expected_n_cpus = $::number_of_logical_cpus + $n_cpus_ok = ("$n_cpus" == "$expected_n_cpus") + + $cmd_ok = check_grub_config($grub_updates) + + if $cmd_ok and $n_cpus_ok { + $ensure = present + notice("CPU and Boot Argument audit passed.") + } else { + $ensure = absent + if !$cmd_ok { + notice("Kernel Boot Argument Mismatch") + include ::platform::compute::grub::recovery + } + } + + file { "/var/run/compute_huge_goenabled": + ensure => $ensure, + owner => 'root', + group => 'root', + mode => '0644', + } + } +} + +class platform::compute::grub::runtime { + include ::platform::compute::grub::update +} + +# Mounts virtual hugetlbfs filesystems for each supported page size +class platform::compute::hugetlbf { + + if str2bool($::is_hugetlbfs_enabled) { + + $fs_list = generate("/bin/bash", "-c", "ls -1d /sys/kernel/mm/hugepages/hugepages-*") + $array = split($fs_list, '\n') + $array.each | String $val | { + $page_name = generate("/bin/bash", "-c", "basename $val") + $page_size = strip(regsubst($page_name, 'hugepages-', '')) + $hugemnt ="/mnt/huge-$page_size" + $options = "pagesize=${page_size}" + + notice("Mounting hugetlbfs at: $hugemnt") + exec { "create $hugemnt": + command => "mkdir -p ${hugemnt}", + onlyif => "test ! -d ${hugemnt}", + } -> + mount { "${hugemnt}": + name => "${hugemnt}", + device => 'none', + fstype => 'hugetlbfs', + ensure => 'mounted', + options => "${options}", + atboot => 'yes', + remounts => true, + } + } + } +} + +class platform::compute::hugepage::params ( + $nr_hugepages_2M = undef, + $nr_hugepages_1G = undef, + $vswitch_2M_pages = '', + $vswitch_1G_pages = '', + $vm_4K_pages = '', + $vm_2M_pages = '', + $vm_1G_pages = '', +) {} + + +define allocate_pages ( + $path, + $page_count, +) { + exec { "Allocate ${page_count} ${path}": + command => "echo $page_count > $path", + onlyif => "test -f $path", + } +} + +# Allocates HugeTLB memory according to the attributes specified in the +# nr_hugepages_2M and nr_hugepages_1G +class platform::compute::allocate + inherits ::platform::compute::hugepage::params { + + # determine the node file system + if str2bool($::is_per_numa_supported) { + $nodefs = '/sys/devices/system/node' + } else { + $nodefs = '/sys/kernel/mm' + } + + if $nr_hugepages_2M != undef { + $nr_hugepages_2M_array = regsubst($nr_hugepages_2M, '[\(\)\"]', '', 'G').split(' ') + $nr_hugepages_2M_array.each | String $val | { + $per_node_2M = $val.split(':') + if size($per_node_2M)== 3 { + $node = $per_node_2M[0] + $page_size = $per_node_2M[1] + allocate_pages { "Start ${node} ${page_size}": + path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages", + page_count => $per_node_2M[2], + } + } + } + } + + if $nr_hugepages_1G != undef { + $nr_hugepages_1G_array = regsubst($nr_hugepages_1G , '[\(\)\"]', '', 'G').split(' ') + $nr_hugepages_1G_array.each | String $val | { + $per_node_1G = $val.split(':') + if size($per_node_1G)== 3 { + $node = $per_node_1G[0] + $page_size = $per_node_1G[1] + allocate_pages { "Start ${node} ${page_size}": + path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages", + page_count => $per_node_1G[2], + } + } + } + } +} + +class platform::compute::extend + inherits ::platform::compute::hugepage::params { + + # nova-compute reads on init, extended nova compute options + # used with nova accounting + file { "/etc/nova/compute_extend.conf": + ensure => 'present', + replace => true, + content => template('platform/compute_extend.conf.erb') + } +} + +# Mount resctrl to allow Cache Allocation Technology per VM +class platform::compute::resctrl { + + if str2bool($::is_resctrl_supported) { + mount { "/sys/fs/resctrl": + name => '/sys/fs/resctrl', + device => 'resctrl', + fstype => 'resctrl', + ensure => 'mounted', + atboot => 'yes', + remounts => true, + } + } +} + +# Set Power Management QoS resume latency constraints for CPUs. +# The PM QoS resume latency limit is set to shallow C-state for vswitch CPUs. +# All other CPUs are allowed to go to the deepest C-state available. +class platform::compute::pmqos ( + $low_wakeup_cpus = '', + $hight_wakeup_cpus = '', +) { + + if str2bool($::is_compute_subfunction) and str2bool($::is_lowlatency_subfunction) { + + $script = "/usr/bin/set-cpu-wakeup-latency.sh" + + # Set low wakeup latency (shallow C-state) for vswitch CPUs using PM QoS interface + exec { "low-wakeup-latency": + command => "${script} low ${low_wakeup_cpus}", + onlyif => "test -f ${script}", + logoutput => true, + } + + #Set high wakeup latency (deep C-state) for non-vswitch CPUs using PM QoS interface + exec { "high-wakeup-latency": + command => "${script} high ${hight_wakeup_cpus}", + onlyif => "test -f ${script}", + logoutput => true, + } + } +} + +class platform::compute { + + Class[$name] -> Class['::platform::vswitch'] + Class[$name] -> Class['::nova::compute'] + + require ::platform::compute::grub::audit + require ::platform::compute::hugetlbf + require ::platform::compute::allocate + require ::platform::compute::pmqos + require ::platform::compute::resctrl + require ::platform::compute::extend +} diff --git a/puppet-manifests/src/modules/platform/templates/compute_extend.conf.erb b/puppet-manifests/src/modules/platform/templates/compute_extend.conf.erb new file mode 100644 index 000000000..d11d1a2e2 --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/compute_extend.conf.erb @@ -0,0 +1,12 @@ +########################################################################### +# +# compute_extend.conf contains compute extended nova options +# +# - This file is managed by Puppet. DO NOT EDIT. +# +########################################################################### +compute_vswitch_2M_pages=<%= @vswitch_2M_pages.gsub!(/\A"|"\Z/, '') %> +compute_vswitch_1G_pages=<%= @vswitch_1G_pages.gsub!(/\A"|"\Z/, '') %> +compute_vm_4K_pages=<%= @vm_4K_pages.gsub!(/\A"|"\Z/, '') %> +compute_vm_2M_pages=<%= @vm_2M_pages.gsub!(/\A"|"\Z/, '') %> +compute_vm_1G_pages=<%= @vm_1G_pages.gsub!(/\A"|"\Z/, '') %>