Integrate host configuration into configuration framework

Integrates the following host configuration into the configuration
framework:
- Host boot parameters
- CPU reservation
- Process affinity
- Memory huge page allocations

Change-Id: I2259e0e93eefd5ce5000271fa32ecaa8d13fa411
Signed-off-by: Matt Peters <matt.peters@windriver.com>

puppet-manifests/src/bin/puppet-manifest-apply.sh

@@ -52,7 +52,23 @@ rm -rf ${PUPPET_TMP}
 mkdir -p ${PUPPET_TMP}/hieradata
 cp /etc/puppet/hieradata/global.yaml ${PUPPET_TMP}/hieradata/global.yaml
 cp /etc/puppet/hieradata/${PERSONALITY}.yaml ${PUPPET_TMP}/hieradata/personality.yaml
-cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml
+
+# When the compute node is first booted and goes online, sysinv-agent reports 
+# host CPU inventory which triggers the first runtime manifest apply that updates
+# the grub. At this time, copying the host file failed due to a timing issue that
+# has not yet been fully understood. Subsequent retries worked. 
+if [ "${PERSONALITY}" = "compute" ]; then
+  n=0
+  until [ $n -ge 3 ]
+  do
+    cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml && break
+    n=$[$n+1]
+    logger -t $0 "Failed to copy /etc/puppet/hieradata/${HOST}.yaml"
+    sleep 15
+  done
+else
+  cp -f ${HIERADATA}/${HOST}.yaml ${PUPPET_TMP}/hieradata/host.yaml
+fi
 cp -f ${HIERADATA}/system.yaml \
     ${HIERADATA}/secure_system.yaml \
     ${HIERADATA}/static.yaml \

puppet-manifests/src/manifests/compute.pp

@@ -13,6 +13,7 @@ include ::platform::sysctl::compute
 include ::platform::dhclient
 include ::platform::partitions
 include ::platform::lvm::compute
+include ::platform::compute
 include ::platform::vswitch
 include ::platform::network
 include ::platform::fstab

puppet-manifests/src/modules/platform/lib/facter/get_cmdline.rb

@@ -0,0 +1,5 @@
+# Returns the current boot parameters
+Facter.add(:get_cmdline) do
+  setcode "cat /proc/cmdline 2>/dev/null"
+end
+

puppet-manifests/src/modules/platform/lib/facter/is_broadwell_processor.rb

@@ -0,0 +1,8 @@
+# Returns true if it is Broadwell processor
+# Broadwell specific flags (model: 79)
+Facter.add("is_broadwell_processor") do
+  setcode do
+      Facter::Core::Execution.exec('grep -q -E "^model\s+:\s+79$" /proc/cpuinfo')
+      $?.exitstatus == 0
+  end
+end

puppet-manifests/src/modules/platform/lib/facter/is_gb_page_supported.rb

@@ -0,0 +1,7 @@
+# Returns true if one GB pages is supported
+Facter.add("is_gb_page_supported") do
+  setcode do
+      Facter::Core::Execution.exec('grep -q pdpe1gb /proc/cpuinfo')
+      $?.exitstatus == 0
+  end
+end

puppet-manifests/src/modules/platform/lib/facter/is_hugetlbfs_enabled.rb

@@ -0,0 +1,7 @@
+# Returns true if hugetlbfs not enabled
+Facter.add("is_hugetlbfs_enabled") do
+  setcode do
+      Facter::Core::Execution.exec('grep -q hugetlbfs /proc/filesystems')
+      $?.exitstatus == 0
+  end
+end

puppet-manifests/src/modules/platform/lib/facter/is_per_numa_supported.rb

@@ -0,0 +1,6 @@
+# Returns true if Resource Control is supported on this node
+Facter.add("is_per_numa_supported") do
+  setcode do
+    Dir.exist?('/sys/devices/system/node/node0')
+  end
+end

puppet-manifests/src/modules/platform/lib/facter/is_resctrl_supported.rb

@@ -0,0 +1,6 @@
+# Returns true if Resource Control is supported on this node
+Facter.add("is_resctrl_supported") do
+  setcode do
+    Dir.exist?('/sys/fs/resctrl')
+  end
+end

puppet-manifests/src/modules/platform/lib/facter/number_of_logical_cpus.rb

@@ -0,0 +1,4 @@
+# Returns number of logical cpus
+Facter.add(:number_of_logical_cpus) do
+  setcode "cat /proc/cpuinfo 2>/dev/null | awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}'"
+end

puppet-manifests/src/modules/platform/lib/facter/number_of_numa_nodes.rb

@@ -0,0 +1,4 @@
+# Returns number of numa nodes
+Facter.add(:number_of_numa_nodes) do
+  setcode "ls -d /sys/devices/system/node/node* 2>/dev/null | wc -l"
+end

puppet-manifests/src/modules/platform/lib/puppet/parser/functions/check_grub_config.rb

@@ -0,0 +1,34 @@
+module Puppet::Parser::Functions
+  newfunction(:check_grub_config,
+              :type => :rvalue,
+              :doc => <<-EOD
+    This internal function checks if a list of arguments are configured
+    in the current boot args based on the input parameters
+
+    EOD
+    ) do |args|
+
+    func_name = "check_grub_config()"
+
+    raise(Puppet::ParseError, "#{func_name}: Requires 1 argument" +
+      "#{args.size} given") if args.size != 1
+
+    expected = args[0]
+    raise(Puppet::ParseError, "#{func_name}: first argument must be a string") \
+      unless expected.instance_of? String
+
+    # get the current boot args
+    cmd = Facter.value(:get_cmdline)
+    cmd_array = cmd.split()
+
+    value = true
+    expected.split().each do |element|
+      value = cmd_array.include?(element)
+      if value == false
+        Puppet.debug("#{element} is not presented in #{cmd}")
+        return value
+      end
+    end
+    value
+  end
+end

puppet-manifests/src/modules/platform/manifests/compute.pp

@@ -0,0 +1,246 @@
+class platform::compute::grub::params (
+  $n_cpus = '',
+  $cpu_options = '',
+  $m_hugepages = 'hugepagesz=2M hugepages=0',
+  $default_pgsz = 'default_hugepagesz=2M',
+  $keys = ['kvm-intel.eptad', 'default_hugepagesz', 'hugepagesz', 'hugepages', 'isolcpus', 'nohz_full', 'rcu_nocbs', 'kthread_cpus', 'irqaffinity'],
+) {
+
+  if $::is_broadwell_processor {
+    $eptad = 'kvm-intel.eptad=0'
+  } else {
+    $eptad = ''
+  }
+
+  if $::is_gb_page_supported {
+    $gb_hugepages = "hugepagesz=1G hugepages=$::number_of_numa_nodes"
+  } else {
+    $gb_hugepages = ''
+  }
+
+  $grub_updates = strip("${eptad} ${$gb_hugepages} ${m_hugepages} ${default_pgsz} ${cpu_options}")
+}
+
+class platform::compute::grub::update
+  inherits ::platform::compute::grub::params {
+
+  notice("Updating grub configuration")
+
+  $to_be_removed = join($keys, " ")
+  exec { "Remove the cpu arguments":
+    command => "grubby --update-kernel=ALL --remove-args='$to_be_removed'",
+  } ->
+  exec { "Add the cpu arguments":
+    command => "grubby --update-kernel=ALL --args='$grub_updates'",
+  }
+}
+
+class platform::compute::grub::recovery {
+
+  notice("Update Grub and Reboot")
+
+  class {'platform::compute::grub::update': } -> Exec['reboot-recovery']
+
+  exec { "reboot-recovery":
+    command => "reboot",
+  }
+}
+
+class platform::compute::grub::audit
+  inherits ::platform::compute::grub::params {
+
+  if ! str2bool($::is_initial_config_primary) {
+
+    notice("Audit CPU and Grub Configuration")
+
+    $expected_n_cpus = $::number_of_logical_cpus
+    $n_cpus_ok = ("$n_cpus" == "$expected_n_cpus")
+
+    $cmd_ok = check_grub_config($grub_updates)
+
+    if $cmd_ok and $n_cpus_ok {
+      $ensure = present
+      notice("CPU and Boot Argument audit passed.")
+    } else {
+      $ensure = absent
+      if !$cmd_ok {
+        notice("Kernel Boot Argument Mismatch")
+        include ::platform::compute::grub::recovery
+      }
+    }
+
+    file { "/var/run/compute_huge_goenabled":
+      ensure  => $ensure,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+    }
+  }
+}
+
+class platform::compute::grub::runtime {
+  include ::platform::compute::grub::update
+}
+
+# Mounts virtual hugetlbfs filesystems for each supported page size
+class platform::compute::hugetlbf {
+
+  if str2bool($::is_hugetlbfs_enabled) {
+
+    $fs_list = generate("/bin/bash", "-c", "ls -1d /sys/kernel/mm/hugepages/hugepages-*")
+    $array = split($fs_list, '\n')
+    $array.each | String $val | {
+      $page_name = generate("/bin/bash", "-c", "basename $val")
+      $page_size = strip(regsubst($page_name, 'hugepages-', ''))
+      $hugemnt ="/mnt/huge-$page_size"
+      $options = "pagesize=${page_size}"
+
+      notice("Mounting hugetlbfs at: $hugemnt")
+      exec { "create $hugemnt":
+        command => "mkdir -p ${hugemnt}",
+        onlyif  => "test ! -d ${hugemnt}",
+      } ->
+      mount { "${hugemnt}":
+        name     => "${hugemnt}",
+        device   => 'none',
+        fstype   => 'hugetlbfs',
+        ensure   => 'mounted',
+        options  => "${options}",
+        atboot   => 'yes',
+        remounts => true,
+      }
+    }
+  }
+}
+
+class platform::compute::hugepage::params (
+  $nr_hugepages_2M = undef,
+  $nr_hugepages_1G = undef,
+  $vswitch_2M_pages = '',
+  $vswitch_1G_pages = '',
+  $vm_4K_pages = '',
+  $vm_2M_pages = '',
+  $vm_1G_pages = '',
+) {}
+
+
+define allocate_pages (
+  $path,
+  $page_count,
+) {
+  exec { "Allocate ${page_count} ${path}":
+    command => "echo $page_count > $path",
+    onlyif => "test -f $path",
+  }
+}
+
+# Allocates HugeTLB memory according to the attributes specified in the
+# nr_hugepages_2M and nr_hugepages_1G
+class platform::compute::allocate
+  inherits ::platform::compute::hugepage::params {
+
+  # determine the node file system
+  if str2bool($::is_per_numa_supported) {
+    $nodefs = '/sys/devices/system/node'
+  } else {
+    $nodefs = '/sys/kernel/mm'
+  }
+
+  if $nr_hugepages_2M != undef {
+    $nr_hugepages_2M_array = regsubst($nr_hugepages_2M, '[\(\)\"]', '', 'G').split(' ')
+    $nr_hugepages_2M_array.each | String $val | {
+      $per_node_2M = $val.split(':')
+      if size($per_node_2M)== 3 {
+        $node = $per_node_2M[0]
+        $page_size = $per_node_2M[1]
+        allocate_pages { "Start ${node} ${page_size}":
+          path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
+          page_count => $per_node_2M[2],
+        }
+      }
+    }
+  }
+
+  if $nr_hugepages_1G  != undef {
+    $nr_hugepages_1G_array = regsubst($nr_hugepages_1G , '[\(\)\"]', '', 'G').split(' ')
+    $nr_hugepages_1G_array.each | String $val | {
+      $per_node_1G = $val.split(':')
+      if size($per_node_1G)== 3 {
+        $node = $per_node_1G[0]
+        $page_size = $per_node_1G[1]
+        allocate_pages { "Start ${node} ${page_size}":
+          path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
+          page_count => $per_node_1G[2],
+        }
+      }
+    }
+  }
+}
+
+class platform::compute::extend
+  inherits ::platform::compute::hugepage::params {
+
+  # nova-compute reads on init, extended nova compute options
+  # used with nova accounting
+  file { "/etc/nova/compute_extend.conf":
+      ensure => 'present',
+      replace => true,
+      content => template('platform/compute_extend.conf.erb')
+  }
+}
+
+# Mount resctrl to allow Cache Allocation Technology per VM
+class platform::compute::resctrl {
+
+  if str2bool($::is_resctrl_supported) {
+    mount { "/sys/fs/resctrl":
+      name     => '/sys/fs/resctrl',
+      device   => 'resctrl',
+      fstype   => 'resctrl',
+      ensure   => 'mounted',
+      atboot   => 'yes',
+      remounts => true,
+    }
+  }
+}
+
+# Set Power Management QoS resume latency constraints for CPUs.
+# The PM QoS resume latency limit is set to shallow C-state for vswitch CPUs.
+# All other CPUs are allowed to go to the deepest C-state available.
+class platform::compute::pmqos (
+  $low_wakeup_cpus = '',
+  $hight_wakeup_cpus = '',
+) {
+
+  if str2bool($::is_compute_subfunction) and str2bool($::is_lowlatency_subfunction) {
+
+    $script = "/usr/bin/set-cpu-wakeup-latency.sh"
+
+    # Set low wakeup latency (shallow C-state) for vswitch CPUs using PM QoS interface
+    exec { "low-wakeup-latency":
+      command => "${script} low ${low_wakeup_cpus}",
+      onlyif => "test -f ${script}",
+      logoutput => true,
+    }
+
+    #Set high wakeup latency (deep C-state) for non-vswitch CPUs using PM QoS interface
+    exec { "high-wakeup-latency":
+      command => "${script} high ${hight_wakeup_cpus}",
+      onlyif => "test -f ${script}",
+      logoutput => true,
+    }
+  }
+}
+
+class platform::compute {
+
+  Class[$name] -> Class['::platform::vswitch']
+  Class[$name] -> Class['::nova::compute']
+
+  require ::platform::compute::grub::audit
+  require ::platform::compute::hugetlbf
+  require ::platform::compute::allocate
+  require ::platform::compute::pmqos
+  require ::platform::compute::resctrl
+  require ::platform::compute::extend
+}

puppet-manifests/src/modules/platform/templates/compute_extend.conf.erb

@@ -0,0 +1,12 @@
+###########################################################################
+#
+# compute_extend.conf contains compute extended nova options
+#
+#   - This file is managed by Puppet. DO NOT EDIT.
+#
+###########################################################################
+compute_vswitch_2M_pages=<%= @vswitch_2M_pages.gsub!(/\A"|"\Z/, '') %>
+compute_vswitch_1G_pages=<%= @vswitch_1G_pages.gsub!(/\A"|"\Z/, '') %>
+compute_vm_4K_pages=<%= @vm_4K_pages.gsub!(/\A"|"\Z/, '') %>
+compute_vm_2M_pages=<%= @vm_2M_pages.gsub!(/\A"|"\Z/, '') %>
+compute_vm_1G_pages=<%= @vm_1G_pages.gsub!(/\A"|"\Z/, '') %>