class platform::compute::params (
  $compute_cpu_list = '',
  $platform_cpu_list = '',
  $reserved_vswitch_cores = '',
  $reserved_platform_cores = '',
  $compute_base_reserved = '',
  $compute_vswitch_reserved = '',
) { }

class platform::compute::config
  inherits ::platform::compute::params {

  file { "/etc/nova/compute_reserved.conf":
      ensure => 'present',
      replace => true,
      content => template('platform/compute_reserved.conf.erb')
  }
}

class platform::compute::grub::params (
  $n_cpus = '',
  $cpu_options = '',
  $m_hugepages = 'hugepagesz=2M hugepages=0',
  $default_pgsz = 'default_hugepagesz=2M',
  $keys = ['kvm-intel.eptad', 'default_hugepagesz', 'hugepagesz', 'hugepages', 'isolcpus', 'nohz_full', 'rcu_nocbs', 'kthread_cpus', 'irqaffinity'],
) {

  if $::is_broadwell_processor {
    $eptad = 'kvm-intel.eptad=0'
  } else {
    $eptad = ''
  }

  if $::is_gb_page_supported {
    $gb_hugepages = "hugepagesz=1G hugepages=$::number_of_numa_nodes"
  } else {
    $gb_hugepages = ''
  }

  $grub_updates = strip("${eptad} ${$gb_hugepages} ${m_hugepages} ${default_pgsz} ${cpu_options}")
}

class platform::compute::grub::update
  inherits ::platform::compute::grub::params {

  notice("Updating grub configuration")

  $to_be_removed = join($keys, " ")
  exec { "Remove the cpu arguments":
    command => "grubby --update-kernel=ALL --remove-args='$to_be_removed'",
  } ->
  exec { "Add the cpu arguments":
    command => "grubby --update-kernel=ALL --args='$grub_updates'",
  }
}

class platform::compute::grub::recovery {

  notice("Update Grub and Reboot")

  class {'platform::compute::grub::update': } -> Exec['reboot-recovery']

  exec { "reboot-recovery":
    command => "reboot",
  }
}

class platform::compute::grub::audit
  inherits ::platform::compute::grub::params {

  if ! str2bool($::is_initial_config_primary) {

    notice("Audit CPU and Grub Configuration")

    $expected_n_cpus = $::number_of_logical_cpus
    $n_cpus_ok = ("$n_cpus" == "$expected_n_cpus")

    $cmd_ok = check_grub_config($grub_updates)

    if $cmd_ok and $n_cpus_ok {
      $ensure = present
      notice("CPU and Boot Argument audit passed.")
    } else {
      $ensure = absent
      if !$cmd_ok {
        notice("Kernel Boot Argument Mismatch")
        include ::platform::compute::grub::recovery
      }
    }

    file { "/var/run/compute_huge_goenabled":
      ensure  => $ensure,
      owner   => 'root',
      group   => 'root',
      mode    => '0644',
    }
  }
}

class platform::compute::grub::runtime {
  include ::platform::compute::grub::update
}

# Mounts virtual hugetlbfs filesystems for each supported page size
class platform::compute::hugetlbf {

  if str2bool($::is_hugetlbfs_enabled) {

    $fs_list = generate("/bin/bash", "-c", "ls -1d /sys/kernel/mm/hugepages/hugepages-*")
    $array = split($fs_list, '\n')
    $array.each | String $val | {
      $page_name = generate("/bin/bash", "-c", "basename $val")
      $page_size = strip(regsubst($page_name, 'hugepages-', ''))
      $hugemnt ="/mnt/huge-$page_size"
      $options = "pagesize=${page_size}"

      # TODO: Once all the code is switched over to use the /dev
      # mount point  we can get rid of this mount point.
      notice("Mounting hugetlbfs at: $hugemnt")
      exec { "create $hugemnt":
        command => "mkdir -p ${hugemnt}",
        onlyif  => "test ! -d ${hugemnt}",
      } ->
      mount { "${hugemnt}":
        name     => "${hugemnt}",
        device   => 'none',
        fstype   => 'hugetlbfs',
        ensure   => 'mounted',
        options  => "${options}",
        atboot   => 'yes',
        remounts => true,
      }

      # The libvirt helm chart expects hugepages to be mounted
      # under /dev so let's do that.
      $hugemnt2 ="/dev/huge-$page_size"
      notice("Mounting hugetlbfs at: $hugemnt2")
      file { "${hugemnt2}":
        ensure => 'directory',
        owner  => 'root',
        group  => 'root',
        mode   => '0755',
      }->
      mount { "${hugemnt2}":
        name     => "${hugemnt2}",
        device   => 'none',
        fstype   => 'hugetlbfs',
        ensure   => 'mounted',
        options  => "${options}",
        atboot   => 'yes',
        remounts => true,
      }
    }

    # The libvirt helm chart also assumes that the default hugepage size
    # will be mounted at /dev/hugepages so let's make that happen too.
    # Once we upstream a fix to the helm chart to automatically determine
    # the mountpoint then we can remove this.
    $page_size = '2M'
    $hugemnt ="/dev/hugepages"
    $options = "pagesize=${page_size}"

    notice("Mounting hugetlbfs at: $hugemnt")
    exec { "create $hugemnt":
      command => "mkdir -p ${hugemnt}",
      onlyif  => "test ! -d ${hugemnt}",
    } ->
    mount { "${hugemnt}":
      name     => "${hugemnt}",
      device   => 'none',
      fstype   => 'hugetlbfs',
      ensure   => 'mounted',
      options  => "${options}",
      atboot   => 'yes',
      remounts => true,
    }
  }
}

class platform::compute::hugepage::params (
  $nr_hugepages_2M = undef,
  $nr_hugepages_1G = undef,
  $vswitch_2M_pages = '',
  $vswitch_1G_pages = '',
  $vm_4K_pages = '',
  $vm_2M_pages = '',
  $vm_1G_pages = '',
) {}


define allocate_pages (
  $path,
  $page_count,
) {
  exec { "Allocate ${page_count} ${path}":
    command => "echo $page_count > $path",
    onlyif => "test -f $path",
  }
}

# Allocates HugeTLB memory according to the attributes specified in the
# nr_hugepages_2M and nr_hugepages_1G
class platform::compute::allocate
  inherits ::platform::compute::hugepage::params {

  # determine the node file system
  if str2bool($::is_per_numa_supported) {
    $nodefs = '/sys/devices/system/node'
  } else {
    $nodefs = '/sys/kernel/mm'
  }

  if $nr_hugepages_2M != undef {
    $nr_hugepages_2M_array = regsubst($nr_hugepages_2M, '[\(\)\"]', '', 'G').split(' ')
    $nr_hugepages_2M_array.each | String $val | {
      $per_node_2M = $val.split(':')
      if size($per_node_2M)== 3 {
        $node = $per_node_2M[0]
        $page_size = $per_node_2M[1]
        allocate_pages { "Start ${node} ${page_size}":
          path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
          page_count => $per_node_2M[2],
        }
      }
    }
  }

  if $nr_hugepages_1G  != undef {
    $nr_hugepages_1G_array = regsubst($nr_hugepages_1G , '[\(\)\"]', '', 'G').split(' ')
    $nr_hugepages_1G_array.each | String $val | {
      $per_node_1G = $val.split(':')
      if size($per_node_1G)== 3 {
        $node = $per_node_1G[0]
        $page_size = $per_node_1G[1]
        allocate_pages { "Start ${node} ${page_size}":
          path => "${nodefs}/${node}/hugepages/hugepages-${page_size}/nr_hugepages",
          page_count => $per_node_1G[2],
        }
      }
    }
  }
}

class platform::compute::extend
  inherits ::platform::compute::hugepage::params {

  # nova-compute reads on init, extended nova compute options
  # used with nova accounting
  file { "/etc/nova/compute_extend.conf":
      ensure => 'present',
      replace => true,
      content => template('platform/compute_extend.conf.erb')
  }
}

# Mount resctrl to allow Cache Allocation Technology per VM
class platform::compute::resctrl {

  if str2bool($::is_resctrl_supported) {
    mount { "/sys/fs/resctrl":
      name     => '/sys/fs/resctrl',
      device   => 'resctrl',
      fstype   => 'resctrl',
      ensure   => 'mounted',
      atboot   => 'yes',
      remounts => true,
    }
  }
}

# Set Power Management QoS resume latency constraints for CPUs.
# The PM QoS resume latency limit is set to shallow C-state for vswitch CPUs.
# All other CPUs are allowed to go to the deepest C-state available.
class platform::compute::pmqos (
  $low_wakeup_cpus = '',
  $hight_wakeup_cpus = '',
) {

  if str2bool($::is_compute_subfunction) and str2bool($::is_lowlatency_subfunction) {

    $script = "/usr/bin/set-cpu-wakeup-latency.sh"

    # Set low wakeup latency (shallow C-state) for vswitch CPUs using PM QoS interface
    exec { "low-wakeup-latency":
      command => "${script} low ${low_wakeup_cpus}",
      onlyif => "test -f ${script}",
      logoutput => true,
    }

    #Set high wakeup latency (deep C-state) for non-vswitch CPUs using PM QoS interface
    exec { "high-wakeup-latency":
      command => "${script} high ${hight_wakeup_cpus}",
      onlyif => "test -f ${script}",
      logoutput => true,
    }
  }
}

class platform::compute {

  Class[$name] -> Class['::platform::vswitch']
  Class[$name] -> Class['::nova::compute']

  require ::platform::compute::grub::audit
  require ::platform::compute::hugetlbf
  require ::platform::compute::allocate
  require ::platform::compute::pmqos
  require ::platform::compute::resctrl
  require ::platform::compute::extend
  require ::platform::compute::config
}