starlingx/config
Code Issues Proposed changes

Allow configuration of isolcpus

Browse Source 
Allow a mixture of floating and pinned workloads by allowing the
application cpu's to be split between isolated and non-isolated

CPUs specified as isolated will be added to the isolcpu set in the grub
config. Lowlatency worker nodes will no longer have all application CPUs
in the isolcpu set.

Kubernetes reserves CPUs in a block, not by numa node or range of CPUs.
Due to this we must restrict the Isolated CPUs to a contiguous block
starting on numa 0 (after the platform cores).

Additional work will be required to support isolcpus with vswitch or the
openstack application.

Change-Id: Ia849967e086bb1dadf27020e400a223c4c7cae5f
Story: 2006565
Task: 36664
Signed-off-by: David Sullivan <david.sullivan@windriver.com>
This commit is contained in:
David Sullivan
2019-09-17 12:29:22 -04:00
parent fdb2159953
commit 37ed87223d
8 changed files with 105 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu.py
View File

@@ -21,10 +21,11 @@ PLATFORM_CPU_TYPE = "Platform"
VSWITCH_CPU_TYPE = "Vswitch"
SHARED_CPU_TYPE = "Shared"
APPLICATION_CPU_TYPE = "Applications"
ISOLATED_CPU_TYPE = "Isolated"
NONE_CPU_TYPE = "None"
CPU_TYPE_LIST = [PLATFORM_CPU_TYPE, VSWITCH_CPU_TYPE,
SHARED_CPU_TYPE, APPLICATION_CPU_TYPE,
SHARED_CPU_TYPE, APPLICATION_CPU_TYPE, ISOLATED_CPU_TYPE,
NONE_CPU_TYPE]
@@ -32,12 +33,14 @@ PLATFORM_CPU_TYPE_FORMAT = _("Platform")
VSWITCH_CPU_TYPE_FORMAT = _("vSwitch")
SHARED_CPU_TYPE_FORMAT = _("Shared")
APPLICATION_CPU_TYPE_FORMAT = _("Applications")
ISOLATED_CPU_TYPE_FORMAT = _("Isolated")
NONE_CPU_TYPE_FORMAT = _("None")
CPU_TYPE_FORMATS = {PLATFORM_CPU_TYPE: PLATFORM_CPU_TYPE_FORMAT,
VSWITCH_CPU_TYPE: VSWITCH_CPU_TYPE_FORMAT,
SHARED_CPU_TYPE: SHARED_CPU_TYPE_FORMAT,
APPLICATION_CPU_TYPE: APPLICATION_CPU_TYPE_FORMAT,
ISOLATED_CPU_TYPE: ISOLATED_CPU_TYPE_FORMAT,
NONE_CPU_TYPE: NONE_CPU_TYPE_FORMAT}

3
sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu_shell.py
View File

@@ -88,7 +88,7 @@ def do_host_cpu_list(cc, args):
help="Name or ID of host")
@utils.arg('-f', '--function',
metavar='<function>',
choices=['vswitch', 'shared', 'platform'],
choices=['vswitch', 'shared', 'platform', 'isolated'],
required=True,
help='The Core Function.')
@utils.arg('-p0', '--num_cores_on_processor0',
@@ -109,7 +109,6 @@ def do_host_cpu_list(cc, args):
help='Number of cores on Processor 3.')
def do_host_cpu_modify(cc, args):
"""Modify cpu core assignments."""
function_list = ['platform', 'vswitch', 'shared']
field_list = ['function', 'allocated_function',
'num_cores_on_processor0', 'num_cores_on_processor1',
'num_cores_on_processor2', 'num_cores_on_processor3']

31
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/cpu.py
View File

@@ -583,6 +583,35 @@ def _update_platform_cpu_counts(host, cpu, counts, capabilities=None):
return counts
def _update_isolated_cpu_counts(host, cpu, counts, capabilities=None):
"""Update the isolated cpu counts based on the requested number of cores
per processor.
"""
for s in range(0, len(host.nodes)):
if capabilities:
count = capabilities.get('num_cores_on_processor%d' % s, None)
else:
count = getattr(cpu, 'num_cores_on_processor%d' % s, None)
if count is None:
continue
count = int(count)
if count < 0:
raise wsme.exc.ClientSideError(
_('Isolated cpus must be non-negative.'))
if host.hyperthreading:
# the data structures track the number of logical cpus and the
# API expects the requested count to refer to the number
# of physical cores requested therefore if HT is enabled then
# multiply the requested number by 2 so that we always reserve a
# full physical core
count *= 2
counts[s][constants.ISOLATED_FUNCTION] = count
# let the remaining values grow/shrink dynamically
counts[s][constants.APPLICATION_FUNCTION] = 0
counts[s][constants.NO_FUNCTION] = 0
return counts
def _check_cpu(cpu, ihost):
if cpu.function:
func = cpu_utils.lookup_function(cpu.function)
@@ -615,6 +644,8 @@ def _check_cpu(cpu, ihost):
cpu_counts = _update_shared_cpu_counts(ihost, cpu, cpu_counts)
if (func.lower() == constants.PLATFORM_FUNCTION.lower()):
cpu_counts = _update_platform_cpu_counts(ihost, cpu, cpu_counts)
if (func.lower() == constants.ISOLATED_FUNCTION.lower()):
cpu_counts = _update_isolated_cpu_counts(ihost, cpu, cpu_counts)
# Semantic check to ensure the minimum/maximum values are enforced
error_string = cpu_utils.check_core_allocations(ihost, cpu_counts, func)

40
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/cpu_utils.py
View File

@@ -16,6 +16,7 @@ CORE_FUNCTIONS = [
constants.VSWITCH_FUNCTION,
constants.SHARED_FUNCTION,
constants.APPLICATION_FUNCTION,
constants.ISOLATED_FUNCTION,
constants.NO_FUNCTION
]
@@ -251,18 +252,22 @@ def check_core_allocations(host, cpu_counts, func):
total_platform_cores = 0
total_vswitch_cores = 0
total_shared_cores = 0
total_isolated_cores = 0
for s in range(0, len(host.nodes)):
available_cores = len(host.cpu_lists[s])
platform_cores = cpu_counts[s][constants.PLATFORM_FUNCTION]
vswitch_cores = cpu_counts[s][constants.VSWITCH_FUNCTION]
shared_cores = cpu_counts[s][constants.SHARED_FUNCTION]
requested_cores = platform_cores + vswitch_cores + shared_cores
isolated_cores = cpu_counts[s][constants.ISOLATED_FUNCTION]
requested_cores = \
platform_cores + vswitch_cores + shared_cores + isolated_cores
if requested_cores > available_cores:
return ("More total logical cores requested than present on "
"'Processor %s' (%s cores)." % (s, available_cores))
total_platform_cores += platform_cores
total_vswitch_cores += vswitch_cores
total_shared_cores += shared_cores
total_isolated_cores += isolated_cores
if func.lower() == constants.PLATFORM_FUNCTION.lower():
if ((constants.CONTROLLER in host.subfunctions) and
(constants.WORKER in host.subfunctions)):
@@ -272,6 +277,10 @@ def check_core_allocations(host, cpu_counts, func):
elif total_platform_cores == 0:
return "%s must have at least one core." % \
constants.PLATFORM_FUNCTION
for s in range(0, len(host.nodes)):
if s > 0 and cpu_counts[s][constants.PLATFORM_FUNCTION] > 0:
return "%s cores can only be allocated on Processor 0" % \
constants.PLATFORM_FUNCTION
if constants.WORKER in (host.subfunctions or host.personality):
if func.lower() == constants.VSWITCH_FUNCTION.lower():
if host.hyperthreading:
@@ -284,6 +293,32 @@ def check_core_allocations(host, cpu_counts, func):
elif total_physical_cores > VSWITCH_MAX_CORES:
return ("The %s function can only be assigned up to %s cores." %
(constants.VSWITCH_FUNCTION.lower(), VSWITCH_MAX_CORES))
# Validate Isolated cores
# We can allocate platform cores on numa 0, otherwise all isolated
# cores must in a contiguous block after the platform cores.
if total_isolated_cores > 0:
if total_vswitch_cores != 0 or total_shared_cores != 0:
return "%s cores can only be configured with %s and %s core " \
"types." % (constants.ISOLATED_FUNCTION,
constants.PLATFORM_FUNCTION,
constants.APPLICATION_FUNCTION)
has_application_cpus = False
for s in range(0, len(host.nodes)):
numa_counts = cpu_counts[s]
isolated_cores_requested = \
numa_counts[constants.ISOLATED_FUNCTION]
if has_application_cpus and isolated_cores_requested:
return "%s cpus must be contiguous" % \
constants.ISOLATED_FUNCTION
platform_cores_requested = \
numa_counts[constants.PLATFORM_FUNCTION]
available_cores = len(host.cpu_lists[s])
if platform_cores_requested + isolated_cores_requested \
!= available_cores:
has_application_cpus = True
reserved_for_vms = len(host.cpus) - total_platform_cores - total_vswitch_cores
if reserved_for_vms <= 0:
return "There must be at least one unused core for %s." % \
@@ -316,6 +351,9 @@ def update_core_allocations(host, cpu_counts):
for i in range(0, cpu_counts[s][constants.SHARED_FUNCTION]):
host.cpu_functions[s][constants.SHARED_FUNCTION].append(
cpu_list.pop(0))
for i in range(0, cpu_counts[s][constants.ISOLATED_FUNCTION]):
host.cpu_functions[s][constants.ISOLATED_FUNCTION].append(
cpu_list.pop(0))
# Assign the remaining cpus to the default function for this host
host.cpu_functions[s][get_default_function(host)] += cpu_list
return

10
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py
View File

@@ -236,10 +236,12 @@ class HostStatesController(rest.RestController):
rank = 1
elif function.lower() == constants.VSWITCH_FUNCTION.lower():
rank = 2
elif function.lower() == constants.APPLICATION_FUNCTION.lower():
elif function.lower() == constants.ISOLATED_FUNCTION.lower():
rank = 3
else:
elif function.lower() == constants.APPLICATION_FUNCTION.lower():
rank = 4
else:
rank = 5
return rank
specified_function = None
@@ -302,6 +304,10 @@ class HostStatesController(rest.RestController):
cpu_counts = cpu_api._update_platform_cpu_counts(ihost, None,
cpu_counts,
capability)
elif (specified_function.lower() ==
constants.ISOLATED_FUNCTION.lower()):
cpu_counts = cpu_api._update_isolated_cpu_counts(
ihost, None, cpu_counts, capability)
# Semantic check to ensure the minimum/maximum values are enforced
error_msg = cpu_utils.check_core_allocations(ihost, cpu_counts,

1
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@@ -115,6 +115,7 @@ PLATFORM_FUNCTION = "Platform"
VSWITCH_FUNCTION = "Vswitch"
SHARED_FUNCTION = "Shared"
APPLICATION_FUNCTION = "Applications"
ISOLATED_FUNCTION = "Isolated"
NO_FUNCTION = "None"
# Host Personality Sub-Types

13
sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py
View File

@@ -182,9 +182,20 @@ class KubernetesPuppet(base.BasePuppet):
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_nodeset = set([c.numa_node for c in platform_cpus])
vswitch_cpus = self._get_host_cpu_list(
host, function=constants.VSWITCH_FUNCTION, threads=True)
vswitch_cpuset = set([c.cpu for c in vswitch_cpus])
# determine set of isolcpus logical cpus and nodes
isol_cpus = self._get_host_cpu_list(
host, function=constants.ISOLATED_FUNCTION, threads=True)
isol_cpuset = set([c.cpu for c in isol_cpus])
# determine platform reserved number of logical cpus
k8s_reserved_cpus = len(platform_cpuset)
k8s_isol_cpus = len(vswitch_cpuset) + len(isol_cpuset)
# determine platform reserved memory
k8s_reserved_mem = 0
host_memory = self.dbapi.imemory_get_by_ihost(host.id)
@@ -229,6 +240,8 @@ class KubernetesPuppet(base.BasePuppet):
"\"%s\"" % k8s_nodeset,
'platform::kubernetes::params::k8s_reserved_cpus':
k8s_reserved_cpus,
'platform::kubernetes::params::k8s_isol_cpus':
k8s_isol_cpus,
'platform::kubernetes::params::k8s_reserved_mem':
k8s_reserved_mem,
})

12
sysinv/sysinv/sysinv/sysinv/puppet/platform.py
View File

@@ -546,9 +546,14 @@ class PlatformPuppet(base.BasePuppet):
non_vswitch_cpuset = host_cpuset - vswitch_cpuset
non_vswitch_ranges = utils.format_range_set(non_vswitch_cpuset)
# isolated logical cpus
app_isolated_cpus = self._get_host_cpu_list(
host, constants.ISOLATED_FUNCTION, threads=True)
app_isolated_cpuset = set([c.cpu for c in app_isolated_cpus])
cpu_options = ""
cpu_ranges = {}
isolcpus_ranges = vswitch_ranges
if constants.LOWLATENCY in host.subfunctions:
config.update({
'platform::compute::pmqos::low_wakeup_cpus':
@@ -558,9 +563,8 @@ class PlatformPuppet(base.BasePuppet):
})
cpu_ranges.update({"nohz_full": rcu_nocbs_ranges})
host_labels = self.dbapi.label_get_by_host(host.id)
if utils.has_openstack_compute(host_labels):
isolcpus_ranges = rcu_nocbs_ranges
isolcpus_ranges = utils.format_range_set(
vswitch_cpuset.union(app_isolated_cpuset))
cpu_ranges.update({
"isolcpus": isolcpus_ranges,