From 499ac5899c6357fd2b48cc78187539d29e85fb6d Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Wed, 19 Aug 2020 17:54:49 -0400 Subject: [PATCH] Enable StarlingX in QEMU/KVM VM To support StarlingX running in Openstack Nova VM (QEMU/KVM environment), there is a new user parameter added in ansible playbook to indicate if the system is running in an Openstack VM. A corresponding service parameter is created in service parameter and to be used to generate a puppet hieradata "virtual_system". A couple of changes required for "virtual system" only: - disable kubelet cpu manager to use full set of cpu and memory resources - set registry.central to system controller's mgmt IP on subcloud to be able to pull images from central cloud registry in DC mode - explicitly set registry.local to subcloud's mgmt IP on subcloud as dcorch synchronizes the dns server with system controller's dns server which cannot be reached on subcloud Change-Id: I3b21b76cc58a71319a32d2ca2e11d35306ba2292 Story: 2007858 Task: 40720 Depends-On: https://review.opendev.org/747478 Signed-off-by: Angie Wang --- .../src/modules/platform/manifests/dns.pp | 2 + .../modules/platform/manifests/kubernetes.pp | 45 +++++++++++-------- .../src/modules/platform/manifests/params.pp | 2 + .../platform/templates/dnsmasq.conf.erb | 18 ++++++++ 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/puppet-manifests/src/modules/platform/manifests/dns.pp b/puppet-manifests/src/modules/platform/manifests/dns.pp index 8c8b8d552..beca86f10 100644 --- a/puppet-manifests/src/modules/platform/manifests/dns.pp +++ b/puppet-manifests/src/modules/platform/manifests/dns.pp @@ -40,6 +40,8 @@ class platform::dns::dnsmasq { $dns_service_ip = $::platform::kubernetes::params::dns_service_ip $distributed_cloud_role = $::platform::params::distributed_cloud_role $sc_address = $::platform::params::system_controller_addr + $sc_mgmt_address = $::platform::params::system_controller_mgmt_addr + $is_virtual_system = $::platform::params::virtual_system file { '/etc/dnsmasq.conf': ensure => 'present', diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 57e2c8bd6..bae6078e1 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -129,28 +129,37 @@ class platform::kubernetes::kubeadm { $k8s_cpu_manager_opts = join([$opts, $opts_sys_res], ' ') } else { - if str2bool($::is_worker_subfunction) - and !('openstack-compute-node' in $host_labels) { - $opts = join(['--feature-gates TopologyManager=true', - "--cpu-manager-policy=${k8s_cpu_mgr_policy}", - "--topology-manager-policy=${k8s_topology_mgr_policy}"], ' ') + if !$::platform::params::virtual_system { + if str2bool($::is_worker_subfunction) + and !('openstack-compute-node' in $host_labels) { + # Enable TopologyManager for hosts with the worker subfunction. + # Exceptions are: + # - DC System controllers + # - Virtualized nodes (lab environment only) - if $k8s_cpu_mgr_policy == 'none' { - $k8s_reserved_cpus = $k8s_platform_cpuset + $opts = join(['--feature-gates TopologyManager=true', + "--cpu-manager-policy=${k8s_cpu_mgr_policy}", + "--topology-manager-policy=${k8s_topology_mgr_policy}"], ' ') + + if $k8s_cpu_mgr_policy == 'none' { + $k8s_reserved_cpus = $k8s_platform_cpuset + } else { + # The union of platform, isolated, and vswitch + $k8s_reserved_cpus = $k8s_all_reserved_cpuset + } + + $opts_res_cpus = "--reserved-cpus=${k8s_reserved_cpus}" + $k8s_cpu_manager_opts = join([$opts, + $opts_sys_res, + $opts_res_cpus], ' ') } else { - # The union of platform, isolated, and vswitch - $k8s_reserved_cpus = $k8s_all_reserved_cpuset + $opts = '--cpu-manager-policy=none' + $k8s_cpu_manager_opts = join([$opts, + $opts_sys_res], ' ') + } - - $opts_res_cpus = "--reserved-cpus=${k8s_reserved_cpus}" - $k8s_cpu_manager_opts = join([$opts, - $opts_sys_res, - $opts_res_cpus], ' ') } else { - $opts = '--cpu-manager-policy=none' - $k8s_cpu_manager_opts = join([$opts, - $opts_sys_res], ' ') - + $k8s_cpu_manager_opts = '--cpu-manager-policy=none' } } diff --git a/puppet-manifests/src/modules/platform/manifests/params.pp b/puppet-manifests/src/modules/platform/manifests/params.pp index d36aed927..6b0c2417c 100644 --- a/puppet-manifests/src/modules/platform/manifests/params.pp +++ b/puppet-manifests/src/modules/platform/manifests/params.pp @@ -25,6 +25,8 @@ class platform::params ( $security_feature = undef, $stx_openstack_applied = false, $system_controller_addr = undef, + $system_controller_mgmt_addr = undef, + $virtual_system = false, ) { $ipv4 = 4 $ipv6 = 6 diff --git a/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb b/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb index bc2e6b930..7a1a7a83d 100644 --- a/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb @@ -124,5 +124,23 @@ cname=registry.central,controller <%- end -%> <%- if @distributed_cloud_role == 'subcloud' -%> +<%- if @is_virtual_system == true and @sc_mgmt_address != nil -%> +# For virtual subcloud (StarlingX running in Openstack Nova VM - QEMU/KVM), +# there is no physical OAM interface (no external network access) to connect +# to central-cloud's local registry, so central registry is exposed on the +# MGMT interface and "registry.central" domain needs to be set to system +# controller's MGMT IP to allow subcloud to pull images from central registry +# via the MGMT interface. +address=/registry.central/<%= @sc_mgmt_address %> +# The "registry.local" domain needs to be explicitly specified with subcloud's +# MGMT IP as dcorch synchronizes the dns server on subcloud with system controller's +# dns server which cannot be reached on subcloud. +address=/registry.local/<%= @mgmt_controller_address %> +<%- else -%> +# For regular DC, central-cloud's local registry is exposed on the OAM interface +# (to provide the ability to push images externally to central registry), so set +# "registry.central" to system controller's OAM IP on subcloud to allow subcloud +# to pull images from central registry via the OAM interface. address=/registry.central/<%= @sc_address %> <%- end -%> +<%- end -%>