starlingx/stx-puppet
Code Issues Proposed changes

Kubernetes Cluster Network Configuration

Browse Source 
The Kubernetes cluster network is introduced and configurable.
The cluster-host interface can be configured on any interface of the
host and is defaulted to the management interface if it is not
specified.
The infrastructure network is no longer used in kubernetes config.
SM and MTCE are setup to monitor the cluster-host if kubernetes is
enabled.
Nova live migration ip is set to use the cluster-host ip.

Tests Performed:
Containerized setup:
AIO-SX: mgmt and cluster-host shared loopback interface
AIO-DX: mgmt and cluster-host shared an interface
AIO-DX: mgmt and cluster-host on different interface
Standard 2+2+2: mgmt and cluster-host shared an interface
Standard 2+2+2: mgmt and cluster-host on different interface
For each of the setup, launch VM and connect to VM console

Non-containerized deployments
AIO-SX sanity
AIO-DX sanity
Standard 2+2 sanity

Story: 2004273
Task: 27826

Change-Id: If6b918665131f01bc62687fbdc7978c5c103e3b7
Signed-off-by: Teresa Ho <teresa.ho@windriver.com>
This commit is contained in:
Teresa Ho
2018-12-21 10:25:42 -05:00
parent f5143f41d6
commit f23637983a
7 changed files with 99 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
puppet-manifests/src/modules/platform/manifests/config.pp
View File

@@ -34,10 +34,17 @@ class platform::config::file {
include ::platform::network::mgmt::params
include ::platform::network::infra::params
include ::platform::network::oam::params
include ::platform::network::cluster_host::params
include ::platform::kubernetes::params
$kubernetes_enabled = $::platform::kubernetes::params::enabled
# dependent template variables
$management_interface = $::platform::network::mgmt::params::interface_name
$infrastructure_interface = $::platform::network::infra::params::interface_name
if $kubernetes_enabled {
$infrastructure_interface = $::platform::network::cluster_host::params::interface_name
} else {
$infrastructure_interface = $::platform::network::infra::params::interface_name
}
$oam_interface = $::platform::network::oam::params::interface_name
$platform_conf = '/etc/platform/platform.conf'

10
puppet-manifests/src/modules/platform/manifests/dns.pp
View File

@@ -63,6 +63,16 @@ class platform::dns::dnsmasq {
}
}
include ::platform::kubernetes::params
$kubernetes_enabled = $::platform::kubernetes::params::enabled
if $kubernetes_enabled {
$service_domain = $::platform::kubernetes::params::service_domain
$dns_service_ip = $::platform::kubernetes::params::dns_service_ip
} else {
$service_domain = undef
$dns_service_ip = undef
}
file { '/etc/dnsmasq.conf':
ensure => 'present',
replace => true,

15
puppet-manifests/src/modules/platform/manifests/kubernetes.pp
View File

@@ -1,8 +1,11 @@
class platform::kubernetes::params (
$enabled = false,
$pod_network_cidr = undef,
$service_network_cidr = undef,
$apiserver_advertise_address = undef,
$etcd_endpoint = undef,
$service_domain = undef,
$dns_service_ip = undef,
$ca_crt = undef,
$ca_key = undef,
$sa_key = undef,
@@ -45,18 +48,6 @@ class platform::kubernetes::kubeadm {
command => 'sysctl --system',
}
# TODO: Update /etc/resolv.conf.k8s to be controlled by sysinv, as is done
# for /etc/resolv.conf. Is should contain all the user-specified DNS
# servers, but not the coredns IP.
# Create custom resolv.conf file for kubelet
-> file { '/etc/resolv.conf.k8s':
ensure => file,
content => 'nameserver 8.8.8.8',
owner => 'root',
group => 'root',
mode => '0644',
}
# Start kubelet.
-> service { 'kubelet':
ensure => 'running',

19
puppet-manifests/src/modules/platform/manifests/network.pp
View File

@@ -81,6 +81,24 @@ class platform::network::oam::params(
$mtu = 1500,
) { }
class platform::network::cluster_host::params(
# shared parametes with base class - required for auto hiera parameter lookup
$interface_name = undef,
$interface_address = undef,
$subnet_version = undef,
$subnet_network = undef,
$subnet_network_url = undef,
$subnet_prefixlen = undef,
$subnet_netmask = undef,
$subnet_start = undef,
$subnet_end = undef,
$gateway_address = undef,
$controller_address = undef, # controller floating
$controller_address_url = undef, # controller floating url address
$controller0_address = undef, # controller unit0
$controller1_address = undef, # controller unit1
$mtu = 1500,
) { }
define network_address (
$address,
@@ -148,6 +166,7 @@ class platform::network (
include ::platform::params
include ::platform::network::mgmt::params
include ::platform::network::infra::params
include ::platform::network::cluster_host::params
include ::platform::network::apply

77
puppet-manifests/src/modules/platform/manifests/sm.pp Executable file → Normal file
View File

@@ -15,6 +15,9 @@ class platform::sm
$system_mode = $::platform::params::system_mode
$system_type = $::platform::params::system_type
include ::platform::kubernetes::params
$kubernetes_enabled = $::platform::kubernetes::params::enabled
include ::platform::network::pxeboot::params
if $::platform::network::pxeboot::params::interface_name {
$pxeboot_ip_interface = $::platform::network::pxeboot::params::interface_name
@@ -30,8 +33,13 @@ class platform::sm
$mgmt_ip_param_ip = $::platform::network::mgmt::params::controller_address
$mgmt_ip_param_mask = $::platform::network::mgmt::params::subnet_prefixlen
include ::platform::network::infra::params
$infra_ip_interface = $::platform::network::infra::params::interface_name
if $kubernetes_enabled {
# Repurposing the infra interface for cluster-host interface
include ::platform::network::cluster_host::params
$infra_ip_interface = $::platform::network::cluster_host::params::interface_name
} else {
$infra_ip_interface = $::platform::network::infra::params::interface_name
}
include ::platform::network::oam::params
$oam_ip_interface = $::platform::network::oam::params::interface_name
@@ -113,9 +121,6 @@ class platform::sm
$rabbitmq_server = '/usr/lib/rabbitmq/bin/rabbitmq-server'
$rabbitmqctl = '/usr/lib/rabbitmq/bin/rabbitmqctl'
include ::platform::kubernetes::params
$kubernetes_enabled = $::platform::kubernetes::params::enabled
include ::platform::mtce::params
$sm_client_port = $::platform::mtce::params::sm_client_port
$sm_server_port = $::platform::mtce::params::sm_server_port
@@ -129,14 +134,14 @@ class platform::sm
$platform_nfs_ip_network_url = $::platform::network::mgmt::params::subnet_network_url
# CGCS NFS network is over the infrastructure network if configured
if $infra_ip_interface {
$cgcs_nfs_ip_interface = $::platform::network::infra::params::interface_name
$cgcs_nfs_ip_param_ip = $::platform::network::infra::params::cgcs_nfs_address
$cgcs_nfs_ip_network_url = $::platform::network::infra::params::subnet_network_url
$cgcs_nfs_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen
if $infra_ip_interface and $kubernetes_enabled != true {
$cgcs_nfs_ip_interface = $::platform::network::infra::params::interface_name
$cgcs_nfs_ip_param_ip = $::platform::network::infra::params::cgcs_nfs_address
$cgcs_nfs_ip_network_url = $::platform::network::infra::params::subnet_network_url
$cgcs_nfs_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen
$cinder_ip_interface = $::platform::network::infra::params::interface_name
$cinder_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen
$cinder_ip_interface = $::platform::network::infra::params::interface_name
$cinder_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen
} else {
$cgcs_nfs_ip_interface = $::platform::network::mgmt::params::interface_name
$cgcs_nfs_ip_param_ip = $::platform::network::mgmt::params::cgcs_nfs_address
@@ -247,6 +252,10 @@ class platform::sm
$hostunit = '0'
$management_my_unit_ip = $::platform::network::mgmt::params::controller0_address
$oam_my_unit_ip = $::platform::network::oam::params::controller_address
if $kubernetes_enabled {
# Repurposing the infra interface for cluster-host interface
$infra_my_unit_ip = $::platform::network::cluster_host::params::controller_address
}
} else {
case $::hostname {
$controller_0_hostname: {
@@ -255,8 +264,14 @@ class platform::sm
$management_peer_unit_ip = $::platform::network::mgmt::params::controller1_address
$oam_my_unit_ip = $::platform::network::oam::params::controller0_address
$oam_peer_unit_ip = $::platform::network::oam::params::controller1_address
$infra_my_unit_ip = $::platform::network::infra::params::controller0_address
$infra_peer_unit_ip = $::platform::network::infra::params::controller1_address
if $kubernetes_enabled {
# Repurposing the infra interface for cluster-host interface
$infra_my_unit_ip = $::platform::network::cluster_host::params::controller0_address
$infra_peer_unit_ip = $::platform::network::cluster_host::params::controller1_address
} else {
$infra_my_unit_ip = $::platform::network::infra::params::controller0_address
$infra_peer_unit_ip = $::platform::network::infra::params::controller1_address
}
}
$controller_1_hostname: {
$hostunit = '1'
@@ -264,8 +279,14 @@ class platform::sm
$management_peer_unit_ip = $::platform::network::mgmt::params::controller0_address
$oam_my_unit_ip = $::platform::network::oam::params::controller1_address
$oam_peer_unit_ip = $::platform::network::oam::params::controller0_address
$infra_my_unit_ip = $::platform::network::infra::params::controller1_address
$infra_peer_unit_ip = $::platform::network::infra::params::controller0_address
if $kubernetes_enabled {
# Repurposing the infra interface for cluster-host interface
$infra_my_unit_ip = $::platform::network::cluster_host::params::controller1_address
$infra_peer_unit_ip = $::platform::network::cluster_host::params::controller0_address
} else {
$infra_my_unit_ip = $::platform::network::infra::params::controller1_address
$infra_peer_unit_ip = $::platform::network::infra::params::controller0_address
}
}
default: {
$hostunit = '2'
@@ -325,13 +346,25 @@ class platform::sm
exec { 'Configure Management Interface':
command => "sm-configure interface controller management-interface ${mgmt_ip_multicast} ${management_my_unit_ip} 2222 2223 \"\" 2222 2223",
}
} else {
exec { 'Configure OAM Interface':
command => "sm-configure interface controller oam-interface \"\" ${oam_my_unit_ip} 2222 2223 ${oam_peer_unit_ip} 2222 2223",
if $kubernetes_enabled {
exec { 'Configure Cluster Host Interface':
command => "sm-configure interface controller infrastructure-interface \"\" ${infra_my_unit_ip} 2222 2223 \"\" 2222 2223",
}
}
} else {
exec { 'Configure OAM Interface':
command => "sm-configure interface controller oam-interface \"\" ${oam_my_unit_ip} 2222 2223 ${oam_peer_unit_ip} 2222 2223",
}
exec { 'Configure Management Interface':
command => "sm-configure interface controller management-interface ${mgmt_ip_multicast} ${management_my_unit_ip} 2222 2223 ${management_peer_unit_ip} 2222 2223",
}
if $kubernetes_enabled or $infra_ip_interface {
exec { 'Configure Infrastructure Interface':
command => "sm-configure interface controller infrastructure-interface ${infra_ip_multicast} ${infra_my_unit_ip} 2222 2223 ${infra_peer_unit_ip} 2222 2223",
}
}
}
exec { 'Configure OAM IP':
@@ -1147,12 +1180,6 @@ class platform::sm
command => "sm-configure service_instance open-ldap open-ldap \"\"",
}
if $infra_ip_interface {
exec { 'Configure Infrastructure Interface':
command => "sm-configure interface controller infrastructure-interface ${infra_ip_multicast} ${infra_my_unit_ip} 2222 2223 ${infra_peer_unit_ip} 2222 2223",
}
}
if $system_mode == 'duplex-direct' or $system_mode == 'duplex' {
exec { 'Configure System Mode':
command => "sm-configure system --cpe_mode ${system_mode}",

5
puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb
View File

@@ -131,3 +131,8 @@ dhcp-leasefile=<%= @config_path %>/dnsmasq.leases
addn-hosts=<%= @config_path %>/dnsmasq.addn_hosts
# File for distributed cloud subcloud ip translation
addn-hosts=<%= @config_path %>/dnsmasq.addn_hosts_dc
# Set server to Kubernetes coredns cluster IP
<%- if @dns_service_ip != nil -%>
server=/<%= @service_domain %>/<%= @dns_service_ip %>
<%- end -%>

3
puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb
View File

@@ -14,7 +14,9 @@ apiServerCertSANs:
- "<%= @apiserver_advertise_address %>"
- "127.0.0.1"
networking:
dnsDomain: <%= @service_domain %>
podSubnet: <%= @pod_network_cidr %>
serviceSubnet: <%= @service_network_cidr %>
controllerManagerExtraArgs:
node-monitor-period: "2s"
node-monitor-grace-period: "20s"
@@ -26,4 +28,3 @@ nodeStatusUpdateFrequency: "4s"
failSwapOn: false
featureGates:
HugePages: false
resolvConf: "/etc/resolv.conf.k8s"