starlingx
/
stx-puppet
Code Issues Proposed changes

Merge "Revert "Add Kata Container support in StarlingX""

This commit is contained in:
Zuul 2020-01-14 21:28:36 +00:00 committed by Gerrit Code Review
parent 36bc8fb667 0f9dd0491b
commit b2d4e41822
9 changed files with 48 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
puppet-manifests/src/manifests/controller.pp
View File

@ -42,7 +42,6 @@ include ::platform::grub
include ::platform::etcd
include ::platform::docker
include ::platform::dockerdistribution
include ::platform::containerd
include ::platform::kubernetes::master
include ::platform::helm

1
puppet-manifests/src/manifests/storage.pp
View File

@ -28,7 +28,6 @@ include ::platform::grub
include ::platform::collectd
include ::platform::filesystem::storage
include ::platform::docker
include ::platform::containerd
include ::platform::ceph::storage
class { '::platform::config::storage::post':

1
puppet-manifests/src/manifests/worker.pp
View File

@ -31,7 +31,6 @@ include ::platform::grub
include ::platform::collectd
include ::platform::filesystem::compute
include ::platform::docker
include ::platform::containerd
include ::platform::dockerdistribution::compute
include ::platform::kubernetes::worker
include ::platform::multipath

94
puppet-manifests/src/modules/platform/manifests/containerd.pp
View File

@ -1,94 +0,0 @@
class platform::containerd::params (
$package_name = 'containerd',
$http_proxy = undef,
$https_proxy = undef,
$no_proxy = undef,
$k8s_registry = undef,
$insecure_registries = undef,
$k8s_cni_bin_dir = '/usr/libexec/cni'
) { }
class platform::containerd::config
inherits ::platform::containerd::params {
include ::platform::docker::params
include ::platform::dockerdistribution::params
include ::platform::kubernetes::params
include ::platform::dockerdistribution::registries
# inherit the proxy setting from docker
$http_proxy = $::platform::docker::params::http_proxy
$https_proxy = $::platform::docker::params::https_proxy
$no_proxy = $::platform::docker::params::no_proxy
$insecure_registries = $::platform::dockerdistribution::registries::insecure_registries
if $http_proxy or $https_proxy {
file { '/etc/systemd/system/containerd.service.d':
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0755',
}
-> file { '/etc/systemd/system/containerd.service.d/http-proxy.conf':
ensure => present,
owner => 'root',
group => 'root',
mode => '0644',
# share the same template as docker, since the conf file is the same
content => template('platform/dockerproxy.conf.erb'),
}
~> exec { 'perform systemctl daemon reload for containerd proxy':
command => 'systemctl daemon-reload',
logoutput => true,
refreshonly => true,
} ~> Service['containerd']
}
Class['::platform::filesystem::docker'] ~> Class[$name]
# get cni bin directory
$k8s_cni_bin_dir = $::platform::kubernetes::params::k8s_cni_bin_dir
file { '/etc/containerd':
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0700',
}
-> file { '/etc/containerd/config.toml':
ensure => present,
owner => 'root',
group => 'root',
mode => '0600',
content => template('platform/config.toml.erb'),
}
-> service { 'containerd':
ensure => 'running',
name => 'containerd',
enable => true,
require => Package['containerd']
}
-> exec { 'enable-containerd':
command => '/usr/bin/systemctl enable containerd.service',
}
-> exec { 'restart-containerd':
# containerd may be already started by docker. Need restart it after configuration
command => '/usr/bin/systemctl restart containerd.service',
}
}
class platform::containerd::install
inherits ::platform::containerd::params {
package { 'containerd':
ensure => 'installed',
name => $package_name,
}
}
class platform::containerd
{
include ::platform::containerd::install
include ::platform::containerd::config
}

19
puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp
View File

@ -270,25 +270,6 @@ class platform::dockerdistribution::compute
mode => '0644',
content => template('platform/insecuredockerregistry.conf.erb'),
}
# containerd requires ca file to access local secure registry
# For self signed cert, ca file is itself.
# cert_file and key_file are not needed when TLS mutual authentication is unused.
$shared_dir = $::platform::params::config_path
$certs_dir = '/etc/ssl/private'
file { $certs_dir:
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0700',
}
-> file { "${certs_dir}/registry-cert.crt":
ensure => 'file',
owner => 'root',
group => 'root',
mode => '0400',
source => "${shared_dir}/registry-cert.crt",
}
}
class platform::dockerdistribution

60
puppet-manifests/src/modules/platform/manifests/kubernetes.pp
View File

@ -223,8 +223,6 @@ class platform::kubernetes::master::init
# This flag is created by Ansible on controller-0;
# - Ansible replay is not impacted by flag creation.
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
# Create necessary certificate files
file { '/etc/kubernetes/pki':
ensure => directory,
@ -281,8 +279,18 @@ class platform::kubernetes::master::init
content => template('platform/kubeadm.yaml.erb'),
}
-> exec { 'pre pull k8s images':
command => "kubeadm config images list --kubernetes-version ${version} --image-repository registry.local:9001/k8s.gcr.io | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
-> exec { 'login local registry':
command => "docker login registry.local:9001 -u ${::platform::dockerdistribution::params::registry_username} -p ${::platform::dockerdistribution::params::registry_password}", # lint:ignore:140chars
logoutput => true,
}
-> exec { 'kubeadm to pre pull images':
command => 'kubeadm config images pull --config /etc/kubernetes/kubeadm.yaml',
logoutput => true,
}
-> exec { 'logout of local registry':
command => 'docker logout registry.local:9001',
logoutput => true,
}
@ -367,7 +375,6 @@ class platform::kubernetes::master
Class['::platform::sysctl::controller::reserve_ports'] -> Class[$name]
Class['::platform::etcd'] -> Class[$name]
Class['::platform::docker::config'] -> Class[$name]
Class['::platform::containerd::config'] -> Class[$name]
# Ensure DNS is configured as name resolution is required when
# kubeadm init is run.
Class['::platform::dns'] -> Class[$name]
@ -386,7 +393,6 @@ class platform::kubernetes::worker::init
inherits ::platform::kubernetes::worker::params {
Class['::platform::docker::config'] -> Class[$name]
Class['::platform::containerd::config'] -> Class[$name]
Class['::platform::filesystem::kubelet'] -> Class[$name]
if str2bool($::is_initial_config) {
@ -399,11 +405,21 @@ class platform::kubernetes::worker::init
$k8s_pause_img = generate('/bin/sh', '-c', $get_k8s_pause_img)
if k8s_pause_img {
exec { 'load k8s pause image by containerd':
command => "crictl pull --creds ${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password} ${k8s_pause_img}", # lint:ignore:140chars
exec { 'login local registry':
command => "docker login registry.local:9001 -u ${::platform::dockerdistribution::params::registry_username} -p ${::platform::dockerdistribution::params::registry_password}", # lint:ignore:140chars
logoutput => true,
}
-> exec { 'load k8s pause image':
command => "docker image pull ${k8s_pause_img}",
logoutput => true,
before => Exec['configure worker node']
}
-> exec { 'logout of local registry':
command => 'docker logout registry.local:9001',
logoutput => true,
}
}
}
@ -598,10 +614,18 @@ class platform::kubernetes::pre_pull_control_plane_images
include ::platform::dockerdistribution::params
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
exec { 'login to local registry':
command => "docker login registry.local:9001 -u ${::platform::dockerdistribution::params::registry_username} -p ${::platform::dockerdistribution::params::registry_password}", # lint:ignore:140chars
logoutput => true,
}
exec { 'pre pull images':
command => "kubeadm config images list --kubernetes-version ${upgrade_to_version} --image-repository=registry.local:9001/k8s.gcr.io | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
-> exec { 'pre pull images':
command => "kubeadm config images pull --kubernetes-version ${upgrade_to_version} --image-repository=registry.local:9001/k8s.gcr.io",
logoutput => true,
}
-> exec { 'logout of local registry':
command => 'docker logout registry.local:9001',
logoutput => true,
}
}
@ -666,11 +690,21 @@ class platform::kubernetes::worker::upgrade_kubelet
$k8s_pause_img = generate('/bin/sh', '-c', $get_k8s_pause_img)
if k8s_pause_img {
exec { 'load k8s pause image':
command => "crictl pull --creds ${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password} ${k8s_pause_img}", # lint:ignore:140chars
exec { 'login local registry':
command => "docker login registry.local:9001 -u ${::platform::dockerdistribution::params::registry_username} -p ${::platform::dockerdistribution::params::registry_password}", # lint:ignore:140chars
logoutput => true,
}
-> exec { 'load k8s pause image':
command => "docker image pull ${k8s_pause_img}",
logoutput => true,
before => Exec['upgrade kubelet']
}
-> exec { 'logout of local registry':
command => 'docker logout registry.local:9001',
logoutput => true,
}
}
exec { 'upgrade kubelet':

119
puppet-manifests/src/modules/platform/templates/config.toml.erb
View File

@ -1,119 +0,0 @@
root = "/var/lib/docker"
state = "/var/run/containerd"
oom_score = 0
[grpc]
address = "/var/run/containerd/containerd.sock"
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[debug]
address = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[plugins]
[plugins.cgroups]
no_prometheus = false
[plugins.cri]
stream_server_address = ""
stream_server_port = "0"
enable_selinux = false
sandbox_image = "registry.local:9001/k8s.gcr.io/pause:3.1"
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
[plugins.cri.containerd]
snapshotter = "overlayfs"
no_pivot = false
default_runtime_name = "runc"
[plugins.cri.containerd.runtimes]
[plugins.cri.containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v1"
[plugins.cri.containerd.runtimes.runc.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "runc"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins.cri.containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins.cri.containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins.cri.containerd.runtimes.katacli.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "/usr/bin/kata-runtime"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins.cri.containerd.runtimes.untrusted]
runtime_type = "io.containerd.kata.v2"
runtime_engine = ""
runtime_root = ""
[plugins.cri.cni]
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = "/etc/cni/net.d"
bin_dir = "<%= @k8s_cni_bin_dir %>"
max_conf_num = 1
conf_template = ""
[plugins.cri.registry]
[plugins.cri.registry.mirrors]
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://registry-1.docker.io"]
# Begin of insecure registries
<%- @insecure_registries.each do |insecure_registry| -%>
[plugins.cri.registry.mirrors."<%= insecure_registry %>"]
endpoint = ["http://<%= insecure_registry %>"]
<%- end -%>
# End of insecure registries
[plugins.cri.registry.configs."registry.local:9001".tls]
ca_file = "/etc/ssl/private/registry-cert.crt"
cert_file = ""
key_file = ""
[plugins.cri.registry.configs."registry.local:9001".auth]
username = ""
password = ""
auth = ""
identitytoken = ""
[plugins.cri.x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins.diff-service]
default = ["walking"]
[plugins.linux]
shim = "containerd-shim"
runtime = "runc"
runtime_root = ""
no_shim = false
shim_debug = false
[plugins.opt]
path = "/opt/containerd"
[plugins.restart]
interval = "10s"
[plugins.scheduler]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"

2
puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb
View File

@ -2,8 +2,6 @@ apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: <%= @apiserver_advertise_address %>
nodeRegistration:
criSocket: "/var/run/containerd/containerd.sock"
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration

2
puppet-manifests/src/modules/platform/templates/kubelet.conf.erb
View File

@ -1,2 +1,2 @@
# Overrides config file for kubelet
KUBELET_EXTRA_ARGS=--cni-bin-dir=<%= @k8s_cni_bin_dir %> --node-ip=<%= @node_ip %> <%= @k8s_cpu_manager_opts %> --container-runtime=remote --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock
KUBELET_EXTRA_ARGS=--cni-bin-dir=<%= @k8s_cni_bin_dir %> --node-ip=<%= @node_ip %> <%= @k8s_cpu_manager_opts %>