Initial kubernetes config on compute

Support the deployment of kubernetes on compute hosts. If the
"--kubernetes" option was given for the config_controller command,
kubernetes will be configured on all compute hosts.

Use a config file when calling kubeadm init, instead of passing
command line parameters.

Change-Id: I43cadd8311280a3e7fd64d637991b4dc046a21ac

puppet-manifests/src/manifests/compute.pp

@@ -29,6 +29,7 @@ include ::platform::ceph
 include ::platform::devices
 include ::platform::grub
 include ::platform::collectd
+include ::platform::kubernetes::worker
 
 include ::openstack::client
 include ::openstack::neutron

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -4,11 +4,77 @@ class platform::kubernetes::params (
   $apiserver_advertise_address = undef,
 ) { }
 
+class platform::kubernetes::kubeadm {
+  $repo_file = "[kubernetes]
+    name=Kubernetes
+    baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+    enabled=1
+    gpgcheck=1
+    repo_gpgcheck=1
+    gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg"
+  $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
+    net.bridge.bridge-nf-call-iptables = 1"
+  $kubeadm_conf = '/etc/systemd/system/kubelet.service.d/kubeadm.conf'
+
+  # Configure the kubernetes repo to allow us to download docker images for
+  # the kubernetes components. This will disappear once we have our own
+  # repo.
+  file { '/etc/yum.repos.d/kubernetes.repo':
+    ensure  => file,
+    content => "$repo_file",
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  } ->
+
+  # Update iptables config. This is required based on:
+  # https://kubernetes.io/docs/tasks/tools/install-kubeadm
+  # This probably belongs somewhere else - initscripts package?
+  file { '/etc/sysctl.d/k8s.conf':
+    ensure  => file,
+    content => "$iptables_file",
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  } ->
+  exec { "update kernel parameters for iptables":
+    command => "sysctl --system",
+  } ->
+
+  # Update kubelet configuration. Should probably just patch the kubelet
+  # package to fix these things. Looks like newer versions of the package
+  # have some of these changes already.
+  file_line { "${kubeadm_conf} KUBELET_EXTRA_ARGS":
+    path => $kubeadm_conf,
+    line => 'Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=cgroupfs"',
+    match => '^Environment="KUBELET_EXTRA_ARGS=',
+  } ->
+  file_line { "${kubeadm_conf} KUBELET_NETWORK_ARGS":
+    path => $kubeadm_conf,
+    line => 'Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"',
+    match => '^Environment="KUBELET_NETWORK_ARGS=',
+  } ->
+  file_line { "${kubeadm_conf} KUBELET_KUBECONFIG_ARGS":
+    path => $kubeadm_conf,
+    line => 'Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --fail-swap-on=false"',
+    match => '^Environment="KUBELET_KUBECONFIG_ARGS=',
+  } ->
+
+  # Start kubelet.
+  service { 'kubelet':
+    ensure => 'running',
+    enable => true,
+  } ->
+  # A seperate enable is required since we have modified the service resource
+  # to never enable services.
+  exec { 'enable-kubelet':
+    command => '/usr/bin/systemctl enable kubelet.service',
+  }
+}
+
 class platform::kubernetes::master::init
   inherits ::platform::kubernetes::params {
 
-  Class['::platform::kubernetes::master'] -> Class[$name]
-
   # This init only needs to be done once. Only controller-0 is supported for
   # now...
   if str2bool($::is_initial_config_primary) {
@@ -21,10 +87,15 @@ class platform::kubernetes::master::init
       line => 'nameserver 8.8.8.8',
     } ->
 
-    # Configure the master node. May want to use a config file instead of
-    # command line parameters.
+    # Configure the master node.
+    file { "/etc/kubernetes/kubeadm.yaml":
+      ensure => 'present',
+      replace => true,
+      content => template('platform/kubeadm.yaml.erb'),
+    } ->
+
     exec { "configure master node":
-      command => "kubeadm init --pod-network-cidr=$pod_network_cidr --apiserver-advertise-address=$apiserver_advertise_address",
+      command => "kubeadm init --config=/etc/kubernetes/kubeadm.yaml",
       logoutput => true,
     } ->
 
@@ -50,63 +121,50 @@ class platform::kubernetes::master
   inherits ::platform::kubernetes::params {
 
   if $enabled {
+    include ::platform::kubernetes::kubeadm
     include ::platform::kubernetes::master::init
 
     Class['::platform::docker::config'] -> Class[$name]
- 
-    $repo_file = "[kubernetes]
-      name=Kubernetes
-      baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
-      enabled=1
-      gpgcheck=1
-      repo_gpgcheck=1
-      gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg"
-    $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
-      net.bridge.bridge-nf-call-iptables = 1"
-    $kubeadm_conf = '/etc/systemd/system/kubelet.service.d/kubeadm.conf'
-
-    # Configure the kubernetes repo to allow us to download docker images for
-    # the kubernetes components. This will disappear once we have our own
-    # repo.
-    file { '/etc/yum.repos.d/kubernetes.repo':
-      ensure  => file,
-      content => "$repo_file",
-      owner   => 'root',
-      group   => 'root',
-      mode    => '0644',
-    } ->
-
-    # Update iptables config. This is required based on:
-    # https://kubernetes.io/docs/tasks/tools/install-kubeadm
-    # This probably belongs somewhere else - initscripts package?
-    file { '/etc/sysctl.d/k8s.conf':
-      ensure  => file,
-      content => "$iptables_file",
-      owner   => 'root',
-      group   => 'root',
-      mode    => '0644',
-    } ->
-    exec { "update kernel parameters for iptables":
-      command => "sysctl --system",
-    } ->
-
-    # Update kubelet configuration. Should probably just patch the kubelet
-    # package to fix these things.
-    file_line { "${kubeadm_conf} KUBELET_EXTRA_ARGS":
-      path => $kubeadm_conf,
-      line => 'Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=cgroupfs"',
-      match => '^Environment="KUBELET_EXTRA_ARGS=',
-    } ->
-    file_line { "${kubeadm_conf} KUBELET_NETWORK_ARGS":
-      path => $kubeadm_conf,
-      line => 'Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"',
-      match => '^Environment="KUBELET_NETWORK_ARGS=',
-    } ->
-
-    # Start kubelet.
-    service { 'kubelet':
-      ensure => 'running',
-      enable => true,
-    }
+    Class['::platform::kubernetes::kubeadm'] ->
+    Class['::platform::kubernetes::master::init']
+  }
+}
+
+class platform::kubernetes::worker::params (
+  $join_cmd = undef,
+) { }
+
+class platform::kubernetes::worker::init
+  inherits ::platform::kubernetes::worker::params {
+
+  # Start docker - will move to another manifest.
+  service { 'docker':
+    ensure => 'running',
+    enable => true,
+  } ->
+  # A seperate enable is required since we have modified the service resource
+  # to never enable services.
+  exec { 'enable-docker':
+    command => '/usr/bin/systemctl enable docker.service',
+  } ->
+
+  # Configure the worker node. Only do this once, so check whether the
+  # kubelet.conf file has already been created (by the join).
+  exec { "configure worker node":
+    command => "$join_cmd",
+    logoutput => true,
+    unless => 'test -f /etc/kubernetes/kubelet.conf',
+  }
+}
+
+class platform::kubernetes::worker
+  inherits ::platform::kubernetes::params {
+
+  if $enabled {
+    include ::platform::kubernetes::kubeadm
+    include ::platform::kubernetes::worker::init
+
+    Class['::platform::kubernetes::kubeadm'] ->
+    Class['::platform::kubernetes::worker::init']
   }
 }

puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb

@@ -0,0 +1,6 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: <%= @apiserver_advertise_address %>
+networking:
+  podSubnet: <%= @pod_network_cidr %>