Trigger application reapply on host unlock/delete

- Triggers an application reapply (including override
  regeneration) on node unlock to pick up a new node being added
  or configuration changes being made.
- The reapply also triggers on node delete if the host
  had a compute node label to remove any per-host overrides.
- Turned on the restriction that nodes must be locked to modify
  labels.  Added an audit task to sync any labels made before
  a node is unlocked and missing from k8s.
- Modified k8s puppet manifest to only launch kubelet on the
  initial configuration, after that the service file is
  modified to have a dependancy on the config gate.
  This is to avoid PLEG errors in kubernetes due to the node
  being overwhelmed during boot.

Change-Id: I1d9ca92f451aa322765da43ffcbb1d95f97f92f2
Story: 2004520
Task: 28826
Signed-off-by: Tyler Smith <tyler.smith@windriver.com>

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -39,7 +39,6 @@ class platform::kubernetes::kubeadm {
   }
   # Start kubelet.
   -> service { 'kubelet':
-    ensure => 'running',
     enable => true,
   }
   # A seperate enable is required since we have modified the service resource
@@ -124,6 +123,21 @@ class platform::kubernetes::master::init
       command   => "kubectl --kubeconfig=/etc/kubernetes/admin.conf taint node ${::platform::params::hostname} node-role.kubernetes.io/master-", # lint:ignore:140chars
       logoutput => true,
     }
+
+    # Add a dependency to kubelet on config so it doesn't enter a bad state on subsequent boots
+    -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
+      ensure  => file,
+      content => template('platform/kube-stx-override.conf.erb'),
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+    }
+
+    # Reload systemd
+    -> exec { 'perform systemctl daemon reload for kubelet override':
+      command   => 'systemctl daemon-reload',
+      logoutput => true,
+    }
   } else {
     if str2bool($::is_initial_config) {
       # For subsequent controller installs, install kubernetes using the
@@ -206,6 +220,21 @@ class platform::kubernetes::master::init
         command   => "kubectl --kubeconfig=/etc/kubernetes/admin.conf taint node ${::platform::params::hostname} node-role.kubernetes.io/master-", # lint:ignore:140chars
         logoutput => true,
       }
+
+      # Add a dependency to kubelet on config so it doesn't enter a bad state on subsequent boots
+      -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
+        ensure  => file,
+        content => template('platform/kube-stx-override.conf.erb'),
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0644',
+      }
+
+      # Reload systemd
+      -> exec { 'perform systemctl daemon reload for kubelet override':
+        command   => 'systemctl daemon-reload',
+        logoutput => true,
+      }
     }
   }
 }
@@ -242,6 +271,21 @@ class platform::kubernetes::worker::init
     logoutput => true,
     unless    => 'test -f /etc/kubernetes/kubelet.conf',
   }
+
+  # Add a dependency to kubelet on config so it doesn't enter a bad state
+  -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
+    ensure  => file,
+    content => template('platform/kube-stx-override.conf.erb'),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }
+
+  # Reload systemd
+  -> exec { 'perform systemctl daemon reload for kubelet override':
+    command   => 'systemctl daemon-reload',
+    logoutput => true,
+  }
 }
 
 class platform::kubernetes::worker

puppet-manifests/src/modules/platform/templates/kube-stx-override.conf.erb

@@ -0,0 +1,2 @@
+[Unit]
+After=config.service