Enable kubernetes secret encryption at rest

This update enabled kubernetes secret resource encryption during
ansible play system deployment. It contains the following changes:

- An extraArgs "encryption-provider-config:
  /etc/kubernetes/encryption-provider.yaml" is added to kubeadm.yaml
  by template. This argument will be fed into kube-apiserver static
  pod manifest so that kube-apiserver will be started with this
  encryption enabling flag.
- A template file is added to generate the actual encryption provider
  configuration file. This generated file is hostPath mounted to
  kube-apiserver pod as kube-apiserver's encryption provider configuration.
- The generated encryption provider configuration file is transfered to
  the second controller by the shared fs for kube-apiserver pod running
  on the second controller.

Change-Id: If1c1e1887e024923e5d876bf589a5bfd312d5da9
Story: 2007243
Task: 38626
Depends-On: https://review.opendev.org/#/c/705975
Signed-off-by: Andy Ning <andy.ning@windriver.com>

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml

@@ -85,6 +85,16 @@
     name: kubelet
     enabled: yes
 
+- name: Create kube api server encryption provider config file
+  vars:
+    aescbc_keys:
+      - name: key1
+        secret: "{{ lookup('password', '/dev/null chars=ascii_letters length=16') | b64encode }}"
+  template:
+    src: "encryption-provider.yaml.j2"
+    dest: "{{ encryption_provider_config }}"
+    mode: 0600
+
 - name: Create Kube admin yaml
   copy:
     src: roles/common/files/kubeadm.yaml.erb
@@ -163,6 +173,16 @@
     - "sed -i -e '/<%= @apiserver_oidc_groups_claim %>/d' /etc/kubernetes/kubeadm.yaml"
   when: apiserver_oidc | length == 3
 
+- name: Update Kube admin yaml with encryption provider config flag
+  command: "{{ item }}"
+  args:
+    warn: false
+  with_items:
+    - "sed -i -e 's|<%= @apiserver_encryption_provider_config %>|'$ENCRYPTION_PROVIDER_CONFIG'|g'
+       /etc/kubernetes/kubeadm.yaml"
+  environment:
+    ENCRYPTION_PROVIDER_CONFIG: "{{ encryption_provider_config }}"
+
 - name: Initializing Kubernetes master
   command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml
 
@@ -308,6 +328,14 @@
     - front-proxy-ca.crt
     - front-proxy-ca.key
 
+- name: Copy kube api server encryption provider config
+  copy:
+    src: "{{ encryption_provider_config }}"
+    dest: "{{ config_permdir }}/kubernetes/"
+    remote_src: yes
+    force: yes
+    mode: 0400
+
 - name: Mark Kubernetes config complete
   file:
     path: /etc/platform/.initial_k8s_config_complete

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/templates/encryption-provider.yaml.j2

@@ -0,0 +1,13 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+  - resources:
+    - secrets
+    providers:
+    - aescbc:
+        keys:
+{% for key in aescbc_keys %}
+        - name: {{key['name']}}
+          secret: {{key['secret']}}
+{% endfor %}
+    - identity: {}

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml

@@ -25,3 +25,6 @@ docker_cert_dir: /etc/docker/certs.d
 cgroup_root: /sys/fs/cgroup
 k8s_cgroup_name: k8s-infra
 kubeadm_pki_dir: /etc/kubernetes/pki
+
+# Kubernetes api server encryption provider configuration file
+encryption_provider_config: /etc/kubernetes/encryption-provider.yaml

playbookconfig/src/playbooks/roles/common/files/kubeadm.yaml.erb

@@ -22,6 +22,13 @@ apiServer:
     oidc-groups-claim: <%= @apiserver_oidc_groups_claim %>
     feature-gates: "SCTPSupport=true,TTLAfterFinished=true"
     event-ttl: "24h"
+    encryption-provider-config: <%= @apiserver_encryption_provider_config %>
+  extraVolumes:
+    - name: "encryption-config"
+      hostPath: <%= @apiserver_encryption_provider_config %>
+      mountPath: <%= @apiserver_encryption_provider_config %>
+      readOnly: true
+      pathType: File
 controllerManager:
   extraArgs:
     node-monitor-period: "2s"