Fix cert-manager migration playbook for subclouds

This review addresses changes required by the insertion of openldap in
the cert-manager migration playbook from previous version. It includes:
- Fixes in the playbook itself:
    - Fix detection of openldap cert;
    - Add the ability to run in subclouds with an old version;
    - Fixed the template that generates the platform certificates,
      adding default values in important fields case user does not
      define them in the inventory file.
- Fixes in bootstrap/rehoming:
    - Fix the overwrite of the ICA set by the user in the old version
      for kubernetes root ca in upgrades, inside the 'system-local-ca'
      secret (*);
    - Not recreate openldap secret if it isn't required, in upgrades;
    - Differentiate between secret types for subclouds in upgrades (it
      has to account for TLS type as well, not only Opaque);
    - Increase some rehoming timeouts;
    - Install the new SystemController system-loca-ca as a trusted CA
      in rehoming and restart kubeapi and openldap servers.
- Minor improvements:
    - Check if country name is limited to two letters in cert subject;
    - The role common/install-trusted-ca creates temporary files in
      SystemController. This could lead to race conditions if more
      than one playbook that uses the role were executed at the same
      time. Changed it to use random components in the filenames.

Test plan:
- Deploy SX, DX and DC with both SX and DX subclouds.
    - Execute cert-manager migration playboook.
- Rehome SX subcloud with Opaque 'system-local-ca' (normal case)
- Rehome SX/DX subclouds with TLS 'system-local-ca' (after
  cert-manager migration playbook is executed)
- Upgrade SX, DX and DC Systems with SX and DX subclouds from 21.12
  and 22.06 to designer iso 22.12. Executed the upgrades both with:
  - Running the cert-manager migration in the FROM side. (**)
  - Not running the cert-manager migration in the FROM side.

P.S.:
(*) Due to the existence of an upgrade start script called in the
    'from' side that will overwrite the secret after this code is
    called, this change will only have effects in upgrades moving
    forward.
(**) Some upgrade scenarios were affected by the issue in (*).
     Re-executing the cert-manager migration playbook in the TO side
     was able to fix this cases.

Closes-Bug: 2012435
Depends-on: https://review.opendev.org/c/starlingx/config/+/878913

Signed-off-by: Marcelo de Castro Loebens <Marcelo.DeCastroLoebens@windriver.com>
Change-Id: If9e56347c530a6556508c87659a24d8e8514624e

examples/migrate/migrate-platform-certificates-to-certmanager-inventory-EXAMPLE.yml

@@ -71,7 +71,7 @@ all:
           dns_domain: xyz.com
           duration: 2160h      # 90d
           renewBefore: 360h    # 15d
-          subject_C: canada
+          subject_C: CA
           subject_ST: ontario
           subject_L: ottawa
           subject_O: myorganization

playbookconfig/src/playbooks/migrate_platform_certificates_to_certmanager.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -77,13 +77,30 @@
 - hosts: localhost
   gather_facts: no
   tasks:
-    - name: Install certificates as system Trusted CA certificates
+    - block:
-      include_role:
+      - name: Install certificates as system Trusted CA certificates
-        name: common/install-trusted-ca
+        include_role:
-      with_items:
+          name: common/install-trusted-ca
-        - { name: system_local_ca_cert, content: "{{ system_local_ca_cert }}" }
+        with_items:
-        - { name: system_root_ca_cert, content: "{{ system_root_ca_cert }}" }
+          - { name: system_local_ca_cert, content: "{{ system_local_ca_cert }}" }
-      loop_control:
+          - { name: system_root_ca_cert, content: "{{ system_root_ca_cert }}" }
-        label: "{{ item.name }}"
+        loop_control:
+          label: "{{ item.name }}"
+
+      - name: Restart kube-apiserver to pick the new certificates
+        include_role:
+          name: common/restart-kube-apiserver
+
+      - name: Restart openldap server to use the new certificates
+        shell: "sm-restart service open-ldap"
+        become: true
+
+      - name: Check openldap service is enabled after restart
+        shell: sm-query service open-ldap | grep -c enabled-active
+        become: true
+        register: service_status
+        until: service_status.stdout == '1'
+        retries: 10
+        delay: 10
 
       when: mode == 'update'

playbookconfig/src/playbooks/roles/bootstrap/install-platform-certificates/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,23 +8,6 @@
 # generate a certificate spec file which is going to be applied to
 # kubernetes at a later step
 #
-
-- name: Read kubernetes Root CA certificate
-  shell: cat "{{ kubeadm_pki_dir }}/ca.crt" | base64 -w0
-  register: kubernetes_root_ca_crt
-  become: true
-
-- name: Read kubernetes Root CA key
-  shell: cat "{{ kubeadm_pki_dir }}/ca.key" | base64 -w0
-  register: kubernetes_root_ca_key
-  become: true
-
-- name: Set Root CA and local CA based on kubernetes Root CA
-  set_fact:
-    system_root_ca_cert: "{{ kubernetes_root_ca_crt.stdout }}"
-    system_local_ca_cert: "{{ kubernetes_root_ca_crt.stdout }}"
-    system_local_ca_key: "{{ kubernetes_root_ca_key.stdout }}"
-
 - block:
   - name: Get distributed_cloud_role if not defined
     shell: |
@@ -35,17 +18,114 @@
   - name: Set distributed_cloud_role value
     set_fact:
       distributed_cloud_role: "{{ dc_role.stdout }}"
+
   when: distributed_cloud_role is undefined
 
-- name: Set which platform certificates to install
+# In upgrades, retrieve data from 'system-local-ca' if it exists
-  set_fact:
-    install_oidc_auth_apps_certificate: false
-    install_system_open_ldap_certificate: "{{ true if distributed_cloud_role != 'subcloud' else false }}"
-    install_system_registry_local_certificate: false
-    install_system_restapi_gui_certificate: false
-
-# Set up certificates for non-subcloud system
 - block:
+  - name: Check if 'system-local-ca' secret exists
+    shell: |
+      kubectl get secret system-local-ca -n cert-manager \
+      --ignore-not-found=true --no-headers=true | \
+      awk '{ if ($0 != "") print "true"; exit}'
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: system_local_ca_exists
+
+  - block:
+    - name: Retrieve system local CA cert from k8s secret (on System Controller)
+      command: kubectl get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.crt}'
+      environment:
+        KUBECONFIG: "/etc/kubernetes/admin.conf"
+      register: cert_result
+
+    - name: Retrieve system local CA key from k8s secret (on System Controller)
+      command: kubectl get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.key}'
+      environment:
+        KUBECONFIG: "/etc/kubernetes/admin.conf"
+      register: key_result
+
+    - name: Set local CA cert/key
+      set_fact:
+        system_root_ca_cert: "{{ cert_result.stdout }}"
+        system_local_ca_cert: "{{ cert_result.stdout }}"
+        system_local_ca_key: "{{ key_result.stdout }}"
+        ca_data_obtained: true
+
+    when: system_local_ca_exists.stdout | bool
+
+  when: mode == 'restore' and
+        migrate_platform_data is defined and
+        migrate_platform_data
+  connection: local
+
+# Else retrieve from kubernetes root CA
+- block:
+  - name: Read kubernetes Root CA certificate
+    shell: cat "{{ kubeadm_pki_dir }}/ca.crt" | base64 -w0
+    register: kubernetes_root_ca_crt
+    become: true
+
+  - name: Read kubernetes Root CA key
+    shell: cat "{{ kubeadm_pki_dir }}/ca.key" | base64 -w0
+    register: kubernetes_root_ca_key
+    become: true
+
+  - name: Set Root CA and local CA based on kubernetes Root CA
+    set_fact:
+      system_root_ca_cert: "{{ kubernetes_root_ca_crt.stdout }}"
+      system_local_ca_cert: "{{ kubernetes_root_ca_crt.stdout }}"
+      system_local_ca_key: "{{ kubernetes_root_ca_key.stdout }}"
+
+  connection: local
+  when: ca_data_obtained is not defined
+
+# If subcloud, we need to know the secret type. Also in restore mode the ca
+# cert must be restored from backup, so we are only changing it otherwise.
+- block:
+  - name: Get 'system-local-ca' secret type for subcloud
+    shell: |
+      kubectl get secret system-local-ca -n cert-manager \
+      --ignore-not-found=true --no-headers=true | \
+      awk '{print $2}'
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: subcloud_local_ca_secret_type
+
+  - name: Set subcloud_local_ca_secret_type value
+    set_fact:
+      subcloud_local_ca_secret_type: "{{ subcloud_local_ca_secret_type.stdout | lower }}"
+      subcloud_local_ca_should_be_altered: "{{ mode == 'bootstrap' or
+                                               (mode == 'restore' and
+                                                migrate_platform_data is defined and
+                                                migrate_platform_data) }}"
+
+  when: distributed_cloud_role == 'subcloud'
+
+
+- block:
+  - name: Initialize platform certificates to not be altered
+    set_fact:
+      install_oidc_auth_apps_certificate: false
+      install_system_open_ldap_certificate: false
+      install_system_registry_local_certificate: false
+      install_system_restapi_gui_certificate: false
+
+  # In non subcloud systems, we may need to create the openldap secret
+  - name: Check if openldap certificate already exists
+    shell: |
+      kubectl get secret system-openldap-local-certificate -n deployment \
+      --ignore-not-found=true --no-headers=true | \
+      awk '{ if ($0 != "") print "true"; exit}'
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: openldap_cert_exists
+
+  - name: Register if openldap certificate should be created
+    set_fact:
+      install_system_open_ldap_certificate: "{{ not (openldap_cert_exists.stdout | bool) }}"
+    when: distributed_cloud_role != 'subcloud'
+
   - name: Generate kubernetes yaml for cert-manager resources
     include_role:
       name: common/generate-platform-certificates-template
@@ -60,15 +140,16 @@
     until: create_k8_apply_ep is not failed
     retries: 10
     delay: 30
-  when: distributed_cloud_role != 'subcloud'
 
-# The following role should be executed during subcloud bootstrap or upgrade,
+  when: distributed_cloud_role != 'subcloud' or
-# not subcloud restore as the ca cert is supposed to be restored from the backup.
+        (subcloud_local_ca_should_be_altered and
+        subcloud_local_ca_secret_type == 'kubernetes.io/tls')
+
+# If the subcloud's secret is not tls (opaque or doesn't exist yet)
+# we send the systemcontroller's. Else, we do nothing.
 - name: Send root CA certificate to subcloud
   include_role:
     name: common/send-ca-cert-to-subcloud
-  when: (distributed_cloud_role == 'subcloud') and
+  when: distributed_cloud_role == 'subcloud' and
-        ((mode == 'bootstrap') or
+        subcloud_local_ca_should_be_altered and
-         (mode == 'restore' and
+        subcloud_local_ca_secret_type != 'kubernetes.io/tls'
-          migrate_platform_data is defined and
-          migrate_platform_data))

playbookconfig/src/playbooks/roles/common/delete-kubernetes-resources/tasks/main.yml

@@ -1,16 +1,16 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# This task provides functionality to delete kubernetes objects
+# Simple util to delete kubernetes resources by name and namespace
 #
-- name: Delete kubernetes objects
+- name: Delete kubernetes resources
   command: >-
     kubectl delete {{ item.type }}
     --ignore-not-found=true
-    {{ item.secret }}
+    {{ item.name }}
     -n {{ item.namespace }}
   environment:
     KUBECONFIG: /etc/kubernetes/admin.conf

playbookconfig/src/playbooks/roles/common/generate-platform-certificates-template/tasks/main.yml

@@ -8,6 +8,9 @@
 # generate a certificate spec file which is going to be applied to
 # kubernetes at a later step
 #
+- name: Validate subject fields
+  include_tasks: validate-subject-fields.yaml
+
 - name: Get address pool information for system
   shell: |
     source /etc/platform/openrc; system addrpool-list --nowrap
@@ -63,7 +66,7 @@
   shell: |
     source /etc/platform/openrc
     system show | grep distributed_cloud_role | awk '{ print $4 }'
-  register: distributed_cloud_role
+  register: dc_role
 
 - name: Get cert-manager api version
   shell: |

playbookconfig/src/playbooks/roles/common/generate-platform-certificates-template/tasks/validate-subject-fields.yaml

@@ -0,0 +1,15 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Validate X509 certificate subject fields
+#
+
+- name: Validate subject_C (Country Name)
+  fail:
+    msg: |
+      [ERROR]: subject_C should be defined using an ISO-3166 two-letter code.
+  when: system_platform_certificate.subject_C is defined and
+        system_platform_certificate.subject_C | length != 2

playbookconfig/src/playbooks/roles/common/generate-platform-certificates-template/templates/platform_certificates.yml.j2

@@ -13,7 +13,7 @@ type: kubernetes.io/tls
 ---
 apiVersion: v1
 items:
-- apiVersion: "{{ cert_manager_api_version.stdout | default('cert-manager.io/v1') }}"
+- apiVersion: "{{ cert_manager_api_version.stdout | default(default.cert_manager_api_version, true) }}"
   kind: ClusterIssuer
   metadata:
     creationTimestamp: null
@@ -24,7 +24,7 @@ items:
   status: {}
 {% if install_system_restapi_gui_certificate | bool %}
 {% set short_certificate_name = 'system-restapi-gui' %}
-- apiVersion: "{{ cert_manager_api_version.stdout | default('cert-manager.io/v1') }}"
+- apiVersion: "{{ cert_manager_api_version.stdout | default(default.cert_manager_api_version, true) }}"
   kind: Certificate
   metadata:
     creationTimestamp: null
@@ -33,8 +33,8 @@ items:
   spec:
     commonName: "{{ system_platform_certificate.subject_CN | default(oam_ip.stdout) }}"
     dnsNames:
-    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain }}"
+    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain | default(default.dns_domain) }}"
-    duration: "{{ system_platform_certificate.duration }}"
+    duration: "{{ system_platform_certificate.duration | default(default.duration) }}"
     ipAddresses:
     - "{{ oam_ip.stdout }}"
     issuerRef:
@@ -47,7 +47,7 @@ items:
 {% endif %}
 {% if install_system_registry_local_certificate | bool %}
 {% set short_certificate_name = 'system-registry-local' %}
-- apiVersion: "{{ cert_manager_api_version.stdout | default('cert-manager.io/v1') }}"
+- apiVersion: "{{ cert_manager_api_version.stdout | default(default.cert_manager_api_version, true) }}"
   kind: Certificate
   metadata:
     creationTimestamp: null
@@ -56,12 +56,12 @@ items:
   spec:
     commonName: "{{ system_platform_certificate.subject_CN | default(oam_ip.stdout) }}"
     dnsNames:
-    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain }}"
+    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain | default(default.dns_domain) }}"
     - registry.local
-{% if distributed_cloud_role.stdout == 'systemcontroller' %}
+{% if dc_role.stdout == 'systemcontroller' %}
     - registry.central
 {% endif %}
-    duration: "{{ system_platform_certificate.duration }}"
+    duration: "{{ system_platform_certificate.duration | default(default.duration) }}"
     ipAddresses:
     - "{{ oam_ip.stdout }}"
     - "{{ management_floating_ip.stdout }}"
@@ -75,7 +75,7 @@ items:
 {% endif %}
 {% if install_oidc_auth_apps_certificate | bool %}
 {% set short_certificate_name = 'system-oidc-dex' %}
-- apiVersion: "{{ cert_manager_api_version.stdout | default('cert-manager.io/v1') }}"
+- apiVersion: "{{ cert_manager_api_version.stdout | default(default.cert_manager_api_version, true) }}"
   kind: Certificate
   metadata:
     creationTimestamp: null
@@ -84,8 +84,8 @@ items:
   spec:
     commonName: "{{ system_platform_certificate.subject_CN | default(oam_ip.stdout) }}"
     dnsNames:
-    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain }}"
+    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain | default(default.dns_domain) }}"
-    duration: "{{ system_platform_certificate.duration }}"
+    duration: "{{ system_platform_certificate.duration | default(default.duration) }}"
     ipAddresses:
     - "{{ oam_ip.stdout }}"
     # Add kubernetes cluster ip to make sure certificate has issuer ip in san list
@@ -103,7 +103,7 @@ items:
 {% endif %}
 {% if install_system_open_ldap_certificate | bool %}
 {% set short_certificate_name = 'system-openldap' %}
-- apiVersion: "{{ cert_manager_api_version.stdout | default('cert-manager.io/v1') }}"
+- apiVersion: "{{ cert_manager_api_version.stdout | default(default.cert_manager_api_version, true) }}"
   kind: Certificate
   metadata:
     creationTimestamp: null
@@ -112,11 +112,11 @@ items:
   spec:
     commonName: "{{ system_platform_certificate.subject_CN | default(short_certificate_name) }}"
     dnsNames:
-    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain }}"
+    - "{{ clean_region_name }}.{{ system_platform_certificate.dns_domain | default(default.dns_domain) }}"
     - controller
     - controller-0
     - controller-1
-    duration: "{{ system_platform_certificate.duration }}"
+    duration: "{{ system_platform_certificate.duration | default(default.duration) }}"
     ipAddresses:
     - "{{ management_floating_ip.stdout }}"
     - "{{ management_c0_ip.stdout }}"

playbookconfig/src/playbooks/roles/common/generate-platform-certificates-template/vars/main.yaml

@@ -0,0 +1,11 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+default:
+  cert_manager_version: 'cert-manager.io/v1'
+  dns_domain: 'starlingx.local'
+  duration: '2160h'     # 90d

playbookconfig/src/playbooks/roles/common/install-trusted-ca/tasks/main.yml

@@ -1,20 +1,43 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 # These tasks provide the functionality to validate ICA duration and
 # install it as a platform Trusted CA
 #
+- name: Create root pem temporary file
+  tempfile:
+    state: file
+    prefix: root_
+    suffix: .pem
+    path: /tmp/
+  register: root_ca_file
+
+- name: Create ca pem temporary file
+  tempfile:
+    state: file
+    prefix: ca_
+    suffix: .pem
+    path: /tmp/
+  register: local_ca_file
+
 - name: Save {{ item.name }} certificate to a file
-  shell: echo "{{ item.content }}" | base64 -d > /tmp/ca.pem
+  copy:
+    dest: "{{ local_ca_file.path }}"
+    content: "{{ item.content | b64decode }}"
+    mode: 0640
 
 - block:
+  - name: Save system_root_ca_cert to a file
+    copy:
+      dest: "{{ root_ca_file.path }}"
+      content: "{{ system_root_ca_cert | b64decode }}"
+      mode: 0640
+
   - name: Check if system_local_ca_cert is signed by system_root_ca_cert or self-signed
-    shell: |
+    command: openssl verify -verbose -CAfile {{ root_ca_file.path }} {{ local_ca_file.path }}
-      echo "{{ system_root_ca_cert }}" | base64 -d > /tmp/root.pem
-      openssl verify -verbose -CAfile /tmp/root.pem /tmp/ca.pem
     register: ca_verification
     # failed_when as false in order to print a better error msg in the task below
     failed_when: false
@@ -30,7 +53,7 @@
 
 - name: Get CA information from certificate
   shell: |
-    cat /tmp/ca.pem | openssl x509 -text -noout | grep "CA:"
+    openssl x509 -in {{ local_ca_file.path }} -text -noout | grep "CA:"
   register: is_ca
 
 - name: Fail when certificate specified is not an actual CA certificate
@@ -44,7 +67,7 @@
 
 - name: Check that CA certificate remaining duration is longer than {{ ca_duration }} years
   shell: |
-      expiration_date=$(cat /tmp/ca.pem | openssl x509 -noout -enddate | cut -d'=' -f2)
+      expiration_date=$(cat {{ local_ca_file.path }} | openssl x509 -noout -enddate | cut -d'=' -f2)
       expiration_date_timestamp=$(date -d "${expiration_date}" +%s)
       date_5years_from_now_timestamp=$(date -d "+{{ ca_duration }} years" +%s)
       time_left_ica=$(expr $expiration_date_timestamp - $date_5years_from_now_timestamp)
@@ -62,8 +85,19 @@
 - name: Install {{ item.name }} certificate as a Trusted CA certificate
   shell: >-
     source /etc/platform/openrc &&
-    system certificate-install -m ssl_ca /tmp/ca.pem
+    system certificate-install -m ssl_ca {{ local_ca_file.path }}
   register: install_cert_output
   until: install_cert_output is not failed
   retries: 3
   delay: 60
+
+- name: Delete temporary .pem files
+  file:
+    path: "{{ file_item }}"
+    state: absent
+  with_items:
+    - "{{ local_ca_file.path }}"
+    - "{{ root_ca_file.path }}"
+  loop_control:
+    loop_var: file_item
+  become: yes

playbookconfig/src/playbooks/roles/common/restart-kube-apiserver/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Restart kube-apiserver process.
+#
+
+- name: Get pid of current kube-apiserver process
+  command: pidof kube-apiserver
+  register: kube_apiserver_pid
+
+- name: Restart kube-apiserver
+  shell: crictl ps | awk '/kube-apiserver/{print$1}' | xargs crictl stop > /dev/null
+
+- name: Wait while kube-apiserver restarts
+  command: pidof kube-apiserver || true
+  register: new_kube_apiserver_pid
+  until: (new_kube_apiserver_pid.stdout | length > 0) and
+         new_kube_apiserver_pid.stdout != kube_apiserver_pid.stdout
+  retries: 20
+  delay: 30

playbookconfig/src/playbooks/roles/common/send-ca-cert-to-subcloud/tasks/main.yml

@@ -1,13 +1,45 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 # These tasks provide the functionality to send root CA certificate
 # to subcloud and store it as a k8s secret.
 #
-- name: Retrieve system-local-ca from k8s secret (on System Controller)
+
+# If the playbook is called from bootstrap/install-platform-certificates
+# this value is already set, and we know it is bootstrap or upgrade.
+# If it isn't set, we expect it to be rehoming.
+- name: Set if it is an bootstrap/upgrade
+  set_fact:
+    ca_send_from_bootstrap: "{{ subcloud_local_ca_secret_type is defined }}"
+
+- block:
+  - name: Get 'system-local-ca' secret type for subcloud
+    shell: |
+      kubectl get secret system-local-ca -n cert-manager \
+      --ignore-not-found=true --no-headers=true | \
+      awk '{print $2}'
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: ca_secret_type
+
+  - name: Set subcloud_local_ca_secret_type value. If inexistent, it will be 'opaque'
+    set_fact:
+      subcloud_local_ca_secret_type: "{{ ca_secret_type.stdout | lower | default('opaque', true) }}"
+
+  - name: Fail if secret exists but the type is neither TLS nor Opaque
+    fail:
+      msg: |
+        "Secret type of system-local-ca is not supported. \
+        Type: {{ subcloud_local_ca_secret_type }}."
+    when: subcloud_local_ca_secret_type != 'opaque' and
+          subcloud_local_ca_secret_type != 'kubernetes.io/tls'
+
+  when: subcloud_local_ca_secret_type is not defined or (subcloud_local_ca_secret_type | length == 0)
+
+- name: Retrieve system local CA cert from k8s secret (on System Controller)
   command: kubectl get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.crt}'
   environment:
     KUBECONFIG: "/etc/kubernetes/admin.conf"
@@ -19,20 +51,142 @@
     msg: "Failed to retrieve system CA certificate from secret."
   when: cert_result.stdout == ''
 
-- name: Set system local CA cert
+- name: Set system root/local CA cert based on 'system-local-ca'
   set_fact:
     system_local_ca_cert: "{{ cert_result.stdout }}"
+    system_root_ca_cert: "{{ cert_result.stdout }}"
 
-- name: Generate kubernetes yaml for the secret resource
+- block:
-  template:
+  - name: Retrieve system local CA key from k8s secret (on System Controller)
-    src: system_local_ca_secret.yml.j2
+    command: kubectl get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.key}'
-    dest: /tmp/system_local_ca_secret.yml
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: key_result
+    connection: local
+
+  - name: Fail if the local CA key returned is empty
+    fail:
+      msg: "Failed to retrieve system CA certificate key from secret."
+    when: key_result.stdout == ''
+
+  - name: Set system local CA key
+    set_fact:
+      system_local_ca_key: "{{ key_result.stdout }}"
+
+  - name: Retrieve root CA from k8s secret (on System Controller)
+    command: kubectl get secret system-local-ca -n cert-manager -o jsonpath='{.data.ca\.crt}'
+    environment:
+      KUBECONFIG: "/etc/kubernetes/admin.conf"
+    register: ca_cert_result
+    connection: local
+
+  - name: Set root CA cert
+    set_fact:
+      system_root_ca_cert: "{{ ca_cert_result.stdout }}"
+    when: ca_cert_result.stdout | length > 0
+
+  - name: Set root CA cert equal local if there isn't one in the secret
+    set_fact:
+      system_root_ca_cert: "{{ system_local_ca_cert }}"
+    when: ca_cert_result.stdout | length == 0
+
+  when: subcloud_local_ca_secret_type == 'kubernetes.io/tls'
+
+- name: Create temporary kubernetes yaml file for the secret resource
+  tempfile:
+    state: file
+    prefix: system_local_ca_secret_
+    suffix: .yml
+    path: /tmp/
+  register: ca_secret_spec_file
+
+- block:
+  - name: Generate kubernetes yaml for the secret resource (for Opaque)
+    template:
+      src: system_local_ca_secret_opaque.yml.j2
+      dest: "{{ ca_secret_spec_file.path }}"
+      mode: '0640'
+    become: yes
+
+  - name: Delete old secret
+    include_role:
+      name: common/delete-kubernetes-resources
+    with_items:
+      - { name: system-local-ca, namespace: cert-manager, type: secret }
+
+  when: subcloud_local_ca_secret_type == 'opaque'
+
+- block:
+  - name: Initialize platform certificates to not be altered
+    set_fact:
+      install_oidc_auth_apps_certificate: false
+      install_system_open_ldap_certificate: false
+      install_system_registry_local_certificate: false
+      install_system_restapi_gui_certificate: false
+
+  - name: Generate kubernetes yaml for the secret resource (for TLS)
+    include_role:
+      name: common/generate-platform-certificates-template
+    vars:
+      destination: "{{ ca_secret_spec_file.path }}"
+
+  when: subcloud_local_ca_secret_type == 'kubernetes.io/tls'
 
 - name: Apply kubernetes yaml to create the secret (on subcloud)
-  command: kubectl apply -f /tmp/system_local_ca_secret.yml
+  command: kubectl apply -f "{{ ca_secret_spec_file.path }}"
   environment:
     KUBECONFIG: /etc/kubernetes/admin.conf
   register: apply_result
   until: apply_result is not failed
   retries: 10
   delay: 30
+
+- name: Delete kubernetes yaml with certificate spec
+  file:
+    path: "{{ ca_secret_spec_file.path }}"
+    state: absent
+  become: yes
+
+# If not bootstrap/upgrade, install the root certificates
+- block:
+  - name: Install certificates (Subcloud)
+    include_role:
+      name: common/install-trusted-ca
+    with_items:
+      - { name: system_root_ca_cert, content: "{{ system_root_ca_cert }}" }
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Restart kube-apiserver to pick the new certificates (Subcloud)
+    include_role:
+      name: common/restart-kube-apiserver
+
+  when: not ca_send_from_bootstrap
+
+- block:
+  - name: Install certificates (SystemController)
+    include_role:
+      name: common/install-trusted-ca
+    with_items:
+      - { name: system_root_ca_cert, content: "{{ system_root_ca_cert }}" }
+    loop_control:
+      label: "{{ item.name }}"
+
+  - name: Restart kube-apiserver to pick the new certificates (SystemController)
+    include_role:
+      name: common/restart-kube-apiserver
+
+  - name: Restart openldap server to trust the new certificates
+    shell: "sm-restart service open-ldap"
+    become: true
+
+  - name: Check openldap service is enabled after restart
+    shell: sm-query service open-ldap | grep -c enabled-active
+    become: true
+    register: service_status
+    until: service_status.stdout == '1'
+    retries: 10
+    delay: 10
+
+  when: not ca_send_from_bootstrap
+  connection: local

playbookconfig/src/playbooks/roles/common/send-ca-cert-to-subcloud/templates/system_local_ca_secret_opaque.yml.j2

@@ -6,4 +6,4 @@ kind: Secret
 metadata:
   name: system-local-ca
   namespace: cert-manager
-type: opaque
+type: Opaque

playbookconfig/src/playbooks/roles/migrate-platform-certificates-to-certmanager/migrate-certificates/tasks/check-certificates-to-be-installed.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -23,11 +23,12 @@
 - name: Check if openldap certificate exists
   shell: |
     source /etc/platform/openrc
-    system certificate-list | grep openldap
+    system certificate-list | grep openldap | \
+    awk '{ if ($0 != "") print "true"; exit}'
   register: openldap_certificate_exists
 
 - set_fact:
     install_oidc_auth_apps_certificate: "{{ true if oidc_applied.stdout | bool else false }}"
-    install_system_open_ldap_certificate: "{{ true if openldap_certificate_exists.stdout | int == 0 else false }}"
+    install_system_open_ldap_certificate: "{{ true if openldap_certificate_exists.stdout | bool else false }}"
     install_system_registry_local_certificate: true
     install_system_restapi_gui_certificate: "{{ true if https_enabled.stdout | bool else false }}"

playbookconfig/src/playbooks/roles/migrate-platform-certificates-to-certmanager/migrate-certificates/tasks/main.yml

@@ -55,27 +55,29 @@
       retries: 3
       delay: 30
 
-    - name: Verify if 'system-local-ca' secret was created by a cert-manager cert that acts as its Root CA
+    - name: Retrieve certificates that may own system-local-ca secret
       shell: >-
         kubectl get certificates -A
         -o=custom-columns='SECRET:spec.secretName','NAME:metadata.name','NAMESPACE:metadata.namespace'
         --no-headers=true |
         awk '$1 == "system-local-ca"' |
-        awk '{ print "{secret: "$2", namespace: "$3", type: certificate}" }'
+        awk '{ print "{name: "$2", namespace: "$3", type: certificate}" }'
       environment:
         KUBECONFIG: /etc/kubernetes/admin.conf
       register: cert_to_remove
 
-    - name: Delete the Root CA that owns the secret 'system-local-ca' if it exists
+    - name: Delete certificate that owns the secret 'system-local-ca' if it exists
-      include_tasks: delete-kubernetes-objects.yml
+      include_role:
+        name: common/delete-kubernetes-resources
       when: cert_to_remove.stdout | length > 0
       loop: "{{ cert_to_remove.stdout_lines | map('from_yaml') | list }}"
 
     - name: Delete the 'system-local-ca' clusterIssuer and secret if they exist
-      include_tasks: delete-kubernetes-objects.yml
+      include_role:
+        name: common/delete-kubernetes-resources
       with_items:
-        - { secret: system-local-ca, namespace: cert-manager, type: clusterissuer }
+        - { name: system-local-ca, namespace: cert-manager, type: clusterissuer }
-        - { secret: system-local-ca, namespace: cert-manager, type: secret }
+        - { name: system-local-ca, namespace: cert-manager, type: secret }
 
     - name: Generate kubernetes yaml for cert-manager resources
       include_role:
@@ -83,25 +85,41 @@
       vars:
         destination: "{{ cert_manager_spec_file }}"
 
-    - name: Select which certificates should be migrated to the new 'system-local-ca' issuer
+    # This list is composed of all certificates issued by the cluster issuer
+    # plus the four platform ones we will always renew
+    - name: Create a list of certificates that should be migrated to the new 'system-local-ca' issuer
       shell: |
         #Collecting certs previously issued by 'system-local-ca'
         {(kubectl get certificates -A \
         -o=custom-columns='SECRET:spec.secretName','NAMESPACE:metadata.namespace','ISSUER:spec.issuerRef.name' \
         --no-headers=true |
         awk '$3 == "system-local-ca"' |
-        awk '{ print "{secret: "$1", namespace: "$2", type: secret}" }' |
+        awk '{ print "{name: "$1", namespace: "$2", type: secret}" }' |
         awk NF);
 
         #Adding certs we will renew by default (if repeated, they will be filtered in renewal task)
-        (printf "{secret: system-openldap-local-certificate, namespace: deployment, type: secret}
+        (printf "{name: system-openldap-local-certificate, namespace: deployment, type: secret}
-        {secret: system-registry-local-certificate, namespace: deployment, type: secret}
+        {name: system-registry-local-certificate, namespace: deployment, type: secret}
-        {secret: system-restapi-gui-certificate, namespace: deployment, type: secret}
+        {name: system-restapi-gui-certificate, namespace: deployment, type: secret}
-        {secret: oidc-auth-apps-certificate, namespace: kube-system, type: secret}\n")}
+        {name: oidc-auth-apps-certificate, namespace: kube-system, type: secret}\n")}
       environment:
         KUBECONFIG: /etc/kubernetes/admin.conf
       register: certs_to_renew
 
+    # This is a workaround for a problem found in upgrades (not really ideal)
+    # It will cause warnings uppon applying the file
+    - name: Remove 'last-applied-configuration' annotation from certs to avoid version problems after upgrades
+      shell: |
+        kubectl annotate certificate "{{ item.certificate }}" -n "{{ item.namespace }}" \
+        kubectl.kubernetes.io/last-applied-configuration- || true
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      with_items:
+        - { certificate: system-openldap-local-certificate, namespace: deployment }
+        - { certificate: system-registry-local-certificate, namespace: deployment }
+        - { certificate: system-restapi-gui-certificate, namespace: deployment }
+        - { certificate: oidc-auth-apps-certificate, namespace: kube-system }
+
     - name: Apply kubernetes yaml to create cert-manager clusterissuer and certificates
       command: kubectl apply -f "{{ cert_manager_spec_file }}"
       environment:
@@ -118,7 +136,8 @@
       become: yes
 
     - name: Force certificate renewals by deleting their secrets
-      include_tasks: delete-kubernetes-objects.yml
+      include_role:
+        name: common/delete-kubernetes-resources
       loop: "{{ certs_to_renew.stdout_lines | map('from_yaml') | unique | list }}"
 
     - name: Install certificates as system Trusted CA certificates
@@ -130,6 +149,10 @@
       loop_control:
         label: "{{ item.name }}"
 
+    - name: Restart kube-apiserver to pick the new certificates
+      include_role:
+        name: common/restart-kube-apiserver
+
     - name: Update oidc-auth-apps in order to use new certificate
       include_tasks: reapply-oidc-auth-app.yml
       when: oidc_applied.stdout | bool
@@ -175,16 +198,18 @@
       delay: 3
 
     - name: Delete certificates
-      include_tasks: delete-kubernetes-objects.yml
+      include_role:
+        name: common/delete-kubernetes-resources
       with_items:
-        - { secret: system-registry-local-certificate, namespace: deployment, type: certificate }
+        - { name: system-registry-local-certificate, namespace: deployment, type: certificate }
-        - { secret: system-restapi-gui-certificate, namespace: deployment, type: certificate }
+        - { name: system-restapi-gui-certificate, namespace: deployment, type: certificate }
 
     - name: Delete secrets
-      include_tasks: delete-kubernetes-objects.yml
+      include_role:
+        name: common/delete-kubernetes-resources
       with_items:
-        - { secret: system-registry-local-certificate, namespace: deployment, type: secret }
+        - { name: system-registry-local-certificate, namespace: deployment, type: secret }
-        - { secret: system-restapi-gui-certificate, namespace: deployment, type: secret }
+        - { name: system-restapi-gui-certificate, namespace: deployment, type: secret }
 
     - debug:
         msg: >-

playbookconfig/src/playbooks/roles/migrate-platform-certificates-to-certmanager/migrate-certificates/tasks/reapply-oidc-auth-app.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,12 +8,21 @@
 # certificate and reapply the app to apply the new configuration.
 #
 
+- name: Retrieve current software version of the host
+  shell: source /etc/build.info; echo $SW_VERSION
+  register: current_software_version
+
+- name: Set host_software_version fact
+  set_fact:
+    host_software_version: "{{ current_software_version.stdout }}"
+
 # Secret system-local-ca-oidc-secret is a copy of system-local-ca
 # we need it because this secret needs to be in the same namespace as oidc-auth-apps
 - name: Delete system-local-ca-oidc-secret before recreating it
-  include_tasks: delete-kubernetes-objects.yml
+  include_role:
+    name: common/delete-kubernetes-resources
   with_items:
-    - { secret: system-local-ca-oidc-secret, namespace: kube-system, type: secret }
+    - { name: system-local-ca-oidc-secret, namespace: kube-system, type: secret }
 
 - name: Create new dex-client secret based of system-local-ca
   shell: >-
@@ -35,24 +44,43 @@
         issuer_root_ca: /home/dex-ca.pem
         issuer_root_ca_secret: system-local-ca-oidc-secret
 
-- name: Merge new volume and volumeMounts overrides with existing ones
+- block:
-  vars:
+  - name: Merge new volume and volumeMounts overrides with existing ones
-    new_overrides: |
+    vars:
-      volumes:
+      new_overrides: |
-      - name: https-tls
+        volumes:
-        secret:
+        - name: https-tls
-          defaultMode: 420
+          secret:
-          secretName: oidc-auth-apps-certificate
+            defaultMode: 420
-      volumeMounts:
+            secretName: oidc-auth-apps-certificate
-      - mountPath: /etc/dex/tls/
+        volumeMounts:
-        name: https-tls
+        - mountPath: /etc/dex/tls/
-  script: merge_certificate_mounts.py "{{ new_overrides }}"
+          name: https-tls
-  become_user: postgres
+    script: merge_certificate_mounts.py "{{ new_overrides }}"
-  become: yes
+    become_user: postgres
-  register: yaml_merge_out
+    become: yes
+    register: yaml_merge_out
 
-- name: Create override file for dex helm chart
+  - name: Create override file for dex helm chart
-  shell: echo "{{ yaml_merge_out.stdout }}" > /tmp/dex-override.yaml
+    shell: echo "{{ yaml_merge_out.stdout }}" > /tmp/dex-override.yaml
+
+  when: host_software_version is version("21.12", '>')
+
+- name: Create override file for oidc-auth-apps-certificate (legacy version)
+  copy:
+    dest: "/tmp/dex-override.yaml"
+    content: |
+      certs:
+        web:
+          secret:
+            tlsName: oidc-auth-apps-certificate
+            caName: oidc-auth-apps-certificate
+        grpc:
+          secret:
+            serverTlsName: oidc-auth-apps-certificate
+            clientTlsName: oidc-auth-apps-certificate
+            caName: oidc-auth-apps-certificate
+  when: host_software_version is version("21.12", '==')
 
 - name: Create override file for secret-observer helm chart
   copy:
@@ -115,3 +143,34 @@
   until: wait_oidc_ep is not failed
   retries: 3
   delay: 90
+  when: host_software_version is version("21.12", '>')
+
+- name: Wait for oidc-auth-apps pods to become active (legacy version)
+  command: >-
+    kubectl wait -n kube-system --for=condition=Ready pods --selector app=dex --timeout=90s
+  environment:
+    KUBECONFIG: /etc/kubernetes/admin.conf
+  register: wait_oidc_ep
+  until: wait_oidc_ep is not failed
+  retries: 3
+  delay: 90
+  when: host_software_version is version("21.12", '==')
+
+- name: Check if kube-apiserver parameters are applied
+  shell: |
+    ps -ef | grep kube-apiserver | grep oidc | grep -v grep | \
+    awk '{ if ($0 != "") print "true"; exit}'
+  register: api_process_oidc_output
+
+- name: Warn the user if the kube-apiserver doesn't have oidc parameters applied
+  fail:
+    msg: |
+      [WARNING]: oidc parameters are not applied to the kube-apiserver.
+      This may cause oidc-auth-apps to not work propperly.
+      Execute:
+          "system service-parameter-list | grep oidc | grep kube_apiserver"
+      to check if the parameters are set correctly, then:
+          "system service-parameter-apply kubernetes"
+      to apply them if they are.
+  when: not (api_process_oidc_output.stdout | bool)
+  ignore_errors: true

playbookconfig/src/playbooks/roles/rehome-subcloud/update-k8s-root-ca/tasks/main.yml

@@ -49,7 +49,7 @@
     shell: source /etc/platform/openrc; sw-manager kube-rootca-update-strategy show
     register: check_rootca_strategy
     until: check_rootca_strategy.stdout is search('applied')
-    retries: 30
+    retries: 50
     delay: 60
 
   - name: Delete rootca strategy

playbookconfig/src/playbooks/roles/rehome-subcloud/update-keystone-data/tasks/migrate_keystone_ids.yml

@@ -78,5 +78,5 @@
     - "cert-alarm"
   register: service_status
   until: service_status.stdout == '1'
-  retries: 10
+  retries: 20
-  delay: 10
+  delay: 15