Support bootstrap replay with networking changes

Currently bootstrap playbook replay will fail if the management or
cluster host networks are changed. To resolve this a couple of changes
are needed:

* Restart the sysinv agent and wait until it is ready. The sysinv agent
  uses the current management ip for the rabbitMQ connection and
  applying runtime manifests. The process needs to be restarted to
  resync that data.

* Copy the etcd certs to the /opt/platform on replay. The etcd-server
  certs are regenerated on replay. When the cluster host network changed
  the SAN in the certs under /opt/platform were out of date resulting in
  kube-apiserver failures on controller-0 unlock.

Closes-Bug: 1925668
Signed-off-by: David Sullivan <david.sullivan@windriver.com>
Change-Id: I228321a2540a0024cd217ed844feb54be9ae3b29

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml

@@ -68,6 +68,29 @@
 - include: one_time_config_tasks.yml
   when: not initial_db_populated
 
+- name: Find etcd certs files
+  find:
+    paths: "/etc/etcd"
+    patterns:
+      - '*.crt'
+      - '*.key'
+  register: etcd_certs_find_output
+
+- name: Copy etcd certificates to etcd certs directory
+  copy:
+    src: "/etc/etcd/{{ item }}"
+    dest: "{{ etcd_certs_dir }}/{{ item }}"
+    remote_src: yes
+    force: yes
+  with_items:
+    - "etcd-server.crt"
+    - "etcd-server.key"
+    - "ca.crt"
+    - "ca.key"
+    - "etcd-client.crt"
+    - "etcd-client.key"
+  when: etcd_certs_find_output.matched != 0
+
 # Banner customization is not part of one_time_config_task.yml as the user may
 # choose to change their banner look and feel and replay.
 - name: Check if custom banner exists

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/one_time_config_tasks.yml

@@ -18,7 +18,6 @@
     pxe_config_dir: "{{ config_permdir + '/pxelinux.cfg' }}"
     branding_config_dir: "{{ config_permdir + '/branding' }}"
     ssl_ca_certs_dir: "{{ config_permdir + '/ssl_ca' }}"
-    etcd_certs_dir: "{{ config_permdir + '/etcd' }}"
 
 - debug:
     msg: >-
@@ -26,7 +25,6 @@
       pxe_config_dir: {{ pxe_config_dir }}
       branding_config_dir: {{ branding_config_dir }}
       ssl_ca_certs_dir: {{ ssl_ca_certs_dir }}
-      etcd_certs_dir: {{ etcd_certs_dir }}
 
 - name: Ensure Postres, PXE config directories exist
   file:
@@ -67,29 +65,6 @@
     remote_src: yes
   with_items: "{{ postgres_result.files }}"
 
-- name: Find etcd certs files
-  find:
-    paths: "/etc/etcd"
-    patterns:
-      - '*.crt'
-      - '*.key'
-  register: etcd_certs_find_output
-
-- name: Copy etcd certificates to etcd certs directory
-  copy:
-    src: "/etc/etcd/{{ item }}"
-    dest: "{{ etcd_certs_dir }}/{{ item }}"
-    remote_src: yes
-    force: yes
-  with_items:
-    - "etcd-server.crt"
-    - "etcd-server.key"
-    - "ca.crt"
-    - "ca.key"
-    - "etcd-client.crt"
-    - "etcd-client.key"
-  when: etcd_certs_find_output.matched != 0
-
 - name: Create a symlink to PXE config files
   file:
     src: "{{ pxe_config_dir }}"

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/update_sysinv_database.yml

@@ -98,22 +98,29 @@
     msg: Timeout waiting for system controller database configuration to complete
   when: distributed_cloud_role == 'systemcontroller'
 
-- block:
-  - name: Update sysinv with new region name
-    replace:
-      path: /etc/sysinv/sysinv.conf
-      regexp: "region_name=.*$"
-      replace: "region_name={{ region_name }}"
 
-  - name: Restart sysinv-agent and sysinv-api to pick up sysinv.conf update
-    command: "{{ item }}"
-    with_items:
-      - /etc/init.d/sysinv-agent restart
-      - /usr/lib/ocf/resource.d/platform/sysinv-api reload
-    environment:
-      OCF_ROOT: "/usr/lib/ocf"
+- name: Update sysinv with new region name
+  replace:
+    path: /etc/sysinv/sysinv.conf
+    regexp: "region_name=.*$"
+    replace: "region_name={{ region_name }}"
   when: distributed_cloud_role == 'subcloud'
 
+- name: Restart sysinv-agent and sysinv-api to pick up sysinv.conf update
+  command: "{{ item }}"
+  with_items:
+    - /etc/init.d/sysinv-agent restart
+    - /usr/lib/ocf/resource.d/platform/sysinv-api reload
+  environment:
+    OCF_ROOT: "/usr/lib/ocf"
+
+- name: Wait for sysinv inventory
+  wait_for:
+    path: /var/run/sysinv/.sysinv_reported
+    state: present
+    timeout: 600
+    msg: Timeout waiting for system inventory to complete
+
 - name: Set flag to mark the initial db population completed milestone
   file:
     path: "{{ initial_db_populated_flag }}"

playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml

@@ -8,3 +8,4 @@ ssl_ca_complete_flag: /etc/platform/.ssl_ca_complete
 region_config: no
 source_device_image_bind_dir: /opt/platform/device_images
 target_device_image_bind_dir: /www/pages/device_images
+etcd_certs_dir: "{{ config_permdir + '/etcd' }}"