Create update_subcloud playbook

This playbook changes the management network to the
admin network on a subcloud and updates the admin
endpoints. The admin address pool and network must be
created beforehand and assigned to an interface.

Test Plan:

Passed in IPv4 AIOSX subcloud:
- Run update_subcloud playbook in the systemcontroller
- Verify admin route was created in the subcloud
- Verify admin endpoints were updated in the subcloud
- Verify no errors was produced in service logs(sysinv,
patching, nfv-vim, fm).
- Verify subcloud was online after the playbook.
- Verify subcloud was in-sync after being managed.
- Sanity test for subcloud-backup create.

Passed in IPv6 AIOSX subcloud:
- Trigger the playbook by "dcmanager subcloud-update"
command.
- Verify admin route was created in the subcloud
- Verify admin endpoints were updated in the subcloud
- Verify no errors was produced in service logs(sysinv,
patching, nfv-vim, fm).
- Verify subcloud was online after the playbook.
- Verify subcloud was in-sync after being managed.

Story: 2010319
Task: 47176

Signed-off-by: Hugo Brito <hugo.brito@windriver.com>
Co-Authored-by: Yuxing Jiang <yuxing.jiang@windriver.com>
Change-Id: If5585544c340b718c75de2efdb35c8de03d30068

playbookconfig/src/playbooks/create_subcloud_backup.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -13,7 +13,7 @@
   become: yes
 
   roles:
-    - common/subcloud-bnr-precheck
+    - common/remote-execution-precheck
     - common/prepare-env
     - subcloud-bnr/backup
 
@@ -21,6 +21,7 @@
     optimized_bnr_supported: true
     override_files_dir: "{{ lookup('env', 'HOME') }}"
     password_change: false
+    playbook_overrides: "{{ subcloud_bnr_overrides }}"
     vault_pass: "{{ ansible_become_pass|list|shuffle|join }}"
     vault_id_file: "/tmp/vaultid_{{ inventory_hostname }}.tmp"
     default_backup_prefix: "{{ inventory_hostname }}_platform_backup"

playbookconfig/src/playbooks/delete_subcloud_backup.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -14,10 +14,11 @@
   become: yes
 
   roles:
-    - common/subcloud-bnr-precheck
+    - common/remote-execution-precheck
     - subcloud-bnr/delete
 
   vars:
     override_files_dir: "{{ lookup('env', 'HOME') }}"
     password_change: false
+    playbook_overrides: "{{ subcloud_bnr_overrides }}"
     local_backup_dir: "/opt/platform-backup/backups"

playbookconfig/src/playbooks/rehome_subcloud.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -16,10 +16,10 @@
     - common/prepare-env
     - common/validate-target
     - { role: rehome-subcloud/validate-before-rehome, become: yes }
-    - { role: rehome-subcloud/check-services-status, become: yes }
+    - { role: common/check-services-status, become: yes }
     - { role: rehome-subcloud/prepare-env, become: yes }
     - { role: rehome-subcloud/update-ca-cert, become: yes }
     - { role: rehome-subcloud/update-network-config, become: yes }
-    - { role: rehome-subcloud/update-sc-cert, become: yes }
+    - { role: common/update-sc-cert, become: yes }
     - { role: rehome-subcloud/update-keystone-data, become: yes }
-    - { role: rehome-subcloud/check-services-status, become: yes }
+    - { role: common/check-services-status, become: yes }

playbookconfig/src/playbooks/restore_subcloud_backup.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -14,7 +14,7 @@
   become: yes
 
   roles:
-    - common/subcloud-bnr-precheck
+    - common/remote-execution-precheck
     - common/prepare-env
     - subcloud-bnr/restore
 
@@ -22,6 +22,7 @@
     optimized_bnr_supported: true
     override_files_dir: "{{ lookup('env', 'HOME') }}"
     password_change: false
+    playbook_overrides: "{{ subcloud_bnr_overrides }}"
     vault_pass: "{{ ansible_become_pass|list|shuffle|join }}"
     vault_id_file: "/tmp/vaultid_{{ inventory_hostname }}.tmp"
     default_backup_prefix: "{{ inventory_hostname }}_platform_backup"

playbookconfig/src/playbooks/roles/common/check-services-status/meta/main.yml

@@ -0,0 +1,8 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+allow_duplicates: true

playbookconfig/src/playbooks/roles/common/check-services-status/tasks/main.yml

@@ -1,12 +1,11 @@
 ---
 #
-# Copyright (c) 2021 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 # ROLE DESCRIPTION:
-#   This role checks related services are all enabled-active before
-#   finishing the rehome playbook
+#   This role checks related services are all enabled-active
 #
 
 - name: Check all services are enabled-active

playbookconfig/src/playbooks/roles/common/remote-execution-precheck/tasks/main.yml

@@ -1,19 +1,19 @@
 ---
 #
-# Copyright (c) 2022 Wind River Systems, Inc.
+# Copyright (c) 2022-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# Precheck to ensure all subcloud backup and restore related playbooks are
-# executed on the system controller and the overrides file is provided.
+#   Precheck to ensure the playbooks are executed on the system controller
+#   and the overrides file is provided.
 #
-- name: Fail if subcloud_bnr_overrides parameter is not specified
+- name: Fail if playbook overrides parameter is not specified
   fail:
     msg: "Location of operation overrides must be specified."
-  when: subcloud_bnr_overrides is undefined
+  when: playbook_overrides is undefined
 
 - name: Load input parameters for operation
-  include_vars: "{{ subcloud_bnr_overrides }}"
+  include_vars: "{{ playbook_overrides }}"
 
 - name: Retrieve the distributed cloud role
   shell: source /etc/platform/platform.conf; echo $distributed_cloud_role

playbookconfig/src/playbooks/roles/common/update-sc-cert/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2021-2022 Wind River Systems, Inc.
+# Copyright (c) 2021-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -22,7 +22,7 @@
   command: >-
     kubectl --kubeconfig=/etc/kubernetes/admin.conf delete ns
     "{{ sc_adminep_ca_cert_ns }}"
-  when: get_sc_adminep_ca_cert_ns
+  when: get_sc_adminep_ca_cert_ns.stdout != ""
 
 - name: Set up subcloud admin endpoints certificates
   import_role:

playbookconfig/src/playbooks/roles/update-subcloud/prepare-env/tasks/main.yml

@@ -0,0 +1,107 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#   This role is to check the target host environment before proceeding to
+#   the next step.
+#
+
+- name: Get subcloud active controller
+  shell: |
+    source /etc/platform/openrc
+    system host-show $(cat /etc/hostname) --column hostname --format value
+  register: controller_name
+
+- name: Set subcloud admin floating address fact
+  set_fact:
+    sc_floating_address: "{{ admin_floating_address }}"
+
+- name: Retrieve software version number
+  # lookup module does not work with /etc/build.info as it does not have ini
+  # format. Resort to shell source.
+  shell: source /etc/build.info; echo $SW_VERSION
+  register: sw_version_result
+
+- name: Fail if software version is not defined
+  fail:
+    msg: "SW_VERSION is missing in /etc/build.info"
+  when: sw_version_result.stdout_lines|length == 0
+
+- name: Retrieve system mode
+  shell: source /etc/platform/platform.conf; echo $system_mode
+  register: system_mode_result
+
+- name: Fail if system mode is not defined
+  fail:
+    msg: "system_mode is missing in /etc/platform/platform.conf"
+  when: system_mode_result.stdout_lines|length == 0
+
+- name: Set host software version, system mode
+  set_fact:
+    software_version: "{{ sw_version_result.stdout }}"
+    system_mode: "{{ system_mode_result.stdout }}"
+
+- name: Set config path fact
+  set_fact:
+    config_permdir: "{{ platform_path + '/config/' + software_version }}"
+
+- name: Check existing admin network
+  shell: >-
+    source /etc/platform/openrc; system network-list | grep admin
+  register: existing_admin_network
+
+- name: Fail if admin network is not configured
+  fail:
+    msg: "admin network is not configured"
+  when: existing_admin_network == ""
+
+- name: Get ifname of the existing admin network of controller-0
+  shell: >-
+    source /etc/platform/openrc; system interface-network-list controller-0 |
+    awk '$8 == "admin" { print $6 }'
+  register: controller_0_admin_network_if
+
+- name: Fail if admin network interface of controller-0 is not assigned
+  fail:
+    msg: "admin network interfaces of controller-0 is not assigned"
+  when: controller_0_admin_network_if.stdout == ""
+
+- name: Set admin_if_c0 fact
+  set_fact:
+    admin_if_c0: "{{ controller_0_admin_network_if.stdout_lines[0] }}"
+
+- block:
+  - name: Get ifname of the existing admin network of controller-1
+    shell: >-
+      source /etc/platform/openrc; system interface-network-list controller-1 |
+      awk '$8 == "admin" { print $6 }'
+    register: controller_1_admin_network_if
+
+  - name: Fail if admin network interface of controller-1 is not assigned
+    fail:
+      msg: "admin network interfaces of controller-1 is not assigned"
+    when: controller_1_admin_network_if.stdout == ""
+
+  - name: Set admin_if_c1 fact
+    set_fact:
+      admin_if_c1: "{{ controller_1_admin_network_if.stdout_lines[0] }}"
+
+  when: system_mode != "simplex"
+
+- name: Get existing admin floating address
+  shell: >-
+    source /etc/platform/openrc; system addrpool-list |
+    awk ' $4 == "admin" && $14 == "{{ admin_floating_address }}" '
+  register: check_existing_admin_floating_address
+
+- name: Get existing admin gateway
+  shell: >-
+    source /etc/platform/openrc; system addrpool-list |
+    awk ' $4 == "admin" && $20 == "{{ admin_gateway_address }}" '
+  register: check_existing_admin_gateway
+
+- name: Set admin gateway and floating address update required facts
+  set_fact:
+    admin_floating_update_required: "{{ true if check_existing_admin_floating_address.stdout | bool else false }}"
+    admin_gateway_update_required: "{{ true if check_existing_admin_gateway.stdout | bool else false }}"

playbookconfig/src/playbooks/roles/update-subcloud/prepare-env/vars/main.yml

@@ -0,0 +1,8 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+platform_path: /opt/platform

playbookconfig/src/playbooks/roles/update-subcloud/restart-services/tasks/main.yaml

@@ -0,0 +1,47 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#   This role is to restart services post admin network reconfiguration to
+#   ensure all the services are healthy
+#
+
+- name: Restart affected services using pmon
+  command: "pmon-restart {{ item }}"
+  loop:
+    - "sw-patch-controller-daemon"
+    - "sm-api"
+    - "fm-api"
+
+- name: Restart sm managed services
+  command: sm-restart-safe service {{ item }}
+  with_items:
+    - "sysinv-inv"
+    - "sysinv-conductor"
+    - "cert-mon"
+    - "cert-alarm"
+    - "fm-mgr"
+    - "dcdbsync-api"
+    - "barbican-keystone-listener"
+
+- name: Wait until services are restarted
+  shell: sm-query service {{ item }} | grep -c enabled-active
+  loop:
+    - "sysinv-inv"
+    - "sysinv-conductor"
+    - "cert-mon"
+    - "cert-alarm"
+    - "fm-mgr"
+    - "dcdbsync-api"
+    - "barbican-keystone-listener"
+  register: service_status
+  until: service_status.stdout == '1'
+  retries: 10
+  delay: 10
+
+- name: Restart vim service
+  # Vim need to restart at the end until the other services are restarted and
+  # return to healthy
+  command: sm-restart service vim

playbookconfig/src/playbooks/roles/update-subcloud/update-admin-endpoints/tasks/check_and_update_endpoint.yml

@@ -0,0 +1,29 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#   These tasks validate and update admin endpoints
+#
+
+- name: "Get {{ service }} admin endpoint UUID"
+  shell: >-
+    source /etc/platform/openrc; openstack endpoint list |
+    awk '($12 == "admin" && $6 == "{{ service }}") { print $2 }'
+  register: admin_endpoint_UUID
+
+- name: "Check {{ service }} admin endpoint URL"
+  shell: >-
+    source /etc/platform/openrc; openstack endpoint list |
+    awk ' $12 == "admin" && $6 == "{{ service }}" ' |
+    grep "{{ parsed_admin_floating_address }}"
+  register: Check_admin_endpoint_URL
+
+- name: "Correct {{ service }} admin endpoint"
+  shell: >-
+    source /etc/platform/openrc; openstack endpoint set
+    --region {{ region_name.stdout }} --interface admin
+    --url https://{{ parsed_admin_floating_address }}:{{ port }}
+    --service {{ service }} --enable admin_endpoint_UUID.stdout
+  when: Check_admin_endpoint_URL.stdout == ''

playbookconfig/src/playbooks/roles/update-subcloud/update-admin-endpoints/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#   This role is to update the openstack admin endpoints to reflect the admin
+#   network reconfiguration change
+#
+
+- name: Wrap admin_floating_address in [] brackets if it's an ipv6 address
+  set_fact:
+    parsed_admin_floating_address: "{{ admin_floating_address | ipwrap }}"
+
+- name: Retrieve admin endpoint region name
+  shell: >-
+    source /etc/platform/openrc; system show |
+    grep region_name | awk '{ print $4 }'
+  register: region_name
+
+- name: Create service endpoint list
+  set_fact:
+    service_list:
+      - { port: "5001", service: "keystone" }
+      - { port: "6386/v1", service: "sysinv" }
+      - { port: "5492", service: "patching" }
+      - { port: "4546", service: "vim" }
+      - { port: "18003", service: "fm" }
+      - { port: "9312", service: "barbican" }
+      - { port: "8220/v1.0", service: "dcdbsync" }
+
+- name: Check and update admin endpoints
+  include_tasks: check_and_update_endpoint.yml
+  vars:
+    service: "{{ item.service }}"
+    port: "{{ item.port }}"
+  loop: "{{ service_list }}"

playbookconfig/src/playbooks/roles/update-subcloud/update-network/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#   This role is to update the admin network and add static route to the
+#   system controller via the admin subnet
+#
+
+- name: Update admin subnet
+  import_tasks: update_admin_subnet.yml
+  when: admin_floating_update_required or admin_gateway_update_required
+
+- name: Check existing host route in controller-0
+  shell: >-
+    source /etc/platform/openrc; system host-route-list controller-0 |
+    awk '/{{ admin_gateway_address }}/'
+  register: existing_route_controller_0
+
+- name: Add static route from controller-0 to system controller
+  shell: >-
+    source /etc/platform/openrc; system host-route-add controller-0
+    {{ admin_if_c0 }}
+    {{ (system_controller_subnet | ipaddr(0)).split('/')[0] }}
+    {{ system_controller_subnet | ipaddr('prefix') }}
+    {{ admin_gateway_address }}
+  when: existing_route_controller_0.stdout | length == 0
+
+- block:
+  - name: Check existing host route in controller-1
+    shell: >-
+      source /etc/platform/openrc; system host-route-list controller-1 |
+      awk '/{{ admin_gateway_address }}/'
+    register: existing_route_controller_1
+
+  - name: Add static route from controller-1 to system controller
+    shell: >-
+      source /etc/platform/openrc; system host-route-add controller-1
+      {{ admin_if_c1 }}
+      {{ (system_controller_subnet | ipaddr(0)).split('/')[0] }}
+      {{ system_controller_subnet | ipaddr('prefix') }}
+      {{ admin_gateway_address }}
+    when: existing_route_controller_1.stdout | length == 0
+  when: system_mode != 'simplex'

playbookconfig/src/playbooks/roles/update-subcloud/update-network/tasks/update_admin_subnet.yml

@@ -0,0 +1,66 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# SUB-TASKS DESCRIPTION:
+#   These tasks update the subcloud's admin network configurations.
+#
+
+- name: Get previous subcloud admin network address pool uuid
+  shell: >-
+    source /etc/platform/openrc;
+    system addrpool-list | awk '/admin/{ print$2 }'
+  register: subcloud_admin_pool_uuid
+
+- name: Delete previous subcloud admin network address pool
+  shell: >-
+    source /etc/platform/openrc; system addrpool-delete
+    {{ subcloud_admin_pool_uuid.stdout }}
+  when: subcloud_admin_pool_uuid.stdout | length > 0
+
+- name: Add subcloud admin network address pool
+  shell: >-
+    source /etc/platform/openrc; system addrpool-add admin
+    {{ (admin_subnet | ipaddr(0)).split('/')[0]  }}
+    {{ admin_subnet | ipaddr('prefix') }}
+    --floating-address {{ admin_floating_address }}
+    --gateway-address {{ admin_gateway_address }} |
+    awk '/uuid/{ print$4 }'
+  register: add_subcloud_admin_pool_uuid
+
+- name: Add new subcloud admin network
+  shell: >-
+    source /etc/platform/openrc;
+    system network-add admin admin false
+    {{ add_subcloud_admin_pool_uuid.stdout }}
+
+- name: Add new subcloud admin interface network association on controller-0
+  shell: >-
+    source /etc/platform/openrc;
+    system interface-network-assign controller-0
+    {{ admin_if_c0 }} admin
+
+- name: Add new subcloud admin interface network association on controller-1
+  shell: >-
+    source /etc/platform/openrc;
+    system interface-network-assign controller-1
+    {{ admin_if_c1 }} admin
+  when: system_mode != 'simplex'
+
+- name: Wait for ip address configured on the interface of controller-0
+  shell: ip address show "{{ admin_if_c0 }}" | grep -wc "{{ admin_floating_address }}"
+  register: check_c0_new_config
+  until: check_c0_new_config != "0"
+  retries: 30
+  delay: 10
+  when: controller_name.stdout == "controller-0"
+
+- name: Wait for ip address configured on the interface of controller-1
+  shell: ip address show "{{ admin_if_c1 }}" | grep -wc "{{ admin_floating_address }}"
+  register: check_c1_new_config
+  until: check_c1_new_config != "0"
+  retries: 30
+  delay: 10
+  when: controller_name.stdout == "controller-1"

playbookconfig/src/playbooks/update_subcloud.yml

@@ -0,0 +1,34 @@
+---
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#   This playbook updates the admin network of a subcloud.
+#
+#   Example command:
+#   ansible-playbook update_subcloud.yml -i <inventory-file>  \
+#     -e 'subcloud_update_overrides=<subcloudname>_update_values.yml'
+#
+
+- hosts: all
+  gather_facts: no
+
+  vars_files:
+    - vars/common/main.yml
+
+  roles:
+    - common/remote-execution-precheck
+    - common/prepare-env
+    - common/validate-target
+    - update-subcloud/prepare-env
+    - update-subcloud/update-network
+    - update-subcloud/update-admin-endpoints
+    - { role: common/update-sc-cert, become: yes }
+    - { role: update-subcloud/restart-services, become: yes }
+    - common/check-services-status
+
+  vars:
+    override_files_dir: "{{ lookup('env', 'HOME') }}"
+    password_change: false
+    playbook_overrides: "{{ subcloud_update_overrides }}"