Add IPsec disable and enable ansible playbook

This change added IPsec disable and enable ansible playbook.
This playbook is supposed to be an engineering tool that can be used
for testing/debugging purpose. It's not intented to be used by end user
as an admin tool.

The playbooks can be run:
ansible-playbook \
/usr/share/ansible/stx-ansible/playbooks/disable-ipsec.yml -e \
"ansible_ssh_user=sysadmin ansible_ssh_pass=<sysadmin passwd> \
ansible_become_pass=<sudo passwd>"

ansible-playbook \
/usr/share/ansible/stx-ansible/playbooks/enable-ipsec.yml -e \
"ansible_ssh_user=sysadmin ansible_ssh_pass=<sysadmin passwd> \
ansible_become_pass=<sudo passwd>"

Test Plan (on a multi node system, eg. DX+ with one worker node):
PASS: Run the IPsec disable ansible playbook to disable IPsec, verify
      strongSwan IPsec service, ipsec-server service are stopped,
      ipsec-config SM service is removed from sm-dump output,
      swanctl configuration files are removed, and no IPsec SAs
      exist, all hosts in enabled|available states.
PASS: Swact active controller, verify swact is successful, all hosts are
      in enabled/available states.
PASS: Lock and unlock the standby controller, verify the controller
      is unlocked successfully, all hosts are in enabled/available
      states after the controller reboots.
PASS: Run the IPsec enable ansible playbook to enable IPsec, verify
      strongSwan IPsec service, ipsec-server service are started,
      ipsec-config SM service presents in sm-dump output,
      swanctl configuration files are generated, IPsec SAs established
      among hosts, and all hosts are in enabled|available states.
PASS: Swact active controller, verify swact is successful, all hosts are
      in enabled/available states.
PASS: Lock and unlock the standby controller, verify the controller
      is unlocked successfully, all hosts are in enabled/available
      states after the controller reboots.

Story: 2010940
Task: 50964

Change-Id: I32533f5220178e967e6a23a2354fb904b660d651
Signed-off-by: Andy Ning <andy.ning@windriver.com>

playbookconfig/src/playbooks/disable-ipsec.yml

@@ -0,0 +1,30 @@
+---
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This playbook provides the capability to disable IPsec on all hosts
+# in a multi node system, such as AIO-DX, Stanadard, etc.
+#
+# The playbook is supposed to run on Active controller.
+#
+# Example to run the playbook:
+# ansible-playbook /usr/share/ansible/stx-ansible/playbooks/roles/disable-ipsec.yml \
+#   -e "ansible_ssh_user=sysadmin ansible_ssh_pass=<ssh_passwd> ansible_become_pass=<sudo passwd>"
+#
+
+- hosts: all
+  gather_facts: no
+
+  roles:
+    - role: configure-ipsec/update_heartbeat_failure_action
+      vars:
+        action: 'alarm'
+    - role: configure-ipsec/disable-ipsec-on-nodes
+      become: yes
+    - role: configure-ipsec/cleanup-after-disable-ipsec
+      become: yes
+    - role: configure-ipsec/update_heartbeat_failure_action
+      vars:
+        action: 'fail'

playbookconfig/src/playbooks/enable-ipsec.yml

@@ -0,0 +1,32 @@
+---
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This playbook provides the capability to enable IPsec on all hosts
+# in a multi node system, such as AIO-DX, Stanadard, etc.
+#
+# The playbook is supposed to run on Active controller.
+#
+# Example to run the playbook:
+# ansible-playbook /usr/share/ansible/stx-ansible/playbooks/roles/enable-ipsec.yml \
+#   -e "ansible_ssh_user=sysadmin ansible_ssh_pass=<ssh_passwd> ansible_become_pass=<sudo passwd>"
+#
+
+- hosts: all
+  gather_facts: no
+
+  roles:
+    - role: configure-ipsec/update_heartbeat_failure_action
+      vars:
+        action: 'alarm'
+    - role: configure-ipsec/prep-enable-ipsec
+      become: yes
+
+    - role: configure-ipsec/enable-ipsec-on-nodes
+      become: yes
+
+    - role: configure-ipsec/update_heartbeat_failure_action
+      vars:
+        action: 'fail'

playbookconfig/src/playbooks/roles/configure-ipsec/cleanup-after-disable-ipsec/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   This role is to do cleanup after IPsec is disabled on all hosts
+#   in the system.
+#
+
+- name: Get system mode
+  shell: source /etc/platform/platform.conf; echo $system_mode
+  register: system_mode
+
+- block:
+  - name: Clear mgmt_ipsec flag in sysinv i_host table
+    script: roles/common/files/clear-mgmt-ipsec-flag.py
+    become_user: postgres
+
+  when: system_mode != "simplex"

playbookconfig/src/playbooks/roles/configure-ipsec/disable-ipsec-on-nodes/tasks/cleanup-services.yml

@@ -26,8 +26,8 @@
 
   - name: Disable strongswan service
     command: >-
-      ansible all -i "{{ item }}," -m command -a "systemctl disable ipsec.service --now" -b
-      -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible all -i "{{ item }}," -m command -a "systemctl disable strongswan-starter.service
+      --now" -b -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
       ansible_become_pass={{ ansible_become_pass }}"
 
   - name: Flush IPsec policies
@@ -36,6 +36,12 @@
       "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
       ansible_become_pass={{ ansible_become_pass }}"
 
+  - name: Flush IPsec states
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "ip xfrm state flush" -b -e
+      "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible_become_pass={{ ansible_become_pass }}"
+
   - name: Fail if strongswan remains active
     command: >-
       ansible all -i "{{ item }}," -m command -a "systemctl is-active ipsec.service" -b -e

playbookconfig/src/playbooks/roles/configure-ipsec/disable-ipsec-on-nodes/tasks/main.yml

@@ -20,6 +20,15 @@
   - set_fact:
       pxeboot_addrs: "{{ ip_addrs_list.stdout }}"
 
+  - name: Deprovision ipsec-config service
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "sm-deprovision service-group-member
+      controller-services ipsec-config --apply" -b -e "ansible_ssh_user={{ ansible_ssh_user }}
+      ansible_ssh_pass={{ ansible_ssh_pass }} ansible_become_pass={{ ansible_become_pass }}"
+    with_items:
+      - controller-0
+      - controller-1
+
   - name: Clean up IPsec services on hosts
     include_tasks: cleanup-services.yml
     loop: "{{ pxeboot_addrs }}"
@@ -41,13 +50,4 @@
     retries: 18
     delay: 10
 
-  - name: Deprovision ipsec-config service
-    command: >-
-      ansible all -i "{{ item }}," -m command -a "sm-deprovision service-group-member
-      controller-services ipsec-config --apply" -b -e "ansible_ssh_user={{ ansible_ssh_user }}
-      ansible_ssh_pass={{ ansible_ssh_pass }} ansible_become_pass={{ ansible_become_pass }}"
-    with_items:
-      - controller-0
-      - controller-1
-
   when: system_mode.stdout != "simplex"

playbookconfig/src/playbooks/roles/configure-ipsec/enable-ipsec-on-nodes/tasks/main.yml

@@ -29,6 +29,21 @@
       pending_hosts: "{{ pending_hosts.stdout }}"
       pxeboot_addrs: "{{ ip_addrs_list.stdout }}"
 
+  # It is found that when enabling IPsec, if the active controller is enabled before
+  # worker node, worker node will no longer have the shared FS (/opt/platform) mounted,
+  # the ipsec-client command will hang forever. Further investigation shows the
+  # puppet apply invoked by ipsec-client is blocked at resolving "file system" facts,
+  # and strace shows statfs() system call is blocked on the shared FS (/opt/platform).
+  # The solution here is to create a facter.conf file to exclude "file system" fact
+  # resolving for puppet apply.
+  - name: Create facter.conf to exclude "file system" fact resolve in puppet apply
+    command: >-
+      ansible all -i "{{ item }}," -m shell -a "mkdir -p /etc/puppetlabs/facter &&
+      echo -e \"facts : {\n  blocklist : [ \"file system\" ],\n}\" >
+      /etc/puppetlabs/facter/facter.conf" -b -e "ansible_ssh_user={{ ansible_ssh_user }}
+      ansible_ssh_pass={{ ansible_ssh_pass }} ansible_become_pass={{ ansible_become_pass }}"
+    loop: "{{ pxeboot_addrs }}"
+
   - name: Execute initial-auth operation on hosts
     include_tasks: execute-initial-auth-operation.yml
     when: 'pending_hosts | length > 0'
@@ -60,4 +75,11 @@
       - controller-0
       - controller-1
 
+  - name: Remove the created facter.conf and directories
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "rm -rf /etc/puppetlabs"
+      -b -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible_become_pass={{ ansible_become_pass }}"
+    loop: "{{ pxeboot_addrs }}"
+
   when: system_mode.stdout != "simplex"

playbookconfig/src/playbooks/roles/configure-ipsec/prep-enable-ipsec/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   This role starts services required to enable IPsec on all hosts
+#   in the system.
+#
+
+- name: Get system mode
+  shell: source /etc/platform/platform.conf; echo $system_mode
+  register: system_mode
+
+- block:
+  - name: Enable and start strongswan IPsec service
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "systemctl enable ipsec.service --now"
+      -b -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible_become_pass={{ ansible_become_pass }}"
+    with_items:
+      - controller-0
+      - controller-1
+
+  - name: Enable and start ipsec-server service
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "systemctl enable ipsec-server.service --now"
+      -b -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible_become_pass={{ ansible_become_pass }}"
+    with_items:
+      - controller-0
+      - controller-1
+
+  - name: Create pmon configuration symlink for ipsec-server
+    command: >-
+      ansible all -i "{{ item }}," -m command -a "ln -sf
+      /usr/share/starlingx/pmon.d/ipsec-server.conf /etc/pmon.d/ipsec-server.conf"
+      -b -e "ansible_ssh_user={{ ansible_ssh_user }} ansible_ssh_pass={{ ansible_ssh_pass }}
+      ansible_become_pass={{ ansible_become_pass }}"
+    with_items:
+      - controller-0
+      - controller-1
+
+  when: system_mode != "simplex"

playbookconfig/src/playbooks/roles/configure-ipsec/update_heartbeat_failure_action/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   This role is to modify mtc heartbeat failure action.
+#
+
+- name: Get system mode
+  shell: source /etc/platform/platform.conf; echo $system_mode
+  register: system_mode
+
+- block:
+  - name: Modify mtc heartbeat failure action
+    shell: >-
+      source /etc/platform/openrc; system service-parameter-modify
+      platform maintenance heartbeat_failure_action={{ action }}
+
+  - name: Apply mtc heartbeat failure action update
+    shell: source /etc/platform/openrc; system service-parameter-apply platform
+
+  when: system_mode != "simplex"