Fix restoring dc-vault on a central controller

At this moment, when we do a restore procedure on a
DC system controller, the /opt/dc-vault directory will be
created under "/" filesystem. It should be created on
a separated filesystem, but that filesystem is available
only after an unlock of the controller.

The proposed solution is to create an additional restore
playbook for the dc-vault that will be manually run after
unlocking controller-0. The backup playbook will create
an additional archive with the contents of dc-vault, and
the dc-vault directory will be removed from the platform
backup.

The new playbook will be used like this:

ansible-playbook
/usr/share/ansible/stx-ansible/playbooks/restore_dc_vault.yml -e
"ansible_become_pass=Li69nux*" -e "admin_password=Li69nux*" -e
"initial_backup_dir=/home/sysadmin" -e
"backup_filename=localhost_dc_vault_backup_2021_02_02_11_46_09.tgz"

Closes-Bug: 1914258
Signed-off-by: Mihnea Saracin <Mihnea.Saracin@windriver.com>
Change-Id: I8fdd5b678e2296cd0ce98ea4dd91e2988beb200f

playbookconfig/src/playbooks/host_vars/backup-restore/default.yml

@@ -75,6 +75,12 @@ openstack_backup_filename_prefix: "{{ inventory_hostname }}_openstack_backup"
 #
 # This variable is used for StarlingX OpenStack application restore only
 #
+
+# The dc_vault backup tarball will be named in this format:
+# <dc_vault_backup_filename_prefix>_<timestamp>.tgz
+#
+dc_vault_backup_filename_prefix: "{{ inventory_hostname }}_dc_vault_backup"
+
 restore_cinder_glance_data: false
 
 # Default directory where the system backup tarballs fetched from the

playbookconfig/src/playbooks/restore_dc_vault.yml

@@ -0,0 +1,16 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+- hosts: all
+  gather_facts: no
+
+  vars_files:
+    - host_vars/backup-restore/default.yml
+
+  roles:
+    - { role: common/prepare-env }
+    - { role: restore-dc-vault/prepare-env }
+    - { role: restore-dc-vault/restore-dc-vault-directory, become: yes }

playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml

@@ -193,7 +193,6 @@
         - "{{ patching_permdir }}"
         - "{{ patching_repo_permdir }}"
         - "{{ extension_permdir }}"
-        - "{{ dc_vault_permdir }}"
         - "{{ deploy_permdir }}"
         - "{{ postgres_dir.path }}"
         - "{{ armada_permdir }}"
@@ -211,6 +210,18 @@
       loop_control:
         label: "{{ item.item }}"
 
+    # For SystemController the dc-vault is part of platform but restored after controller-0 unlock
+    # Create a separate archive for it
+    - block:
+        - name: Check the size (in KiB) of directories that will be backed up for dc-vault
+          shell: "du -sh -k  {{ dc_vault_permdir }} | awk '{print $1}'"
+          register: size_output_dc_vault
+
+        - name: Estimate the total required disk size for platform backup archive
+          set_fact:
+            total_platform_size_estimation: "{{ total_platform_size_estimation|int + size_output_dc_vault.stdout|int }}"
+      when: check_dc_controller.rc == 0
+
     - name: Check the free space in the archive dir
       shell: "df -k {{ backup_dir }} --output=avail | tail -1"
       register: df_output
@@ -347,12 +358,14 @@
         platform_backup_file: "{{ platform_backup_filename_prefix }}_{{ backup_timestamp }}.tgz"
         docker_local_registry_backup_file: "{{ docker_local_registry_backup_filename_prefix }}_{{ backup_timestamp }}.tgz"
         openstack_backup_file: "{{ openstack_backup_filename_prefix }}_{{ backup_timestamp }}.tgz"
+        dc_vault_backup_file: "{{ dc_vault_backup_filename_prefix }}_{{ backup_timestamp }}.tgz"
 
     - name: Set backup files absolute path
       set_fact:
         platform_backup_file_path: "{{ backup_dir }}/{{ platform_backup_file }}"
         docker_local_registry_backup_file_path: "{{ backup_dir }}/{{ docker_local_registry_backup_file }}"
         openstack_backup_file_path: "{{ backup_dir }}/{{ openstack_backup_file }}"
+        dc_vault_backup_file_path: "{{ backup_dir }}/{{ dc_vault_backup_file }}"
 
     - name: Save user uploaded images from local registry to an archive
       import_tasks: export-user-local-registry-images.yml
@@ -378,7 +391,6 @@
            {{ patching_permdir }} \
            {{ patching_repo_permdir }} \
            {{ extension_permdir }} \
-           {{ dc_vault_permdir }} \
            {{ deploy_permdir }} \
            {{ crushmap_file | default(\"\") }} \
            {{ etcd_snapshot_file }} \
@@ -390,6 +402,13 @@
       args:
         warn: false
 
+    - name: Create a tgz archive for dc-vault backup
+      shell: "tar -czf {{ dc_vault_backup_file_path }} $(ls -d \
+           {{ dc_vault_permdir }} 2>/dev/null)"
+      args:
+        warn: false
+      when: check_dc_controller.rc == 0
+
     - name: Create a tgz archive for OpenStack backup
       shell: "tar -czf {{ openstack_backup_file_path }} $(ls -d \
            {{ armada_permdir }}/stx-openstack \

playbookconfig/src/playbooks/roles/restore-dc-vault/prepare-env/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   This role verifies if dc-vault restore should proceed
+#
+- name: Perform generic user input validation for restore
+  import_role:
+    name: backup-restore/validate-input
+
+- name: Perform generic B&R environment validation
+  import_role:
+    name: backup-restore/prepare-env
+
+- name: Check if restoring dc vault is already in progress
+  stat:
+    path: "{{ restore_dc_vault_in_progress_flag }}"
+  register: restore_dc_vault_in_progress_flag_file
+
+- name: Fail if restoring dc vault is already in progress
+  fail:
+    msg: "Restoring dc-vault is already in progress!"
+  when: restore_dc_vault_in_progress_flag_file.stat.exists
+
+- name: Check if the system is a DC controller
+  command: >-
+    grep -i "distributed_cloud_role\s*=\s*systemcontroller"
+    {{ platform_conf_path }}/platform.conf
+  register: check_dc_controller
+  failed_when: false
+
+- name: Fail if controller is not DC controller
+  fail:
+    msg: "Controller must be a DC systemcontroller in order to restore the dc-vault"
+  when: check_dc_controller.rc != 0
+
+- name: Check if controller is unlocked
+  shell: source /etc/platform/openrc ; system host-show $(cat /etc/hostname) --format value --column administrative
+  register: check_unlocked
+
+- name: Fail if controller is not unlocked
+  fail:
+    msg: "Controller must be in an unlocked state before restoring dc-vault"
+  when: '"unlocked" not in (check_unlocked.stdout | lower)'
+
+# Set the restore staging directory to scratch so it's a consistent
+# behavior regardless of where the restore playbook is executed (locally vs remotely)
+- name: Set restore staging directory to /scratch
+  set_fact:
+    target_backup_dir: /scratch
+
+- name: Transfer backup tarball to target if the file is off-box
+  include_role:
+    name: backup-restore/transfer-file
+  when: on_box_data|bool == false
+
+- name: Copy the backup tarball to {{ target_backup_dir }} if the file is already on-box
+  copy:
+    src: "{{ initial_backup_dir }}/{{ backup_filename }}"
+    dest: "{{ target_backup_dir }}"
+    remote_src: yes
+  when: on_box_data|bool == true
+  become: yes
+  become_user: root

playbookconfig/src/playbooks/roles/restore-dc-vault/restore-dc-vault-directory/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   This role exists to restore the dc-vault directory
+#   It will run only if the node is an unlocked system controller
+#
+- block:
+    - name: Set dc-vault backup file path
+      set_fact:
+        dc_vault_backup_file: "{{ target_backup_dir }}/{{ backup_filename }}"
+
+    - name: Check if dc-vault backup file exists
+      stat:
+        path: "{{ dc_vault_backup_file }}"
+      register: dc_vault_backup_file_result
+
+    - name: Fail if dc-vault backup is missing
+      fail:
+        msg: "dc-vault backup file is missing"
+      when: not dc_vault_backup_file_result.stat.exists
+
+    - name: 'Create {{ restore_dc_vault_in_progress_flag }} flag file'
+      file:
+        path: "{{ restore_dc_vault_in_progress_flag }}"
+        state: touch
+
+    - name: Set dc-vault path in the archive
+      set_fact:
+        archive_dc_vault_permdir: "{{ dc_vault_permdir | regex_replace('^\\/', '') }}"
+
+    - name: Look for dc-vault filesystem
+      shell: "tar -tf {{ dc_vault_backup_file }} | grep '{{ dc_vault_permdir|basename }}'"
+      args:
+        warn: false
+      failed_when: false
+      register: search_result
+
+    - name: Restore dc-vault filesystem
+      command: >-
+        tar -C / --overwrite -xpf {{ dc_vault_backup_file }}
+        {{ archive_dc_vault_permdir }}
+      args:
+        warn: false
+      when: search_result.rc == 0
+
+  always:
+    - name: 'Remove the {{ restore_dc_vault_in_progress_flag }} file'
+      file:
+        path: "{{ restore_dc_vault_in_progress_flag }}"
+        state: absent

playbookconfig/src/playbooks/roles/restore-dc-vault/restore-dc-vault-directory/vars/main.yml

@@ -0,0 +1,9 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+platform_conf_path: /etc/platform
+restore_dc_vault_in_progress_flag: "{{ platform_conf_path }}/.restore_dc_vault_in_progress_flag"
+dc_vault_permdir: /opt/dc-vault

playbookconfig/src/playbooks/roles/restore-platform/restore-more-data/tasks/main.yml

@@ -11,7 +11,6 @@
 
 - name: Set parameters for archive paths
   set_fact:
-    archive_dc_vault_permdir: "{{ dc_vault_permdir | regex_replace('^\\/', '') }}"
     archive_platform_conf_path: "{{ platform_conf_path | regex_replace('^\\/', '') }}"
     archive_ceph_backend_flag: "{{ ceph_backend_flag | regex_replace('^\\/', '') }}"
 
@@ -294,21 +293,6 @@
 
   when: ceph_backend.rc == 0
 
-- name: Look for dc-vault filesystem
-  shell: "tar -tf {{ restore_data_file }} | grep '{{ dc_vault_permdir|basename }}'"
-  args:
-    warn: false
-  failed_when: false
-  register: search_result
-
-- name: Restore dc-vault filesystem
-  command: >-
-    tar -C / --overwrite -xpf {{ restore_data_file }}
-    {{ archive_dc_vault_permdir }}
-  args:
-    warn: false
-  when: search_result.rc == 0
-
 - name: Look for deploy files
   shell: "tar -tf {{ restore_data_file }} |
          grep {{ archive_deploy_permdir }}"