ansible-playbooks/playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml

---
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# ROLE DESCRIPTION:
#   This role is to backup StarlingX platform data and
#   StarlingX OpenStack Application data if the app exists.
#   The backup data are stored in two separate tar files.
#
- name: Do StarlingX backup
  block:
    - name: Generate backup_in_progress alarm
      script: fm_alarm.py "--set" "--backup"
      register: alarm_result
      failed_when: false

    - name: Fail if alarm script throws an exception
      fail:
        msg: "Failed to generate backup-in-progress alarm."
      when: alarm_result.rc != 0

    - name: Create temp dir
      tempfile:
        path: "{{ backup_dir }}"
        state: directory
      register: tempdir

    - name: Create postgres temp dir
      file:
        path: "{{ tempdir.path }}/postgres"
        state: directory
      register: postgres_dir

    - name: Backup roles, table spaces and schemas for databases.
      shell: >-
        sudo -u postgres pg_dumpall
        --clean --schema-only > {{ postgres_dir.path }}/postgres.postgreSql.config        
      args:
        warn: false

    - name: Backup postgres, template1, sysinv, barbican db data
      shell: >-
        sudo -u postgres pg_dump --format=plain --inserts --disable-triggers --data-only
        {{ item }} > {{ postgres_dir.path }}/{{ item }}.postgreSql.data        
      args:
        warn: false
      with_items:
        - postgres
        - template1
        - sysinv
        - barbican

    - name: Backup fm db data
      shell: >-
        sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
        --data-only fm --exclude-table=alarm > {{ postgres_dir.path }}/fm.postgreSql.data        
      args:
        warn: false

    - name: Check if it is the primary region
      command: grep -i "region_config\s*=\s*no" {{ platform_conf_path }}/platform.conf
      register: check_region
      failed_when: false

    - name: Backup keystone db for primary region
      shell: >-
        sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
        --data-only keystone > {{ postgres_dir.path }}/keystone.postgreSql.data        
      args:
        warn: false
      when: check_region.rc == 0

    - name: Check if it is dc controller
      command: >-
        grep -i "distributed_cloud_role\s*=\s*systemcontroller"
        {{ platform_conf_path }}/platform.conf        
      register: check_dc_controller
      failed_when: false

    - block:
      - name: Backup dcmanager db for dc controller
        shell: >-
          sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
          --data-only dcmanager > {{ postgres_dir.path }}/dcmanager.postgreSql.data          
        args:
          warn: false

      - name: Backup dcorch db for dc controller
        set_fact:
          dcorch_db: "sudo -u postgres pg_dump --format=plain --inserts --disable-triggers --data-only dcorch "

      - name: Update dcorch tables that will be excluded from backup
        set_fact:
          dcorch_db: "{{ dcorch_db }} --exclude-table={{ item }}"
        with_items:
          - orch_job
          - orch_request
          - resource
          - subcloud_resource

      - name: Backup dcorch db
        shell: "{{ dcorch_db }} > {{ postgres_dir.path }}/dcorch.postgreSql.data"

      when: check_dc_controller.rc == 0

    - name: Create mariadb temp dir
      file:
        path: "{{ tempdir.path }}/mariadb"
        state: directory
      register: mariadb_dir

    - name: Check if mariadb pod is running
      shell: >-
        kubectl --kubeconfig={{ kube_config_dir }} get pods -n openstack | grep {{ mariadb_pod }} | grep -i 'running'        
      failed_when: false
      register: check_mariadb_pod

    - block:
      - name: Set k8s cmd prefix
        set_fact:
          kube_cmd_prefix: "kubectl --kubeconfig={{ kube_config_dir }} exec -i {{ mariadb_pod }} -n openstack -- bash -c "

      - name: Show databases
        shell: "{{ kube_cmd_prefix }} 'exec mysql -uroot -p\"$MYSQL_DBADMIN_PASSWORD\" -e\"show databases\"'"
        register: databases

      - name: Backup mariadb
        shell: >-
          {{ kube_cmd_prefix }} 'exec mysqldump -uroot -p"$MYSQL_DBADMIN_PASSWORD" {{ item }}' >
          {{ mariadb_dir.path }}/{{ item }}.mariadb.data          
        with_items: "{{ databases.stdout_lines | difference(skip_os_dbs) }}"

      when: check_mariadb_pod.rc == 0

    - block:

      - name: Get stx-openstack status
        shell: >-
          source /etc/platform/openrc; system application-show stx-openstack --column status --format value          
        failed_when: false
        register: openstack_status

      - name: Fail the backup if MariaDB is not running
        fail:
          msg: "WARNING: {{ mariadb_pod }} is not running. Cannot backup mariadb data."
        when: openstack_status.stdout == "applied"
      when: check_mariadb_pod.rc != 0

    # In order to restore only the stx-openstack, the user needs to manually
    # remove, delete and upload the application.
    # This procedure will make the helm overrides data that is
    # present in the sysinv DB and the stx-openstack DB to be inconsistent.
    # A dump file containing the overrides data is used here to solve this inconsistency issue.
    # Note that this file will contain only 'update' sql commands because the
    # overrides are already created when we uploaded the application.
    - name: Create Helm overrides temp dir
      file:
        path: "{{ tempdir.path }}/helm_overrides_dir"
        state: directory
      register: helm_overrides_dir

    - name: Get the openstack Helm overrides from the from the database
      shell: >-
        psql -c "copy(select row_to_json(t) from (select name, user_overrides, system_overrides
        from helm_overrides where namespace='openstack') as t) to stdout" sysinv | sed -e 's/\\\\/\\/g'        
      become_user: postgres
      register: helm_overrides_list

    - name: Generate postgres update commands for Helm overrides
      set_fact:
        updates_list: >
          {{ updates_list | default('') }}update helm_overrides set
          system_overrides={% if item.system_overrides %}'{{ item.system_overrides }}'{% else %}NULL{% endif %},
          user_overrides={% if item.user_overrides %}'{{ item.user_overrides }}'{% else %}NULL{% endif %}
          where name='{{ item.name }}' and namespace='openstack';          
      with_items: "{{ helm_overrides_list.stdout_lines | map('from_json') | list }}"

    - name: Backup Helm overrides
      copy:
        dest: "{{ helm_overrides_dir.path }}/helm_overrides_dump.sql"
        mode: 0755
        content: '{{ updates_list | default("") }}'


    # Now Postgres data and MariaDB data are stored in staging dir, we can estimate
    # the disk size requirement for the backup archive.
    - name: Check the size (in kilobyte) of directories that will be backed up
      shell: "du -sh -k  {{ item }} | awk '{print $1}'"
      with_items:
        - /etc
        - /home
        - "{{ config_permdir }}"
        - "{{ sysinv_permdir }}"
        - "{{ puppet_permdir }}/hieradata"
        - "{{ keyring_permdir }}"
        - "{{ patching_permdir }}"
        - "{{ patching_repo_permdir }}"
        - "{{ extension_permdir }}"
        - "{{ patch_vault_permdir }}"
        - "{{ postgres_dir.path }}"
        - "{{ armada_permdir }}"
        - "{{ helm_charts_permdir }}"
        - "{{ mariadb_dir.path }}"
        - "{{ helm_overrides_dir.path }}"
      register: size_output

    # Estimate the backup size. We add 128M overhead for things like ceph crushmap,
    # ldap data, etc. that will be generated and stored in the staging dir later on.
    - name: Estimate the total required disk size for backup archive
      set_fact:
        total_size_estimation: "{{ total_size_estimation|default(1024*128)|int + item.stdout|int }}"
      with_items: "{{ size_output.results }}"
      loop_control:
        label: "{{ item.item }}"

    - name: Check if there is enough free space in the archive dir to create backup
      shell: "df -k /opt/backups --output=avail | tail -1"
      register: available_disk_size

    - name: Fail if there is not enough free space in the archive dir to create backup
      fail:
        msg: >-
           Not enough free space in {{ backup_dir }}. It has {{ available_disk_size }}K.
           It needs at least {{ total_size_estimation }}K.           
      when: available_disk_size < total_size_estimation

    - name: Create ldap temp dir
      file:
        path: "{{ tempdir.path }}/ldap"
        state: directory
      register: ldap_dir

    - name: Name ldap db backup
      set_fact:
        ldap_db_backup: "{{ ldap_dir.path }}/ldap.db"

    - name: Backup ldap db
      command: "slapcat -d 0 -F /etc/openldap/schema -l {{ ldap_db_backup }}"

    - name: Create ceph temp dir
      file:
        path: "{{ tempdir.path }}/ceph"
        state: directory
      register: ceph_dir

    - name: Name ceph crushmap backup
      set_fact:
        crushmap_file: "{{ ceph_dir.path }}/crushmap.bin.backup"

    - name: Create ceph crushmap backup
      command: "ceph osd getcrushmap -o {{ crushmap_file }}"

    - name: Create temp dir for override backup file
      file:
        path: "{{ tempdir.path }}/override"
        state: directory
      register: override_dir

    - name: Name override backup file
      set_fact:
        override_backup_file: "{{ override_dir.path }}/{{ host_override_backup_file }}"

    - name: Create the override backup file
      command: "/usr/bin/sysinv-utils create-host-overrides {{ override_backup_file }}"

    - name: Get docker registries information
      include_role:
        name: common/push-docker-images
        tasks_from: get_docker_registries

    - name: Append registries configuration
      blockinfile:
        path: "{{ override_backup_file }}"
        marker: ""
        block: "{{ registries|default({}) | to_nice_yaml(indent=2) }}"

    - name: Attach timestamp to the platform backup filename
      set_fact:
        platform_backup_file: "{{ platform_backup_filename_prefix }}_{{ lookup('pipe', 'date +%Y_%m_%d_%H_%M_%S') }}.tgz"

    - name: Set platform backup file absolute path
      set_fact:
        platform_backup_file_path: "{{ backup_dir }}/{{ platform_backup_file }}"

    # Archive module has a known bug that doesn't handle empty symbolic links
    # well. Restore to tar command. Can add -P option to keep the leading
    # '/'s in file names in the tar file, so that the tasks that strip leading
    # '/' from the directory names before untar won't be required.
    - name: Create a tgz archive for platform backup
      shell: "tar -czf {{ platform_backup_file_path }} $(ls -d \
           {{ override_backup_file }} \
           /etc \
           /home \
           {{ config_permdir }} \
           {{ sysinv_permdir }} \
           {{ puppet_permdir }}/hieradata \
           {{ keyring_permdir }} \
           {{ patching_permdir }} \
           {{ patching_repo_permdir }} \
           {{ extension_permdir }} \
           {{ patch_vault_permdir }} \
           {{ crushmap_file }} \
           {{ ldap_db_backup }} \
           {{ postgres_dir.path }} \
           {{ armada_permdir }}/platform-integ-apps \
           {{ armada_permdir }}/admin.conf \
           {{ helm_charts_permdir }}/stx-platform 2>/dev/null)"
      args:
        warn: false

    - block:
      - name: Attach timestamp to the OpenStack backup filename
        set_fact:
          openstack_backup_file: "{{ openstack_backup_filename_prefix }}_{{ lookup('pipe', 'date +%Y_%m_%d_%H_%M_%S') }}.tgz"

      - name: Set SpenStack backup file absolute path
        set_fact:
          openstack_backup_file_path: "{{ backup_dir }}/{{ openstack_backup_file }}"

      - name: Create a tgz archive for OpenStack backup
        shell: "tar -czf {{ openstack_backup_file_path }} $(ls -d \
             {{ armada_permdir }}/stx-openstack \
             {{ helm_charts_permdir }}/starlingx \
             {{ mariadb_dir.path }} \
             {{ helm_overrides_dir.path }} 2>/dev/null)"
        args:
          warn: false
      when: check_mariadb_pod.rc == 0 or openstack_status.stdout == "uploaded"

    - name: Notify the user backup tar file(s) are available
      debug:
        msg: >-
          Backup tar file(s) are now available in {{ backup_dir }} on the active controller.          

    - block:
      - name: Transfer platform backup tar file to the local machine
        fetch:
          src: "{{ platform_backup_file_path }}"
          dest: "{{ host_backup_dir }}/"
          flat: yes

      - name: Transfer openstack backup tar files to the local machine if it exists
        fetch:
          src: "{{ openstack_backup_file_path}}"
          dest: "{{ host_backup_dir }}/"
          flat: yes
        when: check_mariadb_pod.rc == 0 or openstack_status.stdout == "uploaded"

      - name: Notify the user where the backup tar file(s) can be found
        debug:
          msg: >-
            Backup tar file(s) have been transferred to {{ host_backup_dir }} on Ansible control host.            
      when: inventory_hostname != 'localhost'

  always:
    - name: Remove the temp dir
      file:
        path: "{{ tempdir.path }}"
        state: absent
      when: tempdir is defined

    - name: Remove the backup in progress flag file
      file:
        path: "{{ backup_in_progress_flag }}"
        state: absent

    - name: Clear backup_in_progress alarm
      script: fm_alarm.py "--clear" "--backup"
      register: alarm_result
      failed_when: false

    - name: Fail if alarm script throws an exception
      fail:
        msg: "Failed to clear backup-in-progress alarm."
      when: alarm_result.rc != 0