496 lines
18 KiB
YAML
496 lines
18 KiB
YAML
---
|
|
#
|
|
# Copyright (c) 2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# ROLE DESCRIPTION:
|
|
# This role is to backup StarlingX platform data and
|
|
# StarlingX OpenStack Application data if the app exists.
|
|
# The backup data are stored in two separate tar files.
|
|
#
|
|
- name: Do StarlingX backup
|
|
block:
|
|
- name: Generate backup_in_progress alarm
|
|
script: fm_alarm.py "--set" "--backup"
|
|
register: alarm_result
|
|
failed_when: false
|
|
|
|
- name: Fail if alarm script throws an exception
|
|
fail:
|
|
msg: "Failed to generate backup-in-progress alarm."
|
|
when: alarm_result.rc != 0
|
|
|
|
- name: Create temp dir
|
|
tempfile:
|
|
path: "{{ backup_dir }}"
|
|
state: directory
|
|
register: tempdir
|
|
|
|
- name: Create postgres temp dir
|
|
file:
|
|
path: "{{ tempdir.path }}/postgres"
|
|
state: directory
|
|
register: postgres_dir
|
|
|
|
- name: Backup roles, table spaces and schemas for databases.
|
|
shell: >-
|
|
sudo -u postgres pg_dumpall
|
|
--clean --schema-only > {{ postgres_dir.path }}/postgres.postgreSql.config
|
|
args:
|
|
warn: false
|
|
|
|
- name: Backup postgres, template1, sysinv, barbican db data
|
|
shell: >-
|
|
sudo -u postgres pg_dump --format=plain --inserts --disable-triggers --data-only
|
|
{{ item }} > {{ postgres_dir.path }}/{{ item }}.postgreSql.data
|
|
args:
|
|
warn: false
|
|
with_items:
|
|
- postgres
|
|
- template1
|
|
- sysinv
|
|
- barbican
|
|
|
|
- name: Backup fm db data
|
|
shell: >-
|
|
sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
|
|
--data-only fm --exclude-table=alarm > {{ postgres_dir.path }}/fm.postgreSql.data
|
|
args:
|
|
warn: false
|
|
|
|
- name: Check if it is the primary region
|
|
command: grep -i "region_config\s*=\s*no" {{ platform_conf_path }}/platform.conf
|
|
register: check_region
|
|
failed_when: false
|
|
|
|
- name: Backup keystone db for primary region
|
|
shell: >-
|
|
sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
|
|
--data-only keystone > {{ postgres_dir.path }}/keystone.postgreSql.data
|
|
args:
|
|
warn: false
|
|
when: check_region.rc == 0
|
|
|
|
- name: Check if it is dc controller
|
|
command: >-
|
|
grep -i "distributed_cloud_role\s*=\s*systemcontroller"
|
|
{{ platform_conf_path }}/platform.conf
|
|
register: check_dc_controller
|
|
failed_when: false
|
|
|
|
- block:
|
|
- name: Backup dcmanager db for dc controller
|
|
shell: >-
|
|
sudo -u postgres pg_dump --format=plain --inserts --disable-triggers
|
|
--data-only dcmanager > {{ postgres_dir.path }}/dcmanager.postgreSql.data
|
|
args:
|
|
warn: false
|
|
|
|
- name: Backup dcorch db for dc controller
|
|
set_fact:
|
|
dcorch_db: "sudo -u postgres pg_dump --format=plain --inserts --disable-triggers --data-only dcorch "
|
|
|
|
- name: Update dcorch tables that will be excluded from backup
|
|
set_fact:
|
|
dcorch_db: "{{ dcorch_db }} --exclude-table={{ item }}"
|
|
with_items:
|
|
- orch_job
|
|
- orch_request
|
|
- resource
|
|
- subcloud_resource
|
|
|
|
- name: Backup dcorch db
|
|
shell: "{{ dcorch_db }} > {{ postgres_dir.path }}/dcorch.postgreSql.data"
|
|
|
|
when: check_dc_controller.rc == 0
|
|
|
|
- name: Create mariadb temp dir
|
|
file:
|
|
path: "{{ tempdir.path }}/mariadb"
|
|
state: directory
|
|
register: mariadb_dir
|
|
|
|
- name: Check if mariadb pod is running
|
|
shell: >-
|
|
kubectl --kubeconfig={{ kube_config_dir }} get pods -n openstack | grep {{ mariadb_pod }} | grep -i 'running'
|
|
failed_when: false
|
|
register: check_mariadb_pod
|
|
|
|
- block:
|
|
- name: Set k8s cmd prefix
|
|
set_fact:
|
|
kube_cmd_prefix: "kubectl --kubeconfig={{ kube_config_dir }} exec -i {{ mariadb_pod }} -n openstack -- bash -c "
|
|
|
|
- name: Show databases
|
|
shell: "{{ kube_cmd_prefix }} 'exec mysql -uroot -p\"$MYSQL_DBADMIN_PASSWORD\" -e\"show databases\"'"
|
|
register: databases
|
|
|
|
- name: Backup mariadb
|
|
shell: >-
|
|
{{ kube_cmd_prefix }} 'exec mysqldump -uroot -p"$MYSQL_DBADMIN_PASSWORD" {{ item }}' >
|
|
{{ mariadb_dir.path }}/{{ item }}.mariadb.data
|
|
with_items: "{{ databases.stdout_lines | difference(skip_os_dbs) }}"
|
|
|
|
when: check_mariadb_pod.rc == 0
|
|
|
|
- block:
|
|
- name: Check if stx-openstack is applied
|
|
shell: >-
|
|
source /etc/platform/openrc; system application-list | grep stx-openstack | grep applied
|
|
failed_when: false
|
|
register: check_openstack_applied
|
|
|
|
- name: Check if stx-openstack is uploaded
|
|
shell: >-
|
|
source /etc/platform/openrc; system application-list | grep stx-openstack | grep uploaded
|
|
failed_when: false
|
|
register: check_openstack_uploaded
|
|
|
|
- name: Fail the backup if MariaDB is not running
|
|
fail:
|
|
msg: "WARNING: {{ mariadb_pod }} is not running. Cannot backup mariadb data."
|
|
when: check_openstack_applied.rc == 0
|
|
when: check_mariadb_pod.rc != 0
|
|
|
|
# Now Postgres data and MariaDB data are stored in staging dir, we can estimate
|
|
# the disk size requirement for the backup archive.
|
|
- name: Check the size (in kilobyte) of directories that will be backed up
|
|
shell: "du -sh -k {{ item }} | awk '{print $1}'"
|
|
with_items:
|
|
- /etc
|
|
- /home
|
|
- "{{ config_permdir }}"
|
|
- "{{ puppet_permdir }}/hieradata"
|
|
- "{{ keyring_permdir }}"
|
|
- "{{ patching_permdir }}"
|
|
- "{{ patching_repo_permdir }}"
|
|
- "{{ extension_permdir }}"
|
|
- "{{ patch_vault_permdir }}"
|
|
- "{{ postgres_dir.path }}"
|
|
- "{{ armada_permdir }}"
|
|
- "{{ helm_charts_permdir }}"
|
|
- "{{ mariadb_dir.path }}"
|
|
register: size_output
|
|
|
|
# Estimate the backup size. We add 128M overhead for things like ceph crushmap,
|
|
# ldap data, etc. that will be generated and stored in the staging dir later on.
|
|
- name: Estimate the total required disk size for backup archive
|
|
set_fact:
|
|
total_size_estimation: "{{ total_size_estimation|default(1024*128)|int + item.stdout|int }}"
|
|
with_items: "{{ size_output.results }}"
|
|
loop_control:
|
|
label: "{{ item.item }}"
|
|
|
|
- name: Check if there is enough free space in the archive dir to create backup
|
|
shell: "df -k /opt/backups --output=avail | tail -1"
|
|
register: available_disk_size
|
|
|
|
- name: Fail if there is not enough free space in the archive dir to create backup
|
|
fail:
|
|
msg: >-
|
|
Not enough free space in {{ backup_dir }}. It has {{ available_disk_size }}K.
|
|
It needs at least {{ total_size_estimation }}K.
|
|
when: available_disk_size < total_size_estimation
|
|
|
|
- name: Create ldap temp dir
|
|
file:
|
|
path: "{{ tempdir.path }}/ldap"
|
|
state: directory
|
|
register: ldap_dir
|
|
|
|
- name: Name ldap db backup
|
|
set_fact:
|
|
ldap_db_backup: "{{ ldap_dir.path }}/ldap.db"
|
|
|
|
- name: Backup ldap db
|
|
command: "slapcat -d 0 -F /etc/openldap/schema -l {{ ldap_db_backup }}"
|
|
|
|
- name: Create ceph temp dir
|
|
file:
|
|
path: "{{ tempdir.path }}/ceph"
|
|
state: directory
|
|
register: ceph_dir
|
|
|
|
- name: Name ceph crushmap backup
|
|
set_fact:
|
|
crushmap_file: "{{ ceph_dir.path }}/crushmap.bin.backup"
|
|
|
|
- name: Create ceph crushmap backup
|
|
command: "ceph osd getcrushmap -o {{ crushmap_file }}"
|
|
|
|
- name: Create temp dir for override backup file
|
|
file:
|
|
path: "{{ tempdir.path }}/override"
|
|
state: directory
|
|
register: override_dir
|
|
|
|
- name: Name override backup file
|
|
set_fact:
|
|
override_backup_file: "{{ override_dir.path }}/{{ host_override_backup_file }}"
|
|
|
|
- name: Retrieve dns_servers from sysinv db
|
|
shell: "source /etc/platform/openrc; system dns-show | grep nameservers | awk '{print $4}'"
|
|
register: dns_servers
|
|
|
|
- name: Create and write dns_servers into override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "dns_servers:"
|
|
create: yes
|
|
|
|
- name: Write each dns server into override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: " - {{ item }}"
|
|
with_items: "{{ dns_servers.stdout.split(',') }}"
|
|
|
|
- name: Query addrpool in sysinv db
|
|
shell:
|
|
source /etc/platform/openrc; system addrpool-list --nowrap > {{ tempdir.path }}/addrpool.txt
|
|
|
|
- name: Retrieve pxeboot_subnet from sysinv db
|
|
shell: "grep pxeboot {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: pxeboot_subnet
|
|
|
|
- name: Retrieve management_subnet from sysinv db
|
|
shell: "grep management {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: management_subnet
|
|
|
|
- name: Retrieve management_start_address from sysinv db
|
|
shell:
|
|
"grep management {{ tempdir.path }}/addrpool.txt | awk 'match($12, /-/) {print substr($12, 3, RSTART-3)}'"
|
|
register: management_start_address
|
|
|
|
- name: Retrieve cluster_host_subnet from sysinv db
|
|
shell:
|
|
"grep cluster-host-subnet {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: cluster_host_subnet
|
|
|
|
- name: Retrieve cluster_pod_subnet from sysinv db
|
|
shell: "grep cluster-pod-subnet {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: cluster_pod_subnet
|
|
|
|
- name: Retrieve cluster_service_subnet from sysinv db
|
|
shell: "grep cluster-service-subnet {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: cluster_service_subnet
|
|
|
|
- name: Retrieve external_oam_subnet from sysinv db
|
|
shell: "grep oam {{ tempdir.path }}/addrpool.txt | awk '{print $6\"/\"$8}'"
|
|
register: external_oam_subnet
|
|
|
|
- name: Retrieve external_oam_gateway_address from sysinv db
|
|
shell: "grep oam {{ tempdir.path }}/addrpool.txt | awk '{print $20}'"
|
|
register: external_oam_gateway_address
|
|
|
|
- name: Retrieve external_oam_floating_address from sysinv db
|
|
shell: "grep oam {{ tempdir.path }}/addrpool.txt | awk '{print $14}'"
|
|
register: external_oam_floating_address
|
|
|
|
- name: Write entries to override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "{{ item }}"
|
|
with_items:
|
|
- "pxeboot_subnet: {{ pxeboot_subnet.stdout }}"
|
|
- "management_subnet: {{ management_subnet.stdout }}"
|
|
- "management_start_address: {{ management_start_address.stdout }}"
|
|
- "cluster_host_subnet: {{ cluster_host_subnet.stdout }}"
|
|
- "cluster_pod_subnet: {{ cluster_pod_subnet.stdout }}"
|
|
- "cluster_service_subnet: {{ cluster_service_subnet.stdout }}"
|
|
- "external_oam_subnet: {{ external_oam_subnet.stdout }}"
|
|
- "external_oam_gateway_address: {{ external_oam_gateway_address.stdout }}"
|
|
- "external_oam_floating_address: {{ external_oam_floating_address.stdout }}"
|
|
|
|
- name: Query service parameters for docker
|
|
shell: >-
|
|
source /etc/platform/openrc; system service-parameter-list --nowrap |
|
|
grep -w docker | awk '{if ($4=="docker") print $line}' > {{ tempdir.path }}/docker.txt
|
|
|
|
- name: Check if there is a unified docker registry
|
|
shell: grep -w insecure_registry {{ tempdir.path }}/docker.txt
|
|
register: check_unified
|
|
failed_when: false
|
|
|
|
- block:
|
|
- name: Get insecure_registry value
|
|
shell: grep -w insecure_registry {{ tempdir.path }}/docker.txt | awk '{print $10}'
|
|
register: insecure
|
|
|
|
- name: Retrieve unified docker registry from sysinv
|
|
shell: grep -w k8s {{ tempdir.path }}/docker.txt | awk '{if ($8=="k8s") print $10}'
|
|
register: unified_docker_registry
|
|
|
|
- name: Write unified docker registry to override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "{{ item }}"
|
|
with_items:
|
|
- "is_secure_registry: {{ not (insecure.stdout|bool) }}"
|
|
- "docker_registries:"
|
|
- " unified: {{ unified_docker_registry.stdout }}"
|
|
when: check_unified.rc == 0
|
|
|
|
- block:
|
|
- name: Search for docker registries
|
|
shell: >-
|
|
grep -w registry {{ tempdir.path }}/docker.txt |
|
|
awk '{if ($6=="registry") print $8 ".io: " $10}'
|
|
failed_when: false
|
|
register: docker_registries
|
|
|
|
- block:
|
|
- name: Write docker_registries
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "docker_registries:"
|
|
|
|
- name: Write docker_registry to override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: " {{ item }}"
|
|
with_items: "{{ docker_registries.stdout_lines }}"
|
|
|
|
- name: Replace k8s.io with k8s.gcr.io in the override backup file
|
|
replace:
|
|
path: "{{ override_backup_file }}"
|
|
regexp: 'k8s\.io'
|
|
replace: 'k8s.gcr.io'
|
|
|
|
when: docker_registries.stdout != ""
|
|
when: check_unified.rc != 0
|
|
|
|
- name: Check if docker no-proxy exists
|
|
shell: >-
|
|
grep -w no_proxy {{ tempdir.path }}/docker.txt |
|
|
awk '{if ($8=="no_proxy") print $10}'
|
|
register: docker_no_proxy
|
|
failed_when: false
|
|
|
|
- block:
|
|
- name: Write no_proxy into override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "docker_no_proxy:"
|
|
|
|
- name: Write each no_proxy address into override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: " - {{ item }}"
|
|
with_items: "{{ docker_no_proxy.stdout.split(',') }}"
|
|
|
|
when: docker_no_proxy.stdout != ""
|
|
|
|
- name: Look for docker proxy entries
|
|
shell: >-
|
|
grep -w proxy {{ tempdir.path }}/docker.txt | grep -w -v no_proxy |
|
|
awk '{if ($6=="proxy") print "docker_" $8 ": " $10}'
|
|
register: check_docker_proxy
|
|
failed_when: false
|
|
|
|
- block:
|
|
- name: Write docker proxy to override backup file
|
|
lineinfile:
|
|
dest: "{{ override_backup_file }}"
|
|
line: "{{ item }}"
|
|
with_items: "{{ check_docker_proxy.stdout }}"
|
|
when: check_docker_proxy.stdout != ""
|
|
|
|
- name: Attach timestamp to the platform backup filename
|
|
set_fact:
|
|
platform_backup_file: "{{ platform_backup_filename_prefix }}_{{ lookup('pipe', 'date +%Y_%m_%d_%H_%M_%S') }}.tgz"
|
|
|
|
- name: Set platform backup file absolute path
|
|
set_fact:
|
|
platform_backup_file_path: "{{ backup_dir }}/{{ platform_backup_file }}"
|
|
|
|
# Archive module has a known bug that doesn't handle empty symbolic links
|
|
# well. Restore to tar command. Can add -P option to keep the leading
|
|
# '/'s in file names in the tar file, so that the tasks that strip leading
|
|
# '/' from the directory names before untar won't be required.
|
|
- name: Create a tgz archive for platform backup
|
|
shell: "tar -czf {{ platform_backup_file_path }} $(ls -d \
|
|
{{ override_backup_file }} \
|
|
/etc \
|
|
/home \
|
|
{{ config_permdir }} \
|
|
{{ puppet_permdir }}/hieradata \
|
|
{{ keyring_permdir }} \
|
|
{{ patching_permdir }} \
|
|
{{ patching_repo_permdir }} \
|
|
{{ extension_permdir }} \
|
|
{{ patch_vault_permdir }} \
|
|
{{ crushmap_file }} \
|
|
{{ ldap_db_backup }} \
|
|
{{ postgres_dir.path }} \
|
|
{{ armada_permdir }}/platform-integ-apps \
|
|
{{ armada_permdir }}/admin.conf \
|
|
{{ helm_charts_permdir }}/stx-platform 2>/dev/null)"
|
|
args:
|
|
warn: false
|
|
|
|
- block:
|
|
- name: Attach timestamp to the OpenStack backup filename
|
|
set_fact:
|
|
openstack_backup_file: "{{ openstack_backup_filename_prefix }}_{{ lookup('pipe', 'date +%Y_%m_%d_%H_%M_%S') }}.tgz"
|
|
|
|
- name: Set SpenStack backup file absolute path
|
|
set_fact:
|
|
openstack_backup_file_path: "{{ backup_dir }}/{{ openstack_backup_file }}"
|
|
|
|
- name: Create a tgz archive for OpenStack backup
|
|
shell: "tar -czf {{ openstack_backup_file_path }} $(ls -d \
|
|
{{ armada_permdir }}/stx-openstack \
|
|
{{ helm_charts_permdir }}/starlingx \
|
|
{{ mariadb_dir.path }} 2>/dev/null)"
|
|
args:
|
|
warn: false
|
|
when: check_mariadb_pod.rc == 0 or check_openstack_uploaded.rc == 0
|
|
|
|
- name: Notify the user backup tar file(s) are available
|
|
debug:
|
|
msg: >-
|
|
Backup tar file(s) are now available in {{ backup_dir }} on the active controller.
|
|
|
|
- block:
|
|
- name: Transfer platform backup tar file to the local machine
|
|
fetch:
|
|
src: "{{ platform_backup_file_path }}"
|
|
dest: "{{ host_backup_dir }}/"
|
|
flat: yes
|
|
|
|
- name: Transfer openstack backup tar files to the local machine if it exists
|
|
fetch:
|
|
src: "{{ openstack_backup_file_path}}"
|
|
dest: "{{ host_backup_dir }}/"
|
|
flat: yes
|
|
when: check_mariadb_pod.rc == 0 or check_openstack_uploaded.rc == 0
|
|
|
|
- name: Notify the user where the backup tar file(s) can be found
|
|
debug:
|
|
msg: >-
|
|
Backup tar file(s) have been transferred to {{ host_backup_dir }} on Ansible control host.
|
|
when: inventory_hostname != 'localhost'
|
|
|
|
always:
|
|
- name: Remove the temp dir
|
|
file:
|
|
path: "{{ tempdir.path }}"
|
|
state: absent
|
|
when: tempdir is defined
|
|
|
|
- name: Remove the backup in progress flag file
|
|
file:
|
|
path: "{{ backup_in_progress_flag }}"
|
|
state: absent
|
|
|
|
- name: Clear backup_in_progress alarm
|
|
script: fm_alarm.py "--clear" "--backup"
|
|
register: alarm_result
|
|
failed_when: false
|
|
|
|
- name: Fail if alarm script throws an exception
|
|
fail:
|
|
msg: "Failed to clear backup-in-progress alarm."
|
|
when: alarm_result.rc != 0
|