starlingx
/
ansible-playbooks
Code Issues Proposed changes

Merge "Change location of the backup tarball when restore platform"

This commit is contained in:
Zuul 2019-09-11 19:23:41 +00:00 committed by Gerrit Code Review
parent 4632f8b448 100fe215d8
commit 2f59acce07
16 changed files with 200 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
playbookconfig/src/playbooks/backup-restore/host_vars/default.yml
View File

@ -10,10 +10,8 @@
#
override_files_dir: "{{ lookup('env', 'HOME') }}"
#
# Default directory where the system backup tarball can be found
# Default directory where the backup tar file(s) can be found
# on the active controller
#
backup_dir: /opt/backups
#
@ -43,12 +41,12 @@ backup_filename:
# platform restore and ceph data needs restored
# False: a partial restore where ceph data remain intact during restore
#
# This variable is used for StarlingX Openstack application restore only
# This variable is used for StarlingX OpenStack application restore only
#
restore_ceph_data: false
#
# Default directory where the system backup tarball fetched from the
# Default directory where the system backup tarballs fetched from the
# active controller can be found
#
host_backup_dir: "{{ lookup('env', 'HOME') }}"

16
playbookconfig/src/playbooks/bootstrap/bootstrap.yml
View File

@ -12,23 +12,9 @@
vars_files:
- host_vars/default.yml
pre_tasks:
- stat:
path: "{{ item }}"
register: files_to_import
with_items:
- "{{ override_files_dir }}/secrets.yml"
- "{{ override_files_dir }}/{{ inventory_hostname }}_secrets.yml"
- "{{ override_files_dir }}/site.yml"
- "{{ override_files_dir }}/{{ inventory_hostname }}.yml"
delegate_to: localhost
- include_vars: "{{ item.item }}"
when: item.stat.exists
with_items: "{{ files_to_import.results }}"
# Main play
roles:
- include-override-files
- prepare-env
- { role: validate-config, when: not skip_play, become: yes }
- { role: store-passwd, when: not skip_play and save_password, become: yes }

13
playbookconfig/src/playbooks/bootstrap/host_vars/default.yml
View File

@ -233,13 +233,10 @@ override_files_dir: "{{ lookup('env', 'HOME') }}"
# BACKUP AND RESTORE
# ==================
#
# Default directory on the host where the backup tarball
# can be found for remote play.
#
host_backup_dir: "{{ lookup('env', 'HOME') }}"
# Location where the platform backup tar file is placed to restore the platform.
# This location must be specified at the command line via ansible-playbook -e option.
initial_backup_dir:
#
# System backup tarball filename. User must provide the filename either
# in the override file or via -e option at the command line.
#
# Platform backup tar file. The filename must be specified at the command line via
# ansible-playbook -e option.
backup_filename:

49
playbookconfig/src/playbooks/bootstrap/restore_platform.yml
View File

@ -10,47 +10,12 @@
vars_files:
- host_vars/default.yml
pre_tasks:
- block:
- name: Fail if backup_filename is not defined or set
fail:
msg: "Mandatory configuration parameter backup_filename is not defined or set."
when: backup_filename is not defined or backup_filename is none
roles:
- include-override-files
- pre-restore-bootstrap
- name: Look for override backup file in the backup tarball
shell: "tar -tf {{ host_backup_dir }}/{{ backup_filename }} | grep '_override_backup.yml'"
args:
warn: false
failed_when: false
register: search_result
- block:
- name: Extract override backup file
shell: >-
tar -C {{ override_files_dir }} -xf {{ host_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ search_result.stdout_lines[0] }}
args:
warn: false
- name: Prepare to rename override file
set_fact:
override_filename: "{{ (search_result.stdout_lines[0] | basename).split('_override_backup')[0] }}.yml"
- name: Rename override file
command: >-
mv -f {{ override_files_dir }}/{{ (search_result.stdout_lines[0] | basename) }}
{{ override_files_dir }}/{{ override_filename }}
when: search_result.rc == 0
- name: Fail if override file is missing
fail:
msg: >-
Cannot find {{ host_backup_dir }}/{{ backup_filename }}
or the override file is missing in the backup tarball!
when: search_result.rc != 0
delegate_to: localhost
vars:
change_password: false
- name: Run bootstrap playbook with restore mode
import_playbook: bootstrap.yml mode='restore'
@ -58,5 +23,9 @@
- hosts: all
gather_facts: no
vars_files:
- host_vars/default.yml
roles:
- include-override-files
- { role: restore-more-data, become: yes }

6
playbookconfig/src/playbooks/bootstrap/roles/apply-bootstrap-manifest/tasks/main.yml
View File

@ -43,13 +43,13 @@
- name: Prefetch static hieradata
command: >-
tar -C {{ hieradata_workdir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ hieradata_workdir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
'{{ short_puppet_permdir }}/*static.yaml'
args:
warn: false
- name: Look for ssh_config dir in the backup tarball
shell: "tar -tf {{ staging_dir }}/{{ backup_filename }} | grep 'opt/platform/config/.*/ssh_config'"
shell: "tar -tf {{ target_backup_dir }}/{{ backup_filename }} | grep 'opt/platform/config/.*/ssh_config'"
args:
warn: false
failed_when: false
@ -66,7 +66,7 @@
- name: Prefetch ssh_config
command: >-
tar -C {{ temp_ssh_config_dir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ temp_ssh_config_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_ssh_config_permdir }}
args:
warn: false

4
playbookconfig/src/playbooks/bootstrap/roles/bringup-essential-services/tasks/refresh_local_dns.yml
View File

@ -49,13 +49,13 @@
short_config_permdir: "{{ config_permdir | regex_replace('^\\/', '') }}"
- name: restore /etc/hosts file
command: tar -C /etc -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,' etc/hosts
command: tar -C /etc -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,' etc/hosts
args:
warn: false
- name: restore hosts in config permdir
command: >-
tar -C {{ config_permdir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ config_permdir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_config_permdir }}/hosts
args:
warn: false

2
playbookconfig/src/playbooks/bootstrap/roles/bringup-essential-services/tasks/setup_registry_certificate_and_keys.yml
View File

@ -49,7 +49,7 @@
- block:
- name: Restore certificate and key files
command: >-
tar -C /etc/ssl/private -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C /etc/ssl/private -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
'etc/ssl/private/registry-cert*'
args:
warn: false

22
playbookconfig/src/playbooks/bootstrap/roles/include-override-files/tasks/main.yml Normal file
View File

@ -0,0 +1,22 @@
---
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# ROLE DESCRIPTION:
# This role is to include user override files for a play.
#
- stat:
path: "{{ item }}"
register: files_to_import
with_items:
- "{{ override_files_dir }}/secrets.yml"
- "{{ override_files_dir }}/{{ inventory_hostname }}_secrets.yml"
- "{{ override_files_dir }}/site.yml"
- "{{ override_files_dir }}/{{ inventory_hostname }}.yml"
delegate_to: localhost
- include_vars: "{{ item.item }}"
when: item.stat.exists
with_items: "{{ files_to_import.results }}"

55
playbookconfig/src/playbooks/bootstrap/roles/pre-restore-bootstrap/tasks/main.yml Normal file
View File

@ -0,0 +1,55 @@
---
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# ROLE DESCRIPTION:
# This role is to retrieve the override file from the backup tarball
# required for the controller bootstrap.
#
- block:
- name: Fail if backup_filename is not defined or set
fail:
msg: "Mandatory configuration parameter backup_filename is not defined or set."
when: backup_filename is not defined or backup_filename is none
- name: Fail if initial_backup_dir is not defined or set
fail:
msg: "Mandatory configuration parameter initial_backup_dir is not defined or set."
when: initial_backup_dir is not defined or initial_backup_dir is none
- name: Look for override backup file in the backup tarball
shell: "tar -tf {{ initial_backup_dir }}/{{ backup_filename }} | grep '_override_backup.yml'"
args:
warn: false
failed_when: false
register: search_result
- block:
- name: Extract override file from backup tarball
shell: >-
tar -C {{ override_files_dir }} -xf {{ initial_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ search_result.stdout_lines[0] }}
args:
warn: false
- name: Prepare to rename override file
set_fact:
override_filename: "{{ (search_result.stdout_lines[0] | basename).split('_override_backup')[0] }}.yml"
- name: Rename override file for bootstrap
command: >-
mv -f {{ override_files_dir }}/{{ (search_result.stdout_lines[0] | basename) }}
{{ override_files_dir }}/{{ override_filename }}
when: search_result.rc == 0
- name: Fail if override file is missing
fail:
msg: >-
Cannot find {{ initial_backup_dir }}/{{ backup_filename }}
or the override file is missing in the backup tarball!
when: search_result.rc != 0
delegate_to: localhost

98
playbookconfig/src/playbooks/bootstrap/roles/prepare-env/tasks/load_patching_tasks.yml
View File

@ -16,19 +16,62 @@
path: "{{ restore_in_progress_flag }}"
state: touch
- name: Create {{ staging_dir }} for restore
file:
path: "{{ staging_dir }}"
state: directory
# For remote play the backup tarball will be transferred to /scratch
- block:
# Check if the backup tarball already exists. If it is the second run
# after the reboot, no need to transfer the backup tarball again.
- name: Check if {{ backup_filename }} has been uploaded already
stat:
path: "/scratch/{{ backup_filename }}"
register: check_backup_tarball
# Put the backup tarball in {{ staging_dir }} on controller-0
- name: Transfer backup tarball to remote controller-0
copy:
src: "{{ host_backup_dir }}/{{ backup_filename }}"
dest: "{{ staging_dir }}"
owner: root
group: root
mode: 0644
- block:
# TODO(wzhou): Considering to break backup tarball into multiple small tarfiles
# During restore upload each small tarfile one at a time to restore a subfunction.
# Because Ansible copy module uses ansible_remote_tmp directory as
# a staging area to transfer file, the default ansible_remote_tmp
# which is set in /tmp (1GB) may be too small for backup tarball,
# we require user to set ansible_remote_tmp to a new directory in
# /home/sysadmin via -e option on the command line. For example:
# -e "ansible_remote_tmp=/home/sysadmin/ansible-restore"
- name: Transfer backup tarball to /scratch on controller-0
copy:
src: "{{ initial_backup_dir }}/{{ backup_filename }}"
dest: /scratch
owner: root
group: root
mode: 0644
# As an alternative to Ansible copy, synchronize module may be
# used to transfer large files. But synchronize is broken in Ansible 2.8
# https://github.com/ansible/ansible/issues/56629.
# - name: Transfer backup tarball to /scratch on controller-0
# synchronize:
# src: "{{ initial_backup_dir }}/{{ backup_filename }}"
# dest: "/scratch/{{ backup_filename }}"
when: not check_backup_tarball.stat.exists
- name: Set target_backup_dir to /scratch
set_fact:
target_backup_dir: /scratch
when: inventory_hostname != "localhost"
- name: For local play set target_backup_dir to initial_backup_dir
set_fact:
target_backup_dir: "{{ initial_backup_dir }}"
when: inventory_hostname == "localhost"
- name: Set fact for patching staging dir
set_fact:
patching_staging_dir: /scratch/patching
- name: Create staging directory {{ patching_staging_dir }} for patch files
file:
path: "{{ patching_staging_dir }}"
state: directory
- block:
- name: Get the checksum of the build.info file of the installed load
@ -39,14 +82,14 @@
- name: Retrieve build.info file from backup
command: >-
tar -C {{ staging_dir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ patching_staging_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
etc/build.info
args:
warn: false
- name: Get the checksum of the build.info file from the backup
stat:
path: "{{ staging_dir }}/build.info"
path: "{{ patching_staging_dir }}/build.info"
get_checksum: yes
register: backup_buildinfo_check
@ -57,13 +100,13 @@
- name: Retrieve platform.conf file from the backup
command: >-
tar -C {{ staging_dir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ patching_staging_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
etc/platform/platform.conf
args:
warn: false
- name: Get subfunction from the backup
shell: grep -F 'subfunction' {{ staging_dir }}/platform.conf
shell: grep -F 'subfunction' {{ patching_staging_dir }}/platform.conf
register: backup_subfunc
- name: Get subfunction set from backup platform.conf
@ -110,7 +153,7 @@
- name: Restore patching
command: >-
tar -C /opt -xpf {{ staging_dir }}/{{ backup_filename }} --strip-components=1
tar -C /opt -xpf {{ target_backup_dir }}/{{ backup_filename }} --strip-components=1
{{ short_patching_permdir }}
args:
warn: false
@ -122,7 +165,7 @@
- name: Restore patching repo
command: >-
tar -C /www/pages -xpf {{ staging_dir }}/{{ backup_filename }} --strip-components=2
tar -C /www/pages -xpf {{ target_backup_dir }}/{{ backup_filename }} --strip-components=2
{{ short_patching_repo_permdir }}
args:
warn: false
@ -170,17 +213,17 @@
path: "{{ restore_in_progress_flag }}"
state: absent
- name: Clean up the {{ staging_dir }}
- name: Remove staging directory {{ patching_staging_dir }} for patch files
file:
path: "{{ staging_dir }}"
path: "{{ patching_staging_dir }}"
state: absent
# For better control of the restore, we don't invoke Ansible
# reboot module to reboot the node. We require user to re-run
# the playbook to restore the platform after reboot is completed.
# TODO: Suport patching without re-run of the restore_platform playbook
# by either invoking Ansible reboot module or defining reboot
# as an async task.
# TODO(wzhou): Suport patching without re-run of the restore_platform
# playbook by either invoking Ansible reboot module or defining reboot
# as an async task.
- name: Reboot the controller
shell: sleep 5 && reboot
failed_when: false
@ -206,6 +249,11 @@
path: "{{ restore_patching_complete_flag }}"
state: absent
- name: Remove staging directory {{ patching_staging_dir }} for patch files
file:
path: "{{ patching_staging_dir }}"
state: absent
rescue:
- block:
- name: Remove the {{ restore_in_progress_flag }} file
@ -213,9 +261,9 @@
path: "{{ restore_in_progress_flag }}"
state: absent
- name: Clean up the {{ staging_dir }}
- name: Remove staging directory {{ patching_staging_dir }} for patch files
file:
path: "{{ staging_dir }}"
path: "{{ patching_staging_dir }}"
state: absent
- name: Fail the platform restore

4
playbookconfig/src/playbooks/bootstrap/roles/prepare-env/tasks/main.yml
View File

@ -567,8 +567,8 @@
file:
path: "{{ ansible_remote_tmp }}"
state: directory
owner: root
group: root
owner: sysadmin
group: sys_protected
mode: 0755
become: yes

6
playbookconfig/src/playbooks/bootstrap/roles/prepare-env/tasks/restore_prep_tasks.yml
View File

@ -27,7 +27,7 @@
- name: Restore branding tar file
command: >-
tar -C /opt/branding -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C /opt/branding -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_branding_permdir }}
args:
warn: false
@ -38,7 +38,7 @@
state: absent
- name: Look for banner directory in the backup tarball
shell: "tar -tf {{ staging_dir }}/{{ backup_filename }} | grep -F 'banner/etc'"
shell: "tar -tf {{ target_backup_dir }}/{{ backup_filename }} | grep -F 'banner/etc'"
args:
warn: false
failed_when: false
@ -52,7 +52,7 @@
- name: Restore banner files if they exist in the backup tarball
command: >-
tar -C /opt/banner -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C /opt/banner -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_banner_permdir }}
args:
warn: false

1
playbookconfig/src/playbooks/bootstrap/roles/prepare-env/vars/main.yml
View File

@ -7,7 +7,6 @@ supported_release_versions:
- "19.09"
patching_permdir: /opt/patching
patching_repo_permdir: /www/pages/updates
staging_dir: /scratch/restore
restore_in_progress_flag: /etc/platform/.restore_in_progress
restore_patching_complete_flag: /etc/platform/.restore_patching_complete
node_is_patched_flag: /var/run/node_is_patched

34
playbookconfig/src/playbooks/bootstrap/roles/restore-more-data/tasks/main.yml
View File

@ -29,7 +29,7 @@
- name: Extract platform.conf from the backup tarball
command: >-
tar -C {{ staging_dir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ staging_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_platform_conf_path }}/platform.conf
args:
warn: false
@ -72,21 +72,21 @@
# Restore resolv.conf and dnsmaq
- name: Extract resolv.conf from backup tarball
command: >-
tar -C /etc -xpf {{ staging_dir }}/{{ backup_filename }} --overwrite
tar -C /etc -xpf {{ target_backup_dir }}/{{ backup_filename }} --overwrite
--transform='s,.*/,,' etc/resolv.conf
args:
warn: false
- name: Restore resolv.conf in config permdir (/opt/platform/config/...)
command: >-
tar -C {{ config_permdir }} -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ config_permdir }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
--overwrite --transform='s,.*/,,' '{{ short_config_permdir }}/resolv.conf'
args:
warn: false
- name: Restore dnsmaq in config permdir (/opt/platform/config/...)
command: >-
tar -C {{ config_permdir }} -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ config_permdir }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
--overwrite --transform='s,.*/,,' '{{ short_config_permdir }}/dnsmasq*'
args:
warn: false
@ -97,14 +97,14 @@
- name: Restore boot files in pxelinux.cfg dir
command: >-
tar -C {{ pxe_config_permdir }} -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ pxe_config_permdir }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
--overwrite --transform='s,.*/,,' '{{ short_pxe_config_permdir }}/*-*-*'
args:
warn: false
- name: Extract ldap.db to staging directory
command: >-
tar -C {{ staging_dir }} -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ staging_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
--transform='s,.*/,,' '*/ldap.db'
args:
warn: false
@ -138,13 +138,13 @@
state: absent
- name: Restore home directory
shell: tar -C / --overwrite -xpf {{ staging_dir }}/{{ backup_filename }} 'home/*'
shell: tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }} 'home/*'
args:
warn: false
become_user: root
- name: Restore Helm charts, armada manifests and extension filesystem
command: tar -C / --overwrite -xpf {{ staging_dir }}/{{ backup_filename }} {{ item }}
command: tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }} {{ item }}
args:
warn: false
become_user: root
@ -169,7 +169,7 @@
- name: Restore ceph crush map
command: >-
tar -C {{ ceph_crushmap_dir }} -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ ceph_crushmap_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
--transform='s,.*/,,' '*/crushmap.bin.backup'
args:
warn: false
@ -190,7 +190,7 @@
short_patch_vault_permdir: "{{ patch_vault_permdir | regex_replace('^\\/', '') }}"
- name: Look for patch-vault filesystem
shell: "tar -tf {{ staging_dir }}/{{ backup_filename }} | grep 'patch-vault'"
shell: "tar -tf {{ target_backup_dir }}/{{ backup_filename }} | grep 'patch-vault'"
args:
warn: false
failed_when: false
@ -198,7 +198,7 @@
- name: Restore patch-vault filesystem
command: >-
tar -C / --overwrite -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }}
{{ short_patch_vault_permdir }}
args:
warn: false
@ -246,7 +246,7 @@
- name: Extract postgres db to staging directory
command: >-
tar -C {{ staging_dir }}/postgres -xpf {{ staging_dir }}/{{ backup_filename }}
tar -C {{ staging_dir }}/postgres -xpf {{ target_backup_dir }}/{{ backup_filename }}
--transform='s,.*/,,' '*/*\.postgreSql\.*'
args:
warn: false
@ -338,6 +338,11 @@
retries: 30
delay: 10
- name: Remove {{ ansible_remote_tmp }} directory
file:
path: "{{ ansible_remote_tmp }}"
state: absent
- name: Inform user that restore_platform is run successfully
debug:
msg: >-
@ -351,8 +356,3 @@
Platform restore was unsuccessful. Please refer to the system administration
guide for next step.
when: check_online.rc != 0
# restore_openstack still needs the backup tarball to restore
# mariadb data
- name: Move backup tarball from {{ staging_dir }} to {{ backup_dir }}
command: mv {{ staging_dir }}/{{ backup_filename }} {{ backup_dir }}/{{ backup_filename }}

9
playbookconfig/src/playbooks/bootstrap/roles/restore-more-data/vars/main.yml
View File

@ -6,11 +6,8 @@ patching_repo_permdir: /www/pages/updates
extension_permdir: /opt/extension
patch_vault_permdir: /opt/patch-vault
ldap_permdir: /var/lib/openldap-data
kube_config_dir: "{{ lookup('env', 'KUBECONFIG') }}"
crushmap_file: crushmap.bin.backup
staging_dir: /scratch/restore
# Default directory where the backup tarball is stored at the end
# of the platform restore. As a continuation of the restore process,
# this backup tarball is needed when restoring the openstack application.
backup_dir: /opt/backups
# The staging area to process the data from the backup tarball
# after bootstrap controller-0 is done during the restore.
staging_dir: /opt/backups

2
playbookconfig/src/playbooks/bootstrap/roles/store-passwd/tasks/main.yml
View File

@ -123,7 +123,7 @@
- name: Prefetch the keyring from the tarball
command: >-
tar -C {{ temp_keyring_dir }} -xpf {{ staging_dir }}/{{ backup_filename }} --transform='s,.*/,,'
tar -C {{ temp_keyring_dir }} -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
{{ short_keyring_permdir }}
args:
warn: false