From 9e85dd4943da47938b14a967f1b9548d202b03f0 Mon Sep 17 00:00:00 2001 From: Juan Badia Payno Date: Wed, 28 Jul 2021 13:33:52 +0200 Subject: [PATCH] BnR: managed ceph properly to take a backup of the node This patch stops ceph services before doing a backup and starts the ceph services after the backup. No matter how ceph was installed (ceph-ansible or cephadm) Change-Id: Iffab9c82f9ecdd5845cc93484ae15f70c9153341 --- .../backup_and_restore/defaults/main.yml | 8 ++- .../roles/backup_and_restore/tasks/ceph.yml | 53 +++++++++++++++++++ .../tasks/ceph_authentication.yml | 46 ++++++++++++---- .../backup_and_restore/tasks/ceph_start.yml | 36 +++++++++++++ .../backup_and_restore/tasks/ceph_stop.yml | 38 +++++++++++++ .../backup_and_restore/tasks/cephadm_stat.yml | 32 +++++++++++ .../roles/backup_and_restore/tasks/main.yml | 12 +++++ .../backup_and_restore/tasks/setup_rear.yml | 2 +- 8 files changed, 215 insertions(+), 12 deletions(-) create mode 100644 tripleo_ansible/roles/backup_and_restore/tasks/ceph.yml create mode 100644 tripleo_ansible/roles/backup_and_restore/tasks/ceph_start.yml create mode 100644 tripleo_ansible/roles/backup_and_restore/tasks/ceph_stop.yml create mode 100644 tripleo_ansible/roles/backup_and_restore/tasks/cephadm_stat.yml diff --git a/tripleo_ansible/roles/backup_and_restore/defaults/main.yml b/tripleo_ansible/roles/backup_and_restore/defaults/main.yml index 17f251279..f4f0c0ee0 100644 --- a/tripleo_ansible/roles/backup_and_restore/defaults/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/defaults/main.yml @@ -45,7 +45,7 @@ tripleo_backup_and_restore_nfs_clients_nets: ['192.168.24.0/24', '10.0.0.0/24', tripleo_backup_and_restore_rear_simulate: false tripleo_backup_and_restore_using_uefi_bootloader: 0 tripleo_backup_and_restore_exclude_paths_common: ['/data/*', '/tmp/*', '{{ tripleo_backup_and_restore_nfs_storage_folder }}/*'] -tripleo_backup_and_restore_exclude_paths_controller_non_bootrapnode: true +tripleo_backup_and_restore_exclude_paths_controller_non_bootstrapnode: false tripleo_backup_and_restore_exclude_paths_controller: ['/var/lib/mysql/*'] tripleo_backup_and_restore_exclude_paths_compute: ['/var/lib/nova/instances/*'] tripleo_backup_and_restore_hiera_config_file: "/etc/puppet/hiera.yaml" @@ -106,3 +106,9 @@ tripleo_backup_and_restore_cron_user: "stack" # Any extra parameters that will be added to the backup command when it is executed by cron tripleo_backup_and_restore_cron_extra: "" + +# The role which handles the ceph on the controllers +tripleo_backup_and_restore_ceph_mon_role: "ceph_mon" + +# The cephadm path +tripleo_backup_and_restore_cephadm_path: "/usr/sbin/cephadm" diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/ceph.yml b/tripleo_ansible/roles/backup_and_restore/tasks/ceph.yml new file mode 100644 index 000000000..4b5a50666 --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/tasks/ceph.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Ceph installed with ceph ansible + tags: bar_create_recover_image + when: + - cephadm_ls.stdout is not defined or cephadm_ls.stdout == "[]" + block: + + - name: Set ceph monitor {{ ceph_systemd_state }} + systemd: + state: "{{ ceph_systemd_state }}" + name: "ceph-mon@{{ inventory_hostname }}" + + - name: Set ceph management {{ ceph_systemd_state }} + systemd: + state: "{{ ceph_systemd_state }}" + name: "ceph-mgr@{{ inventory_hostname }}" + + - name: Get the node names of the mds nodes + command: hiera -c '{{ tripleo_backup_and_restore_hiera_config_file }}' 'ceph_mds_short_node_names' + register: bar_ceph_mds_nodes + + - name: Set ceph mds {{ ceph_systemd_state }} + systemd: + state: "{{ ceph_systemd_state }}" + name: "ceph-mds@{{ inventory_hostname }}" + when: + - bar_ceph_mds_nodes.stdout != "nil" + - inventory_hostname in bar_ceph_mds_nodes.stdout + + - name: Get the node names of the rgw nodes + command: hiera -c '{{ tripleo_backup_and_restore_hiera_config_file }}' 'ceph_rgw_short_node_names' + register: bar_ceph_rgw_nodes + + - name: Set ceph rgw {{ ceph_systemd_state }} + command: systemctl start "ceph-radosgw@rgw.{{ inventory_hostname }}.*" + when: + - bar_ceph_rgw_nodes != "nil" + - inventory_hostname in bar_ceph_rgw_nodes.stdout diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/ceph_authentication.yml b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_authentication.yml index 2e2dfb20e..0a457d9ee 100644 --- a/tripleo_ansible/roles/backup_and_restore/tasks/ceph_authentication.yml +++ b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_authentication.yml @@ -32,24 +32,50 @@ tags: - always -- name: Ensure cephmon container is running - command: "{{ tripleo_container_cli }} unpause ceph-mon-{{ ansible_facts['hostname'] }}" - failed_when: false +- name: Stat cephadm file + stat: + path: "{{ tripleo_backup_and_restore_cephadm_path }}" + register: stat_cephadm + become: true + ignore_errors: true tags: - bar_create_recover_image -- name: Export ceph authentication - shell: | - set -o pipefail - {{ tripleo_container_cli }} exec ceph-mon-{{ ansible_facts['hostname'] }} bash -c "ceph auth export" - register: bar_ceph_authentication_content +- name: List Ceph daemon instances on this host + shell: "{{ tripleo_backup_and_restore_cephadm_path }} ls --no-detail" + register: cephadm_ls + become: true + when: stat_cephadm.stat.exists tags: - bar_create_recover_image +- name: Ceph installed with ceph ansible + when: + - cephadm_ls.stdout is not defined or cephadm_ls.stdout == "[]" + tags: bar_create_recover_image + block: + - name: Ensure cephmon container is running + command: "{{ tripleo_container_cli }} unpause ceph-mon-{{ inventory_hostname }}" + failed_when: false + + - name: Export ceph authentication + shell: | + set -o pipefail + {{ tripleo_container_cli }} exec ceph-mon-{{ inventory_hostname }} bash -c "ceph auth export" + register: bar_ceph_authentication_content + +- name: Ceph installed with cephadm + when: + - cephadm_ls.stdout is defined + - cephadm_ls.stdout != '[]' + tags: bar_create_recover_image + block: + - name: Export ceph authentication + command: cephadm shell -- ceph auth export + register: bar_ceph_authentication_content + - name: Save ceph authentication at {{ tripleo_backup_and_restore_ceph_auth_file }} copy: dest: "{{ tripleo_backup_and_restore_ceph_auth_file }}" content: | {{ bar_ceph_authentication_content.stdout }} - tags: - - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/ceph_start.yml b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_start.yml new file mode 100644 index 000000000..edc2ae4fe --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_start.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Stat cephadm file + import_tasks: cephadm_stat.yml + +- name: Start ceph with ceph-ansible + import_tasks: ceph.yml + vars: + ceph_systemd_state: started + +- name: Ceph installed with cephadm + when: + - cephadm_ls.stdout is defined + - cephadm_ls.stdout != "[]" + tags: bar_create_recover_image + block: + + - name: Start ceph + systemd: + state: started + name: "ceph.target" + become: true diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/ceph_stop.yml b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_stop.yml new file mode 100644 index 000000000..0e1a49a4e --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/tasks/ceph_stop.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Stat cephadm file + import_tasks: cephadm_stat.yml + +- name: Stop ceph with ceph-ansible + import_tasks: ceph.yml + vars: + ceph_systemd_state: stopped + +- name: Ceph installed with cephadm + when: + - cephadm_ls.stdout is defined + - cephadm_ls.stdout != "[]" + tags: bar_create_recover_image + block: + + - name: Stop ceph\*.service + command: systemctl stop ceph\*.service + become: true + + - name: Stop ceph\*.target + command: systemctl stop ceph\*.target + become: true diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/cephadm_stat.yml b/tripleo_ansible/roles/backup_and_restore/tasks/cephadm_stat.yml new file mode 100644 index 000000000..93d9fdc23 --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/tasks/cephadm_stat.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Stat cephadm file + stat: + path: "{{ tripleo_backup_and_restore_cephadm_path }}" + register: stat_cephadm + become: true + ignore_errors: true + tags: + - bar_create_recover_image + +- name: List Ceph daemon instances on this host + shell: "{{ tripleo_backup_and_restore_cephadm_path }} ls --no-detail" + register: cephadm_ls + become: true + when: stat_cephadm.stat.exists + tags: + - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/main.yml b/tripleo_ansible/roles/backup_and_restore/tasks/main.yml index 2433234c0..d5d4e358e 100644 --- a/tripleo_ansible/roles/backup_and_restore/tasks/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/tasks/main.yml @@ -52,6 +52,12 @@ - name: Perform backup block: + - name: Stop ceph services + import_tasks: ceph_stop.yml + when: + - tripleo_backup_and_restore_ceph_mon_role in groups + - inventory_hostname in groups[tripleo_backup_and_restore_ceph_mon_role] + - name: Create recovery images with ReaR import_tasks: run_backup.yml always: @@ -66,3 +72,9 @@ when: - pacemaker_enabled - tripleo_backup_and_restore_enable_snapshots|bool + + - name: Start ceph services + import_tasks: ceph_start.yml + when: + - tripleo_backup_and_restore_ceph_mon_role in groups + - inventory_hostname in groups[tripleo_backup_and_restore_ceph_mon_role] diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/setup_rear.yml b/tripleo_ansible/roles/backup_and_restore/tasks/setup_rear.yml index 6c799a9d7..672b8b3e4 100644 --- a/tripleo_ansible/roles/backup_and_restore/tasks/setup_rear.yml +++ b/tripleo_ansible/roles/backup_and_restore/tasks/setup_rear.yml @@ -77,7 +77,7 @@ - "{{ (tripleo_backup_and_restore_bootstrap_nodeid.stdout != tripleo_backup_and_restore_hostname.stdout and tripleo_backup_and_restore_enabled_services.stdout is search('pacemaker') and tripleo_backup_and_restore_enabled_services.stdout is search('mysql') and - tripleo_backup_and_restore_exclude_paths_controller_non_bootrapnode|bool) | + tripleo_backup_and_restore_exclude_paths_controller_non_bootstrapnode|bool) | ternary(tripleo_backup_and_restore_exclude_paths_controller, []) }}" tags: - bar_setup_rear