Only do snapshot on inactive nodes

The nodes must be taken out from Pacemaker cluster before taking a snapshot, one node at a time to not cause any downtimes. This will ensure that the database and the Ceph snapshots are consistent and we do not take the snapshot just during a write.

This behaviour can be deactivated using a parameter.

Also added the molecule data to use puppet lookup command

Change-Id: I3ea96ff2219733b3d8dd38680f2fa9eb50bb546b

tripleo_ansible/playbooks/cli-overcloud-snapshot.yaml

@@ -13,11 +13,24 @@
 #   License for the specific language governing permissions and limitations
 #   under the License.
 
-- name: TripleO Snapshots
-  hosts: "{{ tripleo_controller_group_name }},{{ tripleo_compute_group_name }}"
+- name: TripleO Compute Snapshots
+  hosts: "{{ tripleo_compute_group_name }}"
   remote_user: "{{ tripleo_target_user | default(lookup('env', 'USER')) }}"
-  vars_files:
-    ../roles/snapshot_and_revert/defaults/main.yml
+  become: true
+  pre_tasks:
+    - name: Gather facts with an active connection
+      setup:
+        gather_subset:
+          - 'devices'
+      tags:
+        - always
+  roles:
+    - role: snapshot_and_revert
+
+- name: TripleO Controller Snapshots
+  hosts: "{{ tripleo_controller_group_name }}"
+  remote_user: "{{ tripleo_target_user | default(lookup('env', 'USER')) }}"
+  serial: "{{ tripleo_snapshot_revert_snapshot_inactive_nodes|bool | ternary('1', omit) }}"
   become: true
   pre_tasks:
     - name: Gather facts with an active connection

tripleo_ansible/roles/snapshot_and_revert/defaults/main.yml

@@ -23,3 +23,8 @@ tripleo_snapshot_revert_log_size: 1G
 tripleo_snapshot_revert_root_size: 2G
 tripleo_snapshot_revert_srv_size: 512M
 tripleo_snapshot_revert_var_size: 3G
+
+# These parameters ensure that nodes are taken out from Pacemaker cluster
+# before taking a snapshot, one node at a time to not cause any downtimes.
+tripleo_snapshot_revert_snapshot_inactive_nodes: "{{ snapshot_inactive_nodes | default('true') }}"
+tripleo_snapshot_revert_pcs_timeout: 3600

tripleo_ansible/roles/snapshot_and_revert/molecule/default/prepare.yml

@@ -0,0 +1,90 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  become: true
+  gather_facts: true
+  pre_tasks:
+    - name: set basic user fact
+      set_fact:
+        ansible_user: "{{ lookup('env', 'USER') }}"
+      when:
+        - ansible_user is undefined
+
+    - name: set basic home fact
+      set_fact:
+        ansible_user_dir: "{{ lookup('env', 'HOME') }}"
+      when:
+        - ansible_user_dir is undefined
+
+    - name: Disable SELinux
+      selinux:
+        state: disabled
+
+  roles:
+    - role: test_deps
+      test_deps_setup_tripleo: true
+      test_deps_tripleo_packages:
+        - hiera
+        - puppet-tripleo
+    - role: env_data
+
+  post_tasks:
+    - name: Create hiera config file
+      file:
+        path: "{{ ansible_user_dir }}/hiera.yaml"
+        state: touch
+
+    - name: Create hieradata directory
+      file:
+        path: "{{ ansible_user_dir }}/hieradata"
+        state: directory
+        mode: '0755'
+
+    - name: Insert some data into hiera.yaml
+      copy:
+        dest: "{{ ansible_user_dir }}/hiera.yaml"
+        content: |
+            ---
+            :backends:
+              - json
+            :json:
+              :datadir: {{ ansible_user_dir }}/hieradata
+            :hierarchy:
+              - service_configs
+              - service_names
+
+    - name: Insert some data into service_configs.json
+      copy:
+        dest: "{{ ansible_user_dir }}/hieradata/service_configs.json"
+        content: |
+            {
+                "mysql::server::root_password": "password"
+            }
+
+    - name: Insert some data into service_names.json
+      copy:
+        dest: "{{ ansible_user_dir }}/hieradata/service_names.json"
+        content: |
+            {
+                "service_names": [
+                    "mysql",
+                    "mysql_client",
+                    "tripleo_validations"
+                ]
+            }

tripleo_ansible/roles/snapshot_and_revert/tasks/main.yml

@@ -109,6 +109,24 @@
   tags:
     - create_snapshots
 
+- name: Read Services running on the host
+  command: puppet lookup --explain 'service_names'
+  when: tripleo_snapshot_revert_snapshot_inactive_nodes|bool
+  register: services_enabled
+  changed_when: false
+  failed_when: false
+  tags:
+    - create_snapshots
+
+- name: Take out the node from Pacemaker cluster before taking a snapshot
+  command: pcs cluster stop --request-timeout={{ tripleo_snapshot_revert_pcs_timeout }}
+  when:
+    - tripleo_snapshot_revert_snapshot_inactive_nodes|bool
+    - services_enabled is defined
+    - '"pacemaker" in services_enabled.stdout'
+  tags:
+    - create_snapshots
+
 - name: Create lvm snapshot if there are at least 8GB of free space in the vg volumegroup
   when:
     - ansible_facts.lvm is defined
@@ -142,3 +160,18 @@
       when: ansible_facts.lvm.lvs.lv_var is defined
   tags:
     - create_snapshots
+
+- name: Pacemaker management
+  when:
+    - services_enabled is defined
+    - tripleo_snapshot_revert_snapshot_inactive_nodes|bool
+    - "'pacemaker' in services_enabled.stdout"
+  block:
+    - name: Add the node to the pacemaker cluster
+      command: pcs cluster start
+
+    - name: Wait until pacemaker has Galera up&running
+      shell: /var/lib/container-config-scripts/pacemaker_wait_bundle.sh galera galera-bundle Master
+      when: "'mysql' in services_enabled.stdout"
+  tags:
+    - create_snapshots