Choose node with largest seqno number for mariadb recovery

When all mariadb nodes are stopped gracefully, mariadb galera will
write it's last executed position into the grastate.dat file. Need find
the node with largest seqno number in that file and recovery from that
node.

Closes-Bug: #1627717
Change-Id: I6e97c190eec99c966bffde0698f783e519ba14bd

ansible/roles/mariadb/tasks/recover_cluster.yml

@@ -14,6 +14,60 @@
   when: container_state.Running | bool
   any_errors_fatal: True
 
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
+- block:
+  - name: Copying grastate.dat file from mariadb container
+    command: docker cp mariadb:/var/lib/mysql/grastate.dat /tmp/kolla_mariadb_grastate.dat
+    changed_when: false
+
+  - name: Print the content of grastate.dat file
+    command: cat /tmp/kolla_mariadb_grastate.dat
+    register: cat_grastate
+    changed_when: false
+
+  - name: Registering mariadb seqno variable
+    set_fact:
+      seqno: "{{ (cat_grastate.stdout|from_yaml).seqno }}"
+    changed_when: false
+
+  - name: Comparing seqno value on all mariadb hosts
+    shell: "if [[ {{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi"
+    with_items: "{{ groups['mariadb'] }}"
+    changed_when: false
+    register: seqno_compare
+
+  - name: Writing hostname of host with the largest seqno to temp file
+    local_action: copy content={{ inventory_hostname }} dest=/tmp/kolla_mariadb_recover_inventory_name mode=0644
+    changed_when: false
+    when: seqno_compare.results | map(attribute='stdout') | join('') == ""
+
+  - name: Registering mariadb_recover_inventory_name from temp file
+    set_fact:
+      mariadb_recover_inventory_name: "{{ lookup('file', '/tmp/kolla_mariadb_recover_inventory_name') }}"
+  when:
+    - mariadb_recover_inventory_name is not defined
+
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
 - name: Starting first MariaDB container
   kolla_docker:
     action: "start_container"

ansible/roles/mariadb/tasks/start.yml

@@ -12,7 +12,9 @@
       - "kolla_logs:/var/log/kolla/"
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
-        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name )
+
 
 # TODO(jeffrey4l), remove the task check when the wair_for bug is fixed
 # https://github.com/ansible/ansible-modules-core/issues/2788

tools/kolla-ansible

@@ -152,7 +152,7 @@ case "$1" in
         ;;
 (mariadb_recovery)
         ACTION="Attempting to restart mariadb cluster"
-        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
+        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy -e common_run=true"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
         ;;
 (destroy)