MariaDB lights out recovery

This playbook only matters for multinode since AIO can recover from
power outage without additional configuration.

DocImpact
Implements: blueprint mariadb-lights-out
Change-Id: I903c3bcd069af39814bcabcef37684b1f043391f
This commit is contained in:
SamYaple 2016-02-29 15:02:15 +00:00
parent f4d6aaf097
commit 2aaaed770e
7 changed files with 123 additions and 66 deletions

View File

@ -0,0 +1,8 @@
---
- hosts: mariadb
roles:
- { role: mariadb,
tags: mariadb,
when: enable_mariadb | bool }
vars:
mariadb_recover: true

View File

@ -1,71 +1,10 @@
---
- name: Cleaning up temp file on localhost
local_action: file path=/tmp/kolla_mariadb_cluster state=absent
changed_when: False
always_run: True
run_once: True
- include: lookup_cluster.yml
- name: Creating temp file on localhost
local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600
changed_when: False
always_run: True
run_once: True
- name: Creating mariadb volume
kolla_docker:
action: "create_volume"
common_options: "{{ docker_common_options }}"
name: "mariadb"
register: mariadb_volume
- name: Writing hostname of host with existing cluster files to temp file
local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600
changed_when: False
always_run: True
when: not mariadb_volume | changed
- name: Registering host from temp file
set_fact:
delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}"
- name: Cleaning up temp file on localhost
local_action: file path=/tmp/kolla_mariadb_cluster state=absent
changed_when: False
always_run: True
run_once: True
- name: Starting MariaDB bootstrap container
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
KOLLA_BOOTSTRAP:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
DB_ROOT_PASSWORD: "{{ database_password }}"
DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
image: "{{ mariadb_image_full }}"
labels:
BOOTSTRAP:
name: "mariadb"
restart_policy: "never"
volumes:
- "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
- "mariadb:/var/lib/mysql"
- "kolla_logs:/var/log/kolla/"
- include: bootstrap_cluster.yml
when:
- delegate_host == 'None'
- inventory_hostname == groups['mariadb'][0]
- name: Waiting for MariaDB service to be ready
command: "docker exec mariadb ls /var/lib/mysql/{{ item }}"
register: bootstrap_result
when:
- delegate_host == 'None'
- inventory_hostname == groups['mariadb'][0]
until: bootstrap_result | success
changed_when: False
retries: 6
delay: 10
with_items:
- cluster.exists
- mariadb.pid
- include: recover_cluster.yml
when: mariadb_recover | default(False)

View File

@ -0,0 +1,31 @@
---
- name: Starting MariaDB bootstrap container
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
KOLLA_BOOTSTRAP:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
DB_ROOT_PASSWORD: "{{ database_password }}"
DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
ARGS: "--wsrep-new-cluster"
image: "{{ mariadb_image_full }}"
labels:
BOOTSTRAP:
name: "mariadb"
restart_policy: "never"
volumes:
- "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
- "kolla_logs:/var/log/kolla/"
- "mariadb:/var/lib/mysql"
- name: Waiting for MariaDB service to be ready
command: "docker exec mariadb ls /var/lib/mysql/{{ item }}"
register: bootstrap_result
until: bootstrap_result | success
changed_when: False
retries: 6
delay: 10
with_items:
- cluster.exists
- mariadb.pid

View File

@ -0,0 +1,35 @@
---
- name: Cleaning up temp file on localhost
local_action: file path=/tmp/kolla_mariadb_cluster state=absent
changed_when: False
always_run: True
run_once: True
- name: Creating temp file on localhost
local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600
changed_when: False
always_run: True
run_once: True
- name: Creating mariadb volume
kolla_docker:
action: "create_volume"
common_options: "{{ docker_common_options }}"
name: "mariadb"
register: mariadb_volume
- name: Writing hostname of host with existing cluster files to temp file
local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600
changed_when: False
always_run: True
when: not mariadb_volume | changed
- name: Registering host from temp file
set_fact:
delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}"
- name: Cleaning up temp file on localhost
local_action: file path=/tmp/kolla_mariadb_cluster state=absent
changed_when: False
always_run: True
run_once: True

View File

@ -0,0 +1,39 @@
---
- fail:
msg: "MariaDB cluster was not found. Is your inventory correct?"
when: delegate_host == 'None'
- name: Checking if and mariadb containers are running
kolla_docker:
name: "mariadb"
action: "get_container_state"
register: container_state
- fail:
msg: "There are running MariaDB nodes, please stop them first."
when: container_state.Running | bool
any_errors_fatal: True
- name: Starting first MariaDB container
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
ARGS: "--wsrep-new-cluster"
image: "{{ mariadb_image_full }}"
labels:
BOOTSTRAP:
name: "mariadb"
restart_policy: "never"
volumes:
- "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
- "kolla_logs:/var/log/kolla/"
- "mariadb:/var/lib/mysql"
when:
- (mariadb_recover_inventory_name is not defined and inventory_hostname == groups['mariadb'][0]) or
(mariadb_recover_inventory_name is defined and inventory_hostname == mariadb_recover_inventory_name)
- name: Reset bootstrap fact
set_fact:
delegate_host: "None"

View File

@ -33,7 +33,6 @@ fi
# This catches all cases of the BOOTSTRAP variable being set, including empty
if [[ "${!KOLLA_BOOTSTRAP[@]}" ]] && [[ ! -e /var/lib/mysql/cluster.exists ]]; then
ARGS="--wsrep-new-cluster"
mysql_install_db
bootstrap_db
touch /var/lib/mysql/cluster.exists

View File

@ -38,6 +38,7 @@ Options:
Commands:
prechecks Do pre-deployment checks for hosts
mariadb_recovery Recover a completely stopped mariadb cluster
deploy Deploy and start all kolla containers
post-deploy Do post deploy on deploy node
pull Pull all images for containers (only pulls, no runnnig container changes)
@ -119,6 +120,11 @@ case "$1" in
ACTION="Pre-deployment checking"
PLAYBOOK="${BASEDIR}/ansible/prechecks.yml"
;;
(mariadb_recovery)
ACTION="Attempting to restart mariadb cluster"
EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
;;
(deploy)
ACTION="Deploying Playbooks"
EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"