From 2aaaed770e158c3dc0d6e1895021c0e0475d200d Mon Sep 17 00:00:00 2001 From: SamYaple Date: Mon, 29 Feb 2016 15:02:15 +0000 Subject: [PATCH] MariaDB lights out recovery This playbook only matters for multinode since AIO can recover from power outage without additional configuration. DocImpact Implements: blueprint mariadb-lights-out Change-Id: I903c3bcd069af39814bcabcef37684b1f043391f --- ansible/mariadb_recovery.yml | 8 +++ ansible/roles/mariadb/tasks/bootstrap.yml | 69 ++----------------- .../roles/mariadb/tasks/bootstrap_cluster.yml | 31 +++++++++ .../roles/mariadb/tasks/lookup_cluster.yml | 35 ++++++++++ .../roles/mariadb/tasks/recover_cluster.yml | 39 +++++++++++ docker/mariadb/extend_start.sh | 1 - tools/kolla-ansible | 6 ++ 7 files changed, 123 insertions(+), 66 deletions(-) create mode 100644 ansible/mariadb_recovery.yml create mode 100644 ansible/roles/mariadb/tasks/bootstrap_cluster.yml create mode 100644 ansible/roles/mariadb/tasks/lookup_cluster.yml create mode 100644 ansible/roles/mariadb/tasks/recover_cluster.yml diff --git a/ansible/mariadb_recovery.yml b/ansible/mariadb_recovery.yml new file mode 100644 index 0000000000..53cf73b42b --- /dev/null +++ b/ansible/mariadb_recovery.yml @@ -0,0 +1,8 @@ +--- +- hosts: mariadb + roles: + - { role: mariadb, + tags: mariadb, + when: enable_mariadb | bool } + vars: + mariadb_recover: true diff --git a/ansible/roles/mariadb/tasks/bootstrap.yml b/ansible/roles/mariadb/tasks/bootstrap.yml index f6a1842615..6647cb598c 100644 --- a/ansible/roles/mariadb/tasks/bootstrap.yml +++ b/ansible/roles/mariadb/tasks/bootstrap.yml @@ -1,71 +1,10 @@ --- -- name: Cleaning up temp file on localhost - local_action: file path=/tmp/kolla_mariadb_cluster state=absent - changed_when: False - always_run: True - run_once: True +- include: lookup_cluster.yml -- name: Creating temp file on localhost - local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600 - changed_when: False - always_run: True - run_once: True - -- name: Creating mariadb volume - kolla_docker: - action: "create_volume" - common_options: "{{ docker_common_options }}" - name: "mariadb" - register: mariadb_volume - -- name: Writing hostname of host with existing cluster files to temp file - local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600 - changed_when: False - always_run: True - when: not mariadb_volume | changed - -- name: Registering host from temp file - set_fact: - delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}" - -- name: Cleaning up temp file on localhost - local_action: file path=/tmp/kolla_mariadb_cluster state=absent - changed_when: False - always_run: True - run_once: True - -- name: Starting MariaDB bootstrap container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - environment: - KOLLA_BOOTSTRAP: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - DB_ROOT_PASSWORD: "{{ database_password }}" - DB_MAX_TIMEOUT: "{{ database_max_timeout }}" - image: "{{ mariadb_image_full }}" - labels: - BOOTSTRAP: - name: "mariadb" - restart_policy: "never" - volumes: - - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" - - "mariadb:/var/lib/mysql" - - "kolla_logs:/var/log/kolla/" +- include: bootstrap_cluster.yml when: - delegate_host == 'None' - inventory_hostname == groups['mariadb'][0] -- name: Waiting for MariaDB service to be ready - command: "docker exec mariadb ls /var/lib/mysql/{{ item }}" - register: bootstrap_result - when: - - delegate_host == 'None' - - inventory_hostname == groups['mariadb'][0] - until: bootstrap_result | success - changed_when: False - retries: 6 - delay: 10 - with_items: - - cluster.exists - - mariadb.pid +- include: recover_cluster.yml + when: mariadb_recover | default(False) diff --git a/ansible/roles/mariadb/tasks/bootstrap_cluster.yml b/ansible/roles/mariadb/tasks/bootstrap_cluster.yml new file mode 100644 index 0000000000..6c3c8537f9 --- /dev/null +++ b/ansible/roles/mariadb/tasks/bootstrap_cluster.yml @@ -0,0 +1,31 @@ +--- +- name: Starting MariaDB bootstrap container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + DB_ROOT_PASSWORD: "{{ database_password }}" + DB_MAX_TIMEOUT: "{{ database_max_timeout }}" + ARGS: "--wsrep-new-cluster" + image: "{{ mariadb_image_full }}" + labels: + BOOTSTRAP: + name: "mariadb" + restart_policy: "never" + volumes: + - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" + - "kolla_logs:/var/log/kolla/" + - "mariadb:/var/lib/mysql" + +- name: Waiting for MariaDB service to be ready + command: "docker exec mariadb ls /var/lib/mysql/{{ item }}" + register: bootstrap_result + until: bootstrap_result | success + changed_when: False + retries: 6 + delay: 10 + with_items: + - cluster.exists + - mariadb.pid diff --git a/ansible/roles/mariadb/tasks/lookup_cluster.yml b/ansible/roles/mariadb/tasks/lookup_cluster.yml new file mode 100644 index 0000000000..b14afedc8d --- /dev/null +++ b/ansible/roles/mariadb/tasks/lookup_cluster.yml @@ -0,0 +1,35 @@ +--- +- name: Cleaning up temp file on localhost + local_action: file path=/tmp/kolla_mariadb_cluster state=absent + changed_when: False + always_run: True + run_once: True + +- name: Creating temp file on localhost + local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600 + changed_when: False + always_run: True + run_once: True + +- name: Creating mariadb volume + kolla_docker: + action: "create_volume" + common_options: "{{ docker_common_options }}" + name: "mariadb" + register: mariadb_volume + +- name: Writing hostname of host with existing cluster files to temp file + local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600 + changed_when: False + always_run: True + when: not mariadb_volume | changed + +- name: Registering host from temp file + set_fact: + delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}" + +- name: Cleaning up temp file on localhost + local_action: file path=/tmp/kolla_mariadb_cluster state=absent + changed_when: False + always_run: True + run_once: True diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml new file mode 100644 index 0000000000..2a19c11acb --- /dev/null +++ b/ansible/roles/mariadb/tasks/recover_cluster.yml @@ -0,0 +1,39 @@ +--- +- fail: + msg: "MariaDB cluster was not found. Is your inventory correct?" + when: delegate_host == 'None' + +- name: Checking if and mariadb containers are running + kolla_docker: + name: "mariadb" + action: "get_container_state" + register: container_state + +- fail: + msg: "There are running MariaDB nodes, please stop them first." + when: container_state.Running | bool + any_errors_fatal: True + +- name: Starting first MariaDB container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + ARGS: "--wsrep-new-cluster" + image: "{{ mariadb_image_full }}" + labels: + BOOTSTRAP: + name: "mariadb" + restart_policy: "never" + volumes: + - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" + - "kolla_logs:/var/log/kolla/" + - "mariadb:/var/lib/mysql" + when: + - (mariadb_recover_inventory_name is not defined and inventory_hostname == groups['mariadb'][0]) or + (mariadb_recover_inventory_name is defined and inventory_hostname == mariadb_recover_inventory_name) + +- name: Reset bootstrap fact + set_fact: + delegate_host: "None" diff --git a/docker/mariadb/extend_start.sh b/docker/mariadb/extend_start.sh index f6de85c362..3d826c4e85 100644 --- a/docker/mariadb/extend_start.sh +++ b/docker/mariadb/extend_start.sh @@ -33,7 +33,6 @@ fi # This catches all cases of the BOOTSTRAP variable being set, including empty if [[ "${!KOLLA_BOOTSTRAP[@]}" ]] && [[ ! -e /var/lib/mysql/cluster.exists ]]; then - ARGS="--wsrep-new-cluster" mysql_install_db bootstrap_db touch /var/lib/mysql/cluster.exists diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 0a1965cdbb..11ed742c77 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -38,6 +38,7 @@ Options: Commands: prechecks Do pre-deployment checks for hosts + mariadb_recovery Recover a completely stopped mariadb cluster deploy Deploy and start all kolla containers post-deploy Do post deploy on deploy node pull Pull all images for containers (only pulls, no runnnig container changes) @@ -119,6 +120,11 @@ case "$1" in ACTION="Pre-deployment checking" PLAYBOOK="${BASEDIR}/ansible/prechecks.yml" ;; +(mariadb_recovery) + ACTION="Attempting to restart mariadb cluster" + EXTRA_OPTS="$EXTRA_OPTS -e action=deploy" + PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml" + ;; (deploy) ACTION="Deploying Playbooks" EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"