From 5cd55bf236adc4c271418bda7306ec2336fa8816 Mon Sep 17 00:00:00 2001 From: caoyuan Date: Tue, 14 Feb 2017 16:07:50 +0800 Subject: [PATCH] Optimize reconfiguration for mariadb Change-Id: I278609f9832955849bc9381120a1b260f5a03f1b Partially-implements: blueprint better-reconfigure --- ansible/roles/mariadb/defaults/main.yml | 13 ++ ansible/roles/mariadb/handlers/main.yml | 120 ++++++++++++++++++ ansible/roles/mariadb/tasks/bootstrap.yml | 5 +- .../roles/mariadb/tasks/bootstrap_cluster.yml | 21 +-- ansible/roles/mariadb/tasks/config.yml | 67 ++++++++-- ansible/roles/mariadb/tasks/deploy.yml | 15 +-- .../roles/mariadb/tasks/lookup_cluster.yml | 4 +- ansible/roles/mariadb/tasks/pull.yml | 7 +- ansible/roles/mariadb/tasks/reconfigure.yml | 66 +--------- .../roles/mariadb/tasks/recover_cluster.yml | 36 ++---- ansible/roles/mariadb/tasks/start.yml | 31 ----- ansible/roles/mariadb/tasks/upgrade.yml | 8 +- 12 files changed, 215 insertions(+), 178 deletions(-) create mode 100644 ansible/roles/mariadb/handlers/main.yml delete mode 100644 ansible/roles/mariadb/tasks/start.yml diff --git a/ansible/roles/mariadb/defaults/main.yml b/ansible/roles/mariadb/defaults/main.yml index 410e86d3e9..917100816c 100644 --- a/ansible/roles/mariadb/defaults/main.yml +++ b/ansible/roles/mariadb/defaults/main.yml @@ -1,6 +1,19 @@ --- project_name: "mariadb" +mariadb_services: + mariadb: + container_name: mariadb + group: mariadb + enabled: true + image: "{{ mariadb_image_full }}" + volumes: + - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "mariadb:/var/lib/mysql" + - "kolla_logs:/var/log/kolla/" + + #################### # Database #################### diff --git a/ansible/roles/mariadb/handlers/main.yml b/ansible/roles/mariadb/handlers/main.yml new file mode 100644 index 0000000000..2e74012878 --- /dev/null +++ b/ansible/roles/mariadb/handlers/main.yml @@ -0,0 +1,120 @@ +--- +- name: Starting first MariaDB container + vars: + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + BOOTSTRAP_ARGS: "--wsrep-new-cluster" + image: "{{ service.image }}" + labels: + BOOTSTRAP: + name: "{{ service.container_name }}" + restart_policy: "never" + volumes: "{{ service.volumes }}" + when: + - bootstrap_host is defined + - bootstrap_host == inventory_hostname + notify: + - wait first mariadb container + - restart slave mariadb + - restart master mariadb + + +# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed +# https://github.com/ansible/ansible-modules-core/issues/2788 +- name: wait first mariadb container + wait_for: + host: "{{ api_interface_address }}" + port: "{{ mariadb_port }}" + connect_timeout: 1 + timeout: 60 + search_regex: "MariaDB" + register: check_mariadb_port + until: check_mariadb_port | success + retries: 10 + delay: 6 + when: + - bootstrap_host is defined + - bootstrap_host == inventory_hostname + +- name: restart slave mariadb + vars: + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" + mariadb_container: "{{ check_mariadb_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - inventory_hostname != master_host + - inventory_hostname in groups[service.group] + - service.enabled | bool + - mariadb_config_json.changed | bool + or mariadb_galera_conf.changed | bool + or mariadb_wsrep_notify.changed | bool + or mariadb_container.changed | bool + or bootstrap_host is defined + notify: + - wait for slave mariadb + +# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed +# https://github.com/ansible/ansible-modules-core/issues/2788 +- name: wait for slave mariadb + wait_for: + host: "{{ api_interface_address }}" + port: "{{ mariadb_port }}" + connect_timeout: 1 + timeout: 60 + search_regex: "MariaDB" + register: check_mariadb_port + until: check_mariadb_port | success + retries: 10 + delay: 6 + when: + - inventory_hostname != master_host + +- name: restart master mariadb + vars: + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" + mariadb_container: "{{ check_mariadb_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - inventory_hostname == master_host + - inventory_hostname in groups[service.group] + - service.enabled | bool + - mariadb_config_json.changed | bool + or mariadb_galera_conf.changed | bool + or mariadb_wsrep_notify.changed | bool + or mariadb_container.changed | bool + or bootstrap_host is defined + notify: + - Waiting for master mariadb + +# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed +# https://github.com/ansible/ansible-modules-core/issues/2788 +- name: Waiting for master mariadb + wait_for: + host: "{{ api_interface_address }}" + port: "{{ mariadb_port }}" + connect_timeout: 1 + timeout: 60 + search_regex: "MariaDB" + register: check_mariadb_port + until: check_mariadb_port | success + retries: 10 + delay: 6 + when: + - inventory_hostname == master_host diff --git a/ansible/roles/mariadb/tasks/bootstrap.yml b/ansible/roles/mariadb/tasks/bootstrap.yml index 6647cb598c..5b6b33f5a5 100644 --- a/ansible/roles/mariadb/tasks/bootstrap.yml +++ b/ansible/roles/mariadb/tasks/bootstrap.yml @@ -1,9 +1,12 @@ --- +- set_fact: + master_host: "{{ groups['mariadb'][0] }}" + - include: lookup_cluster.yml - include: bootstrap_cluster.yml when: - - delegate_host == 'None' + - not has_cluster | bool - inventory_hostname == groups['mariadb'][0] - include: recover_cluster.yml diff --git a/ansible/roles/mariadb/tasks/bootstrap_cluster.yml b/ansible/roles/mariadb/tasks/bootstrap_cluster.yml index 51915f4e32..56dabcc66b 100644 --- a/ansible/roles/mariadb/tasks/bootstrap_cluster.yml +++ b/ansible/roles/mariadb/tasks/bootstrap_cluster.yml @@ -19,21 +19,8 @@ - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" - "mariadb:/var/lib/mysql" + notify: + - Starting first MariaDB container -- name: Starting first MariaDB container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - environment: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - BOOTSTRAP_ARGS: "--wsrep-new-cluster" - image: "{{ mariadb_image_full }}" - labels: - BOOTSTRAP: - name: "mariadb" - restart_policy: "never" - volumes: - - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" - - "/etc/localtime:/etc/localtime:ro" - - "kolla_logs:/var/log/kolla/" - - "mariadb:/var/lib/mysql" +- set_fact: + bootstrap_host: "{{ inventory_hostname }}" diff --git a/ansible/roles/mariadb/tasks/config.yml b/ansible/roles/mariadb/tasks/config.yml index 5a0f36d5f8..01a9790dc2 100644 --- a/ansible/roles/mariadb/tasks/config.yml +++ b/ansible/roles/mariadb/tasks/config.yml @@ -1,34 +1,73 @@ --- - name: Ensuring config directories exist file: - path: "{{ node_config_directory }}/{{ item }}" + path: "{{ node_config_directory }}/{{ item.key }}" state: "directory" recurse: yes - with_items: - - "mariadb" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ mariadb_services }}" - name: Copying over config.json files for services + vars: + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" template: - src: "{{ item }}.json.j2" - dest: "{{ node_config_directory }}/{{ item }}/config.json" - with_items: - - "mariadb" + src: "{{ service_name }}.json.j2" + dest: "{{ node_config_directory }}/{{ service_name }}/config.json" + register: mariadb_config_json + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + notify: + - restart slave mariadb + - restart master mariadb - name: Copying over galera.cnf vars: - service_name: "{{ item }}" + service_name: "mariadb" + service: "{{ mariadb_services[service_name] }}" merge_configs: sources: - "{{ role_path }}/templates/galera.cnf.j2" - "{{ node_custom_config }}/galera.cnf" - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/galera.cnf" - dest: "{{ node_config_directory }}/{{ item }}/galera.cnf" - with_items: - - "mariadb" + dest: "{{ node_config_directory }}/{{ service_name }}/galera.cnf" + register: mariadb_galera_conf + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + notify: + - restart slave mariadb + - restart master mariadb - name: Copying over wsrep-notify.sh template: src: "{{ role_path }}/templates/wsrep-notify.sh.j2" - dest: "{{ node_config_directory }}/{{ item }}/wsrep-notify.sh" - with_items: - - "mariadb" + dest: "{{ node_config_directory }}/{{ item.key }}/wsrep-notify.sh" + register: mariadb_wsrep_notify + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ mariadb_services }}" + notify: + - restart slave mariadb + - restart master mariadb + +- name: Check mariadb containers + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + register: check_mariadb_containers + when: + - action != "config" + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ mariadb_services }}" + notify: + - restart slave mariadb + - restart master mariadb diff --git a/ansible/roles/mariadb/tasks/deploy.yml b/ansible/roles/mariadb/tasks/deploy.yml index 93f19edce4..bc8177dea4 100644 --- a/ansible/roles/mariadb/tasks/deploy.yml +++ b/ansible/roles/mariadb/tasks/deploy.yml @@ -3,20 +3,9 @@ - include: bootstrap.yml -- include: start.yml +- name: Flush handlers + meta: flush_handlers -# We use register as a test to see whether the database is active and ready to -# communicate. This run on all hosts that have a database and attempts to talk -# to the local database rather than the vip, -- include: register.yml - -# This will restart the container we initially used to bootstrap the cluster to -# make it match the other containers environment-wise. This also prevents a -# change from showing up when rerunning the playbooks -- include: start.yml - -# Since the last start.yml may have recreated some containers we must wait and -# check the health again to ensure the hosts are active. - include: register.yml # Test haproxy user through VIP diff --git a/ansible/roles/mariadb/tasks/lookup_cluster.yml b/ansible/roles/mariadb/tasks/lookup_cluster.yml index efca14ee08..0634573fa0 100644 --- a/ansible/roles/mariadb/tasks/lookup_cluster.yml +++ b/ansible/roles/mariadb/tasks/lookup_cluster.yml @@ -6,7 +6,7 @@ run_once: True - name: Creating temp file on localhost - local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0644 + local_action: copy content='' dest=/tmp/kolla_mariadb_cluster mode=0644 changed_when: False check_mode: no run_once: True @@ -26,7 +26,7 @@ - name: Registering host from temp file set_fact: - delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}" + has_cluster: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') | length > 0 }}" - name: Cleaning up temp file on localhost local_action: file path=/tmp/kolla_mariadb_cluster state=absent diff --git a/ansible/roles/mariadb/tasks/pull.yml b/ansible/roles/mariadb/tasks/pull.yml index 13d27693b3..1fc2cc41f3 100644 --- a/ansible/roles/mariadb/tasks/pull.yml +++ b/ansible/roles/mariadb/tasks/pull.yml @@ -3,5 +3,8 @@ kolla_docker: action: "pull_image" common_options: "{{ docker_common_options }}" - image: "{{ mariadb_image_full }}" - when: inventory_hostname in groups['mariadb'] + image: "{{ item.value.image }}" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ mariadb_services }}" diff --git a/ansible/roles/mariadb/tasks/reconfigure.yml b/ansible/roles/mariadb/tasks/reconfigure.yml index 92bf538c47..e078ef1318 100644 --- a/ansible/roles/mariadb/tasks/reconfigure.yml +++ b/ansible/roles/mariadb/tasks/reconfigure.yml @@ -1,66 +1,2 @@ --- -- name: Ensuring the containers up - kolla_docker: - name: "{{ item.name }}" - action: "get_container_state" - register: container_state - failed_when: container_state.Running == false - when: inventory_hostname in groups[item.group] - with_items: - - { name: mariadb, group: mariadb } - -- include: config.yml - -- name: Check the configs - command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check - changed_when: false - failed_when: false - register: check_results - when: inventory_hostname in groups[item.group] - with_items: - - { name: mariadb, group: mariadb } - -# NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS' -# and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE', -# just remove the container and start again -- name: Containers config strategy - kolla_docker: - name: "{{ item.name }}" - action: "get_container_env" - register: container_envs - when: inventory_hostname in groups[item.group] - with_items: - - { name: mariadb, group: mariadb } - -- name: Remove the containers - kolla_docker: - name: "{{ item[0]['name'] }}" - action: "remove_container" - register: remove_containers - when: - - config_strategy == "COPY_ONCE" or item[1]['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE' - - item[2]['rc'] == 1 - - inventory_hostname in groups[item[0]['group']] - with_together: - - [{ name: mariadb, group: mariadb }] - - "{{ container_envs.results }}" - - "{{ check_results.results }}" - -- include: start.yml - when: remove_containers.changed - -- name: Restart containers - kolla_docker: - name: "{{ item[0]['name'] }}" - action: "restart_container" - when: - - config_strategy == 'COPY_ALWAYS' - - item[1]['KOLLA_CONFIG_STRATEGY'] != 'COPY_ONCE' - - item[2]['rc'] == 1 - - inventory_hostname in groups[item[0]['group']] - with_together: - - [{ name: mariadb, group: mariadb }] - - "{{ container_envs.results }}" - - "{{ check_results.results }}" - -- include: check.yml +- include: deploy.yml diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml index bf62a85f4c..707d745c7e 100644 --- a/ansible/roles/mariadb/tasks/recover_cluster.yml +++ b/ansible/roles/mariadb/tasks/recover_cluster.yml @@ -1,7 +1,7 @@ --- - fail: msg: "MariaDB cluster was not found. Is your inventory correct?" - when: delegate_host == 'None' + when: not has_cluster | bool - name: Checking if and mariadb containers are running kolla_docker: @@ -57,6 +57,15 @@ when: - mariadb_recover_inventory_name is not defined +- set_fact: + bootstrap_host: "{{ mariadb_recover_inventory_name }}" + master_host: "{{ mariadb_recover_inventory_name }}" + changed_when: true + notify: + - Starting first MariaDB container + - restart slave mariadb + - restart master mariadb + - name: Cleaning up temp file on mariadb hosts file: path=/tmp/kolla_mariadb_grastate.dat state=absent changed_when: false @@ -67,28 +76,3 @@ changed_when: false check_mode: no run_once: true - -- name: Starting first MariaDB container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - environment: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - BOOTSTRAP_ARGS: "--wsrep-new-cluster" - image: "{{ mariadb_image_full }}" - labels: - BOOTSTRAP: - name: "mariadb" - restart_policy: "never" - volumes: - - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" - - "/etc/localtime:/etc/localtime:ro" - - "kolla_logs:/var/log/kolla/" - - "mariadb:/var/lib/mysql" - when: - - (mariadb_recover_inventory_name is not defined and inventory_hostname == groups['mariadb'][0]) or - (mariadb_recover_inventory_name is defined and inventory_hostname == mariadb_recover_inventory_name) - -- name: Reset bootstrap fact - set_fact: - delegate_host: "None" diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml deleted file mode 100644 index bec5f29ba7..0000000000 --- a/ansible/roles/mariadb/tasks/start.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Starting mariadb container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ mariadb_image_full }}" - name: "mariadb" - volumes: - - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" - - "/etc/localtime:/etc/localtime:ro" - - "mariadb:/var/lib/mysql" - - "kolla_logs:/var/log/kolla/" - when: delegate_host != 'None' or - ( groups['mariadb'] | length ) == 1 or - ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or - ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name ) - - -# TODO(jeffrey4l), remove the task check when the wair_for bug is fixed -# https://github.com/ansible/ansible-modules-core/issues/2788 -- name: Waiting for MariaDB service to be ready - wait_for: - host: "{{ api_interface_address }}" - port: "{{ mariadb_port }}" - connect_timeout: 1 - timeout: 60 - search_regex: "MariaDB" - register: check_mariadb_port - until: check_mariadb_port | success - retries: 10 - delay: 6 diff --git a/ansible/roles/mariadb/tasks/upgrade.yml b/ansible/roles/mariadb/tasks/upgrade.yml index fafc44ff38..e078ef1318 100644 --- a/ansible/roles/mariadb/tasks/upgrade.yml +++ b/ansible/roles/mariadb/tasks/upgrade.yml @@ -1,8 +1,2 @@ --- -- include: config.yml - -- include: lookup_cluster.yml - -- include: start.yml - -- include: check.yml +- include: deploy.yml