diff --git a/playbooks/galera-install.yml b/playbooks/galera-install.yml index a2d481ba25..7f0d2acbe2 100644 --- a/playbooks/galera-install.yml +++ b/playbooks/galera-install.yml @@ -13,8 +13,57 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Check galera cluster status + hosts: galera_all + max_fail_percentage: 0 + tasks: + - name: Check if mysql is running + command: /usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf ping + ignore_errors: true + register: mysql_running + + - fail: + msg: "The cluster may be broken, mysql is not running but appears to be installed. Fix it before re-running the playbook or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'." + when: + - not galera_ignore_cluster_state | bool + - mysql_running.rc == 1 + - mysql_running.stderr | search("Check that mysqld is running and that the socket") + + - name: Gather mysql facts + mysql_status_facts: + ignore_errors: true + + - fail: + msg: "The cluster may be broken, mysql is running but unable to gather mysql facts. Fix it before re-running the playbook or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'." + when: + - not galera_ignore_cluster_state | bool + - "{{ mysql_running.rc == 0 }}" + - "{{ 'mysql_status' not in hostvars[inventory_hostname] }}" + + - fail: + msg: "wsrep_incoming_addresses does not match across the cluster. Please fix before re-running the playbooks or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'." + with_items: "{{ play_hosts }}" + when: + - not galera_ignore_cluster_state | bool + - "{{ mysql_running.rc == 0 }}" + - "{{ hostvars[item]['mysql_running']['rc'] == 0 }}" + - "{{ hostvars[inventory_hostname]['mysql_status']['wsrep_incoming_addresses'] != hostvars[item]['mysql_status']['wsrep_incoming_addresses'] }}" + + - set_fact: + galera_existing_cluster: false + + - set_fact: + galera_existing_cluster: true + with_items: "{{ play_hosts }}" + when: "{{ hostvars[item].mysql_running.rc == 0 }}" + vars: + galera_ignore_cluster_state: false + tags: + - discover-galera-cluster-state + - name: Install galera server hosts: galera_all + serial: 1 max_fail_percentage: 20 user: root pre_tasks: diff --git a/playbooks/library/mysql_status_facts b/playbooks/library/mysql_status_facts new file mode 100644 index 0000000000..facd173305 --- /dev/null +++ b/playbooks/library/mysql_status_facts @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCUMENTATION = """ +--- +module: mysql_status_facts +short_description: + - A module for gathering mysql status facts. +description: + - A module for gathering mysql status facts. +author: Rcbops +""" + +EXAMPLES = """ +- name: Gather mysql status facts + mysql_status_facts: +""" + +import subprocess + + +class MysqlStatusFacts(object): + def __init__(self, module): + self.state_change = False + self.module = module + + def gather_facts(self): + """Get information about mysql status.""" + try: + output = subprocess.check_output(["mysql", "-e", "show status"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + message = 'Mysql fact collection failed: "%s".' % e.output.strip() + self.module.fail_json(msg=message) + else: + lines = output.split('\n') + facts = dict(l.split('\t') for l in lines if l) + self.module.exit_json( + changed=self.state_change, + ansible_facts={'mysql_status': facts}) + + +def main(): + module = AnsibleModule( + argument_spec=dict( + ), + supports_check_mode=False + ) + mysql_facts = MysqlStatusFacts(module) + mysql_facts.gather_facts() + +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/playbooks/roles/galera_server/defaults/main.yml b/playbooks/roles/galera_server/defaults/main.yml index 959317852b..8a93b09fea 100644 --- a/playbooks/roles/galera_server/defaults/main.yml +++ b/playbooks/roles/galera_server/defaults/main.yml @@ -18,6 +18,8 @@ is_metal: true galera_cluster_name: openstack_galera_cluster +galera_existing_cluster: true + galera_monitoring_user: monitoring galera_monitoring_user_password: "" galera_root_user: root diff --git a/playbooks/roles/galera_server/handlers/main.yml b/playbooks/roles/galera_server/handlers/main.yml new file mode 100644 index 0000000000..e020d1d032 --- /dev/null +++ b/playbooks/roles/galera_server/handlers/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Restart mysql + service: + name: mysql + state: restarted + args: "{{ (not galera_existing_cluster | bool and inventory_hostname == groups['galera_all'][0]) | ternary('--wsrep-new-cluster', '') }}" diff --git a/playbooks/roles/galera_server/tasks/galera_add_node.yml b/playbooks/roles/galera_server/tasks/galera_add_node.yml deleted file mode 100644 index e97d22e21e..0000000000 --- a/playbooks/roles/galera_server/tasks/galera_add_node.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check if mysql is running - shell: "pgrep -fl [m]ysqld" - register: mysql_running - changed_when: mysql_running.rc != 0 - ignore_errors: True - tags: - - galera-bootstrap - -- name: Get incoming addresses from cluster - shell: | - mysql -e 'show status like "wsrep_incoming_addresses"\G'|awk '/Value/{print $2}' - register: wsrep_incoming_addresses - changed_when: "not wsrep_incoming_addresses.stdout|search('{{ ansible_ssh_host }}')" - tags: - - galera-add-node - - galera-bootstrap - -- name: Restart mysql - service: - name: mysql - state: restarted - when: wsrep_incoming_addresses|changed or mysql_running|changed - tags: - - galera-add-node - - galera-bootstrap diff --git a/playbooks/roles/galera_server/tasks/galera_bootstrap.yml b/playbooks/roles/galera_server/tasks/galera_bootstrap.yml deleted file mode 100644 index f58fe86b4c..0000000000 --- a/playbooks/roles/galera_server/tasks/galera_bootstrap.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check if mysql is running - shell: "pgrep -fl [m]ysqld" - register: mysql_running - changed_when: mysql_running.rc != 0 - ignore_errors: True - tags: - - galera-bootstrap - -- name: Check wsrep cluster size - shell: | - mysql -e 'show status like "wsrep_cluster_size%"\G'|awk '/Value/{print $2}' - register: wsrep_cluster_size - changed_when: wsrep_cluster_size.stdout|search("1") - when: not mysql_running|changed - tags: - - galera-bootstrap - -- name: Stop mariadb - service: - name: mysql - state: stopped - pattern: mysqld - register: mysqlstopped - when: > - (not mysql_running|changed) and - (wsrep_cluster_size|changed or wsrep_cluster_size.stderr | search("ERROR")) - tags: - - galera-bootstrap - -- name: Check if mysql is running - shell: "pgrep -fl [m]ysqld" - register: mysql_running - changed_when: mysql_running.rc != 0 - ignore_errors: True - tags: - - galera-bootstrap - -- name: bootstrap cluster (initialise mysql with --wsrep-new-cluster) - service: - name: mysql - state: restarted - args: --wsrep-new-cluster - when: wsrep_cluster_size|changed or mysqlstopped|changed or mysql_running|changed - tags: - - galera-bootstrap diff --git a/playbooks/roles/galera_server/tasks/galera_install.yml b/playbooks/roles/galera_server/tasks/galera_install.yml index 624893ceae..66f3d8b75d 100644 --- a/playbooks/roles/galera_server/tasks/galera_install.yml +++ b/playbooks/roles/galera_server/tasks/galera_install.yml @@ -43,6 +43,7 @@ retries: 5 delay: 2 with_items: galera_apt_packages + notify: Restart mysql tags: - galera-apt-packages @@ -57,3 +58,10 @@ with_items: galera_pip_packages tags: - galera-pip-packages + +- name: Check if mysql is running + command: /usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf ping + ignore_errors: true + changed_when: mysql_started.rc != 0 + register: mysql_started + notify: Restart mysql diff --git a/playbooks/roles/galera_server/tasks/galera_post_install.yml b/playbooks/roles/galera_server/tasks/galera_post_install.yml index 89d2462fae..dcdbc31020 100644 --- a/playbooks/roles/galera_server/tasks/galera_post_install.yml +++ b/playbooks/roles/galera_server/tasks/galera_post_install.yml @@ -38,6 +38,7 @@ - { src: cluster.cnf.j2, dest: /etc/mysql/conf.d/cluster.cnf } - { src: debian.cnf.j2, dest: /etc/mysql/debian.cnf } - { src: mysql_defaults.j2, dest: /etc/default/mysql } + notify: Restart mysql tags: - galera-config @@ -48,6 +49,7 @@ mode: "{{ item.mode|default('0644') }}" with_items: - { src: mysql_init.sh, dest: /etc/init.d/mysql, mode: "0755" } + notify: Restart mysql tags: - galera-config diff --git a/playbooks/roles/galera_server/tasks/galera_pre_install.yml b/playbooks/roles/galera_server/tasks/galera_pre_install.yml index 7c09058229..38cfe657e0 100644 --- a/playbooks/roles/galera_server/tasks/galera_pre_install.yml +++ b/playbooks/roles/galera_server/tasks/galera_pre_install.yml @@ -98,5 +98,6 @@ template: src: "limits.conf.j2" dest: "/etc/security/limits.conf" + notify: Restart mysql tags: - galera-config diff --git a/playbooks/roles/galera_server/tasks/main.yml b/playbooks/roles/galera_server/tasks/main.yml index 38fa3d7a72..e7cb7fe4a5 100644 --- a/playbooks/roles/galera_server/tasks/main.yml +++ b/playbooks/roles/galera_server/tasks/main.yml @@ -17,13 +17,8 @@ - include: galera_install.yml - include: galera_post_install.yml -- include: galera_bootstrap.yml - when: inventory_hostname == groups['galera_all'][0] -- include: galera_setup.yml - when: inventory_hostname == groups['galera_all'][0] - -- include: galera_add_node.yml - when: inventory_hostname != groups['galera_all'][0] - - name: Flush handlers meta: flush_handlers + +- include: galera_setup.yml + when: inventory_hostname == groups['galera_all'][0] diff --git a/scripts/run-aio-build.sh b/scripts/run-aio-build.sh index 0ec18989cc..1b6e02da8d 100755 --- a/scripts/run-aio-build.sh +++ b/scripts/run-aio-build.sh @@ -59,8 +59,7 @@ echo "" echo "If this server has been rebooted, you will need to re-bootstrap" echo "Galera to get the cluster operational. To do this execute:" echo "" -echo "cd $(pwd)/playbooks" -echo "openstack-ansible galera-install --tags galera-bootstrap" +echo "This requires you to identify the most advanced node. For details see http://galeracluster.com/documentation-webpages/quorumreset.html echo "" EOF chmod +x /etc/update-motd.d/21-galera