Restart mysql when config changed

Currently mysql is only (re)started when creating a cluster, adding a
node or the service isn't running. This means changes to configuration
files are not used.

This commit moves the restart functionality to a handler and sets
notifies on the the appropriate tasks so that when the config is changes
mysql is restarted.

This commit modifies the galera play to run in serial so that only one
instance of mysql can be restarted at a time. This is to prevent the
possibility of them all being restarted in one go. An additional play
'Check galera cluster status' has been added, this makes an attempt to
check that the cluster is healthy before proceeding with any possible
config changes. The checks can be overridden by setting
'galera_ignore_cluster_state=true' when running the playbook.

A facts module has been added to gather information about the status of
the cluster for use by the galera-install playbook.

DocImpact
Closes-bug: #1449958

Change-Id: Id83ba642c114d1f5cac04cc219be151a82a1023f

playbooks/galera-install.yml

@@ -13,8 +13,57 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Check galera cluster status
+  hosts: galera_all
+  max_fail_percentage: 0
+  tasks:
+    - name: Check if mysql is running
+      command: /usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf ping
+      ignore_errors: true
+      register: mysql_running
+
+    - fail:
+        msg: "The cluster may be broken, mysql is not running but appears to be installed. Fix it before re-running the playbook or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'."
+      when:
+        - not galera_ignore_cluster_state | bool
+        - mysql_running.rc == 1
+        - mysql_running.stderr | search("Check that mysqld is running and that the socket")
+
+    - name: Gather mysql facts
+      mysql_status_facts:
+      ignore_errors: true
+
+    - fail:
+        msg: "The cluster may be broken, mysql is running but unable to gather mysql facts. Fix it before re-running the playbook or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'."
+      when:
+        - not galera_ignore_cluster_state | bool
+        - "{{ mysql_running.rc == 0 }}"
+        - "{{ 'mysql_status' not in hostvars[inventory_hostname] }}"
+
+    - fail:
+        msg: "wsrep_incoming_addresses does not match across the cluster. Please fix before re-running the playbooks or override with 'openstack-ansible -e galera_ignore_cluster_state=true galera-install.yml'."
+      with_items: "{{ play_hosts }}"
+      when:
+        - not galera_ignore_cluster_state | bool
+        - "{{ mysql_running.rc == 0 }}"
+        - "{{ hostvars[item]['mysql_running']['rc'] == 0 }}"
+        - "{{ hostvars[inventory_hostname]['mysql_status']['wsrep_incoming_addresses'] != hostvars[item]['mysql_status']['wsrep_incoming_addresses'] }}"
+
+    - set_fact:
+        galera_existing_cluster: false
+
+    - set_fact:
+        galera_existing_cluster: true
+      with_items: "{{ play_hosts }}"
+      when: "{{ hostvars[item].mysql_running.rc == 0 }}"
+  vars:
+    galera_ignore_cluster_state: false
+  tags:
+    - discover-galera-cluster-state
+
 - name: Install galera server
   hosts: galera_all
+  serial: 1
   max_fail_percentage: 20
   user: root
   pre_tasks:

playbooks/library/mysql_status_facts

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCUMENTATION = """
+---
+module: mysql_status_facts
+short_description:
+    - A module for gathering mysql status facts.
+description:
+    - A module for gathering mysql status facts.
+author: Rcbops
+"""
+
+EXAMPLES = """
+- name: Gather mysql status facts
+  mysql_status_facts:
+"""
+
+import subprocess
+
+
+class MysqlStatusFacts(object):
+    def __init__(self, module):
+        self.state_change = False
+        self.module = module
+
+    def gather_facts(self):
+        """Get information about mysql status."""
+        try:
+            output = subprocess.check_output(["mysql", "-e", "show status"],
+                                             stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            message = 'Mysql fact collection failed: "%s".' % e.output.strip()
+            self.module.fail_json(msg=message)
+        else:
+            lines = output.split('\n')
+            facts = dict(l.split('\t') for l in lines if l)
+            self.module.exit_json(
+                changed=self.state_change,
+                ansible_facts={'mysql_status': facts})
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+        ),
+        supports_check_mode=False
+    )
+    mysql_facts = MysqlStatusFacts(module)
+    mysql_facts.gather_facts()
+
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()

playbooks/roles/galera_server/defaults/main.yml

@@ -18,6 +18,8 @@ is_metal: true
 
 galera_cluster_name: openstack_galera_cluster
 
+galera_existing_cluster: true
+
 galera_monitoring_user: monitoring
 galera_monitoring_user_password: ""
 galera_root_user: root

playbooks/roles/galera_server/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Restart mysql
+  service:
+    name: mysql
+    state: restarted
+    args: "{{ (not galera_existing_cluster | bool and inventory_hostname == groups['galera_all'][0]) | ternary('--wsrep-new-cluster', '') }}"

playbooks/roles/galera_server/tasks/galera_add_node.yml

@@ -1,40 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Check if mysql is running
-  shell: "pgrep -fl [m]ysqld"
-  register: mysql_running
-  changed_when: mysql_running.rc != 0
-  ignore_errors: True
-  tags:
-    - galera-bootstrap
-
-- name: Get incoming addresses from cluster
-  shell: |
-    mysql -e 'show status like "wsrep_incoming_addresses"\G'|awk '/Value/{print $2}'
-  register: wsrep_incoming_addresses
-  changed_when: "not wsrep_incoming_addresses.stdout|search('{{ ansible_ssh_host }}')"
-  tags:
-    - galera-add-node
-    - galera-bootstrap
-
-- name: Restart mysql
-  service:
-    name: mysql
-    state: restarted
-  when: wsrep_incoming_addresses|changed or mysql_running|changed
-  tags:
-    - galera-add-node
-    - galera-bootstrap

playbooks/roles/galera_server/tasks/galera_bootstrap.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Check if mysql is running
-  shell: "pgrep -fl [m]ysqld"
-  register: mysql_running
-  changed_when: mysql_running.rc != 0
-  ignore_errors: True
-  tags:
-    - galera-bootstrap
-
-- name: Check wsrep cluster size
-  shell: |
-    mysql -e 'show status like "wsrep_cluster_size%"\G'|awk '/Value/{print $2}'
-  register: wsrep_cluster_size
-  changed_when: wsrep_cluster_size.stdout|search("1")
-  when: not mysql_running|changed
-  tags:
-    - galera-bootstrap
-
-- name: Stop mariadb
-  service:
-    name: mysql
-    state: stopped
-    pattern: mysqld
-  register: mysqlstopped
-  when: >
-    (not mysql_running|changed) and
-    (wsrep_cluster_size|changed or wsrep_cluster_size.stderr | search("ERROR"))
-  tags:
-    - galera-bootstrap
-
-- name: Check if mysql is running
-  shell: "pgrep -fl [m]ysqld"
-  register: mysql_running
-  changed_when: mysql_running.rc != 0
-  ignore_errors: True
-  tags:
-    - galera-bootstrap
-
-- name: bootstrap cluster (initialise mysql with --wsrep-new-cluster)
-  service:
-    name: mysql
-    state: restarted
-    args: --wsrep-new-cluster
-  when: wsrep_cluster_size|changed or mysqlstopped|changed or mysql_running|changed
-  tags:
-    - galera-bootstrap

playbooks/roles/galera_server/tasks/galera_install.yml

@@ -43,6 +43,7 @@
   retries: 5
   delay: 2
   with_items: galera_apt_packages
+  notify: Restart mysql
   tags:
     - galera-apt-packages
 
@@ -57,3 +58,10 @@
   with_items: galera_pip_packages
   tags:
     - galera-pip-packages
+
+- name: Check if mysql is running
+  command: /usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf ping
+  ignore_errors: true
+  changed_when: mysql_started.rc != 0
+  register: mysql_started
+  notify: Restart mysql

playbooks/roles/galera_server/tasks/galera_post_install.yml

@@ -38,6 +38,7 @@
     - { src: cluster.cnf.j2, dest: /etc/mysql/conf.d/cluster.cnf }
     - { src: debian.cnf.j2, dest: /etc/mysql/debian.cnf }
     - { src: mysql_defaults.j2, dest: /etc/default/mysql }
+  notify: Restart mysql
   tags:
     - galera-config
 
@@ -48,6 +49,7 @@
     mode: "{{ item.mode|default('0644') }}"
   with_items:
     - { src: mysql_init.sh, dest: /etc/init.d/mysql, mode: "0755" }
+  notify: Restart mysql
   tags:
     - galera-config
 

playbooks/roles/galera_server/tasks/galera_pre_install.yml

@@ -98,5 +98,6 @@
   template:
     src: "limits.conf.j2"
     dest: "/etc/security/limits.conf"
+  notify: Restart mysql
   tags:
     - galera-config

playbooks/roles/galera_server/tasks/main.yml

@@ -17,13 +17,8 @@
 - include: galera_install.yml
 - include: galera_post_install.yml
 
-- include: galera_bootstrap.yml
-  when: inventory_hostname == groups['galera_all'][0]
-- include: galera_setup.yml
-  when: inventory_hostname == groups['galera_all'][0]
-
-- include: galera_add_node.yml
-  when: inventory_hostname != groups['galera_all'][0]
-
 - name: Flush handlers
   meta: flush_handlers
+
+- include: galera_setup.yml
+  when: inventory_hostname == groups['galera_all'][0]

scripts/run-aio-build.sh

@@ -59,8 +59,7 @@ echo ""
 echo "If this server has been rebooted, you will need to re-bootstrap"
 echo "Galera to get the cluster operational. To do this execute:"
 echo ""
-echo "cd $(pwd)/playbooks"
-echo "openstack-ansible galera-install --tags galera-bootstrap"
+echo "This requires you to identify the most advanced node. For details see http://galeracluster.com/documentation-webpages/quorumreset.html
 echo ""
 EOF
 chmod +x /etc/update-motd.d/21-galera