Custom haproxy script for monitoring galera

Depends-On: https://review.opendev.org/710217/

Change-Id: I85652f23e487c40192106d23f2cdd45a3077deca

ansible/group_vars/all.yml

@@ -328,6 +328,7 @@ mariadb_port: "{{ database_port }}"
 mariadb_wsrep_port: "4567"
 mariadb_ist_port: "4568"
 mariadb_sst_port: "4444"
+mariadb_clustercheck_port: "4569"
 
 masakari_api_port: "15868"
 

ansible/roles/common/templates/conf/input/02-mariadb.conf.j2

@@ -10,3 +10,17 @@
   format1 /^(?<Payload>.*)/
   enable_watch_timer false
 </source>
+<source>
+  @type tail
+  path /var/log/kolla/mariadb/xinetd.log
+  pos_file /var/run/{{ fluentd_binary }}/mariadb-xinetd.pos
+  tag infra.mariadb.xinetd
+  format multiline
+  format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /
+  format1 /^(?<Timestamp>\S+) (?<Payload>.*)?$/
+  time_key Timestamp
+  keep_time_key true
+  time_format %y/%-m/%-d@%T
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>

ansible/roles/mariadb/defaults/main.yml

@@ -21,7 +21,7 @@ mariadb_services:
         backend_tcp_extra:
           - "option srvtcpka"
           - "timeout server 3600s"
-          - "option mysql-check user haproxy post-41"
+          - "option {% if enable_mariadb_clustercheck | bool %}httpchk{% else %}mysql-check user haproxy post-41{% endif %}"
         custom_member_list: "{{ internal_haproxy_members.split(';') }}"
       mariadb_external_lb:
         enabled: "{{ enable_external_mariadb_load_balancer|bool }}"
@@ -35,6 +35,18 @@ mariadb_services:
           - "option srvtcpka"
           - "timeout server 3600s"
         custom_member_list: "{{ external_haproxy_members.split(';') }}"
+  mariadb-clustercheck:
+    container_name: mariadb_clustercheck
+    group: mariadb
+    enabled: "{{ enable_mariadb_clustercheck | bool }}"
+    image: "{{ mariadb_clustercheck_image_full }}"
+    volumes: "{{ mariadb_clustercheck_default_volumes + mariadb_clustercheck_extra_volumes }}"
+    dimensions: "{{ mariadb_clustercheck_dimensions }}"
+    environment:
+      MYSQL_USERNAME: "haproxy"
+      MYSQL_PASSWORD: ""
+      MYSQL_HOST: "{{ api_interface_address }}"
+      AVAILABLE_WHEN_DONOR: "1"
 
 ####################
 # Database
@@ -45,8 +57,8 @@ database_max_timeout: 120
 ####################
 # HAProxy
 ####################
-internal_haproxy_members: "{% for host in groups['mariadb'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
-external_haproxy_members: "{% for host in groups['mariadb'] %}server {{ host }} {{ host }}:{{ mariadb_port }} check inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
+internal_haproxy_members: "{% for host in groups['mariadb'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port }} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
+external_haproxy_members: "{% for host in groups['mariadb'] %}server {{ host }} {{ host }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port}} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
 
 ####################
 # Docker
@@ -55,7 +67,14 @@ mariadb_install_type: "{{ kolla_install_type }}"
 mariadb_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_install_type }}-mariadb"
 mariadb_tag: "{{ openstack_tag }}"
 mariadb_image_full: "{{ mariadb_image }}:{{ mariadb_tag }}"
+
+mariadb_clustercheck_install_type: "{{ kolla_install_type }}"
+mariadb_clustercheck_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_clustercheck_install_type }}-mariadb-clustercheck"
+mariadb_clustercheck_tag: "{{ openstack_tag }}"
+mariadb_clustercheck_image_full: "{{ mariadb_clustercheck_image }}:{{ mariadb_clustercheck_tag }}"
+
 mariadb_dimensions: "{{ default_container_dimensions }}"
+mariadb_clustercheck_dimensions: "{{ default_container_dimensions }}"
 
 mariadb_default_volumes:
   - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
@@ -63,7 +82,14 @@ mariadb_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
   - "mariadb:/var/lib/mysql"
   - "kolla_logs:/var/log/kolla/"
+mariadb_clustercheck_default_volumes:
+  - "{{ node_config_directory }}/mariadb-clustercheck/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+
 mariadb_extra_volumes: "{{ default_extra_volumes }}"
+mariadb_clustercheck_extra_volumes: "{{ default_extra_volumes }}"
 
 ########################################
 # Vars used within recover_cluster.yml
@@ -87,3 +113,8 @@ mariadb_backup_database_schema: "PERCONA_SCHEMA"
 mariadb_backup_database_user: "backup"
 mariadb_backup_database_address: "{{ database_address }}"
 mariadb_backup_type: "full"
+
+####################
+# Clustercheck
+####################
+enable_mariadb_clustercheck: "yes"

ansible/roles/mariadb/handlers/main.yml

@@ -89,3 +89,19 @@
 - name: Ensure MariaDB is running normally on bootstrap host
   include_tasks: 'restart_services.yml'
   listen: Bootstrap MariaDB cluster
+
+- name: Restart mariadb-clustercheck container
+  vars:
+    service_name: "mariadb-clustercheck"
+    service: "{{ mariadb_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ service.image }}"
+    name: "{{ service.container_name }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+    environment: "{{ service.environment }}"
+  listen:
+    - restart mariadb-clustercheck

ansible/roles/mariadb/tasks/config.yml

@@ -40,19 +40,17 @@
     - inventory_hostname == mariadb_backup_host
 
 - name: Copying over config.json files for services
-  vars:
-    service_name: "mariadb"
-    service: "{{ mariadb_services[service_name] }}"
   template:
-    src: "{{ service_name }}.json.j2"
-    dest: "{{ node_config_directory }}/{{ service_name }}/config.json"
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
     mode: "0660"
   become: true
   when:
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ mariadb_services }}"
   notify:
-    - restart mariadb
+    - "restart {{ item.key }}"
 
 - name: Copying over config.json files for mariabackup
   vars:
@@ -97,5 +95,20 @@
   notify:
     - restart mariadb
 
+- name: Copying over xinetd clustercheck.conf
+  vars:
+    service_name: "mariadb-clustercheck"
+    service: "{{ mariadb_services[service_name] }}"
+  template:
+    src: "{{ role_path }}/templates/xinetd-clustercheck.j2"
+    dest: "{{ node_config_directory }}/{{ service_name }}/clustercheck"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - restart mariadb-clustercheck
+
 - include_tasks: check-containers.yml
   when: kolla_action != "config"

ansible/roles/mariadb/tasks/recover_cluster.yml

@@ -185,15 +185,17 @@
     - bootstrap_host is defined
     - bootstrap_host == inventory_hostname
 
-- name: Restart slave MariaDB container
+- name: Restart slave MariaDB container(s)
   become: true
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ mariadb_service.container_name }}"
-    image: "{{ mariadb_service.image }}"
-    volumes: "{{ mariadb_service.volumes }}"
-    dimensions: "{{ mariadb_service.dimensions }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+    dimensions: "{{ item.value.dimensions }}"
+    environment: "{{ item.value.environment | default({}) }}"
+  with_dict: "{{ mariadb_services }}"
   when:
     - bootstrap_host is defined
     - bootstrap_host != inventory_hostname
@@ -213,15 +215,17 @@
     - bootstrap_host is defined
     - bootstrap_host != inventory_hostname
 
-- name: Restart master MariaDB container
+- name: Restart master MariaDB container(s)
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ mariadb_service.container_name }}"
-    image: "{{ mariadb_service.image }}"
-    volumes: "{{ mariadb_service.volumes }}"
-    dimensions: "{{ mariadb_service.dimensions }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+    dimensions: "{{ item.value.dimensions }}"
+    environment: "{{ item.value.environment | default({}) }}"
+  with_dict: "{{ mariadb_services }}"
   when:
     - bootstrap_host is defined
     - bootstrap_host == inventory_hostname

ansible/roles/mariadb/templates/mariadb-clustercheck.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "/usr/sbin/xinetd -dontfork -filelog /var/log/kolla/mariadb/xinetd.log",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/clustercheck",
+            "dest": "/etc/xinetd.d/clustercheck",
+            "owner": "root",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/mariadb",
+            "owner": "mysql:mysql",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/mariadb/templates/xinetd-clustercheck.j2

@@ -0,0 +1,16 @@
+# default: on
+# description: mysqlchk
+service mysqlchk
+{
+        disable = no
+        flags = REUSE
+        socket_type = stream
+        type = UNLISTED
+        port = {{ mariadb_clustercheck_port }}
+        wait = no
+        user = mysql
+        server = /usr/bin/clustercheck
+        log_type       = FILE /var/log/kolla/mariadb/xinetd.log
+        log_on_failure = HOST ATTEMPT
+        per_source = UNLIMITED
+}

releasenotes/notes/mariadb-clustercheck-39b5837c09868f1e.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Enable Galera node state checking by using ``clustercheck`` script that is
+    used by HAProxy to define node up/down state.