Gracefully shutdown Mysql before upgrade.

When upgrading from MySQL 10.1 to 10.3 a bug appears if no
shutdown is being performed, as the redo log format has
changed in version 10.3.2 [0].

Make sure we always stop the MySQL server cleanly before
upgrading to a new version, to avoid redo log issue.

Note: to be idempotent, we need to stop the mysql container
rather than delete it; to be able to stop the container, we
amend the restart policy of the mysql container.

[0] - https://jira.mariadb.org/browse/MDEV-14848

Change-Id: Ia07b7755867858c74c7334424e8e6579ace495db
Co-Authored-By: Damien Ciabrini <dciabrin@redhat.com>
Closes-Bug: #1810136

docker/services/database/mysql.yaml

@@ -191,7 +191,7 @@ outputs:
           mysql:
             start_order: 2
             image: *mysql_image
-            restart: always
+            restart: unless-stopped
             net: host
             healthcheck:
               test: /openstack/healthcheck
@@ -240,7 +240,64 @@ outputs:
               Log files from mysql containers can be found under
               /var/log/containers/mysql.
           ignore_errors: true
-      upgrade_tasks: []
+      upgrade_tasks:
+          # LP 1810136
+          # After upgrade, the new mariadb (e.g. 10.3) might not be able
+          # to replay the redo log of an older one (e.g. 10.1) if mysql
+          # stopped unexpectedly. Force it to stop and clean the redo log
+          # here to avoid issue after upgrade.
+        - name: Stop MySQL server and ensure redo log is cleaned up before upgrade
+          when: step|int == 2
+          block:
+            - name: Retrieve mysql container configuration from paunch
+              shell:
+                str_replace:
+                  # Use a Jinja template below to force str_replace
+                  # to quote the resulting string
+                  # TODO: remove the grep filter once #1811384 is fixed
+                  template: &mysql_template '{{ "paunch" }} list --default-runtime RUNTIME -f json -c container -c image | grep -v "isn''t supported by" | jq -r ''.[] | select(.container=="mysql") + {"cli":"RUNTIME"}'''
+                  params:
+                    RUNTIME: "{{ container_cli }}"
+              register: mysql_paunch
+            - name: Retrieve mysql container configuration from paunch (docker)
+              shell:
+                str_replace:
+                  template: *mysql_template
+                  params:
+                    RUNTIME: "docker"
+              register: mysql_paunch_docker
+              when: '"mysql" not in mysql_paunch.stdout'
+            - name: Mysql container facts
+              set_fact:
+                mysql_json: "{{ mysql_paunch_docker.stdout if mysql_paunch_docker.changed else mysql_paunch.stdout | from_json }}"
+            - name: Redo log clean-up script
+              set_fact:
+                # The purpose of this script is to start mysql so that it
+                # replays the redo log, and shutdown mysql cleanly
+                mysql_clean_up_script:
+                  list_join:
+                    - ' '
+                    - - 'kolla_set_configs;'
+                      - 'mysqld_safe --user=mysql --skip-networking &'
+                      - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
+                      - 'mysqladmin shutdown'
+            - name: Bind mounts for temporary clean-up container
+              set_fact:
+                mysql_clean_up_volumes: *mysql_volumes
+            - name: Stop the current mysql container
+              shell: |
+                {{ mysql_json.cli }} update --restart=unless-stopped mysql
+                {{ mysql_json.cli }} stop mysql
+            - name: Clean up redo log by running a transient mysql server
+              shell:
+                str_replace:
+                  template:
+                    "{{ mysql_json.cli }} run --rm -u root --net=host ENV VOLUMES \"IMAGE\" /bin/bash -ecx \"SCRIPT\""
+                  params:
+                    ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
+                    IMAGE: "{{ mysql_json.image }}"
+                    VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}"
+                    SCRIPT: "{{ mysql_clean_up_script }}"
       post_upgrade_tasks:
         - when: step|int == 1
           import_role: