Browse Source

Gracefully shutdown Mysql before upgrade.

When upgrading from MySQL 10.1 to 10.3 a bug appears if no
shutdown is being performed, as the redo log format has
changed in version 10.3.2 [0].

Make sure we always stop the MySQL server cleanly before
upgrading to a new version, to avoid redo log issue.

Note: to be idempotent, we need to stop the mysql container
rather than delete it; to be able to stop the container, we
amend the restart policy of the mysql container.

[0] - https://jira.mariadb.org/browse/MDEV-14848

Change-Id: Ia07b7755867858c74c7334424e8e6579ace495db
Co-Authored-By: Damien Ciabrini <dciabrin@redhat.com>
Closes-Bug: #1810136
tags/10.4.0
Jose Luis Franco Arza 4 months ago
parent
commit
0015cc7441
1 changed files with 59 additions and 2 deletions
  1. 59
    2
      docker/services/database/mysql.yaml

+ 59
- 2
docker/services/database/mysql.yaml View File

@@ -191,7 +191,7 @@ outputs:
191 191
           mysql:
192 192
             start_order: 2
193 193
             image: *mysql_image
194
-            restart: always
194
+            restart: unless-stopped
195 195
             net: host
196 196
             healthcheck:
197 197
               test: /openstack/healthcheck
@@ -240,7 +240,64 @@ outputs:
240 240
               Log files from mysql containers can be found under
241 241
               /var/log/containers/mysql.
242 242
           ignore_errors: true
243
-      upgrade_tasks: []
243
+      upgrade_tasks:
244
+          # LP 1810136
245
+          # After upgrade, the new mariadb (e.g. 10.3) might not be able
246
+          # to replay the redo log of an older one (e.g. 10.1) if mysql
247
+          # stopped unexpectedly. Force it to stop and clean the redo log
248
+          # here to avoid issue after upgrade.
249
+        - name: Stop MySQL server and ensure redo log is cleaned up before upgrade
250
+          when: step|int == 2
251
+          block:
252
+            - name: Retrieve mysql container configuration from paunch
253
+              shell:
254
+                str_replace:
255
+                  # Use a Jinja template below to force str_replace
256
+                  # to quote the resulting string
257
+                  # TODO: remove the grep filter once #1811384 is fixed
258
+                  template: &mysql_template '{{ "paunch" }} list --default-runtime RUNTIME -f json -c container -c image | grep -v "isn''t supported by" | jq -r ''.[] | select(.container=="mysql") + {"cli":"RUNTIME"}'''
259
+                  params:
260
+                    RUNTIME: "{{ container_cli }}"
261
+              register: mysql_paunch
262
+            - name: Retrieve mysql container configuration from paunch (docker)
263
+              shell:
264
+                str_replace:
265
+                  template: *mysql_template
266
+                  params:
267
+                    RUNTIME: "docker"
268
+              register: mysql_paunch_docker
269
+              when: '"mysql" not in mysql_paunch.stdout'
270
+            - name: Mysql container facts
271
+              set_fact:
272
+                mysql_json: "{{ mysql_paunch_docker.stdout if mysql_paunch_docker.changed else mysql_paunch.stdout | from_json }}"
273
+            - name: Redo log clean-up script
274
+              set_fact:
275
+                # The purpose of this script is to start mysql so that it
276
+                # replays the redo log, and shutdown mysql cleanly
277
+                mysql_clean_up_script:
278
+                  list_join:
279
+                    - ' '
280
+                    - - 'kolla_set_configs;'
281
+                      - 'mysqld_safe --user=mysql --skip-networking &'
282
+                      - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
283
+                      - 'mysqladmin shutdown'
284
+            - name: Bind mounts for temporary clean-up container
285
+              set_fact:
286
+                mysql_clean_up_volumes: *mysql_volumes
287
+            - name: Stop the current mysql container
288
+              shell: |
289
+                {{ mysql_json.cli }} update --restart=unless-stopped mysql
290
+                {{ mysql_json.cli }} stop mysql
291
+            - name: Clean up redo log by running a transient mysql server
292
+              shell:
293
+                str_replace:
294
+                  template:
295
+                    "{{ mysql_json.cli }} run --rm -u root --net=host ENV VOLUMES \"IMAGE\" /bin/bash -ecx \"SCRIPT\""
296
+                  params:
297
+                    ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
298
+                    IMAGE: "{{ mysql_json.image }}"
299
+                    VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}"
300
+                    SCRIPT: "{{ mysql_clean_up_script }}"
244 301
       post_upgrade_tasks:
245 302
         - when: step|int == 1
246 303
           import_role:

Loading…
Cancel
Save