tripleo-heat-templates/deployment/database/mysql-pacemaker-puppet.yaml
Damien Ciabrini 9bdf4b1685 Upgrade mariadb storage during upgrade tasks
When a tripleo major upgrade or FFU causes an update or mariadb
to a new major version (e.g. 10.1 -> 10.3), some internal DB
tables must be upgraded (myisam tables), and sometimes some
the existing user tables may be migrated to new mariadb defaults.

Move the db-specific upgrade steps into a dedicated script and
make sure that it is called at the right time while upgrading
the undercloud and/or the overcloud.

Closes-Bug: #1913438

Change-Id: I92353622994b28c895d95bdcbe348a73b6c6bb99
(cherry picked from commit ad35169510)
2021-03-10 16:17:25 +01:00

645 lines
28 KiB
YAML

heat_template_version: rocky
description: >
MySQL service deployment with pacemaker bundle
parameters:
ContainerMysqlImage:
description: image
type: string
ContainerMysqlConfigImage:
description: The container image to use for the mysql config_volume
type: string
ClusterCommonTag:
default: false
description: When set to false, a pacemaker service is configured
to use a floating tag for its container image name,
e.g. 'REGISTRY/NAMESPACE/IMAGENAME:pcmklatest'. When
set to true, the service uses a floating prefix as
well, e.g. 'cluster.common.tag/IMAGENAME:pcmklatest'.
type: boolean
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
MysqlRootPassword:
type: string
hidden: true
default: ''
MysqlClustercheckPassword:
type: string
hidden: true
MysqlUpgradePersist:
type: boolean
default: false
MysqlUpgradeTransfer:
type: boolean
default: true
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EnableInternalTLS:
type: boolean
default: false
InternalTLSCAFile:
default: '/etc/ipa/ca.crt'
type: string
description: Specifies the default CA cert to use if TLS is used for
services in the internal network.
ConfigDebug:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
DeployIdentifier:
default: ''
type: string
description: >
Setting this to a unique value will re-run any deployment tasks which
perform configuration on a Heat stack-update.
ContainerCli:
type: string
default: 'podman'
description: CLI tool used to manage containers.
constraints:
- allowed_values: ['docker', 'podman']
resources:
ContainersCommon:
type: ../containers-common.yaml
MysqlBase:
type: ./mysql-base.yaml
properties:
EndpointMap: {get_param: EndpointMap}
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
conditions:
puppet_debug_enabled: {get_param: ConfigDebug}
internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
docker_enabled: {equals: [{get_param: ContainerCli}, 'docker']}
common_tag_enabled: {equals: [{get_param: ClusterCommonTag}, true]}
outputs:
role_data:
description: Containerized service MySQL using composable services.
value:
service_name: {get_attr: [MysqlBase, role_data, service_name]}
config_settings:
map_merge:
- get_attr: [MysqlBase, role_data, config_settings]
- tripleo::profile::pacemaker::database::mysql::bind_address:
str_replace:
template:
"%{hiera('fqdn_$NETWORK')}"
params:
$NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]}
# NOTE: bind IP is found in hiera replacing the network name with the
# local node IP for the given network; replacement examples
# (eg. for internal_api):
# internal_api -> IP
# internal_api_uri -> [IP]
# internal_api_subnet - > IP/CIDR
tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr:
str_replace:
template:
"%{hiera('$NETWORK')}"
params:
$NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]}
tripleo::profile::pacemaker::database::mysql::ca_file:
get_param: InternalTLSCAFile
tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image: &mysql_image_pcmklatest
yaql:
data:
if:
- common_tag_enabled
- yaql:
data: {get_param: ContainerMysqlImage}
expression: concat("cluster.common.tag/", $.data.rightSplit(separator => "/", maxSplits => 1)[1])
- {get_param: ContainerMysqlImage}
expression: concat($.data.rightSplit(separator => ":", maxSplits => 1)[0], ":pcmklatest")
tripleo::profile::pacemaker::database::mysql_bundle::control_port: 3123
tripleo::profile::pacemaker::database::mysql_bundle::container_backend: {get_param: ContainerCli}
tripleo::mysql::firewall_rules:
'104 mysql galera-bundle':
dport:
- 873
- 3123
- 3306
- 4444
- 4567
- 4568
- 9200
tripleo::profile::pacemaker::database::mysql_bundle::bind_address:
str_replace:
template:
"%{hiera('fqdn_$NETWORK')}"
params:
$NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]}
-
if:
- internal_tls_enabled
-
tripleo::profile::pacemaker::database::mysql_bundle::ca_file:
get_param: InternalTLSCAFile
- {}
# BEGIN DOCKER SETTINGS #
puppet_config:
config_volume: mysql
puppet_tags: file # set this even though file is the default
step_config:
list_join:
- "\n"
- - "['Mysql_datadir', 'Mysql_user', 'Mysql_database', 'Mysql_grant', 'Mysql_plugin'].each |String $val| { noop_resource($val) }"
- "exec {'wait-for-settle': command => '/bin/true' }"
- "include ::tripleo::profile::pacemaker::database::mysql_bundle"
config_image: {get_param: ContainerMysqlConfigImage}
kolla_config:
/var/lib/kolla/config_files/mysql.json:
command: /usr/sbin/pacemaker_remoted
config_files:
- dest: /etc/libqb/force-filesystem-sockets
source: /dev/null
owner: root
perm: '0644'
- source: "/var/lib/kolla/config_files/src/*"
dest: "/"
merge: true
preserve_properties: true
- source: "/var/lib/kolla/config_files/src-tls/*"
dest: "/"
merge: true
optional: true
preserve_properties: true
permissions:
- path: /var/log/mysql
owner: mysql:mysql
recurse: true
- path: /etc/pki/tls/certs/mysql.crt
owner: mysql:mysql
perm: '0600'
optional: true
- path: /etc/pki/tls/private/mysql.key
owner: mysql:mysql
perm: '0600'
optional: true
container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]}
docker_config:
step_1:
mysql_data_ownership:
start_order: 0
detach: false
image: {get_param: ContainerMysqlImage}
net: host
user: root
# Kolla does only non-recursive chown
command: ['chown', '-R', 'mysql:', '/var/lib/mysql']
volumes:
- /var/lib/mysql:/var/lib/mysql:z
mysql_bootstrap:
start_order: 1
detach: false
image: {get_param: ContainerMysqlImage}
net: host
user: root
# Kolla bootstraps aren't idempotent, explicitly checking if bootstrap was done
command:
- 'bash'
- '-ec'
-
list_join:
- "\n"
- - 'if [ -e /var/lib/mysql/mysql ]; then exit 0; fi'
- 'echo -e "\n[mysqld]\nwsrep_provider=none" >> /etc/my.cnf'
- 'kolla_set_configs'
- 'sudo -u mysql -E kolla_extend_start'
- 'timeout ${DB_MAX_TIMEOUT} /bin/bash -c ''while pgrep -af /usr/bin/mysqld_safe | grep -q -v grep; do sleep 1; done'''
- 'mysqld_safe --skip-networking --wsrep-on=OFF &'
- 'timeout ${DB_MAX_TIMEOUT} /bin/bash -c ''until mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" ping 2>/dev/null; do sleep 1; done'''
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "CREATE USER ''clustercheck''@''localhost'' IDENTIFIED BY ''${DB_CLUSTERCHECK_PASSWORD}'';"'
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"'
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"'
- 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown'
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
- &mysql_volumes
- /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z
- /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z
- /var/lib/mysql:/var/lib/mysql:rw,z
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
KOLLA_BOOTSTRAP: true
DB_MAX_TIMEOUT: 60
DB_CLUSTERCHECK_PASSWORD: {get_param: MysqlClustercheckPassword}
DB_ROOT_PASSWORD:
yaql:
expression: $.data.passwords.where($ != '').first()
data:
passwords:
- {get_param: MysqlRootPassword}
- {get_param: [DefaultPasswords, mysql_root_password]}
step_2:
mysql_wait_bundle:
start_order: 0
detach: false
net: host
ipc: host
user: root
command: # '/container_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"'
list_concat:
- - '/container_puppet_apply.sh'
- '2'
- 'file,file_line,concat,augeas,galera_ready,mysql_database,mysql_grant,mysql_user'
- 'include tripleo::profile::pacemaker::database::mysql_bundle'
- if:
- puppet_debug_enabled
- - '--debug'
- - ''
image: {get_param: ContainerMysqlImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, container_puppet_apply_volumes]}
- - /var/lib/mysql:/var/lib/mysql:rw,z
- /var/lib/config-data/puppet-generated/mysql/root:/root:rw
- if:
- docker_enabled
- - /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- null
environment:
# NOTE: this should force this container to re-run on each
# update (scale-out, etc.)
TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
host_prep_tasks:
- name: create persistent directories
file:
path: "{{ item.path }}"
state: directory
setype: "{{ item.setype }}"
mode: "{{ item.mode|default(omit) }}"
with_items:
- {'path': /var/log/containers/mysql, 'setype': 'svirt_sandbox_file_t', 'mode': '0750'}
- {'path': /var/lib/mysql, 'setype': 'svirt_sandbox_file_t'}
- {'path': /var/log/mariadb, 'setype': 'svirt_sandbox_file_t', 'mode': '0750'}
metadata_settings:
get_attr: [MysqlBase, role_data, metadata_settings]
deploy_steps_tasks:
- name: MySQL tag container image for pacemaker
when: step|int == 1
import_role:
name: tripleo-container-tag
vars:
container_image: {get_param: ContainerMysqlImage}
container_image_latest: *mysql_image_pcmklatest
- name: MySQL HA Wrappers Step
when: step|int == 2
block: &mysql_puppet_bundle
- name: Mysql puppet bundle
import_role:
name: tripleo_ha_wrapper
vars:
tripleo_ha_wrapper_service_name: mysql
tripleo_ha_wrapper_resource_name: galera
tripleo_ha_wrapper_bundle_name: galera-bundle
tripleo_ha_wrapper_resource_state: Master
tripleo_ha_wrapper_puppet_config_volume: mysql
tripleo_ha_wrapper_puppet_execute: '["Mysql_datadir", "Mysql_user", "Mysql_database", "Mysql_grant", "Mysql_plugin"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::database::mysql_bundle'
tripleo_ha_wrapper_puppet_tags: 'pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation'
tripleo_ha_wrapper_puppet_debug: {get_param: ConfigDebug}
update_tasks:
- name: Mariadb fetch and retag container image for pacemaker
when: step|int == 2
block: &mysql_fetch_retag_container_tasks
- name: Get container galera image
set_fact:
galera_image: {get_param: ContainerMysqlImage}
galera_image_latest: *mysql_image_pcmklatest
- name: Pull latest galera images
command: "{{container_cli}} pull {{galera_image}}"
register: result
retries: 3
delay: 3
until: result.rc == 0
- name: Get previous galera image id
shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image_latest}}"
register: old_galera_image_id
failed_when: false
- name: Get new galera image id
shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image}}"
register: new_galera_image_id
- name: Retag pcmklatest to latest galera image
include_role:
name: tripleo-container-tag
vars:
container_image: "{{galera_image}}"
container_image_latest: "{{galera_image_latest}}"
when:
- old_galera_image_id.stdout != new_galera_image_id.stdout
post_update_tasks:
- name: Mysql bundle post update
when: step|int == 1
block: *mysql_puppet_bundle
vars:
tripleo_ha_wrapper_minor_update: true
upgrade_tasks:
- vars:
mysql_upgrade_persist: {get_param: MysqlUpgradePersist}
when:
- step|int == 3
- mysql_upgrade_persist
tags:
- never
- system_upgrade
- system_upgrade_prepare
block:
- name: Persist mysql data
include_role:
name: tripleo-persist
tasks_from: persist.yml
vars:
tripleo_persist_dir: /var/lib/mysql
- vars:
mysql_upgrade_persist: {get_param: MysqlUpgradePersist}
when:
- step|int == 5
- mysql_upgrade_persist
tags:
- never
- system_upgrade
- system_upgrade_run
block:
- name: Restore mysql data
include_role:
name: tripleo-persist
tasks_from: restore.yml
vars:
tripleo_persist_dir: /var/lib/mysql
- name: Prepare switch of galera image name
when:
- step|int == 0
block:
- name: Get galera image id currently used by pacemaker
shell: "pcs resource config galera-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'"
register: galera_image_current_res
failed_when: false
- name: Image facts for galera
set_fact:
galera_image_latest: *mysql_image_pcmklatest
galera_image_current: "{{galera_image_current_res.stdout}}"
- name: Temporarily tag the current galera image id with the upgraded image name
import_role:
name: tripleo-container-tag
vars:
container_image: "{{galera_image_current}}"
container_image_latest: "{{galera_image_latest}}"
pull_image: false
when:
- galera_image_current != ''
- galera_image_current != galera_image_latest
# During an OS Upgrade, the cluster may not exist so we use
# the shell module instead.
# TODO(odyssey4me):
# Fix the pacemaker_resource module to handle the exception
# for a non-existant cluster more gracefully.
- name: Check galera cluster resource status
shell: pcs resource config galera-bundle
failed_when: false
changed_when: false
register: galera_pcs_res_result
- name: Set fact galera_pcs_res
set_fact:
galera_pcs_res: "{{galera_pcs_res_result.rc == 0}}"
- name: set is_mysql_bootstrap_node fact
tags: common
set_fact: is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}
- name: Update galera pcs resource bundle for new container image
when:
- step|int == 1
- is_mysql_bootstrap_node|bool
- galera_pcs_res|bool
- galera_image_current != galera_image_latest
block:
- name: Disable the galera cluster resource before container upgrade
pacemaker_resource:
resource: galera
state: disable
wait_for_resource: true
register: output
retries: 5
until: output.rc == 0
- name: Move Mysql logging to /var/log/containers
block:
- name: Check Mysql logging configuration in pacemaker
command: cibadmin --query --xpath "//storage-mapping[@id='mysql-log']"
failed_when: false
register: mysql_logs_moved
- name: Change Mysql logging configuration in pacemaker
# rc == 6 means the configuration doesn't exist in the CIB
when: mysql_logs_moved.rc == 6
block:
- name: Add a bind mount for logging in the galera bundle
command: pcs resource bundle update galera-bundle storage-map add id=mysql-log source-dir=/var/log/containers/mysql target-dir=/var/log/mysql options=rw
- name: Reconfigure Mysql log file in the galera resource agent
command: pcs resource update galera log=/var/log/mysql/mysqld.log
- name: Update the galera bundle to use the new container image name
command: "pcs resource bundle update galera-bundle container image={{galera_image_latest}}"
- name: Enable the galera cluster resource
pacemaker_resource:
resource: galera
state: enable
wait_for_resource: true
register: output
retries: 5
until: output.rc == 0
- name: Create hiera data to upgrade mysql in a stepwise manner.
when:
- step|int == 1
- cluster_recreate|bool
block:
- name: set mysql upgrade node facts in a single-node environment
set_fact:
mysql_short_node_names_upgraded: "{{ mysql_short_node_names }}"
mysql_node_names_upgraded: "{{ mysql_node_names }}"
cacheable: no
when: groups['mysql'] | length <= 1
- name: set mysql upgrade node facts from the limit option
set_fact:
mysql_short_node_names_upgraded: "{{ mysql_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
mysql_node_names_upgraded: "{{ mysql_node_names_upgraded|default([]) + [item] }}"
cacheable: no
when:
- groups['mysql'] | length > 1
- item.split('.')[0] in ansible_limit.split(':')
loop: "{{ mysql_node_names }}"
- fail:
msg: >
You can't upgrade galera without staged upgrade.
You need to use the limit option in order to do so.
when: >-
mysql_short_node_names_upgraded is not defined or
mysql_short_node_names_upgraded | length == 0 or
mysql_node_names_upgraded is not defined or
mysql_node_names_upgraded | length == 0
- debug:
msg: "Prepare galera upgrade for {{ mysql_short_node_names_upgraded }}"
- name: remove mysql init container on upgrade-scaleup to force re-init
include_role:
name: tripleo-container-rm
vars:
tripleo_containers_to_rm:
- mysql_wait_bundle
when:
- mysql_short_node_names_upgraded | length > 1
- name: add the mysql short name to hiera data for the upgrade.
include_role:
name: tripleo-upgrade-hiera
tasks_from: set.yml
vars:
tripleo_upgrade_key: mysql_short_node_names_override
tripleo_upgrade_value: "{{mysql_short_node_names_upgraded}}"
- name: add the mysql long name to hiera data for the upgrade
include_role:
name: tripleo-upgrade-hiera
tasks_from: set.yml
vars:
tripleo_upgrade_key: mysql_node_names_override
tripleo_upgrade_value: "{{mysql_node_names_upgraded}}"
- name: remove the extra hiera data needed for the upgrade.
include_role:
name: tripleo-upgrade-hiera
tasks_from: remove.yml
vars:
tripleo_upgrade_key: "{{item}}"
loop:
- mysql_short_node_names_override
- mysql_node_names_override
when: mysql_short_node_names_upgraded | length == mysql_node_names | length
- name: Retag the pacemaker image if containerized
when:
- step|int == 3
block: *mysql_fetch_retag_container_tasks
- name: Check and upgrade Mysql database after major version upgrade
# Note: during upgrade to Stein, a new pacemaker cluster is recreated,
# controller nodes added sequentially to this new cluster, and the upgrade
# workflow (upgrade tasks, deploy/convertge) is ran once per controller.
# This mysql upgrade block must run only once per controller, before
# the controller is added into the cluster (by mysql_init_bundle) and
# before pacemaker has a chance to start galera on that controller.
# So do not exercise the upgrade if mysql is already running.
when: step|int == 3
block:
# mariadb package changes ownership of /var/lib/mysql on package
# update, so update here rather than in tripleo-package, to
# guarantee that ownership is fixed at the end of step 3
- name: Update host mariadb packages
when: step|int == 3
package: name=mariadb-server-galera state=latest
- name: Mysql upgrade script
set_fact:
mysql_upgrade_script:
list_join:
- "\n"
- # start a temporary mariadb server for running the upgrade
- |
kolla_set_configs
if mysqladmin ping --silent; then exit 0; fi
chown -R mysql:mysql /var/lib/mysql
chown -R mysql:mysql /var/log/mysql
mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log &
# an idempotent script takes care of all upgrade steps
# we inline the content here because at the time this is executed,
# the script is not yet copied in /var/lib/container-config-scripts
- { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
# stop the temporary server
- mysqladmin shutdown
- name: Bind mounts for temporary container
set_fact:
mysql_upgrade_db_bind_mounts:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
- *mysql_volumes
- - /var/log/containers/mysql:/var/log/mysql:rw,z
- name: Upgrade Mysql database from a temporary container
shell:
str_replace:
template:
'CONTAINER_CLI run --rm CONTAINER_CLI_EXTRA_ARGS \
-u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"'
params:
CONTAINER_CLI_EXTRA_ARGS:
if:
- docker_enabled
- '--log-driver=syslog'
- '--log-driver=k8s-file --log-opt path=/var/log/containers/mysql/db-upgrade.log'
UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
UPGRADE_IMAGE: *mysql_image_pcmklatest
UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}"
CONTAINER_CLI: "{{ container_cli }}"
environment:
UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}"
external_upgrade_tasks:
# https://bugs.launchpad.net/tripleo/+bug/1753247
# OC pacemaker part, UC or non pacemaker part is in deployment steps
- name: Rename old neutron database to ovs_neutron
shell: >
if [ -d /var/lib/mysql/neutron ] ; then
mysql -e "CREATE DATABASE IF NOT EXISTS \`ovs_neutron\`;"
for table in `mysql -B -N -e "SHOW TABLES;" neutron`
do
mysql -e "RENAME TABLE \`neutron\`.\`$table\` to \`ovs_neutron\`.\`$table\`"
done
mysql -e "DROP DATABASE \`neutron\`;"
fi
become: true
tags:
- never
- system_upgrade_stop_services
when:
- step|int == 2
- vars:
mysql_upgrade_transfer: {get_param: MysqlUpgradeTransfer}
when:
- step|int == 2
- mysql_upgrade_transfer
tags:
- never
- system_upgrade_transfer_data
block:
- name: Transfer mysql data
include_role:
name: tripleo-transfer
vars:
tripleo_transfer_src_dir: /var/lib/mysql
tripleo_transfer_src_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_node_names'][1]}}"
tripleo_transfer_dest_dir: /var/lib/mysql
tripleo_transfer_dest_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_bootstrap_node_name']}}"
tripleo_transfer_flag_file: /var/lib/tripleo/transfer-flags/var-lib-mysql