heat_template_version: rocky description: > MySQL service deployment with pacemaker bundle parameters: ContainerMysqlImage: description: image type: string ContainerMysqlConfigImage: description: The container image to use for the mysql config_volume type: string ClusterCommonTag: default: false description: When set to false, a pacemaker service is configured to use a floating tag for its container image name, e.g. 'REGISTRY/NAMESPACE/IMAGENAME:pcmklatest'. When set to true, the service uses a floating prefix as well, e.g. 'cluster.common.tag/IMAGENAME:pcmklatest'. type: boolean ClusterFullTag: default: false description: When set to true, the pacemaker service uses a fully constant tag for its container image name, e.g. 'cluster.common.tag/SERVICENAME:pcmklatest'. type: boolean EndpointMap: default: {} description: Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. type: json ServiceData: default: {} description: Dictionary packing service data type: json ServiceNetMap: default: {} description: Mapping of service_name -> network name. Typically set via parameter_defaults in the resource registry. This mapping overrides those in ServiceNetMapDefaults. type: json DefaultPasswords: default: {} type: json MysqlRootPassword: type: string hidden: true default: '' MysqlClustercheckPassword: type: string hidden: true MysqlUpgradePersist: type: boolean default: false MysqlUpgradeTransfer: type: boolean default: true RoleName: default: '' description: Role name on which the service is applied type: string RoleParameters: default: {} description: Parameters specific to the role type: json EnableInternalTLS: type: boolean default: false InternalTLSCAFile: default: '/etc/ipa/ca.crt' type: string description: Specifies the default CA cert to use if TLS is used for services in the internal network. ConfigDebug: default: false description: Whether to run config management (e.g. Puppet) in debug mode. type: boolean DeployIdentifier: default: '' type: string description: > Setting this to a unique value will re-run any deployment tasks which perform configuration on a Heat stack-update. ContainerCli: type: string default: 'podman' description: CLI tool used to manage containers. constraints: - allowed_values: ['docker', 'podman'] resources: ContainersCommon: type: ../containers-common.yaml MysqlBase: type: ./mysql-base.yaml properties: EndpointMap: {get_param: EndpointMap} ServiceData: {get_param: ServiceData} ServiceNetMap: {get_param: ServiceNetMap} DefaultPasswords: {get_param: DefaultPasswords} RoleName: {get_param: RoleName} RoleParameters: {get_param: RoleParameters} conditions: puppet_debug_enabled: {get_param: ConfigDebug} internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} docker_enabled: {equals: [{get_param: ContainerCli}, 'docker']} common_tag_enabled: {equals: [{get_param: ClusterCommonTag}, true]} common_tag_full: {equals: [{get_param: ClusterFullTag}, true]} outputs: role_data: description: Containerized service MySQL using composable services. value: service_name: {get_attr: [MysqlBase, role_data, service_name]} firewall_rules: '104 mysql galera-bundle': dport: - 873 - 3123 - 3306 - 4444 - 4567 - 4568 - 9200 config_settings: map_merge: - get_attr: [MysqlBase, role_data, config_settings] - tripleo::profile::pacemaker::database::mysql::bind_address: str_replace: template: "%{hiera('fqdn_$NETWORK')}" params: $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]} # NOTE: bind IP is found in hiera replacing the network name with the # local node IP for the given network; replacement examples # (eg. for internal_api): # internal_api -> IP # internal_api_uri -> [IP] # internal_api_subnet - > IP/CIDR tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr: str_replace: template: "%{hiera('$NETWORK')}" params: $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]} tripleo::profile::pacemaker::database::mysql::ca_file: get_param: InternalTLSCAFile tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image: &mysql_image_pcmklatest if: - common_tag_full - "cluster.common.tag/mariadb:pcmklatest" - yaql: data: if: - common_tag_enabled - yaql: data: {get_param: ContainerMysqlImage} expression: concat("cluster.common.tag/", $.data.rightSplit(separator => "/", maxSplits => 1)[1]) - {get_param: ContainerMysqlImage} expression: concat($.data.rightSplit(separator => ":", maxSplits => 1)[0], ":pcmklatest") tripleo::profile::pacemaker::database::mysql_bundle::control_port: 3123 tripleo::profile::pacemaker::database::mysql_bundle::container_backend: {get_param: ContainerCli} tripleo::profile::pacemaker::database::mysql_bundle::bind_address: str_replace: template: "%{hiera('fqdn_$NETWORK')}" params: $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]} - if: - internal_tls_enabled - tripleo::profile::pacemaker::database::mysql_bundle::ca_file: get_param: InternalTLSCAFile - {} # BEGIN DOCKER SETTINGS # puppet_config: config_volume: mysql puppet_tags: file # set this even though file is the default step_config: list_join: - "\n" - - "['Mysql_datadir', 'Mysql_user', 'Mysql_database', 'Mysql_grant', 'Mysql_plugin'].each |String $val| { noop_resource($val) }" - "exec {'wait-for-settle': command => '/bin/true' }" - "include tripleo::profile::pacemaker::database::mysql_bundle" config_image: {get_param: ContainerMysqlConfigImage} kolla_config: /var/lib/kolla/config_files/mysql.json: command: /usr/sbin/pacemaker_remoted config_files: - dest: /etc/libqb/force-filesystem-sockets source: /dev/null owner: root perm: '0644' - source: "/var/lib/kolla/config_files/src/*" dest: "/" merge: true preserve_properties: true - source: "/var/lib/kolla/config_files/src-tls/*" dest: "/" merge: true optional: true preserve_properties: true permissions: - path: /var/log/mysql owner: mysql:mysql recurse: true - path: /etc/pki/tls/certs/mysql.crt owner: mysql:mysql perm: '0600' optional: true - path: /etc/pki/tls/private/mysql.key owner: mysql:mysql perm: '0600' optional: true container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]} docker_config: step_1: mysql_data_ownership: start_order: 0 detach: false image: {get_param: ContainerMysqlImage} net: host user: root # Kolla does only non-recursive chown command: ['chown', '-R', 'mysql:', '/var/lib/mysql'] volumes: - /var/lib/mysql:/var/lib/mysql:z mysql_bootstrap: start_order: 1 detach: false image: {get_param: ContainerMysqlImage} net: host user: root # Kolla bootstraps aren't idempotent, explicitly checking if bootstrap was done command: - 'bash' - '-ec' - list_join: - "\n" - - 'if [ -e /var/lib/mysql/mysql ]; then exit 0; fi' - 'echo -e "\n[mysqld]\nwsrep_provider=none" >> /etc/my.cnf' - 'kolla_set_configs' - 'sudo -u mysql -E kolla_extend_start' - 'timeout ${DB_MAX_TIMEOUT} /bin/bash -c ''while pgrep -af /usr/bin/mysqld_safe | grep -q -v grep; do sleep 1; done''' - 'mysqld_safe --skip-networking --wsrep-on=OFF &' - 'timeout ${DB_MAX_TIMEOUT} /bin/bash -c ''until mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" ping 2>/dev/null; do sleep 1; done''' - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "CREATE USER ''clustercheck''@''localhost'' IDENTIFIED BY ''${DB_CLUSTERCHECK_PASSWORD}'';"' - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"' - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"' - 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown' volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - &mysql_volumes - /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z - /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z - /var/lib/mysql:/var/lib/mysql:rw,z environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS KOLLA_BOOTSTRAP: true DB_MAX_TIMEOUT: 60 DB_CLUSTERCHECK_PASSWORD: {get_param: MysqlClustercheckPassword} DB_ROOT_PASSWORD: yaql: expression: $.data.passwords.where($ != '').first() data: passwords: - {get_param: MysqlRootPassword} - {get_param: [DefaultPasswords, mysql_root_password]} step_2: mysql_wait_bundle: start_order: 0 detach: false net: host ipc: host user: root command: # '/container_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"' list_concat: - - '/container_puppet_apply.sh' - '2' - 'file,file_line,concat,augeas,galera_ready,mysql_database,mysql_grant,mysql_user' - 'include tripleo::profile::pacemaker::database::mysql_bundle' - if: - puppet_debug_enabled - - '--debug' - - '' image: {get_param: ContainerMysqlImage} volumes: list_concat: - {get_attr: [ContainersCommon, container_puppet_apply_volumes]} - - /var/lib/mysql:/var/lib/mysql:rw,z - /var/lib/config-data/puppet-generated/mysql/root:/root:rw - if: - docker_enabled - - /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro - null environment: # NOTE: this should force this container to re-run on each # update (scale-out, etc.) TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier} host_prep_tasks: - name: create persistent directories file: path: "{{ item.path }}" state: directory setype: "{{ item.setype }}" mode: "{{ item.mode|default(omit) }}" with_items: - {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'} - {'path': /var/lib/mysql, 'setype': 'container_file_t'} - {'path': /var/log/mariadb, 'setype': 'container_file_t', 'mode': '0750'} metadata_settings: get_attr: [MysqlBase, role_data, metadata_settings] deploy_steps_tasks: - name: MySQL tag container image for pacemaker when: step|int == 1 import_role: name: tripleo_container_tag vars: container_image: {get_param: ContainerMysqlImage} container_image_latest: *mysql_image_pcmklatest - name: MySQL HA Wrappers Step when: step|int == 2 block: &mysql_puppet_bundle - name: Mysql puppet bundle import_role: name: tripleo_ha_wrapper vars: tripleo_ha_wrapper_service_name: mysql tripleo_ha_wrapper_resource_name: galera tripleo_ha_wrapper_bundle_name: galera-bundle tripleo_ha_wrapper_resource_state: Master tripleo_ha_wrapper_puppet_config_volume: mysql tripleo_ha_wrapper_puppet_execute: '["Mysql_datadir", "Mysql_user", "Mysql_database", "Mysql_grant", "Mysql_plugin"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::database::mysql_bundle' tripleo_ha_wrapper_puppet_tags: 'pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation' tripleo_ha_wrapper_puppet_debug: {get_param: ConfigDebug} update_tasks: - name: Tear-down non-HA mysql container when: - step|int == 1 block: &mysql_teardown_nonha - name: "stat mysql container" command: "podman container exists mysql" failed_when: false changed_when: false register: stat_mysql_container - name: Create clustercheck user and permissions command: argv: "{{ mysql_exec_data | container_exec_cmd }}" changed_when: true when: - stat_mysql_container.rc == 0 vars: mysql_exec_data: environment: CLUSTERCHECK_PASSWORD: {get_param: MysqlClustercheckPassword} command: - 'mysql' - '/bin/sh' - '-c' - mysql -e "CREATE USER IF NOT EXISTS 'clustercheck'@'localhost' IDENTIFIED BY '${CLUSTERCHECK_PASSWORD}'; GRANT PROCESS ON *.* TO 'clustercheck'@'localhost' WITH GRANT OPTION;" - name: Remove non-HA mysql container include_role: name: tripleo_container_rm vars: tripleo_container_cli: "{{ container_cli }}" tripleo_containers_to_rm: - mysql - name: Mariadb fetch and retag container image for pacemaker when: step|int == 2 block: &mysql_fetch_retag_container_tasks - name: Get container galera image set_fact: galera_image: {get_param: ContainerMysqlImage} galera_image_latest: *mysql_image_pcmklatest - name: Pull latest galera images command: "{{container_cli}} pull {{galera_image}}" register: result retries: 3 delay: 3 until: result.rc == 0 - name: Get previous galera image id shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image_latest}}" register: old_galera_image_id failed_when: false - name: Get new galera image id shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image}}" register: new_galera_image_id - name: Retag pcmklatest to latest galera image include_role: name: tripleo_container_tag vars: container_image: "{{galera_image}}" container_image_latest: "{{galera_image_latest}}" when: - old_galera_image_id.stdout != new_galera_image_id.stdout post_update_tasks: - name: Mysql bundle post update when: step|int == 1 block: *mysql_puppet_bundle vars: tripleo_ha_wrapper_minor_update: true upgrade_tasks: - name: Tear-down non-HA mysql container when: - step|int == 0 block: *mysql_teardown_nonha - vars: mysql_upgrade_persist: {get_param: MysqlUpgradePersist} when: - step|int == 3 - mysql_upgrade_persist tags: - never - system_upgrade - system_upgrade_prepare block: - name: Persist mysql data include_role: name: tripleo_persist tasks_from: persist.yml vars: tripleo_persist_dir: /var/lib/mysql - vars: mysql_upgrade_persist: {get_param: MysqlUpgradePersist} when: - step|int == 5 - mysql_upgrade_persist tags: - never - system_upgrade - system_upgrade_run block: - name: Restore mysql data include_role: name: tripleo_persist tasks_from: restore.yml vars: tripleo_persist_dir: /var/lib/mysql - name: Prepare switch of galera image name when: - step|int == 0 block: - name: Get galera image id currently used by pacemaker shell: "pcs resource config galera-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" register: galera_image_current_res failed_when: false - name: Image facts for galera set_fact: galera_image_latest: *mysql_image_pcmklatest galera_image_current: "{{galera_image_current_res.stdout}}" - name: Temporarily tag the current galera image id with the upgraded image name import_role: name: tripleo_container_tag vars: container_image: "{{galera_image_current}}" container_image_latest: "{{galera_image_latest}}" pull_image: false when: - galera_image_current != '' - galera_image_current != galera_image_latest # During an OS Upgrade, the cluster may not exist so we use # the shell module instead. # TODO(odyssey4me): # Fix the pacemaker_resource module to handle the exception # for a non-existant cluster more gracefully. - name: Check galera cluster resource status shell: pcs resource config galera-bundle failed_when: false changed_when: false register: galera_pcs_res_result - name: Set fact galera_pcs_res set_fact: galera_pcs_res: "{{galera_pcs_res_result.rc == 0}}" - name: set is_mysql_bootstrap_node fact tags: common set_fact: is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_hostname|lower}} - name: Update galera pcs resource bundle for new container image when: - step|int == 1 - is_mysql_bootstrap_node|bool - galera_pcs_res|bool - galera_image_current != galera_image_latest block: - name: Disable the galera cluster resource before container upgrade pacemaker_resource: resource: galera state: disable wait_for_resource: true register: output retries: 5 until: output.rc == 0 - name: Move Mysql logging to /var/log/containers block: - name: Check Mysql logging configuration in pacemaker command: cibadmin --query --xpath "//storage-mapping[@id='mysql-log']" failed_when: false register: mysql_logs_moved - name: Change Mysql logging configuration in pacemaker # rc == 6 means the configuration doesn't exist in the CIB when: mysql_logs_moved.rc == 6 block: - name: Add a bind mount for logging in the galera bundle command: pcs resource bundle update galera-bundle storage-map add id=mysql-log source-dir=/var/log/containers/mysql target-dir=/var/log/mysql options=rw - name: Reconfigure Mysql log file in the galera resource agent command: pcs resource update galera log=/var/log/mysql/mysqld.log - name: Update the galera bundle to use the new container image name command: "pcs resource bundle update galera-bundle container image={{galera_image_latest}}" - name: Enable the galera cluster resource pacemaker_resource: resource: galera state: enable wait_for_resource: true register: output retries: 5 until: output.rc == 0 - name: Create hiera data to upgrade mysql in a stepwise manner. when: - step|int == 1 - cluster_recreate|bool block: - name: set mysql upgrade node facts in a single-node environment set_fact: mysql_short_node_names_upgraded: "{{ mysql_short_node_names }}" mysql_node_names_upgraded: "{{ mysql_node_names }}" cacheable: no when: groups['mysql'] | length <= 1 - name: set mysql upgrade node facts from the limit option set_fact: mysql_short_node_names_upgraded: "{{ mysql_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}" mysql_node_names_upgraded: "{{ mysql_node_names_upgraded|default([]) + [item] }}" cacheable: no when: - groups['mysql'] | length > 1 - item.split('.')[0] in ansible_limit.split(':') loop: "{{ mysql_node_names | default([]) }}" - fail: msg: > You can't upgrade galera without staged upgrade. You need to use the limit option in order to do so. when: >- mysql_short_node_names_upgraded is not defined or mysql_short_node_names_upgraded | length == 0 or mysql_node_names_upgraded is not defined or mysql_node_names_upgraded | length == 0 - debug: msg: "Prepare galera upgrade for {{ mysql_short_node_names_upgraded }}" - name: remove mysql init container on upgrade-scaleup to force re-init include_role: name: tripleo_container_rm vars: tripleo_containers_to_rm: - mysql_wait_bundle when: - mysql_short_node_names_upgraded | length > 1 - name: add the mysql short name to hiera data for the upgrade. include_role: name: tripleo_upgrade_hiera tasks_from: set.yml vars: tripleo_upgrade_key: mysql_short_node_names_override tripleo_upgrade_value: "{{mysql_short_node_names_upgraded}}" - name: add the mysql long name to hiera data for the upgrade include_role: name: tripleo_upgrade_hiera tasks_from: set.yml vars: tripleo_upgrade_key: mysql_node_names_override tripleo_upgrade_value: "{{mysql_node_names_upgraded}}" - name: remove the extra hiera data needed for the upgrade. include_role: name: tripleo_upgrade_hiera tasks_from: remove.yml vars: tripleo_upgrade_key: "{{item}}" loop: - mysql_short_node_names_override - mysql_node_names_override when: mysql_short_node_names_upgraded | length == mysql_node_names | length - name: Retag the pacemaker image if containerized when: - step|int == 3 block: *mysql_fetch_retag_container_tasks - name: Check and upgrade Mysql database after major version upgrade # Note: during upgrade to Stein, a new pacemaker cluster is recreated, # controller nodes added sequentially to this new cluster, and the upgrade # workflow (upgrade tasks, deploy/convertge) is ran once per controller. # This mysql upgrade block must run only once per controller, before # the controller is added into the cluster (by mysql_init_bundle) and # before pacemaker has a chance to start galera on that controller. # So do not exercise the upgrade if mysql is already running. when: step|int == 3 block: # mariadb package changes ownership of /var/lib/mysql on package # update, so update here rather than in tripleo-package, to # guarantee that ownership is fixed at the end of step 3 - name: Update host mariadb packages when: step|int == 3 package: name=mariadb-server-galera state=latest - name: Mysql upgrade script set_fact: mysql_upgrade_script: list_join: - "\n" - # start a temporary mariadb server for running the upgrade - | kolla_set_configs if mysqladmin ping --silent; then exit 0; fi chown -R mysql:mysql /var/lib/mysql chown -R mysql:mysql /var/log/mysql mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log & # an idempotent script takes care of all upgrade steps # we inline the content here because at the time this is executed, # the script is not yet copied in /var/lib/container-config-scripts - { get_file: ../../container_config_scripts/mysql_upgrade_db.sh } # stop the temporary server - mysqladmin shutdown - name: Bind mounts for temporary container set_fact: mysql_upgrade_db_bind_mounts: list_concat: - {get_attr: [ContainersCommon, volumes]} - *mysql_volumes - - /var/log/containers/mysql:/var/log/mysql:rw,z - name: Upgrade Mysql database from a temporary container shell: str_replace: template: 'CONTAINER_CLI run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \ -u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"' params: UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' UPGRADE_IMAGE: *mysql_image_pcmklatest UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}" CONTAINER_CLI: "{{ container_cli }}" LOG_DIR: '/var/log/containers/mysql' environment: UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}" external_upgrade_tasks: - vars: mysql_upgrade_transfer: {get_param: MysqlUpgradeTransfer} when: - step|int == 2 - mysql_upgrade_transfer tags: - never - system_upgrade_transfer_data block: - name: Transfer mysql data include_role: name: tripleo_transfer vars: tripleo_transfer_src_dir: /var/lib/mysql tripleo_transfer_src_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_node_names'][1]}}" tripleo_transfer_dest_dir: /var/lib/mysql tripleo_transfer_dest_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_bootstrap_node_name']}}" tripleo_transfer_flag_file: /var/lib/tripleo/transfer-flags/var-lib-mysql