Add way to backup from MariaDB replica server

ProxySQL reports its version to MySQL clients,
causing the backup process to fail when ProxySQL is used.

This patch adds a new script that handles backups from
the replica server directly, without modifying the original
backup script. The new script retrieves the server address
via the load balancer and connects directly to the replica
server to execute the backup.

Moreover, this reduces the risk that the backup will be inconsistent
or that it will block OpenStack MySQL clients, because they are
connected to a different server, and no one is connected to the replica,
or that it will overload the load balancer (such as ProxySQL or HAProxy)
which listens on the VIP.

Closes-Bug: #2080818
Change-Id: Ibf7e40e8b059d733e114963022df06180249c650
This commit is contained in:
Michal Arbet 2024-09-11 19:20:54 +02:00
parent 068790cddc
commit ec4c4310dd
3 changed files with 144 additions and 1 deletions

View File

@ -39,9 +39,12 @@ COPY mariadb_sudoers /etc/sudoers.d/kolla_mariadb_sudoers
COPY extend_start.sh /usr/local/bin/kolla_extend_start COPY extend_start.sh /usr/local/bin/kolla_extend_start
COPY security_reset.expect /usr/local/bin/kolla_security_reset COPY security_reset.expect /usr/local/bin/kolla_security_reset
COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh COPY backup.sh /usr/local/bin/kolla_mariadb_backup.sh
COPY backup_replica.sh /usr/local/bin/kolla_mariadb_backup_replica.sh
RUN chmod 644 /usr/local/bin/kolla_extend_start \ RUN chmod 644 /usr/local/bin/kolla_extend_start \
&& chmod 755 /usr/local/bin/kolla_security_reset /usr/local/bin/kolla_mariadb_backup.sh \ && chmod 755 /usr/local/bin/kolla_security_reset \
/usr/local/bin/kolla_mariadb_backup.sh \
/usr/local/bin/kolla_mariadb_backup_replica.sh \
&& chmod 750 /etc/sudoers.d \ && chmod 750 /etc/sudoers.d \
&& chmod 440 /etc/sudoers.d/kolla_mariadb_sudoers \ && chmod 440 /etc/sudoers.d/kolla_mariadb_sudoers \
&& rm -rf /var/lib/mysql/* && rm -rf /var/lib/mysql/*

View File

@ -0,0 +1,135 @@
#!/usr/bin/env bash
set -eu
set -o pipefail
BACKUP_DIR=/backup/
DEFAULT_MY_CNF="/etc/mysql/my.cnf"
REPLICA_MY_CNF="$(mktemp)"
RETRY_INTERVAL=5 # Interval between retries (in seconds)
MAX_RETRIES=12 # Max retries (12 retries * 5 seconds = 60 seconds)
# Cleanup function to remove the REPLICA_MY_CNF file
cleanup() {
rm -f "${REPLICA_MY_CNF}"
}
# Set trap to ensure cleanup occurs on exit or error
trap cleanup EXIT
cd "${BACKUP_DIR}"
# Execute a full backup
backup_full() {
echo "Taking a full backup"
LAST_FULL_DATE=$(date +%d-%m-%Y-%s)
mariabackup \
--defaults-file="${REPLICA_MY_CNF}" \
--backup \
--stream=mbstream \
--history="${LAST_FULL_DATE}" | gzip > \
"${BACKUP_DIR}/mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz" && \
echo "${LAST_FULL_DATE}" > "${BACKUP_DIR}/last_full_date"
}
# Execute an incremental backup
backup_incremental() {
if [ -r "${BACKUP_DIR}/last_full_date" ]; then
LAST_FULL_DATE=$(cat "${BACKUP_DIR}/last_full_date")
else
LAST_FULL_DATE=""
fi
if [ ! -z "${LAST_FULL_DATE}" ]; then
echo "Taking an incremental backup"
mariabackup \
--defaults-file="${REPLICA_MY_CNF}" \
--backup \
--stream=mbstream \
--incremental-history-name="${LAST_FULL_DATE}" \
--history="${LAST_FULL_DATE}" | gzip > \
"${BACKUP_DIR}/incremental-$(date +%H)-mysqlbackup-${LAST_FULL_DATE}.qp.mbc.mbs.gz"
else
echo "Error: Full backup don't exist."
exit 1
fi
}
# Retry logic for database queries
retry_mysql_query() {
local query="$1"
local result=""
local attempt=1
while [ ${attempt} -le ${MAX_RETRIES} ]; do
result=$(mysql -h "${HOST}" -u "${USER}" -p"${PASS}" -s -N -e "${query}" 2>/dev/null || true)
if [ -n "${result}" ]; then
echo "${result}"
return 0
fi
echo "Attempt ${attempt}/${MAX_RETRIES} failed. Retrying in ${RETRY_INTERVAL} seconds..."
sleep "${RETRY_INTERVAL}"
attempt=$((attempt + 1))
done
echo "Error: Failed to execute the query after ${MAX_RETRIES} attempts."
return 1
}
get_and_set_replica_server() {
HOST="$(grep '^host' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
USER="$(grep '^user' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
PASS="$(grep '^password' "${DEFAULT_MY_CNF}" | awk -F '=' '{print $2}' | xargs)"
ALL_HOSTS_SELECT="SELECT REGEXP_REPLACE(VARIABLE_VALUE, ':[0-9]*','') FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_incoming_addresses';"
ALL_HOSTS=$(retry_mysql_query "${ALL_HOSTS_SELECT}")
if [ -z "${ALL_HOSTS}" ]; then
echo "Backup failed due to inability to fetch a list of servers."
exit 1
fi
ACTIVE_HOST_SELECT='SELECT @@hostname;'
ACTIVE_HOST=$(retry_mysql_query "${ACTIVE_HOST_SELECT}" | xargs getent hosts | awk '{print $1}')
if [ -z "${ACTIVE_HOST}" ]; then
echo "Backup failed due to inability to fetch active host."
exit 1
fi
# Multinode
if echo "${ALL_HOSTS}" | grep -q ','; then
for server in $(echo "${ALL_HOSTS}" | tr ',' '\n'); do
if [[ "${server}" != "${ACTIVE_HOST}" ]]; then
REPLICA_HOST="${server}"
break
fi
done
# Single node
else
REPLICA_HOST="${ALL_HOSTS}"
fi
if [ -z "${REPLICA_HOST}" ]; then
echo "Backup failed due to inability to determine replica host."
exit 1
fi
cp "${DEFAULT_MY_CNF}" "${REPLICA_MY_CNF}"
sed -i "s/${HOST}/${REPLICA_HOST}/g" "${REPLICA_MY_CNF}"
}
if [ -n "${BACKUP_TYPE}" ]; then
get_and_set_replica_server
case "${BACKUP_TYPE}" in
"full")
backup_full
;;
"incremental")
backup_incremental
;;
*)
echo "Only full or incremental options are supported."
exit 1
;;
esac
else
echo "You need to specify either full or incremental backup options."
exit 1
fi

View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixes MariaDB backup failure when ProxySQL is used
`LP#2080818 <https://launchpad.net/bugs/2080818>`__