398 lines
16 KiB
Smarty
Executable File
398 lines
16 KiB
Smarty
Executable File
{{- define "helm-toolkit.scripts.db-backup-restore.backup_main" }}
|
|
#!/bin/bash
|
|
|
|
# This file contains a database backup framework which database scripts
|
|
# can use to perform a backup. The idea here is that the database-specific
|
|
# functions will be implemented by the various databases using this script
|
|
# (like mariadb, postgresql or etcd for example). The database-specific
|
|
# script will need to first "source" this file like this:
|
|
# source /tmp/backup_main.sh
|
|
#
|
|
# Then the script should call the main backup function (backup_databases):
|
|
# backup_databases [scope]
|
|
# [scope] is an optional parameter, defaulted to "all". If only one specific
|
|
# database is required to be backed up then this parameter will
|
|
# contain the name of the database; otherwise all are backed up.
|
|
#
|
|
# The framework will require the following variables to be exported:
|
|
#
|
|
# export DB_NAMESPACE Namespace where the database(s) reside
|
|
# export DB_NAME Name of the database system
|
|
# export LOCAL_DAYS_TO_KEEP Number of days to keep the local backups
|
|
# export REMOTE_DAYS_TO_KEEP Number of days to keep the remote backups
|
|
# export ARCHIVE_DIR Local location where the backup tarballs should
|
|
# be stored. (full directory path)
|
|
# export BACK_UP_MODE Determines the mode of backup taken.
|
|
# export REMOTE_BACKUP_ENABLED "true" if remote backup enabled; false
|
|
# otherwise
|
|
# export CONTAINER_NAME Name of the container on the RGW to store
|
|
# the backup tarball.
|
|
# export STORAGE_POLICY Name of the storage policy defined on the
|
|
# RGW which is intended to store backups.
|
|
# RGW access variables:
|
|
# export OS_REGION_NAME Name of the region the RGW resides in
|
|
# export OS_AUTH_URL Keystone URL associated with the RGW
|
|
# export OS_PROJECT_NAME Name of the project associated with the
|
|
# keystone user
|
|
# export OS_USERNAME Name of the keystone user
|
|
# export OS_PASSWORD Password of the keystone user
|
|
# export OS_USER_DOMAIN_NAME Keystone domain the project belongs to
|
|
# export OS_PROJECT_DOMAIN_NAME Keystone domain the user belongs to
|
|
# export OS_IDENTITY_API_VERSION Keystone API version to use
|
|
#
|
|
# The following variables are optional:
|
|
# export RGW_TIMEOUT Number of seconds to wait for the
|
|
# connection to the RGW to be available
|
|
# when sending a backup to the RGW. Default
|
|
# is 1800 (30 minutes).
|
|
#
|
|
# The database-specific functions that need to be implemented are:
|
|
# dump_databases_to_directory <directory> <err_logfile> [scope]
|
|
# where:
|
|
# <directory> is the full directory path to dump the database files
|
|
# into. This is a temporary directory for this backup only.
|
|
# <err_logfile> is the full directory path where error logs are to be
|
|
# written by the application.
|
|
# [scope] set to "all" if all databases are to be backed up; or
|
|
# set to the name of a specific database to be backed up.
|
|
# This optional parameter is defaulted to "all".
|
|
# returns: 0 if no errors; 1 if any errors occurred
|
|
#
|
|
# This function is expected to dump the database file(s) to the specified
|
|
# directory path. If this function completes successfully (returns 0), the
|
|
# framework will automatically tar/zip the files in that directory and
|
|
# name the tarball appropriately according to the proper conventions.
|
|
#
|
|
# The functions in this file will take care of:
|
|
# 1) Calling "dump_databases_to_directory" and then compressing the files,
|
|
# naming the tarball properly, and then storing it locally at the specified
|
|
# local directory.
|
|
# 2) Sending the tarball built to the remote gateway, to be stored in the
|
|
# container configured to store database backups.
|
|
# 3) Removing local backup tarballs which are older than the number of days
|
|
# specified by the "LOCAL_DAYS_TO_KEEP" variable.
|
|
# 4) Removing remote backup tarballs (from the remote gateway) which are older
|
|
# than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
|
|
#
|
|
|
|
# Note: not using set -e in this script because more elaborate error handling
|
|
# is needed.
|
|
set -x
|
|
|
|
log_backup_error_exit() {
|
|
MSG=$1
|
|
ERRCODE=$2
|
|
log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
|
|
rm -f $ERR_LOG_FILE
|
|
rm -rf $TMP_DIR
|
|
exit $ERRCODE
|
|
}
|
|
|
|
log() {
|
|
#Log message to a file or stdout
|
|
#TODO: This can be convert into mail alert of alert send to a monitoring system
|
|
#Params: $1 log level
|
|
#Params: $2 service
|
|
#Params: $3 message
|
|
#Params: $4 Destination
|
|
LEVEL=$1
|
|
SERVICE=$2
|
|
MSG=$3
|
|
DEST=$4
|
|
DATE=$(date +"%m-%d-%y %H:%M:%S")
|
|
if [[ -z "$DEST" ]]; then
|
|
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}"
|
|
else
|
|
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST
|
|
fi
|
|
}
|
|
|
|
#Get the day delta since the archive file backup
|
|
seconds_difference() {
|
|
ARCHIVE_DATE=$( date --date="$1" +%s )
|
|
if [[ $? -ne 0 ]]; then
|
|
SECOND_DELTA=0
|
|
fi
|
|
CURRENT_DATE=$( date +%s )
|
|
SECOND_DELTA=$(($CURRENT_DATE-$ARCHIVE_DATE))
|
|
if [[ "$SECOND_DELTA" -lt 0 ]]; then
|
|
SECOND_DELTA=0
|
|
fi
|
|
echo $SECOND_DELTA
|
|
}
|
|
|
|
# Send the specified tarball file at the specified filepath to the
|
|
# remote gateway.
|
|
send_to_remote_server() {
|
|
FILEPATH=$1
|
|
FILE=$2
|
|
|
|
# Grab the list of containers on the remote site
|
|
RESULT=$(openstack container list 2>&1)
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
echo $RESULT | grep $CONTAINER_NAME
|
|
if [[ $? -ne 0 ]]; then
|
|
# Find the swift URL from the keystone endpoint list
|
|
SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
|
|
|
|
# Get a token from keystone
|
|
TOKEN=$(openstack token issue -f value -c id)
|
|
|
|
# Create the container
|
|
RES_FILE=$(mktemp -p /tmp)
|
|
curl -g -i -X PUT ${SWIFT_URL}/${CONTAINER_NAME} \
|
|
-H "X-Auth-Token: ${TOKEN}" \
|
|
-H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
|
|
|
|
if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Error creating container ${CONTAINER_NAME}"
|
|
cat $RES_FILE
|
|
rm -f $RES_FILE
|
|
return 1
|
|
fi
|
|
rm -f $RES_FILE
|
|
|
|
swift stat $CONTAINER_NAME
|
|
if [[ $? -ne 0 ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Error retrieving container ${CONTAINER_NAME} details after creation."
|
|
return 1
|
|
fi
|
|
fi
|
|
else
|
|
echo $RESULT | grep "HTTP 401"
|
|
if [[ $? -eq 0 ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
|
|
return 1
|
|
else
|
|
echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable"
|
|
if [[ $? -eq 0 ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
|
|
# In this case, keystone or the site/node may be temporarily down.
|
|
# Return slightly different error code so the calling code can retry
|
|
return 2
|
|
else
|
|
log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Create an object to store the file
|
|
openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE || log ERROR "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
|
|
openstack object show $CONTAINER_NAME $FILE
|
|
if [[ $? -ne 0 ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Error retrieving container object $FILE after creation."
|
|
return 1
|
|
fi
|
|
|
|
log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
|
|
return 0
|
|
}
|
|
|
|
# This function attempts to store the built tarball to the remote gateway,
|
|
# with built-in logic to handle error cases like:
|
|
# 1) Network connectivity issues - retries for a specific amount of time
|
|
# 2) Authorization errors - immediately logs an ERROR and returns
|
|
store_backup_remotely() {
|
|
FILEPATH=$1
|
|
FILE=$2
|
|
|
|
# If the RGW_TIMEOUT has already been set, use that value, otherwise give it
|
|
# a default value.
|
|
if [[ -z $RGW_TIMEOUT ]]; then
|
|
RGW_TIMEOUT=1800
|
|
fi
|
|
|
|
ERROR_SEEN=false
|
|
DONE=false
|
|
TIMEOUT_EXP=$(( $(date +%s) + $RGW_TIMEOUT ))
|
|
while [[ $DONE == "false" ]]; do
|
|
# Store the new archive to the remote backup storage facility.
|
|
send_to_remote_server $FILEPATH $FILE
|
|
|
|
# Check if successful
|
|
if [[ $? -eq 0 ]]; then
|
|
log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to RGW."
|
|
DONE=true
|
|
elif [[ $? -eq 2 ]]; then
|
|
# Temporary failure occurred. We need to retry if we have not timed out
|
|
log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to RGW due to connection issue."
|
|
DELTA=$(( TIMEOUT_EXP - $(date +%s) ))
|
|
if [[ $DELTA -lt 0 ]]; then
|
|
DONE=true
|
|
log ERROR "${DB_NAME}_backup" "Timed out waiting for RGW to become available."
|
|
ERROR_SEEN=true
|
|
else
|
|
log INFO "${DB_NAME}_backup" "Sleeping 30 seconds waiting for RGW to become available..."
|
|
sleep 30
|
|
log INFO "${DB_NAME}_backup" "Retrying..."
|
|
fi
|
|
else
|
|
log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW."
|
|
ERROR_SEEN=true
|
|
DONE=true
|
|
fi
|
|
done
|
|
|
|
if [[ $ERROR_SEEN == "true" ]]; then
|
|
log ERROR "${DB_NAME}_backup" "Errors encountered. Exiting."
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
remove_old_local_archives() {
|
|
log INFO "${DB_NAME}_backup" "Deleting backups older than ${LOCAL_DAYS_TO_KEEP} days"
|
|
if [[ -d $ARCHIVE_DIR ]]; then
|
|
for ARCHIVE_FILE in $(ls -1 $ARCHIVE_DIR/*.gz); do
|
|
ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
|
|
if [[ "$(seconds_difference $ARCHIVE_DATE)" -gt "$(($LOCAL_DAYS_TO_KEEP*86400))" ]]; then
|
|
log INFO "${DB_NAME}_backup" "Deleting file $ARCHIVE_FILE."
|
|
rm -rf $ARCHIVE_FILE
|
|
if [[ $? -ne 0 ]]; then
|
|
# Log error but don't exit so we can finish the script
|
|
# because at this point we haven't sent backup to RGW yet
|
|
log ERROR "${DB_NAME}_backup" "Cannot remove ${ARCHIVE_FILE}"
|
|
fi
|
|
else
|
|
log INFO "${DB_NAME}_backup" "Keeping file ${ARCHIVE_FILE}."
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
remove_old_remote_archives() {
|
|
log INFO "${DB_NAME}_backup" "Deleting backups older than ${REMOTE_DAYS_TO_KEEP} days"
|
|
BACKUP_FILES=$(mktemp -p /tmp)
|
|
DB_BACKUP_FILES=$(mktemp -p /tmp)
|
|
|
|
openstack object list $CONTAINER_NAME > $BACKUP_FILES
|
|
if [[ $? -ne 0 ]]; then
|
|
log_backup_error_exit "Could not obtain a list of current backup files in the RGW" 1
|
|
fi
|
|
|
|
# Filter out other types of backup files
|
|
cat $BACKUP_FILES | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $DB_BACKUP_FILES
|
|
|
|
for ARCHIVE_FILE in $(cat $DB_BACKUP_FILES); do
|
|
ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
|
|
if [[ "$(seconds_difference ${ARCHIVE_DATE})" -gt "$((${REMOTE_DAYS_TO_KEEP}*86400))" ]]; then
|
|
log INFO "${DB_NAME}_backup" "Deleting file ${ARCHIVE_FILE} from the RGW"
|
|
openstack object delete $CONTAINER_NAME $ARCHIVE_FILE || log_backup_error_exit "Cannot delete container object ${ARCHIVE_FILE}!" 1
|
|
fi
|
|
done
|
|
|
|
# Cleanup now that we're done.
|
|
rm -f $BACKUP_FILES $DB_BACKUP_FILES
|
|
}
|
|
|
|
# Main function to backup the databases. Calling functions need to supply:
|
|
# 1) The directory where the final backup will be kept after it is compressed.
|
|
# 2) A temporary directory to use for placing database files to be compressed.
|
|
# Note: this temp directory will be deleted after backup is done.
|
|
# 3) Optional "scope" parameter indicating what database to back up. Defaults
|
|
# to "all".
|
|
backup_databases() {
|
|
SCOPE=${1:-"all"}
|
|
|
|
# Create necessary directories if they do not exist.
|
|
mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
|
|
export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
|
|
|
|
# Create temporary log file
|
|
export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"
|
|
|
|
# It is expected that this function will dump the database files to the $TMP_DIR
|
|
dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE
|
|
|
|
# If successful, there should be at least one file in the TMP_DIR
|
|
if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
|
|
cat $ERR_LOG_FILE
|
|
log_backup_error_exit "Backup of the ${DB_NAME} database failed and needs attention."
|
|
fi
|
|
|
|
log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..."
|
|
|
|
NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
|
|
if [[ -z "${BACK_UP_MODE}" ]]; then
|
|
TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz"
|
|
else
|
|
TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${BACK_UP_MODE}.${NOW}.tar.gz"
|
|
fi
|
|
|
|
cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"
|
|
|
|
#Archive the current database files
|
|
tar zcvf $ARCHIVE_DIR/$TARBALL_FILE *
|
|
if [[ $? -ne 0 ]]; then
|
|
log_backup_error_exit "Backup tarball could not be created."
|
|
fi
|
|
|
|
# Get the size of the file
|
|
ARCHIVE_SIZE=$(ls -l $ARCHIVE_DIR/$TARBALL_FILE | awk '{print $5}')
|
|
|
|
log INFO "${DB_NAME}_backup" "Tarball $TARBALL_FILE created successfully."
|
|
|
|
cd $ARCHIVE_DIR
|
|
|
|
# Remove the temporary directory and files as they are no longer needed.
|
|
rm -rf $TMP_DIR
|
|
rm -f $ERR_LOG_FILE
|
|
|
|
#Only delete the old archive after a successful archive
|
|
export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g')
|
|
if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
|
|
remove_old_local_archives
|
|
fi
|
|
|
|
REMOTE_BACKUP=$(echo $REMOTE_BACKUP_ENABLED | sed 's/"//g')
|
|
if $REMOTE_BACKUP; then
|
|
store_backup_remotely $ARCHIVE_DIR $TARBALL_FILE
|
|
if [[ $? -ne 0 ]]; then
|
|
# This error should print first, then print the summary as the last
|
|
# thing that the user sees in the output.
|
|
log ERROR "${DB_NAME}_backup" "Backup ${TARBALL_FILE} could not be sent to remote RGW."
|
|
set +x
|
|
echo "=================================================================="
|
|
echo "Local backup successful, but could not send to remote RGW."
|
|
echo "Backup archive name: $TARBALL_FILE"
|
|
echo "Backup archive size: $ARCHIVE_SIZE"
|
|
echo "=================================================================="
|
|
set -x
|
|
# Because the local backup was successful, exit with 0 so the pod will not
|
|
# continue to restart and fill the disk with more backups. The ERRORs are
|
|
# logged and alerting system should catch those errors and flag the operator.
|
|
exit 0
|
|
fi
|
|
|
|
#Only delete the old archive after a successful archive
|
|
export REMOTE_DAYS_TO_KEEP=$(echo $REMOTE_DAYS_TO_KEEP | sed 's/"//g')
|
|
if [[ "$REMOTE_DAYS_TO_KEEP" -gt 0 ]]; then
|
|
remove_old_remote_archives
|
|
fi
|
|
|
|
# Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
|
|
set +x
|
|
echo "=================================================================="
|
|
echo "Local backup and backup to remote RGW successful!"
|
|
echo "Backup archive name: $TARBALL_FILE"
|
|
echo "Backup archive size: $ARCHIVE_SIZE"
|
|
echo "=================================================================="
|
|
set -x
|
|
else
|
|
# Remote backup is not enabled. This is ok; at least we have a local backup.
|
|
log INFO "${DB_NAME}_backup" "Skipping remote backup, as it is not enabled."
|
|
|
|
# Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
|
|
set +x
|
|
echo "=================================================================="
|
|
echo "Local backup successful!"
|
|
echo "Backup archive name: $TARBALL_FILE"
|
|
echo "Backup archive size: $ARCHIVE_SIZE"
|
|
echo "=================================================================="
|
|
set -x
|
|
fi
|
|
}
|
|
{{- end }}
|