openstack/openstack-helm
Code Issues Proposed changes

Add remote backups failover support

Browse Source 
This PS adds a support for a failover location to
upload and retrieve remote backups.

Change-Id: If5a6d8c562409f9fb0d7bbe039b49f0fc154d1a3
This commit is contained in:
Sergiy Markin
2025-04-15 13:15:12 -05:00
parent 5f5cd11849
commit 85569c271a
10 changed files with 437 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
View File

@@ -167,7 +167,7 @@ send_to_remote_server() {
RESULT=$(openstack container list 2>&1)
if [[ $? -eq 0 ]]; then
echo $RESULT | grep $CONTAINER_NAME
printf "%s\n" "$RESULT" | grep "$CONTAINER_NAME"
if [[ $? -ne 0 ]]; then
# Find the swift URL from the keystone endpoint list
SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
@@ -207,7 +207,7 @@ send_to_remote_server() {
echo $RESULT | grep -E "HTTP 401|HTTP 403"
if [[ $? -eq 0 ]]; then
log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
return 1
return 2
else
echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
if [[ $? -eq 0 ]]; then
@@ -217,7 +217,7 @@ send_to_remote_server() {
return 2
else
log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
return 1
return 2
fi
fi
fi
@@ -242,7 +242,7 @@ send_to_remote_server() {
RESULT=$(openstack container list 2>&1)
if [[ $? -eq 0 ]]; then
echo $RESULT | grep $THROTTLE_CONTAINER_NAME
printf "%s\n" "$RESULT" | grep "$THROTTLE_CONTAINER_NAME"
if [[ $? -ne 0 ]]; then
# Find the swift URL from the keystone endpoint list
SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
@@ -282,7 +282,7 @@ send_to_remote_server() {
echo $RESULT | grep -E "HTTP 401|HTTP 403"
if [[ $? -eq 0 ]]; then
log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
return 1
return 2
else
echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
if [[ $? -eq 0 ]]; then
@@ -292,7 +292,7 @@ send_to_remote_server() {
return 2
else
log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
return 1
return 2
fi
fi
fi
@@ -387,12 +387,12 @@ send_to_remote_server() {
# with built-in logic to handle error cases like:
# 1) Network connectivity issues - retries for a specific amount of time
# 2) Authorization errors - immediately logs an ERROR and returns
store_backup_remotely() {
FILEPATH=$1
FILE=$2
function store_backup_remotely() {
local FILEPATH=$1
local FILE=$2
local count=0
count=1
while [[ ${count} -le ${REMOTE_BACKUP_RETRIES} ]]; do
while [[ ${count} -lt ${REMOTE_BACKUP_RETRIES} ]]; do
# Store the new archive to the remote backup storage facility.
send_to_remote_server $FILEPATH $FILE
SEND_RESULT="$?"
@@ -422,10 +422,66 @@ store_backup_remotely() {
count=$((count+1))
done
log INFO "${DB_NAME}_backup" "Switching to failover RGW..."
# If initial attempts failed and failover variables are defined, retry with failover environment variables
if [[ $SEND_RESULT -ne 0 ]] && \
[[ -n "${OS_AUTH_URL_FAILOVER}" ]] && \
[[ -n "${OS_REGION_NAME_FAILOVER}" ]] && \
[[ -n "${OS_INTERFACE_FAILOVER}" ]] && \
[[ -n "${OS_PROJECT_DOMAIN_NAME_FAILOVER}" ]] && \
[[ -n "${OS_PROJECT_NAME_FAILOVER}" ]] && \
[[ -n "${OS_USER_DOMAIN_NAME_FAILOVER}" ]] && \
[[ -n "${OS_USERNAME_FAILOVER}" ]] && \
[[ -n "${OS_PASSWORD_FAILOVER}" ]]; then
log INFO "${DB_NAME}_backup" "Initial attempts failed. Retrying with failover environment variables..."
# Redefine OS_* variables with OS_*_FAILOVER ones
export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
export OS_USERNAME=${OS_USERNAME_FAILOVER}
export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
count=0
while [[ ${count} -lt ${REMOTE_BACKUP_RETRIES} ]]; do
# Store the new archive to the remote backup storage facility.
send_to_remote_server $FILEPATH $FILE
SEND_RESULT="$?"
# Check if successful
if [[ $SEND_RESULT -eq 0 ]]; then
log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to failover RGW."
return 0
elif [[ $SEND_RESULT -eq 2 ]]; then
if [[ ${count} -ge ${REMOTE_BACKUP_RETRIES} ]]; then
log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the failover RGW in " \
"${REMOTE_BACKUP_RETRIES} retries. Errors encountered. Exiting."
break
fi
# Temporary failure occurred. We need to retry
log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to failover RGW due to connection issue."
sleep_time=$(random_number)
log INFO "${DB_NAME}_backup" "Sleeping ${sleep_time} seconds waiting for failover RGW to become available..."
sleep ${sleep_time}
log INFO "${DB_NAME}_backup" "Retrying..."
else
log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the failover RGW. Errors encountered. Exiting."
break
fi
# Increment the counter
count=$((count+1))
done
fi
return 1
}
function get_archive_date(){
# get_archive_date function returns correct archive date
# for different formats of archives' names

310
helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
View File

@@ -147,6 +147,25 @@ usage() {
clean_and_exit $ret_val ""
}
log() {
#Log message to a file or stdout
#TODO: This can be convert into mail alert of alert send to a monitoring system
#Params: $1 log level
#Params: $2 service
#Params: $3 message
#Params: $4 Destination
LEVEL=$1
SERVICE=$2
MSG=$3
DEST=$4
DATE=$(date +"%m-%d-%y %H:%M:%S")
if [[ -z "$DEST" ]]; then
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}"
else
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST
fi
}
#Exit cleanly with some message and return code
clean_and_exit() {
RETCODE=$1
@@ -167,29 +186,29 @@ determine_resulting_error_code() {
echo ${RESULT} | grep "HTTP 404"
if [[ $? -eq 0 ]]; then
echo "Could not find the archive: ${RESULT}"
log ERROR "${DB_NAME}_restore" "Could not find the archive: ${RESULT}"
return 1
else
echo ${RESULT} | grep "HTTP 401"
if [[ $? -eq 0 ]]; then
echo "Could not access the archive: ${RESULT}"
log ERROR "${DB_NAME}_restore" "Could not access the archive: ${RESULT}"
return 1
else
echo ${RESULT} | grep "HTTP 503"
if [[ $? -eq 0 ]]; then
echo "RGW service is unavailable. ${RESULT}"
log WARN "${DB_NAME}_restore" "RGW service is unavailable. ${RESULT}"
# In this case, the RGW may be temporarily down.
# Return slightly different error code so the calling code can retry
return 2
else
echo ${RESULT} | grep "ConnectionError"
if [[ $? -eq 0 ]]; then
echo "Could not reach the RGW: ${RESULT}"
log WARN "${DB_NAME}_restore" "Could not reach the RGW: ${RESULT}"
# In this case, keystone or the site/node may be temporarily down.
# Return slightly different error code so the calling code can retry
return 2
else
echo "Archive ${ARCHIVE} could not be retrieved: ${RESULT}"
log ERROR "${DB_NAME}_restore" "Archive ${ARCHIVE} could not be retrieved: ${RESULT}"
return 1
fi
fi
@@ -199,51 +218,249 @@ determine_resulting_error_code() {
}
# Retrieve a list of archives from the RGW.
retrieve_remote_listing() {
RESULT=$(openstack container show $CONTAINER_NAME 2>&1)
if [[ $? -eq 0 ]]; then
# Get the list, ensureing that we only pick up the right kind of backups from the
# requested namespace
openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $TMP_DIR/archive_list
if [[ $? -ne 0 ]]; then
echo "Container object listing could not be obtained."
return 1
function retrieve_remote_listing() {
# List archives from PRIMARY RGW
log INFO "${DB_NAME}_restore" "Listing archives from PRIMARY RGW..."
list_archives_from_rgw "PRIMARY"
local primary_result=$?
# Check if failover environment variables are defined
if [[ -n "${OS_AUTH_URL_FAILOVER}" ]] && \
[[ -n "${OS_REGION_NAME_FAILOVER}" ]] && \
[[ -n "${OS_INTERFACE_FAILOVER}" ]] && \
[[ -n "${OS_PROJECT_DOMAIN_NAME_FAILOVER}" ]] && \
[[ -n "${OS_PROJECT_NAME_FAILOVER}" ]] && \
[[ -n "${OS_USER_DOMAIN_NAME_FAILOVER}" ]] && \
[[ -n "${OS_USERNAME_FAILOVER}" ]] && \
[[ -n "${OS_PASSWORD_FAILOVER}" ]]; then
# Redefine OS_* variables with OS_*_FAILOVER ones
log INFO "${DB_NAME}_restore" "Listing archives from FAILOVER RGW..."
# Saving original OS_* variables as OS_*_PRIMARY
export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
export OS_USERNAME_PRIMARY=${OS_USERNAME}
export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
export OS_USERNAME=${OS_USERNAME_FAILOVER}
export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
list_archives_from_rgw "FAILOVER"
local failover_result=$?
# Restore original OS_* variables from OS_*_PRIMARY
export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
export OS_USERNAME=${OS_USERNAME_PRIMARY}
export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
# Return success if either primary or failover listing was successful
if [[ $primary_result -eq 0 || $failover_result -eq 0 ]]; then
return 0
else
echo "Archive listing successfully retrieved."
return 1
fi
else
determine_resulting_error_code "${RESULT}"
return $?
# Return the result of the primary listing if failover variables are not defined
return $primary_result
fi
}
function list_archives_from_rgw() {
local prefix=$1
local RESULT
log INFO "${DB_NAME}_restore" "Obtaining list of archives from ${prefix} RGW..."
RESULT=$(openstack container show $CONTAINER_NAME 2>&1)
if [[ $? -eq 0 ]]; then
openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk -v prefix="$prefix" '{print prefix ":" $2}' >> $TMP_DIR/archive_list
if [[ $? -ne 0 ]]; then
log ERROR "${DB_NAME}_restore" "Container object listing could not be obtained from ${prefix} RGW."
return 1
else
log INFO "${DB_NAME}_restore" "Archive listing successfully retrieved from ${prefix} RGW."
fi
else
log ERROR "${DB_NAME}_restore" "Failed to obtain container show from ${prefix} RGW: ${RESULT}"
return 1
fi
return 0
}
# Retrieve a single archive from the RGW.
retrieve_remote_archive() {
ARCHIVE=$1
local archive=$1
local prefix=$(echo $archive | awk -F: '{print $1}')
local filename
RESULT=$(openstack object save --file $TMP_DIR/$ARCHIVE $CONTAINER_NAME $ARCHIVE 2>&1)
if [[ $? -ne 0 ]]; then
if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
else
filename=$archive
fi
if [[ $prefix == "PRIMARY" ]]; then
log INFO "${DB_NAME}_restore" "Retrieving archive ${filename} from PRIMARY RGW..."
retrieve_archive_from_rgw $filename
return $?
elif [[ $prefix == "FAILOVER" ]]; then
log INFO "${DB_NAME}_restore" "Retrieving archive ${filename} from FAILOVER RGW..."
# Saving original OS_* variables as OS_*_PRIMARY
export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
export OS_USERNAME_PRIMARY=${OS_USERNAME}
export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
# Redefine OS_* variables with OS_*_FAILOVER ones
export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
export OS_USERNAME=${OS_USERNAME_FAILOVER}
export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
retrieve_archive_from_rgw $filename
local result=$?
# Restore original OS_* variables from OS_*_PRIMARY
export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
export OS_USERNAME=${OS_USERNAME_PRIMARY}
export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
return $result
else
log ERROR "${DB_NAME}_restore" "Invalid prefix ${prefix} for archive ${archive}."
return 1
fi
}
# Function to retrieve an archive from RGW
retrieve_archive_from_rgw() {
local filename=$1
local RESULT
log INFO "${DB_NAME}_restore" "Obtaining archive ${filename} from RGW..."
RESULT=$(openstack object save --file $TMP_DIR/${filename} $CONTAINER_NAME ${filename} 2>&1)
if [[ $? -eq 0 ]]; then
log INFO "${DB_NAME}_restore" "Archive ${filename} successfully retrieved."
return 0
else
log ERROR "${DB_NAME}_restore" "Failed to retrieve archive ${filename}."
determine_resulting_error_code "${RESULT}"
return $?
else
echo "Archive $ARCHIVE successfully retrieved."
fi
return 0
}
# Delete an archive from the RGW.
# Delete a single archive from the RGW.
delete_remote_archive() {
ARCHIVE=$1
local archive=$1
local prefix=$(echo $archive | awk -F: '{print $1}')
local filename
RESULT=$(openstack object delete ${CONTAINER_NAME} ${ARCHIVE} 2>&1)
if [[ $? -ne 0 ]]; then
if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
else
filename=$archive
fi
if [[ $prefix == "PRIMARY" ]]; then
log INFO "${DB_NAME}_restore" "Deleting archive ${filename} from PRIMARY RGW..."
delete_archive_from_rgw $filename
return $?
elif [[ $prefix == "FAILOVER" ]]; then
log INFO "${DB_NAME}_restore" "Deleting archive ${filename} from FAILOVER RGW..."
# Saving original OS_* variables as OS_*_PRIMARY
export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
export OS_USERNAME_PRIMARY=${OS_USERNAME}
export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
# Redefine OS_* variables with OS_*_FAILOVER ones
export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
export OS_USERNAME=${OS_USERNAME_FAILOVER}
export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
delete_archive_from_rgw $filename
local result=$?
# Restore original OS_* variables from OS_*_PRIMARY
export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
export OS_USERNAME=${OS_USERNAME_PRIMARY}
export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
return $result
else
log ERROR "${DB_NAME}_restore" "Invalid prefix ${prefix} for archive ${archive}."
return 1
fi
}
# Function to delete an archive from RGW
delete_archive_from_rgw() {
local filename=$1
local RESULT
RESULT=$(openstack object delete $CONTAINER_NAME $filename 2>&1)
if [[ $? -eq 0 ]]; then
log INFO "${DB_NAME}_restore" "Archive ${filename} successfully deleted."
return 0
else
log ERROR "${DB_NAME}_restore" "Failed to delete archive ${filename}."
determine_resulting_error_code "${RESULT}"
return $?
else
echo "Archive ${ARCHIVE} successfully deleted."
fi
return 0
}
# Display all archives
@@ -283,18 +500,26 @@ list_archives() {
# Retrieve the archive from the desired location and decompress it into
# the restore directory
get_archive() {
ARCHIVE_FILE=$1
local archive=$1
local prefix=$(echo $archive | awk -F: '{print $1}')
local filename
if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
else
filename=$archive
fi
REMOTE=$2
if [[ "x$REMOTE" == "xremote" ]]; then
echo "Retrieving archive ${ARCHIVE_FILE} from the remote RGW..."
retrieve_remote_archive $ARCHIVE_FILE
log INFO "${DB_NAME}_restore" "Retrieving archive ${prefix}:${filename} from the remote RGW..."
retrieve_remote_archive ${prefix}:${filename}
if [[ $? -ne 0 ]]; then
clean_and_exit 1 "ERROR: Could not retrieve remote archive: $ARCHIVE_FILE"
clean_and_exit 1 "ERROR: Could not retrieve remote archive: ${prefix}:${filename}"
fi
elif [[ "x$REMOTE" == "x" ]]; then
if [[ -e $ARCHIVE_DIR/$ARCHIVE_FILE ]]; then
cp $ARCHIVE_DIR/$ARCHIVE_FILE $TMP_DIR/$ARCHIVE_FILE
if [[ -e $ARCHIVE_DIR/$filename ]]; then
cp $ARCHIVE_DIR/$filename $TMP_DIR/$filename
if [[ $? -ne 0 ]]; then
clean_and_exit 1 "ERROR: Could not copy local archive to restore directory."
fi
@@ -305,9 +530,10 @@ get_archive() {
usage 1
fi
echo "Decompressing archive $ARCHIVE_FILE..."
log INFO "${DB_NAME}_restore" "Decompressing archive $filename..."
cd $TMP_DIR
tar zxvf - < $TMP_DIR/$ARCHIVE_FILE 1>/dev/null
tar zxvf - < $TMP_DIR/$filename 1>/dev/null
if [[ $? -ne 0 ]]; then
clean_and_exit 1 "ERROR: Archive decompression failed."
fi
@@ -464,7 +690,7 @@ delete_archive() {
fi
fi
echo "Successfully deleted archive ${ARCHIVE_FILE} from ${WHERE} storage."
log INFO "${DB_NAME}_restore" "Successfully deleted archive ${ARCHIVE_FILE} from ${WHERE} storage."
}
@@ -578,19 +804,19 @@ cli_main() {
fi
fi
echo "Restoring Database $DB_SPEC And Grants"
log INFO "${DB_NAME}_restore" "Restoring Database $DB_SPEC And Grants"
restore_single_db $DB_SPEC $TMP_DIR
if [[ "$?" -eq 0 ]]; then
echo "Single database restored successfully."
log INFO "${DB_NAME}_restore" "Single database restored successfully."
else
clean_and_exit 1 "ERROR: Single database restore failed."
fi
clean_and_exit 0 ""
else
echo "Restoring All The Databases. This could take a few minutes..."
log INFO "${DB_NAME}_restore" "Restoring All The Databases. This could take a few minutes..."
restore_all_dbs $TMP_DIR
if [[ "$?" -eq 0 ]]; then
echo "All databases restored successfully."
log INFO "${DB_NAME}_restore" "All databases restored successfully."
else
clean_and_exit 1 "ERROR: Database restore failed."
fi

20
mariadb-backup/templates/secret-rgw.yaml
View File

@@ -25,7 +25,8 @@ This manifest results in two secrets being created:
{{- if .Values.conf.backup.remote_backup.enabled }}
{{- $envAll := . }}
{{- $userClass := "mariadb-server" }}
{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
---
apiVersion: v1
@@ -48,6 +49,23 @@ data:
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
{{- if $failoverIdentityClass }}
{{- if $failoverIdentityClass.auth_url }}
OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
{{- end }}
...
{{- if .Values.manifests.job_ks_user }}
{{- $userClass := "admin" }}

2
mariadb-backup/values.yaml
View File

@@ -241,6 +241,8 @@ conf:
lock_expire_after: 7200
retry_after: 3600
container_name: throttle-backups-manager
primary_user_class: mariadb-server
failover_user_class: mariadb-server_failover
secrets:
identity:

1
mariadb/templates/bin/_start.py.tpl
View File

@@ -258,7 +258,6 @@ def stop_mysqld():
logger.info("Mysqld stopped: pid = {0}, "
"exit status = {1}".format(pid, status))
def mysqld_write_cluster_conf(mode='run'):
"""Write out dynamic cluster config.

20
mariadb/templates/secret-rgw.yaml
View File

@@ -25,7 +25,8 @@ This manifest results in two secrets being created:
{{- if .Values.conf.backup.remote_backup.enabled }}
{{- $envAll := . }}
{{- $userClass := "mariadb" }}
{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
---
apiVersion: v1
@@ -48,6 +49,23 @@ data:
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
{{- if $failoverIdentityClass }}
{{- if $failoverIdentityClass.auth_url }}
OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
{{- end }}
...
{{- if .Values.manifests.job_ks_user }}
{{- $userClass := "admin" }}

2
mariadb/values.yaml
View File

@@ -348,6 +348,8 @@ conf:
lock_expire_after: 7200
retry_after: 3600
container_name: throttle-backups-manager
primary_user_class: mariadb
failover_user_class: mariadb_failover
galera:
cluster_leader_ttl: 60
database:

20
postgresql/templates/secret-rgw.yaml
View File

@@ -11,7 +11,8 @@ This manifest results in two secrets being created:
{{- if .Values.conf.backup.remote_backup.enabled }}
{{- $envAll := . }}
{{- $userClass := "postgresql" }}
{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
---
apiVersion: v1
@@ -34,6 +35,23 @@ data:
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
{{- if $failoverIdentityClass }}
{{- if $failoverIdentityClass.auth_url }}
OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
{{- end }}
...
{{- if .Values.manifests.job_ks_user }}
{{- $userClass := "admin" }}

2
postgresql/values.yaml
View File

@@ -335,6 +335,8 @@ conf:
lock_expire_after: 7200
retry_after: 3600
container_name: throttle-backups-manager
primary_user_class: postgresql
failover_user_class: postgresql_failover
exporter:
queries:

38
values_overrides/mariadb/remote_backups.yaml Normal file
View File

@@ -0,0 +1,38 @@
---
conf:
backup:
enabled: true
remote_backup:
enabled: true
volume:
backup:
enabled: true
endpoints:
identity:
auth:
mariadb:
# Auth URL of null indicates local authentication
# HTK will form the URL unless specified here
auth_url: https://rgw-1.test.local
region_name: RegionOne
username: mariadb-backup-user
password: password
project_name: service
user_domain_name: service
project_domain_name: service
mariadb_failover:
# # Auth URL of null indicates local authentication
# # HTK will form the URL unless specified here
auth_url: https://rgw-2.test.local
region_name: RegionOne
username: mariadb-backup-user
password: password
project_name: service
user_domain_name: service
project_domain_name: service
manifests:
pvc_backup: true
job_ks_user: true
cron_job_mariadb_backup: true
secret_backup_restore: true
...