Mariadb backup/restore enhancements

Below enhancements are made to Mariadb backup:
1) Used new helm-toolkit function to send/retrieve Mariadb
   backups to/from RGW via OpenStack Swift API.
2) Modified the backup script such that the database backup
   tarball can be sent to RGW.
3) Added a keystone user for RGW access.
4) Added a secret for OpenStack Swift API access.
5) Changed the cronjob image and runAsUser
6) Modified the restore script so that archives stored remotely
   on RGW can be used for the restore data source.
7) Added functions to the restore script to retrieve data
   from an archive for tables, table rows and table schema of a databse
8) Added a secret containing all the backup/restore related
   configuration needed for invoking the backup/restore operation
   from a different application or namespace.

Change-Id: Iadb9438fe419cded374897b43337039609077e61
This commit is contained in:
Huang, Sophie (sh879n) 2020-05-05 22:37:13 +00:00 committed by Sophie Huang
parent 1da7a5b0f8
commit 573ac49939
8 changed files with 504 additions and 334 deletions

View File

@ -12,104 +12,76 @@
# License for the specific language governing permissions and limitations
# under the License.
set -x
BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current
ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
--connect-timeout 10"
source /tmp/backup_main.sh
MYSQLDUMP="mysqldump \
--defaults-file=/etc/mysql/admin_user.cnf"
# Export the variables required by the framework
# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already
# exported.
export DB_NAMESPACE=${MARIADB_POD_NAMESPACE}
export DB_NAME="mariadb"
export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP}
export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP}
export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
seconds_difference() {
archive_date=$( date --date="$1" +%s )
if [ "$?" -ne 0 ]
# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE
dump_databases_to_directory() {
TMP_DIR=$1
LOG_FILE=$2
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
--connect-timeout 10"
MYSQLDUMP="mysqldump \
--defaults-file=/etc/mysql/admin_user.cnf"
MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
"show databases;" | \
egrep -vi 'information_schema|performance_schema') )
#check if there is a database to backup, otherwise exit
if [[ -z "${MYSQL_DBNAMES// }" ]]
then
second_delta=0
log INFO "There is no database to backup"
return 0
fi
current_date=$( date +%s )
second_delta=$(($current_date-$archive_date))
if [ "$second_delta" -lt 0 ]
#Create a list of Databases
printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list
#Retrieve and create the GRANT files per DB
for db in "${MYSQL_DBNAMES[@]}"
do
echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
from mysql.db where ucase(db)=ucase('$db');") | \
sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \
$MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql
if [ "$?" -eq 0 ]
then
sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql
else
log ERROR "Failed to create GRANT files for ${db}"
fi
done
#Dumping the database
DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all
TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz
$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \
> $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE
if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]]
then
second_delta=0
log INFO "Databases dumped successfully."
return 0
else
log ERROR "Backup failed and need attention."
return 1
fi
echo $second_delta
}
DBNAME=( $($MYSQL --silent --skip-column-names -e \
"show databases;" | \
egrep -vi 'information_schema|performance_schema|mysql') )
#check if there is a database to backup, otherwise exit
if [[ -z "${DBNAME// }" ]]
then
echo "There is no database to backup"
exit 0
fi
#Create archive and backup directories.
mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
#Create a list of Databases
printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list
#Retrieve and create the GRANT files per DB
for db in "${DBNAME[@]}"
do
echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
from mysql.db where ucase(db)=ucase('$db');") | \
sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \
$MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql
if [ "$?" -eq 0 ]
then
sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql
else
cat grant_err.log
fi
done
#Dumping the database
#DATE=$(date +"%Y_%m_%d_%H_%M_%S")
DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}" \
> $BACKUPS_DIR/mariadb.all.sql 2>dberror.log
if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]]
then
#Archive the current db files
pushd $BACKUPS_DIR 1>/dev/null
tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz *
ARCHIVE_RET=$?
popd 1>/dev/null
else
#TODO: This can be convert into mail alert of alert send to a monitoring system
echo "Backup failed and need attention."
cat dberror.log
exit 1
fi
#Remove the current backup
if [ -d $BACKUPS_DIR ]
then
rm -rf $BACKUPS_DIR/*.sql
fi
#Only delete the old archive after a successful archive
if [ $ARCHIVE_RET -eq 0 ]
then
if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ]
then
echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days"
if [ -d $ARCHIVE_DIR ]
then
for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
do
archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
if [ "$(seconds_difference $archive_date)" -gt "$(($MARIADB_BACKUP_DAYS_TO_KEEP*86400))" ]
then
rm -rf $archive_file
fi
done
fi
fi
fi
# Call main program to start the database backup
backup_databases

View File

@ -12,22 +12,26 @@
# License for the specific language governing permissions and limitations
# under the License.
log_error() {
echo $1
exit 1
}
ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore
# Capture the user's command line arguments
ARGS=("$@")
if [[ -s /tmp/restore_main.sh ]]; then
source /tmp/restore_main.sh
else
echo "File /tmp/restore_main.sh does not exist."
exit 1
fi
# Export the variables needed by the framework
export DB_NAME="mariadb"
export DB_NAMESPACE=${MARIADB_POD_NAMESPACE}
export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
RESTORE_USER='restoreuser'
RESTORE_PW=$(pwgen 16 1)
RESTORE_LOG='/tmp/restore_error.log'
rm -f $RESTORE_LOG
#Create Restore Directory
mkdir -p $RESTORE_DIR
# This is for commands which require admin access
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
@ -42,67 +46,89 @@ RESTORE_CMD="mysql \
--host=$MARIADB_SERVER_SERVICE_HOST \
--connect-timeout 10"
#Delete file
delete_files() {
files_to_delete=("$@")
for f in "${files_to_delete[@]}"
do
if [ -f $f ]
then
rm -rf $f
fi
done
}
#Display all archives
list_archives() {
if [ -d ${ARCHIVE_DIR} ]
then
archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
echo "All Archives"
echo "=================================="
for archive in $archives
do
echo $archive | cut -d '/' -f 8
done
else
log_error "Archive directory is not available."
fi
# Get a single database data from the SQL file.
# $1 - database name
# $2 - sql file path
current_db_desc() {
PATTERN="-- Current Database:"
sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2
}
#Return all database from an archive
get_databases() {
archive_file=$1
if [ -e ${ARCHIVE_DIR}/${archive_file} ]
TMP_DIR=$1
DB_FILE=$2
if [[ -e ${TMP_DIR}/db.list ]]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -e ${RESTORE_DIR}/db.list ]
then
DBS=$(cat ${RESTORE_DIR}/db.list )
else
DBS=" "
fi
DBS=$(cat ${TMP_DIR}/db.list )
else
DBS=" "
fi
echo $DBS > $DB_FILE
}
#Display all database from an archive
list_databases() {
archive_file=$1
get_databases $archive_file
#echo $DBS
if [ -n "$DBS" ]
then
echo " "
echo "Databases in the archive $archive_file"
echo "================================================================="
for db in $DBS
do
echo $db
done
# Extract all tables of a database from an archive and put them in the requested
# file.
get_tables() {
DATABASE=$1
TMP_DIR=$2
TABLE_FILE=$3
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
| grep "^CREATE TABLE" | awk -F '`' '{print $2}' \
> $TABLE_FILE
else
# Error, cannot report the tables
echo "No SQL file found - cannot extract the tables"
return 1
fi
}
# Extract all rows in the given table of a database from an archive and put
# them in the requested file.
get_rows() {
DATABASE=$1
TABLE=$2
TMP_DIR=$3
ROW_FILE=$4
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
| grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE
else
# Error, cannot report the rows
echo "No SQL file found - cannot extract the rows"
return 1
fi
}
# Extract the schema for the given table in the given database belonging to
# the archive file found in the TMP_DIR.
get_schema() {
DATABASE=$1
TABLE=$2
TMP_DIR=$3
SCHEMA_FILE=$4
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
DB_FILE=$(mktemp -p /tmp)
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE}
sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE}
if [[ ! (-s ${SCHEMA_FILE}) ]]; then
sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \
> ${SCHEMA_FILE}
fi
rm -f ${DB_FILE}
else
# Error, cannot report the rows
echo "No SQL file found - cannot extract the schema"
return 1
fi
}
@ -116,17 +142,19 @@ create_restore_user() {
delete_restore_user "dont_exit_on_error"
$MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG
if [ "$?" -eq 0 ]
if [[ "$?" -eq 0 ]]
then
$MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG
if [ "$?" -ne 0 ]
if [[ "$?" -ne 0 ]]
then
cat $RESTORE_LOG
log_error "Failed to grant restore user ALL permissions on database ${restore_db}"
echo "Failed to grant restore user ALL permissions on database ${restore_db}"
return 1
fi
else
cat $RESTORE_LOG
log_error "Failed to grant restore user select permissions on all databases"
echo "Failed to grant restore user select permissions on all databases"
return 1
fi
}
@ -135,208 +163,125 @@ delete_restore_user() {
error_handling=$1
$MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG
if [ "$?" -ne 0 ]
if [[ "$?" -ne 0 ]]
then
if [ "$error_handling" == "exit_on_error" ]
then
cat $RESTORE_LOG
log_error "Failed to delete temporary restore user - needs attention to avoid a security hole"
echo "Failed to delete temporary restore user - needs attention to avoid a security hole"
return 1
fi
fi
}
#Restore a single database
restore_single_db() {
single_db_name=$1
if [ -z "$single_db_name" ]
SINGLE_DB_NAME=$1
TMP_DIR=$2
if [[ -z "$SINGLE_DB_NAME" ]]
then
log_error "Restore single DB called but with wrong parameter."
echo "Restore single DB called but with wrong parameter."
return 1
fi
if [ -f ${ARCHIVE_DIR}/${archive_file} ]
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -f ${TMP_DIR}/$SQL_FILE ]]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
# Restoring a single database requires us to create a temporary user
# which has capability to only restore that ONE database. One gotcha
# is that the mysql command to restore the database is going to throw
# errors because of all the other databases that it cannot access. So
# because of this reason, the --force option is used to prevent the
# command from stopping on an error.
create_restore_user $SINGLE_DB_NAME
if [[ $? -ne 0 ]]
then
# Restoring a single database requires us to create a temporary user
# which has capability to only restore that ONE database. One gotcha
# is that the mysql command to restore the database is going to throw
# errors because of all the other databases that it cannot access. So
# because of this reason, the --force option is used to prevent the
# command from stopping on an error.
create_restore_user $single_db_name
$RESTORE_CMD --force < ${RESTORE_DIR}/mariadb.all.sql 2>>$RESTORE_LOG
if [ "$?" -eq 0 ]
echo "Restore $SINGLE_DB_NAME failed create restore user."
return 1
fi
$RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
echo "Database $SINGLE_DB_NAME Restore successful."
else
cat $RESTORE_LOG
delete_restore_user "exit_on_error"
echo "Database $SINGLE_DB_NAME Restore failed."
return 1
fi
delete_restore_user "exit_on_error"
if [[ $? -ne 0 ]]
then
echo "Restore $SINGLE_DB_NAME failed delete restore user."
return 1
fi
if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ]
then
$MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
echo "Database $single_db_name Restore successful."
echo "Database $SINGLE_DB_NAME Permission Restore successful."
else
cat $RESTORE_LOG
delete_restore_user "exit_on_error"
log_error "Database $single_db_name Restore failed."
fi
delete_restore_user "exit_on_error"
if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ]
then
$MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql 2>>$RESTORE_LOG
if [ "$?" -eq 0 ]
then
echo "Database $single_db_name Permission Restore successful."
else
cat $RESTORE_LOG
log_error "Database $single_db_name Permission Restore failed."
fi
else
log_error "There is no permission file available for $single_db_name"
echo "Database $SINGLE_DB_NAME Permission Restore failed."
return 1
fi
else
log_error "There is no database file available to restore from"
echo "There is no permission file available for $SINGLE_DB_NAME"
return 1
fi
else
log_error "Archive does not exist"
echo "There is no database file available to restore from"
return 1
fi
return 0
}
#Restore all the databases
restore_all_dbs() {
if [ -f ${ARCHIVE_DIR}/${archive_file} ]
TMP_DIR=$1
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -f ${TMP_DIR}/$SQL_FILE ]]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
$MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
$MYSQL < ${RESTORE_DIR}/mariadb.all.sql 2>$RESTORE_LOG
if [ "$?" -eq 0 ]
then
echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
else
cat $RESTORE_LOG
log_error "Databases $( echo $DBS | tr -d '\n') Restore failed."
fi
if [ -n "$DBS" ]
then
for db in $DBS
do
if [ -f ${RESTORE_DIR}/${db}_grant.sql ]
then
$MYSQL < ${RESTORE_DIR}/${db}_grant.sql 2>>$RESTORE_LOG
if [ "$?" -eq 0 ]
then
echo "Database $db Permission Restore successful."
else
cat $RESTORE_LOG
log_error "Database $db Permission Restore failed."
fi
else
log_error "There is no permission file available for $db"
fi
done
echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
else
log_error "There is no database file available to restore from"
cat $RESTORE_LOG
echo "Databases $( echo $DBS | tr -d '\n') Restore failed."
return 1
fi
else
log_error "Archive does not exist"
fi
fi
}
usage() {
ret_val=$1
echo "Usage:"
echo "Restore command options"
echo "============================="
echo "help"
echo "list_archives"
echo "list_databases <archive_filename>"
echo "restore <archive_filename> [<db_name> | ALL]"
exit $ret_val
}
is_Option() {
opts=$1
param=$2
find=0
for opt in $opts
do
if [ "$opt" == "$param" ]
if [ -n "$DBS" ]
then
find=1
fi
done
echo $find
}
#Main
if [ ${#ARGS[@]} -gt 3 ]
then
usage 1
elif [ ${#ARGS[@]} -eq 1 ]
then
if [ "${ARGS[0]}" == "list_archives" ]
then
list_archives
elif [ "${ARGS[0]}" == "help" ]
then
usage 0
else
usage 1
fi
elif [ ${#ARGS[@]} -eq 2 ]
then
if [ "${ARGS[0]}" == "list_databases" ]
then
list_databases ${ARGS[1]}
else
usage 1
fi
elif [ ${#ARGS[@]} -eq 3 ]
then
if [ "${ARGS[0]}" != "restore" ]
then
usage 1
else
if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
then
#Get all the databases in that archive
get_databases ${ARGS[1]}
#check if the requested database is available in the archive
if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
then
echo "Creating database ${ARGS[2]} if it does not exist"
$MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`" 2>>$RESTORE_LOG
if [ "$?" -ne 0 ]
for db in $DBS
do
if [ -f ${TMP_DIR}/${db}_grant.sql ]
then
cat $RESTORE_LOG
log_error "Database ${ARGS[2]} could not be created."
fi
echo "Restoring database ${ARGS[2]} and grants...this could take a few minutes."
restore_single_db ${ARGS[2]}
elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
then
echo "Creating databases if they do not exist"
for db in $DBS
do
$MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`"
if [ "$?" -ne 0 ]
$MYSQL < ${TMP_DIR}/${db}_grant.sql 2>>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
echo "Database $db Permission Restore successful."
else
cat $RESTORE_LOG
log_error "Database ${db} could not be created."
echo "Database $db Permission Restore failed."
return 1
fi
done
echo "Restoring all databases and grants...this could take a few minutes."
restore_all_dbs
else
echo "Database ${ARGS[2]} does not exist."
fi
else
echo "Archive file not found"
else
echo "There is no permission file available for $db"
return 1
fi
done
fi
else
echo "There is no database file available to restore from"
return 1
fi
else
usage 1
fi
return 0
}
exit 0
# Call the CLI interpreter, providing the archive directory path and the
# user arguments passed in
cli_main ${ARGS[@]}

View File

@ -42,5 +42,13 @@ data:
{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
restore_mariadb.sh: |
{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
backup_main.sh: |
{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }}
restore_main.sh: |
{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }}
{{- end }}
{{- if .Values.manifests.job_ks_user }}
ks-user.sh: |
{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
{{- end }}
{{- end }}

View File

@ -27,6 +27,12 @@ metadata:
labels:
{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
{{- if .Values.jobs.backup_mariadb.backoffLimit }}
backoffLimit: {{ .Values.jobs.backup_mariadb.backoffLimit }}
{{- end }}
{{- if .Values.jobs.backup_mariadb.activeDeadlineSeconds }}
activeDeadlineSeconds: {{ .Values.jobs.backup_mariadb.activeDeadlineSeconds }}
{{- end }}
schedule: {{ .Values.jobs.backup_mariadb.cron | quote }}
successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }}
failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }}
@ -36,7 +42,7 @@ spec:
labels:
{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
template:
metadata:
@ -48,7 +54,24 @@ spec:
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
- name: backup-perms
{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
command:
- chown
- -R
- "65534:65534"
- $(MARIADB_BACKUP_BASE_DIR)
env:
- name: MARIADB_BACKUP_BASE_DIR
value: {{ .Values.conf.backup.base_path | quote }}
volumeMounts:
- mountPath: /tmp
name: pod-tmp
- mountPath: {{ .Values.conf.backup.base_path }}
name: mariadb-backup-dir
containers:
- name: mariadb-backup
command:
@ -58,14 +81,28 @@ spec:
value: {{ .Values.conf.backup.base_path | quote }}
- name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS
value: {{ .Values.conf.backup.mysqldump_options | quote }}
- name: MARIADB_BACKUP_DAYS_TO_KEEP
value: {{ .Values.conf.backup.days_of_backup_to_keep | quote }}
- name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP
value: {{ .Values.conf.backup.days_to_keep | quote }}
- name: MARIADB_POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: REMOTE_BACKUP_ENABLED
value: "{{ .Values.conf.backup.remote_backup.enabled }}"
{{- if .Values.conf.backup.remote_backup.enabled }}
- name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP
value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }}
- name: CONTAINER_NAME
value: {{ .Values.conf.backup.remote_backup.container_name | quote }}
- name: STORAGE_POLICY
value: "{{ .Values.conf.backup.remote_backup.storage_policy }}"
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.remote_rgw_user }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
{{- end }}
{{- end }}
{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
@ -73,6 +110,10 @@ spec:
name: mariadb-bin
readOnly: true
subPath: backup_mariadb.sh
- mountPath: /tmp/backup_main.sh
name: mariadb-bin
readOnly: true
subPath: backup_main.sh
- mountPath: {{ .Values.conf.backup.base_path }}
name: mariadb-backup-dir
- name: mariadb-secrets
@ -88,7 +129,7 @@ spec:
- name: mariadb-secrets
secret:
secretName: mariadb-secrets
defaultMode: 384
defaultMode: 420
- configMap:
defaultMode: 365
name: mariadb-bin

View File

@ -0,0 +1,20 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_ks_user }}
{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }}
{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }}
{{- $ksUserJob := dict "envAll" . "serviceName" "mariadb" "configMapBin" "mariadb-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}}
{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
{{- end }}

View File

@ -0,0 +1,27 @@
{{/*
This manifest results a secret being created which has the key information
needed for backing up and restoring the Mariadb databases.
*/}}
{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }}
{{- $envAll := . }}
{{- $userClass := "backup_restore" }}
{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }}
BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }}
LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }}
MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }}
REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }}
REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }}
REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }}
REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }}
...
{{- end }}

View File

@ -0,0 +1,78 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This manifest results in two secrets being created:
1) Keystone "remote_rgw_user" secret, which is needed to access the cluster
(remote or same cluster) for storing mariadb backups. If the
cluster is remote, the auth_url would be non-null.
2) Keystone "remote_ks_admin" secret, which is needed to create the
"remote_rgw_user" keystone account mentioned above. This may not
be needed if the account is in a remote cluster (auth_url is non-null
in that case).
*/}}
{{- if .Values.conf.backup.remote_backup.enabled }}
{{- $envAll := . }}
{{- $userClass := "remote_rgw_user" }}
{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
{{- if $identityClass.auth_url }}
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
...
{{- if .Values.manifests.job_ks_user }}
{{- $userClass := "remote_ks_admin" }}
{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
{{- if $identityClass.auth_url }}
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
...
{{- end }}
{{- end }}

View File

@ -29,7 +29,8 @@ images:
prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
image_repo_sync: docker.io/docker:17.07.0
mariadb_backup: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031
mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
scripted_test: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031
pull_policy: "IfNotPresent"
local_registry:
@ -109,6 +110,17 @@ pod:
main:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
mariadb_backup:
pod:
runAsUser: 65534
container:
backup_perms:
runAsUser: 0
readOnlyRootFilesystem: true
mariadb_backup:
runAsUser: 65534
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
tests:
pod:
runAsUser: 999
@ -190,6 +202,13 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
ks_user:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
dependencies:
dynamic:
@ -208,8 +227,9 @@ dependencies:
services:
- endpoint: error_pages
service: oslo_db
mariadb:
jobs: null
backup_mariadb:
jobs:
- mariadb-ks-user
services: null
prometheus_create_mysql_user:
services:
@ -260,10 +280,17 @@ jobs:
backoffLimit: 87600
activeDeadlineSeconds: 3600
backup_mariadb:
# activeDeadlineSeconds == 0 means no deadline
activeDeadlineSeconds: 0
backoffLimit: 6
cron: "0 0 * * *"
history:
success: 3
failed: 1
ks_user:
# activeDeadlineSeconds == 0 means no deadline
activeDeadlineSeconds: 0
backoffLimit: 6
conf:
tests:
@ -284,12 +311,17 @@ conf:
ingress_conf:
worker-processes: "auto"
backup:
enabled: true
enabled: false
base_path: /var/backup
mysqldump_options: >
--single-transaction --quick --add-drop-database
--add-drop-table --add-locks --databases
days_of_backup_to_keep: 3
days_to_keep: 3
remote_backup:
enabled: false
container_name: mariadb
days_to_keep: 14
storage_policy: default-placement
database:
my: |
[mysqld]
@ -407,6 +439,13 @@ monitoring:
mysqld_exporter:
scrape: true
secrets:
identity:
remote_ks_admin: keystone-admin-user
remote_rgw_user: mariadb-backup-user
mariadb:
backup_restore: mariadb-backup-restore
# typically overridden by environmental
# values, but should include all endpoints
# required by this chart
@ -498,6 +537,44 @@ endpoints:
dns:
default: 53
protocol: UDP
identity:
name: backup-storage-auth
namespace: openstack
auth:
remote_ks_admin:
# Auth URL of null indicates local authentication
# HTK will form the URL unless specified here
auth_url: null
region_name: RegionOne
username: admin
password: password
project_name: admin
user_domain_name: default
project_domain_name: default
remote_rgw_user:
# Auth URL of null indicates local authentication
# HTK will form the URL unless specified here
auth_url: null
role: admin
region_name: RegionOne
username: mariadb-backup-user
password: password
project_name: service
user_domain_name: service
project_domain_name: service
hosts:
default: keystone
internal: keystone-api
host_fqdn_override:
default: null
path:
default: /v3
scheme:
default: 'http'
port:
api:
default: 80
internal: 5000
network_policy:
mariadb:
@ -521,6 +598,7 @@ manifests:
deployment_ingress: true
job_image_repo_sync: true
cron_job_mariadb_backup: false
job_ks_user: false
pvc_backup: false
monitoring:
prometheus:
@ -536,6 +614,7 @@ manifests:
secret_dbadmin_password: true
secret_sst_password: true
secret_dbaudit_password: true
secret_backup_restore: false
secret_etc: true
service_discovery: true
service_ingress: true