diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl index b0bea9b16..00517de17 100644 --- a/mariadb/templates/bin/_backup_mariadb.sh.tpl +++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl @@ -12,104 +12,76 @@ # License for the specific language governing permissions and limitations # under the License. set -x -BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current -ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive -MYSQL="mysql \ - --defaults-file=/etc/mysql/admin_user.cnf \ - --connect-timeout 10" +source /tmp/backup_main.sh -MYSQLDUMP="mysqldump \ - --defaults-file=/etc/mysql/admin_user.cnf" +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already +# exported. +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export DB_NAME="mariadb" +export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP} +export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive -seconds_difference() { - archive_date=$( date --date="$1" +%s ) - if [ "$?" -ne 0 ] +# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + + MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --connect-timeout 10" + + MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf" + + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + egrep -vi 'information_schema|performance_schema') ) + + #check if there is a database to backup, otherwise exit + if [[ -z "${MYSQL_DBNAMES// }" ]] then - second_delta=0 + log INFO "There is no database to backup" + return 0 fi - current_date=$( date +%s ) - second_delta=$(($current_date-$archive_date)) - if [ "$second_delta" -lt 0 ] + + #Create a list of Databases + printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list + + #Retrieve and create the GRANT files per DB + for db in "${MYSQL_DBNAMES[@]}" + do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \ + $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql + else + log ERROR "Failed to create GRANT files for ${db}" + fi + done + + #Dumping the database + DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all + TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz + + $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ + > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] then - second_delta=0 + log INFO "Databases dumped successfully." + return 0 + else + log ERROR "Backup failed and need attention." + return 1 fi - echo $second_delta } -DBNAME=( $($MYSQL --silent --skip-column-names -e \ - "show databases;" | \ - egrep -vi 'information_schema|performance_schema|mysql') ) - -#check if there is a database to backup, otherwise exit -if [[ -z "${DBNAME// }" ]] -then - echo "There is no database to backup" - exit 0 -fi - -#Create archive and backup directories. -mkdir -p $BACKUPS_DIR $ARCHIVE_DIR - -#Create a list of Databases -printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list - -#Retrieve and create the GRANT files per DB -for db in "${DBNAME[@]}" -do - echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ - from mysql.db where ucase(db)=ucase('$db');") | \ - sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \ - $MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql - if [ "$?" -eq 0 ] - then - sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql - else - cat grant_err.log - fi -done - -#Dumping the database -#DATE=$(date +"%Y_%m_%d_%H_%M_%S") -DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') -$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}" \ - > $BACKUPS_DIR/mariadb.all.sql 2>dberror.log -if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]] -then - #Archive the current db files - pushd $BACKUPS_DIR 1>/dev/null - tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz * - ARCHIVE_RET=$? - popd 1>/dev/null -else - #TODO: This can be convert into mail alert of alert send to a monitoring system - echo "Backup failed and need attention." - cat dberror.log - exit 1 -fi - -#Remove the current backup -if [ -d $BACKUPS_DIR ] -then - rm -rf $BACKUPS_DIR/*.sql -fi - -#Only delete the old archive after a successful archive -if [ $ARCHIVE_RET -eq 0 ] - then - if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ] - then - echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days" - if [ -d $ARCHIVE_DIR ] - then - for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) - do - archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) - if [ "$(seconds_difference $archive_date)" -gt "$(($MARIADB_BACKUP_DAYS_TO_KEEP*86400))" ] - then - rm -rf $archive_file - fi - done - fi - fi -fi +# Call main program to start the database backup +backup_databases diff --git a/mariadb/templates/bin/_restore_mariadb.sh.tpl b/mariadb/templates/bin/_restore_mariadb.sh.tpl index 3d4d2394a..1e8841189 100755 --- a/mariadb/templates/bin/_restore_mariadb.sh.tpl +++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl @@ -12,22 +12,26 @@ # License for the specific language governing permissions and limitations # under the License. -log_error() { - echo $1 - exit 1 -} - -ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive -RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore +# Capture the user's command line arguments ARGS=("$@") + +if [[ -s /tmp/restore_main.sh ]]; then + source /tmp/restore_main.sh +else + echo "File /tmp/restore_main.sh does not exist." + exit 1 +fi + +# Export the variables needed by the framework +export DB_NAME="mariadb" +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + RESTORE_USER='restoreuser' RESTORE_PW=$(pwgen 16 1) RESTORE_LOG='/tmp/restore_error.log' rm -f $RESTORE_LOG -#Create Restore Directory -mkdir -p $RESTORE_DIR - # This is for commands which require admin access MYSQL="mysql \ --defaults-file=/etc/mysql/admin_user.cnf \ @@ -42,67 +46,89 @@ RESTORE_CMD="mysql \ --host=$MARIADB_SERVER_SERVICE_HOST \ --connect-timeout 10" -#Delete file -delete_files() { - files_to_delete=("$@") - for f in "${files_to_delete[@]}" - do - if [ -f $f ] - then - rm -rf $f - fi - done -} - -#Display all archives -list_archives() { - if [ -d ${ARCHIVE_DIR} ] - then - archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print) - echo "All Archives" - echo "==================================" - for archive in $archives - do - echo $archive | cut -d '/' -f 8 - done - else - log_error "Archive directory is not available." - fi +# Get a single database data from the SQL file. +# $1 - database name +# $2 - sql file path +current_db_desc() { + PATTERN="-- Current Database:" + sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2 } #Return all database from an archive get_databases() { - archive_file=$1 - if [ -e ${ARCHIVE_DIR}/${archive_file} ] + TMP_DIR=$1 + DB_FILE=$2 + + if [[ -e ${TMP_DIR}/db.list ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -e ${RESTORE_DIR}/db.list ] - then - DBS=$(cat ${RESTORE_DIR}/db.list ) - else - DBS=" " - fi + DBS=$(cat ${TMP_DIR}/db.list ) else DBS=" " fi + + echo $DBS > $DB_FILE } -#Display all database from an archive -list_databases() { - archive_file=$1 - get_databases $archive_file - #echo $DBS - if [ -n "$DBS" ] - then - echo " " - echo "Databases in the archive $archive_file" - echo "=================================================================" - for db in $DBS - do - echo $db - done +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ + > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put +# them in the requested file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to +# the archive file found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + DB_FILE=$(mktemp -p /tmp) + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE} + sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} + if [[ ! (-s ${SCHEMA_FILE}) ]]; then + sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ + > ${SCHEMA_FILE} + fi + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 fi } @@ -116,17 +142,19 @@ create_restore_user() { delete_restore_user "dont_exit_on_error" $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] + if [[ "$?" -eq 0 ]] then $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + if [[ "$?" -ne 0 ]] then cat $RESTORE_LOG - log_error "Failed to grant restore user ALL permissions on database ${restore_db}" + echo "Failed to grant restore user ALL permissions on database ${restore_db}" + return 1 fi else cat $RESTORE_LOG - log_error "Failed to grant restore user select permissions on all databases" + echo "Failed to grant restore user select permissions on all databases" + return 1 fi } @@ -135,208 +163,125 @@ delete_restore_user() { error_handling=$1 $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + if [[ "$?" -ne 0 ]] then if [ "$error_handling" == "exit_on_error" ] then cat $RESTORE_LOG - log_error "Failed to delete temporary restore user - needs attention to avoid a security hole" + echo "Failed to delete temporary restore user - needs attention to avoid a security hole" + return 1 fi fi } #Restore a single database restore_single_db() { - single_db_name=$1 - if [ -z "$single_db_name" ] + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + if [[ -z "$SINGLE_DB_NAME" ]] then - log_error "Restore single DB called but with wrong parameter." + echo "Restore single DB called but with wrong parameter." + return 1 fi - if [ -f ${ARCHIVE_DIR}/${archive_file} ] + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -f ${TMP_DIR}/$SQL_FILE ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + # Restoring a single database requires us to create a temporary user + # which has capability to only restore that ONE database. One gotcha + # is that the mysql command to restore the database is going to throw + # errors because of all the other databases that it cannot access. So + # because of this reason, the --force option is used to prevent the + # command from stopping on an error. + create_restore_user $SINGLE_DB_NAME + if [[ $? -ne 0 ]] then - # Restoring a single database requires us to create a temporary user - # which has capability to only restore that ONE database. One gotcha - # is that the mysql command to restore the database is going to throw - # errors because of all the other databases that it cannot access. So - # because of this reason, the --force option is used to prevent the - # command from stopping on an error. - create_restore_user $single_db_name - $RESTORE_CMD --force < ${RESTORE_DIR}/mariadb.all.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] + echo "Restore $SINGLE_DB_NAME failed create restore user." + return 1 + fi + $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Database $SINGLE_DB_NAME Restore successful." + else + cat $RESTORE_LOG + delete_restore_user "exit_on_error" + echo "Database $SINGLE_DB_NAME Restore failed." + return 1 + fi + delete_restore_user "exit_on_error" + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed delete restore user." + return 1 + fi + if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ] + then + $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then - echo "Database $single_db_name Restore successful." + echo "Database $SINGLE_DB_NAME Permission Restore successful." else cat $RESTORE_LOG - delete_restore_user "exit_on_error" - log_error "Database $single_db_name Restore failed." - fi - delete_restore_user "exit_on_error" - - if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ] - then - $MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Database $single_db_name Permission Restore successful." - else - cat $RESTORE_LOG - log_error "Database $single_db_name Permission Restore failed." - fi - else - log_error "There is no permission file available for $single_db_name" + echo "Database $SINGLE_DB_NAME Permission Restore failed." + return 1 fi else - log_error "There is no database file available to restore from" + echo "There is no permission file available for $SINGLE_DB_NAME" + return 1 fi else - log_error "Archive does not exist" + echo "There is no database file available to restore from" + return 1 fi + return 0 } #Restore all the databases restore_all_dbs() { - if [ -f ${ARCHIVE_DIR}/${archive_file} ] + TMP_DIR=$1 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -f ${TMP_DIR}/$SQL_FILE ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then - $MYSQL < ${RESTORE_DIR}/mariadb.all.sql 2>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Databases $( echo $DBS | tr -d '\n') Restore successful." - else - cat $RESTORE_LOG - log_error "Databases $( echo $DBS | tr -d '\n') Restore failed." - fi - if [ -n "$DBS" ] - then - for db in $DBS - do - if [ -f ${RESTORE_DIR}/${db}_grant.sql ] - then - $MYSQL < ${RESTORE_DIR}/${db}_grant.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Database $db Permission Restore successful." - else - cat $RESTORE_LOG - log_error "Database $db Permission Restore failed." - fi - else - log_error "There is no permission file available for $db" - fi - done + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." else - log_error "There is no database file available to restore from" + cat $RESTORE_LOG + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + return 1 fi - else - log_error "Archive does not exist" - fi - fi -} - -usage() { - ret_val=$1 - echo "Usage:" - echo "Restore command options" - echo "=============================" - echo "help" - echo "list_archives" - echo "list_databases " - echo "restore [ | ALL]" - exit $ret_val -} - -is_Option() { - opts=$1 - param=$2 - find=0 - for opt in $opts - do - if [ "$opt" == "$param" ] + if [ -n "$DBS" ] then - find=1 - fi - done - echo $find -} - -#Main -if [ ${#ARGS[@]} -gt 3 ] -then - usage 1 -elif [ ${#ARGS[@]} -eq 1 ] -then - if [ "${ARGS[0]}" == "list_archives" ] - then - list_archives - elif [ "${ARGS[0]}" == "help" ] - then - usage 0 - else - usage 1 - fi -elif [ ${#ARGS[@]} -eq 2 ] -then - if [ "${ARGS[0]}" == "list_databases" ] - then - list_databases ${ARGS[1]} - else - usage 1 - fi -elif [ ${#ARGS[@]} -eq 3 ] -then - if [ "${ARGS[0]}" != "restore" ] - then - usage 1 - else - if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ] - then - #Get all the databases in that archive - get_databases ${ARGS[1]} - - #check if the requested database is available in the archive - if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ] - then - echo "Creating database ${ARGS[2]} if it does not exist" - $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + for db in $DBS + do + if [ -f ${TMP_DIR}/${db}_grant.sql ] then - cat $RESTORE_LOG - log_error "Database ${ARGS[2]} could not be created." - fi - echo "Restoring database ${ARGS[2]} and grants...this could take a few minutes." - restore_single_db ${ARGS[2]} - elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ] - then - echo "Creating databases if they do not exist" - for db in $DBS - do - $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`" - if [ "$?" -ne 0 ] + $MYSQL < ${TMP_DIR}/${db}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then + echo "Database $db Permission Restore successful." + else cat $RESTORE_LOG - log_error "Database ${db} could not be created." + echo "Database $db Permission Restore failed." + return 1 fi - done - echo "Restoring all databases and grants...this could take a few minutes." - restore_all_dbs - else - echo "Database ${ARGS[2]} does not exist." - fi - else - echo "Archive file not found" + else + echo "There is no permission file available for $db" + return 1 + fi + done fi + else + echo "There is no database file available to restore from" + return 1 fi -else - usage 1 -fi + return 0 +} -exit 0 +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml index 81e1713db..4af240877 100644 --- a/mariadb/templates/configmap-bin.yaml +++ b/mariadb/templates/configmap-bin.yaml @@ -42,5 +42,13 @@ data: {{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} restore_mariadb.sh: | {{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }} + restore_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} {{- end }} {{- end }} diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml index e226c268e..80ecdfa2e 100644 --- a/mariadb/templates/cron-job-backup-mariadb.yaml +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -27,6 +27,12 @@ metadata: labels: {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: +{{- if .Values.jobs.backup_mariadb.backoffLimit }} + backoffLimit: {{ .Values.jobs.backup_mariadb.backoffLimit }} +{{- end }} +{{- if .Values.jobs.backup_mariadb.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.jobs.backup_mariadb.activeDeadlineSeconds }} +{{- end }} schedule: {{ .Values.jobs.backup_mariadb.cron | quote }} successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }} failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }} @@ -36,7 +42,7 @@ spec: labels: {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: -{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: template: metadata: @@ -48,7 +54,24 @@ spec: nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }} +{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + - name: backup-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - $(MARIADB_BACKUP_BASE_DIR) + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir containers: - name: mariadb-backup command: @@ -58,14 +81,28 @@ spec: value: {{ .Values.conf.backup.base_path | quote }} - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS value: {{ .Values.conf.backup.mysqldump_options | quote }} - - name: MARIADB_BACKUP_DAYS_TO_KEEP - value: {{ .Values.conf.backup.days_of_backup_to_keep | quote }} + - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.days_to_keep | quote }} - name: MARIADB_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: REMOTE_BACKUP_ENABLED + value: "{{ .Values.conf.backup.remote_backup.enabled }}" +{{- if .Values.conf.backup.remote_backup.enabled }} + - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }} + - name: CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.container_name | quote }} + - name: STORAGE_POLICY + value: "{{ .Values.conf.backup.remote_backup.storage_policy }}" +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.remote_rgw_user }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }} +{{- end }} +{{- end }} {{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -73,6 +110,10 @@ spec: name: mariadb-bin readOnly: true subPath: backup_mariadb.sh + - mountPath: /tmp/backup_main.sh + name: mariadb-bin + readOnly: true + subPath: backup_main.sh - mountPath: {{ .Values.conf.backup.base_path }} name: mariadb-backup-dir - name: mariadb-secrets @@ -88,7 +129,7 @@ spec: - name: mariadb-secrets secret: secretName: mariadb-secrets - defaultMode: 384 + defaultMode: 420 - configMap: defaultMode: 365 name: mariadb-bin diff --git a/mariadb/templates/job-ks-user.yaml b/mariadb/templates/job-ks-user.yaml new file mode 100644 index 000000000..99b384d6c --- /dev/null +++ b/mariadb/templates/job-ks-user.yaml @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }} +{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }} +{{- $ksUserJob := dict "envAll" . "serviceName" "mariadb" "configMapBin" "mariadb-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/mariadb/templates/secret-backup-restore.yaml b/mariadb/templates/secret-backup-restore.yaml new file mode 100644 index 000000000..7886b1a7e --- /dev/null +++ b/mariadb/templates/secret-backup-restore.yaml @@ -0,0 +1,27 @@ +{{/* +This manifest results a secret being created which has the key information +needed for backing up and restoring the Mariadb databases. +*/}} + +{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }} + +{{- $envAll := . }} +{{- $userClass := "backup_restore" }} +{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }} + BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }} + LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }} + MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }} + REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }} + REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }} + REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }} + REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }} +... +{{- end }} diff --git a/mariadb/templates/secret-rgw.yaml b/mariadb/templates/secret-rgw.yaml new file mode 100644 index 000000000..7b960f8ac --- /dev/null +++ b/mariadb/templates/secret-rgw.yaml @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +This manifest results in two secrets being created: + 1) Keystone "remote_rgw_user" secret, which is needed to access the cluster + (remote or same cluster) for storing mariadb backups. If the + cluster is remote, the auth_url would be non-null. + 2) Keystone "remote_ks_admin" secret, which is needed to create the + "remote_rgw_user" keystone account mentioned above. This may not + be needed if the account is in a remote cluster (auth_url is non-null + in that case). +*/}} + +{{- if .Values.conf.backup.remote_backup.enabled }} + +{{- $envAll := . }} +{{- $userClass := "remote_rgw_user" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- if .Values.manifests.job_ks_user }} +{{- $userClass := "remote_ks_admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- end }} +{{- end }} diff --git a/mariadb/values.yaml b/mariadb/values.yaml index ff5ab4173..cabf6d136 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -29,7 +29,8 @@ images: prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 - mariadb_backup: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031 + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic + ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic scripted_test: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031 pull_policy: "IfNotPresent" local_registry: @@ -109,6 +110,17 @@ pod: main: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + mariadb_backup: + pod: + runAsUser: 65534 + container: + backup_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb_backup: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false tests: pod: runAsUser: 999 @@ -190,6 +202,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" dependencies: dynamic: @@ -208,8 +227,9 @@ dependencies: services: - endpoint: error_pages service: oslo_db - mariadb: - jobs: null + backup_mariadb: + jobs: + - mariadb-ks-user services: null prometheus_create_mysql_user: services: @@ -260,10 +280,17 @@ jobs: backoffLimit: 87600 activeDeadlineSeconds: 3600 backup_mariadb: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 cron: "0 0 * * *" history: success: 3 failed: 1 + ks_user: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 conf: tests: @@ -284,12 +311,17 @@ conf: ingress_conf: worker-processes: "auto" backup: - enabled: true + enabled: false base_path: /var/backup mysqldump_options: > --single-transaction --quick --add-drop-database --add-drop-table --add-locks --databases - days_of_backup_to_keep: 3 + days_to_keep: 3 + remote_backup: + enabled: false + container_name: mariadb + days_to_keep: 14 + storage_policy: default-placement database: my: | [mysqld] @@ -407,6 +439,13 @@ monitoring: mysqld_exporter: scrape: true +secrets: + identity: + remote_ks_admin: keystone-admin-user + remote_rgw_user: mariadb-backup-user + mariadb: + backup_restore: mariadb-backup-restore + # typically overridden by environmental # values, but should include all endpoints # required by this chart @@ -498,6 +537,44 @@ endpoints: dns: default: 53 protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + remote_ks_admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + remote_rgw_user: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 network_policy: mariadb: @@ -521,6 +598,7 @@ manifests: deployment_ingress: true job_image_repo_sync: true cron_job_mariadb_backup: false + job_ks_user: false pvc_backup: false monitoring: prometheus: @@ -536,6 +614,7 @@ manifests: secret_dbadmin_password: true secret_sst_password: true secret_dbaudit_password: true + secret_backup_restore: false secret_etc: true service_discovery: true service_ingress: true