Add remote backups failover support

This PS adds a support for a failover location to
upload and retrieve remote backups.

Change-Id: If5a6d8c562409f9fb0d7bbe039b49f0fc154d1a3

helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl

@@ -167,7 +167,7 @@ send_to_remote_server() {
   RESULT=$(openstack container list 2>&1)
 
   if [[ $? -eq 0 ]]; then
-    echo $RESULT | grep $CONTAINER_NAME
+    printf "%s\n" "$RESULT" | grep "$CONTAINER_NAME"
     if [[ $? -ne 0 ]]; then
       # Find the swift URL from the keystone endpoint list
       SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
@@ -207,7 +207,7 @@ send_to_remote_server() {
     echo $RESULT | grep -E "HTTP 401|HTTP 403"
     if [[ $? -eq 0 ]]; then
       log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
-      return 1
+      return 2
     else
       echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
       if [[ $? -eq 0 ]]; then
@@ -217,7 +217,7 @@ send_to_remote_server() {
         return 2
       else
         log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
-        return 1
+        return 2
       fi
     fi
   fi
@@ -242,7 +242,7 @@ send_to_remote_server() {
     RESULT=$(openstack container list 2>&1)
 
     if [[ $? -eq 0 ]]; then
-      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      printf "%s\n" "$RESULT" | grep "$THROTTLE_CONTAINER_NAME"
       if [[ $? -ne 0 ]]; then
         # Find the swift URL from the keystone endpoint list
         SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
@@ -282,7 +282,7 @@ send_to_remote_server() {
       echo $RESULT | grep -E "HTTP 401|HTTP 403"
       if [[ $? -eq 0 ]]; then
         log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
-        return 1
+        return 2
       else
         echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
         if [[ $? -eq 0 ]]; then
@@ -292,7 +292,7 @@ send_to_remote_server() {
           return 2
         else
           log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
-          return 1
+          return 2
         fi
       fi
     fi
@@ -387,12 +387,12 @@ send_to_remote_server() {
 # with built-in logic to handle error cases like:
 #   1) Network connectivity issues - retries for a specific amount of time
 #   2) Authorization errors - immediately logs an ERROR and returns
-store_backup_remotely() {
+function store_backup_remotely() {
-  FILEPATH=$1
+  local FILEPATH=$1
-  FILE=$2
+  local FILE=$2
+  local count=0
 
-  count=1
+  while [[ ${count} -lt ${REMOTE_BACKUP_RETRIES} ]]; do
-  while [[ ${count} -le ${REMOTE_BACKUP_RETRIES} ]]; do
     # Store the new archive to the remote backup storage facility.
     send_to_remote_server $FILEPATH $FILE
     SEND_RESULT="$?"
@@ -422,10 +422,66 @@ store_backup_remotely() {
     count=$((count+1))
   done
 
+  log INFO "${DB_NAME}_backup" "Switching to failover RGW..."
+
+  # If initial attempts failed and failover variables are defined, retry with failover environment variables
+  if [[ $SEND_RESULT -ne 0 ]] && \
+    [[ -n "${OS_AUTH_URL_FAILOVER}" ]] && \
+    [[ -n "${OS_REGION_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_INTERFACE_FAILOVER}" ]] && \
+    [[ -n "${OS_PROJECT_DOMAIN_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_PROJECT_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_USER_DOMAIN_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_USERNAME_FAILOVER}" ]] && \
+    [[ -n "${OS_PASSWORD_FAILOVER}" ]]; then
+    log INFO "${DB_NAME}_backup" "Initial attempts failed. Retrying with failover environment variables..."
+
+    # Redefine OS_* variables with OS_*_FAILOVER ones
+    export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
+    export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
+    export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
+    export OS_USERNAME=${OS_USERNAME_FAILOVER}
+    export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
+
+    count=0
+    while [[ ${count} -lt ${REMOTE_BACKUP_RETRIES} ]]; do
+      # Store the new archive to the remote backup storage facility.
+      send_to_remote_server $FILEPATH $FILE
+      SEND_RESULT="$?"
+
+      # Check if successful
+      if [[ $SEND_RESULT -eq 0 ]]; then
+        log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to failover RGW."
+        return 0
+      elif [[ $SEND_RESULT -eq 2 ]]; then
+        if [[ ${count} -ge ${REMOTE_BACKUP_RETRIES} ]]; then
+          log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the failover RGW in " \
+          "${REMOTE_BACKUP_RETRIES} retries. Errors encountered. Exiting."
+          break
+        fi
+        # Temporary failure occurred. We need to retry
+        log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to failover RGW due to connection issue."
+        sleep_time=$(random_number)
+        log INFO "${DB_NAME}_backup" "Sleeping ${sleep_time} seconds waiting for failover RGW to become available..."
+        sleep ${sleep_time}
+        log INFO "${DB_NAME}_backup" "Retrying..."
+      else
+        log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the failover RGW. Errors encountered. Exiting."
+        break
+      fi
+
+      # Increment the counter
+      count=$((count+1))
+    done
+  fi
+
   return 1
 }
 
-
 function get_archive_date(){
 # get_archive_date function returns correct archive date
 # for different formats of archives' names

helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl

@@ -147,6 +147,25 @@ usage() {
   clean_and_exit $ret_val ""
 }
 
+log() {
+  #Log message to a file or stdout
+  #TODO: This can be convert into mail alert of alert send to a monitoring system
+  #Params: $1 log level
+  #Params: $2 service
+  #Params: $3 message
+  #Params: $4 Destination
+  LEVEL=$1
+  SERVICE=$2
+  MSG=$3
+  DEST=$4
+  DATE=$(date +"%m-%d-%y %H:%M:%S")
+  if [[ -z "$DEST" ]]; then
+    echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}"
+  else
+    echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST
+  fi
+}
+
 #Exit cleanly with some message and return code
 clean_and_exit() {
   RETCODE=$1
@@ -167,29 +186,29 @@ determine_resulting_error_code() {
 
   echo ${RESULT} | grep "HTTP 404"
   if [[ $? -eq 0 ]]; then
-    echo "Could not find the archive: ${RESULT}"
+    log ERROR "${DB_NAME}_restore" "Could not find the archive: ${RESULT}"
     return 1
   else
     echo ${RESULT} | grep "HTTP 401"
     if [[ $? -eq 0 ]]; then
-      echo "Could not access the archive: ${RESULT}"
+      log ERROR "${DB_NAME}_restore" "Could not access the archive: ${RESULT}"
       return 1
     else
       echo ${RESULT} | grep "HTTP 503"
       if [[ $? -eq 0 ]]; then
-        echo "RGW service is unavailable. ${RESULT}"
+        log WARN "${DB_NAME}_restore" "RGW service is unavailable. ${RESULT}"
         # In this case, the RGW may be temporarily down.
         # Return slightly different error code so the calling code can retry
         return 2
       else
         echo ${RESULT} | grep "ConnectionError"
         if [[ $? -eq 0 ]]; then
-          echo "Could not reach the RGW: ${RESULT}"
+          log WARN "${DB_NAME}_restore" "Could not reach the RGW: ${RESULT}"
           # In this case, keystone or the site/node may be temporarily down.
           # Return slightly different error code so the calling code can retry
           return 2
         else
-          echo "Archive ${ARCHIVE} could not be retrieved: ${RESULT}"
+          log ERROR "${DB_NAME}_restore" "Archive ${ARCHIVE} could not be retrieved: ${RESULT}"
           return 1
         fi
       fi
@@ -199,51 +218,249 @@ determine_resulting_error_code() {
 }
 
 # Retrieve a list of archives from the RGW.
-retrieve_remote_listing() {
+function retrieve_remote_listing() {
+  # List archives from PRIMARY RGW
+  log INFO "${DB_NAME}_restore" "Listing archives from PRIMARY RGW..."
+  list_archives_from_rgw "PRIMARY"
+  local primary_result=$?
+
+  # Check if failover environment variables are defined
+  if [[ -n "${OS_AUTH_URL_FAILOVER}" ]] && \
+    [[ -n "${OS_REGION_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_INTERFACE_FAILOVER}" ]] && \
+    [[ -n "${OS_PROJECT_DOMAIN_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_PROJECT_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_USER_DOMAIN_NAME_FAILOVER}" ]] && \
+    [[ -n "${OS_USERNAME_FAILOVER}" ]] && \
+    [[ -n "${OS_PASSWORD_FAILOVER}" ]]; then
+    # Redefine OS_* variables with OS_*_FAILOVER ones
+    log INFO "${DB_NAME}_restore" "Listing archives from FAILOVER RGW..."
+
+    # Saving original OS_* variables as OS_*_PRIMARY
+    export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
+    export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
+    export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
+    export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
+    export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
+    export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
+    export OS_USERNAME_PRIMARY=${OS_USERNAME}
+    export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
+    export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
+
+    export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
+    export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
+    export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
+    export OS_USERNAME=${OS_USERNAME_FAILOVER}
+    export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
+
+    list_archives_from_rgw "FAILOVER"
+    local failover_result=$?
+
+    # Restore original OS_* variables from OS_*_PRIMARY
+    export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
+    export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
+    export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
+    export OS_USERNAME=${OS_USERNAME_PRIMARY}
+    export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
+
+    # Return success if either primary or failover listing was successful
+    if [[ $primary_result -eq 0 || $failover_result -eq 0 ]]; then
+      return 0
+    else
+      return 1
+    fi
+  else
+    # Return the result of the primary listing if failover variables are not defined
+    return $primary_result
+  fi
+}
+
+function list_archives_from_rgw() {
+  local prefix=$1
+  local RESULT
+
+  log INFO "${DB_NAME}_restore" "Obtaining list of archives from ${prefix} RGW..."
   RESULT=$(openstack container show $CONTAINER_NAME 2>&1)
   if [[ $? -eq 0 ]]; then
-    # Get the list, ensureing that we only pick up the right kind of backups from the
+    openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk -v prefix="$prefix" '{print prefix ":" $2}' >> $TMP_DIR/archive_list
-    # requested namespace
-    openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $TMP_DIR/archive_list
     if [[ $? -ne 0 ]]; then
-      echo "Container object listing could not be obtained."
+      log ERROR "${DB_NAME}_restore" "Container object listing could not be obtained from ${prefix} RGW."
       return 1
     else
-      echo "Archive listing successfully retrieved."
+      log INFO "${DB_NAME}_restore" "Archive listing successfully retrieved from ${prefix} RGW."
     fi
   else
-    determine_resulting_error_code "${RESULT}"
+    log ERROR "${DB_NAME}_restore" "Failed to obtain container show from ${prefix} RGW: ${RESULT}"
-    return $?
+    return 1
   fi
+
   return 0
 }
 
 # Retrieve a single archive from the RGW.
 retrieve_remote_archive() {
-  ARCHIVE=$1
+  local archive=$1
+  local prefix=$(echo $archive | awk -F: '{print $1}')
+  local filename
 
-  RESULT=$(openstack object save --file $TMP_DIR/$ARCHIVE $CONTAINER_NAME $ARCHIVE 2>&1)
+  if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
-  if [[ $? -ne 0 ]]; then
+    filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
+  else
+    filename=$archive
+  fi
+
+  if [[ $prefix == "PRIMARY" ]]; then
+    log INFO "${DB_NAME}_restore" "Retrieving archive ${filename} from PRIMARY RGW..."
+    retrieve_archive_from_rgw $filename
+    return $?
+  elif [[ $prefix == "FAILOVER" ]]; then
+    log INFO "${DB_NAME}_restore" "Retrieving archive ${filename} from FAILOVER RGW..."
+
+    # Saving original OS_* variables as OS_*_PRIMARY
+    export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
+    export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
+    export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
+    export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
+    export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
+    export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
+    export OS_USERNAME_PRIMARY=${OS_USERNAME}
+    export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
+    export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
+
+    # Redefine OS_* variables with OS_*_FAILOVER ones
+    export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
+    export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
+    export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
+    export OS_USERNAME=${OS_USERNAME_FAILOVER}
+    export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
+
+    retrieve_archive_from_rgw $filename
+    local result=$?
+
+    # Restore original OS_* variables from OS_*_PRIMARY
+    export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
+    export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
+    export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
+    export OS_USERNAME=${OS_USERNAME_PRIMARY}
+    export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
+
+    return $result
+  else
+    log ERROR "${DB_NAME}_restore" "Invalid prefix ${prefix} for archive ${archive}."
+    return 1
+  fi
+}
+
+# Function to retrieve an archive from RGW
+retrieve_archive_from_rgw() {
+  local filename=$1
+  local RESULT
+
+  log INFO "${DB_NAME}_restore" "Obtaining archive ${filename} from RGW..."
+  RESULT=$(openstack object save --file $TMP_DIR/${filename} $CONTAINER_NAME ${filename}  2>&1)
+  if [[ $? -eq 0 ]]; then
+    log INFO "${DB_NAME}_restore" "Archive ${filename} successfully retrieved."
+    return 0
+  else
+    log ERROR "${DB_NAME}_restore" "Failed to retrieve archive ${filename}."
     determine_resulting_error_code "${RESULT}"
     return $?
-  else
-    echo "Archive $ARCHIVE successfully retrieved."
   fi
-  return 0
 }
 
 # Delete an archive from the RGW.
+# Delete a single archive from the RGW.
 delete_remote_archive() {
-  ARCHIVE=$1
+  local archive=$1
+  local prefix=$(echo $archive | awk -F: '{print $1}')
+  local filename
 
-  RESULT=$(openstack object delete ${CONTAINER_NAME} ${ARCHIVE} 2>&1)
+  if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
-  if [[ $? -ne 0 ]]; then
+    filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
+  else
+    filename=$archive
+  fi
+
+  if [[ $prefix == "PRIMARY" ]]; then
+    log INFO "${DB_NAME}_restore" "Deleting archive ${filename} from PRIMARY RGW..."
+    delete_archive_from_rgw $filename
+    return $?
+  elif [[ $prefix == "FAILOVER" ]]; then
+    log INFO "${DB_NAME}_restore" "Deleting archive ${filename} from FAILOVER RGW..."
+
+    # Saving original OS_* variables as OS_*_PRIMARY
+    export OS_AUTH_URL_PRIMARY=${OS_AUTH_URL}
+    export OS_REGION_NAME_PRIMARY=${OS_REGION_NAME}
+    export OS_INTERFACE_PRIMARY=${OS_INTERFACE}
+    export OS_PROJECT_DOMAIN_NAME_PRIMARY=${OS_PROJECT_DOMAIN_NAME}
+    export OS_PROJECT_NAME_PRIMARY=${OS_PROJECT_NAME}
+    export OS_USER_DOMAIN_NAME_PRIMARY=${OS_USER_DOMAIN_NAME}
+    export OS_USERNAME_PRIMARY=${OS_USERNAME}
+    export OS_PASSWORD_PRIMARY=${OS_PASSWORD}
+    export OS_DEFAULT_DOMAIN_PRIMARY=${OS_DEFAULT_DOMAIN}
+
+    # Redefine OS_* variables with OS_*_FAILOVER ones
+    export OS_AUTH_URL=${OS_AUTH_URL_FAILOVER}
+    export OS_REGION_NAME=${OS_REGION_NAME_FAILOVER}
+    export OS_INTERFACE=${OS_INTERFACE_FAILOVER}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_FAILOVER}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_FAILOVER}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_FAILOVER}
+    export OS_USERNAME=${OS_USERNAME_FAILOVER}
+    export OS_PASSWORD=${OS_PASSWORD_FAILOVER}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_FAILOVER}
+
+    delete_archive_from_rgw $filename
+    local result=$?
+
+    # Restore original OS_* variables from OS_*_PRIMARY
+    export OS_AUTH_URL=${OS_AUTH_URL_PRIMARY}
+    export OS_REGION_NAME=${OS_REGION_NAME_PRIMARY}
+    export OS_INTERFACE=${OS_INTERFACE_PRIMARY}
+    export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME_PRIMARY}
+    export OS_PROJECT_NAME=${OS_PROJECT_NAME_PRIMARY}
+    export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME_PRIMARY}
+    export OS_USERNAME=${OS_USERNAME_PRIMARY}
+    export OS_PASSWORD=${OS_PASSWORD_PRIMARY}
+    export OS_DEFAULT_DOMAIN=${OS_DEFAULT_DOMAIN_PRIMARY}
+
+    return $result
+  else
+    log ERROR "${DB_NAME}_restore" "Invalid prefix ${prefix} for archive ${archive}."
+    return 1
+  fi
+}
+
+# Function to delete an archive from RGW
+delete_archive_from_rgw() {
+  local filename=$1
+  local RESULT
+
+  RESULT=$(openstack object delete $CONTAINER_NAME $filename 2>&1)
+  if [[ $? -eq 0 ]]; then
+    log INFO "${DB_NAME}_restore" "Archive ${filename} successfully deleted."
+    return 0
+  else
+    log ERROR "${DB_NAME}_restore" "Failed to delete archive ${filename}."
     determine_resulting_error_code "${RESULT}"
     return $?
-  else
-    echo "Archive ${ARCHIVE} successfully deleted."
   fi
-  return 0
 }
 
 # Display all archives
@@ -283,18 +500,26 @@ list_archives() {
 # Retrieve the archive from the desired location and decompress it into
 # the restore directory
 get_archive() {
-  ARCHIVE_FILE=$1
+  local archive=$1
+  local prefix=$(echo $archive | awk -F: '{print $1}')
+  local filename
+
+  if [[ $prefix == "PRIMARY" || $prefix == "FAILOVER" ]]; then
+    filename=$(echo $archive | awk -F: '{print $2":"$3":"$4}')
+  else
+    filename=$archive
+  fi
   REMOTE=$2
 
   if [[ "x$REMOTE" == "xremote" ]]; then
-    echo "Retrieving archive ${ARCHIVE_FILE} from the remote RGW..."
+    log INFO "${DB_NAME}_restore" "Retrieving archive ${prefix}:${filename} from the remote RGW..."
-    retrieve_remote_archive $ARCHIVE_FILE
+    retrieve_remote_archive ${prefix}:${filename}
     if [[ $? -ne 0 ]]; then
-      clean_and_exit 1 "ERROR: Could not retrieve remote archive: $ARCHIVE_FILE"
+      clean_and_exit 1 "ERROR: Could not retrieve remote archive: ${prefix}:${filename}"
     fi
   elif [[ "x$REMOTE" == "x" ]]; then
-    if [[ -e $ARCHIVE_DIR/$ARCHIVE_FILE ]]; then
+    if [[ -e $ARCHIVE_DIR/$filename ]]; then
-      cp $ARCHIVE_DIR/$ARCHIVE_FILE $TMP_DIR/$ARCHIVE_FILE
+      cp $ARCHIVE_DIR/$filename $TMP_DIR/$filename
       if [[ $? -ne 0 ]]; then
         clean_and_exit 1 "ERROR: Could not copy local archive to restore directory."
       fi
@@ -305,9 +530,10 @@ get_archive() {
     usage 1
   fi
 
-  echo "Decompressing archive $ARCHIVE_FILE..."
+  log INFO "${DB_NAME}_restore" "Decompressing archive $filename..."
+
   cd $TMP_DIR
-  tar zxvf - < $TMP_DIR/$ARCHIVE_FILE 1>/dev/null
+  tar zxvf - < $TMP_DIR/$filename 1>/dev/null
   if [[ $? -ne 0 ]]; then
     clean_and_exit 1 "ERROR: Archive decompression failed."
   fi
@@ -464,7 +690,7 @@ delete_archive() {
     fi
   fi
 
-  echo "Successfully deleted archive ${ARCHIVE_FILE} from ${WHERE} storage."
+  log INFO "${DB_NAME}_restore" "Successfully deleted archive ${ARCHIVE_FILE} from ${WHERE} storage."
 }
 
 
@@ -578,19 +804,19 @@ cli_main() {
           fi
         fi
 
-        echo "Restoring Database $DB_SPEC And Grants"
+        log INFO "${DB_NAME}_restore" "Restoring Database $DB_SPEC And Grants"
         restore_single_db $DB_SPEC $TMP_DIR
         if [[ "$?" -eq 0 ]]; then
-          echo "Single database restored successfully."
+          log INFO "${DB_NAME}_restore" "Single database restored successfully."
         else
           clean_and_exit 1 "ERROR: Single database restore failed."
         fi
         clean_and_exit 0 ""
       else
-        echo "Restoring All The Databases. This could take a few minutes..."
+        log INFO "${DB_NAME}_restore" "Restoring All The Databases. This could take a few minutes..."
         restore_all_dbs $TMP_DIR
         if [[ "$?" -eq 0 ]]; then
-          echo "All databases restored successfully."
+          log INFO "${DB_NAME}_restore" "All databases restored successfully."
         else
           clean_and_exit 1 "ERROR: Database restore failed."
         fi

mariadb-backup/templates/secret-rgw.yaml

@@ -25,7 +25,8 @@ This manifest results in two secrets being created:
 {{- if .Values.conf.backup.remote_backup.enabled }}
 
 {{- $envAll := . }}
-{{- $userClass := "mariadb-server" }}
+{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
+{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
 {{- $secretName := index $envAll.Values.secrets.identity $userClass }}
 ---
 apiVersion: v1
@@ -48,6 +49,23 @@ data:
   OS_USERNAME: {{ $identityClass.username | b64enc }}
   OS_PASSWORD: {{ $identityClass.password | b64enc }}
   OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+
+{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
+{{- if $failoverIdentityClass }}
+{{- if $failoverIdentityClass.auth_url }}
+  OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
+  OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
+  OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
+  OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
+{{- end }}
 ...
 {{- if .Values.manifests.job_ks_user }}
 {{- $userClass := "admin" }}

mariadb-backup/values.yaml

@@ -241,6 +241,8 @@ conf:
         lock_expire_after: 7200
         retry_after: 3600
         container_name: throttle-backups-manager
+      primary_user_class: mariadb-server
+      failover_user_class: mariadb-server_failover
 
 secrets:
   identity:

mariadb/templates/bin/_start.py.tpl

@@ -258,7 +258,6 @@ def stop_mysqld():
     logger.info("Mysqld stopped: pid = {0}, "
                 "exit status = {1}".format(pid, status))
 
-
 def mysqld_write_cluster_conf(mode='run'):
     """Write out dynamic cluster config.
 

mariadb/templates/secret-rgw.yaml

@@ -25,7 +25,8 @@ This manifest results in two secrets being created:
 {{- if .Values.conf.backup.remote_backup.enabled }}
 
 {{- $envAll := . }}
-{{- $userClass := "mariadb" }}
+{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
+{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
 {{- $secretName := index $envAll.Values.secrets.identity $userClass }}
 ---
 apiVersion: v1
@@ -48,6 +49,23 @@ data:
   OS_USERNAME: {{ $identityClass.username | b64enc }}
   OS_PASSWORD: {{ $identityClass.password | b64enc }}
   OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+
+{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
+{{- if $failoverIdentityClass }}
+{{- if $failoverIdentityClass.auth_url }}
+  OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
+  OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
+  OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
+  OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
+{{- end }}
 ...
 {{- if .Values.manifests.job_ks_user }}
 {{- $userClass := "admin" }}

mariadb/values.yaml

@@ -348,6 +348,8 @@ conf:
         lock_expire_after: 7200
         retry_after: 3600
         container_name: throttle-backups-manager
+      primary_user_class: mariadb
+      failover_user_class: mariadb_failover
   galera:
     cluster_leader_ttl: 60
   database:

postgresql/templates/secret-rgw.yaml

@@ -11,7 +11,8 @@ This manifest results in two secrets being created:
 {{- if .Values.conf.backup.remote_backup.enabled }}
 
 {{- $envAll := . }}
-{{- $userClass := "postgresql" }}
+{{- $userClass := .Values.conf.backup.remote_backup.primary_user_class }}
+{{- $failoverUserClass := .Values.conf.backup.remote_backup.failover_user_class }}
 {{- $secretName := index $envAll.Values.secrets.identity $userClass }}
 ---
 apiVersion: v1
@@ -34,6 +35,23 @@ data:
   OS_USERNAME: {{ $identityClass.username | b64enc }}
   OS_PASSWORD: {{ $identityClass.password | b64enc }}
   OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+
+{{- $failoverIdentityClass := index .Values.endpoints.identity.auth $failoverUserClass }}
+{{- if $failoverIdentityClass }}
+{{- if $failoverIdentityClass.auth_url }}
+  OS_AUTH_URL_FAILOVER: {{ $failoverIdentityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL_FAILOVER: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME_FAILOVER: {{ $failoverIdentityClass.region_name | b64enc }}
+  OS_INTERFACE_FAILOVER: {{ $failoverIdentityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME_FAILOVER: {{ $failoverIdentityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME_FAILOVER: {{ $failoverIdentityClass.user_domain_name | b64enc }}
+  OS_USERNAME_FAILOVER: {{ $failoverIdentityClass.username | b64enc }}
+  OS_PASSWORD_FAILOVER: {{ $failoverIdentityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN_FAILOVER: {{ $failoverIdentityClass.default_domain_id | default "default" | b64enc }}
+{{- end }}
 ...
 {{- if .Values.manifests.job_ks_user }}
 {{- $userClass := "admin" }}

postgresql/values.yaml

@@ -335,6 +335,8 @@ conf:
         lock_expire_after: 7200
         retry_after: 3600
         container_name: throttle-backups-manager
+      primary_user_class: postgresql
+      failover_user_class: postgresql_failover
 
   exporter:
     queries:

values_overrides/mariadb/remote_backups.yaml

@@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: true
+volume:
+  backup:
+    enabled: true
+endpoints:
+  identity:
+    auth:
+      mariadb:
+        # Auth URL of null indicates local authentication
+        # HTK will form the URL unless specified here
+        auth_url: https://rgw-1.test.local
+        region_name: RegionOne
+        username: mariadb-backup-user
+        password: password
+        project_name: service
+        user_domain_name: service
+        project_domain_name: service
+      mariadb_failover:
+        # # Auth URL of null indicates local authentication
+        # # HTK will form the URL unless specified here
+        auth_url: https://rgw-2.test.local
+        region_name: RegionOne
+        username: mariadb-backup-user
+        password: password
+        project_name: service
+        user_domain_name: service
+        project_domain_name: service
+manifests:
+  pvc_backup: true
+  job_ks_user: true
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...