From 5c4056ad341afcc577e63902b6ddbfb222d757e1 Mon Sep 17 00:00:00 2001
From: "Markin, Sergiy (sm515x)" <sm515x@att.com>
Date: Sat, 13 Aug 2022 00:24:46 +0000
Subject: [PATCH] [DATABASE] Add verify databases backup

HTK - added verify_databases_backup_in_directory function that is
going to be defined inside mariadb/postgresql/etcd charts.

Mariadb chart - added verify_databases_backup_archives function
implementation.

Added mariadb-verify container to mariadb-backup cronjob to run
verification process.

Added remove backup verification pocess - comparition of local and remote file md5 hashes.

PostgreSQL chart - added empty implementation of verify_databases_backup_archives() function. This is a subject for future realization.

Change-Id: I361cdb92c66b0b27539997d697adfd1e93c9a29d
---
 helm-toolkit/Chart.yaml                       |   2 +-
 .../db-backup-restore/_backup_main.sh.tpl     |  73 ++-
 mariadb/Chart.yaml                            |   4 +-
 mariadb/templates/bin/_backup_mariadb.sh.tpl  | 467 ++++++++++++++++++
 .../bin/_start_mariadb_verify_server.sh.tpl   |  28 ++
 mariadb/templates/configmap-bin.yaml          |   2 +
 .../templates/cron-job-backup-mariadb.yaml    |  72 ++-
 mariadb/values.yaml                           |  13 +
 mariadb/values_overrides/apparmor.yaml        |   1 +
 postgresql/Chart.yaml                         |   2 +-
 .../templates/bin/_backup_postgresql.sh.tpl   |   8 +
 releasenotes/notes/helm-toolkit.yaml          |   1 +
 releasenotes/notes/mariadb.yaml               |   1 +
 releasenotes/notes/postgresql.yaml            |   1 +
 14 files changed, 662 insertions(+), 13 deletions(-)
 create mode 100644 mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl

diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml
index acb03fa84..22ca47fac 100644
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.47
+version: 0.2.48
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 516d79ee7..687851eb4 100755
--- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -66,6 +66,14 @@
 #       framework will automatically tar/zip the files in that directory and
 #       name the tarball appropriately according to the proper conventions.
 #
+#   verify_databases_backup_archives [scope]
+#       returns: 0 if no errors; 1 if any errors occurred
+#
+#       This function is expected to verify the database backup archives. If this function
+#        completes successfully (returns 0), the
+#       framework will automatically starts remote backup upload.
+#
+#
 # The functions in this file will take care of:
 #   1) Calling "dump_databases_to_directory" and then compressing the files,
 #      naming the tarball properly, and then storing it locally at the specified
@@ -90,6 +98,16 @@ log_backup_error_exit() {
   exit $ERRCODE
 }
 
+log_verify_backup_exit() {
+  MSG=$1
+  ERRCODE=${2:-0}
+  log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
+  rm -f $ERR_LOG_FILE
+  # rm -rf $TMP_DIR
+  exit $ERRCODE
+}
+
+
 log() {
   #Log message to a file or stdout
   #TODO: This can be convert into mail alert of alert send to a monitoring system
@@ -201,12 +219,36 @@ send_to_remote_server() {
     log WARN "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
     return 2
   fi
+
   openstack object show $CONTAINER_NAME $FILE
   if [[ $? -ne 0 ]]; then
     log WARN "${DB_NAME}_backup" "Unable to retrieve container object $FILE after creation."
     return 2
   fi
 
+  # Calculation remote file SHA256 hash
+  REMOTE_FILE=$(mktemp -p /tmp)
+  openstack object save --file ${REMOTE_FILE} $CONTAINER_NAME $FILE
+  if [[ $? -ne 0 ]]; then
+    log WARN "${DB_NAME}_backup" "Unable to save container object $FILE for SHA256 hash verification."
+    rm -rf ${REMOTE_FILE}
+    return 1
+  fi
+
+  # Remote backup verification
+  SHA256_REMOTE=$(cat ${REMOTE_FILE} | sha256sum | awk '{print $1}')
+  SHA256_LOCAL=$(cat ${FILEPATH}/${FILE} | sha256sum | awk '{print $1}')
+  log INFO "${DB_NAME}_backup" "Calculated SHA256 hashes for the file $FILE in container $CONTAINER_NAME."
+  log INFO "${DB_NAME}_backup" "Local SHA256 hash is ${SHA256_LOCAL}."
+  log INFO "${DB_NAME}_backup" "Remote SHA256 hash is ${SHA256_REMOTE}."
+  if [[ "${SHA256_LOCAL}" == "${SHA256_REMOTE}" ]]; then
+      log INFO "${DB_NAME}_backup" "The local backup & remote backup SHA256 hash values are matching for file $FILE in container $CONTAINER_NAME."
+  else
+      log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup sha256 hash values"
+      return 1
+  fi
+  rm -rf ${REMOTE_FILE}
+
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
 }
@@ -382,8 +424,8 @@ remove_old_remote_archives() {
 
   # Cleanup now that we're done.
   for fd in ${BACKUP_FILES} ${DB_BACKUP_FILES}; do
-    if [[ -f fd ]]; then
-      rm -f fd
+    if [[ -f ${fd} ]]; then
+      rm -f ${fd}
     else
       log WARN "${DB_NAME}_backup" "Can not delete a temporary file ${fd}"
     fi
@@ -444,10 +486,6 @@ backup_databases() {
 
   cd $ARCHIVE_DIR
 
-  # Remove the temporary directory and files as they are no longer needed.
-  rm -rf $TMP_DIR
-  rm -f $ERR_LOG_FILE
-
   #Only delete the old archive after a successful archive
   export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g')
   if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
@@ -459,6 +497,25 @@ backup_databases() {
     done
   fi
 
+  # Local backup verification process
+
+  # It is expected that this function will verify the database backup files
+  if verify_databases_backup_archives ${SCOPE}; then
+    log INFO "${DB_NAME}_backup_verify" "Databases backup verified successfully. Uploading verified backups to remote location..."
+  else
+    # If successful, there should be at least one file in the TMP_DIR
+    if [[ $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
+      cat $ERR_LOG_FILE
+    fi
+    log_verify_backup_exit "Verify of the ${DB_NAME} database backup failed and needs attention."
+    exit 1
+  fi
+
+  # Remove the temporary directory and files as they are no longer needed.
+  rm -rf $TMP_DIR
+  rm -f $ERR_LOG_FILE
+
+  # Remote backup
   REMOTE_BACKUP=$(echo $REMOTE_BACKUP_ENABLED | sed 's/"//g')
   if $REMOTE_BACKUP; then
     # Remove Quotes from the constants which were added due to reading
@@ -490,7 +547,7 @@ backup_databases() {
       get_backup_prefix $(cat $DB_BACKUP_FILES)
       for ((i=0; i<${#PREFIXES[@]}; i++)); do
         echo "Working with prefix: ${PREFIXES[i]}"
-        create_hash_table $(cat $DB_BACKUP_FILES | grep ${PREFIXES[i]})
+        create_hash_table $(cat ${DB_BACKUP_FILES} | grep ${PREFIXES[i]})
         remove_old_remote_archives
       done
     fi
@@ -511,4 +568,4 @@ backup_databases() {
     echo "=================================================================="
   fi
 }
-{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml
index 11a1b12d2..432abca0a 100644
--- a/mariadb/Chart.yaml
+++ b/mariadb/Chart.yaml
@@ -12,10 +12,10 @@
 
 ---
 apiVersion: v1
-appVersion: v10.2.31
+appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.27
+version: 0.2.28
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:
diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl
index 499337595..dba8ddb56 100644
--- a/mariadb/templates/bin/_backup_mariadb.sh.tpl
+++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl
@@ -34,6 +34,7 @@ dump_databases_to_directory() {
   LOG_FILE=$2
   SCOPE=${3:-"all"}
 
+
   MYSQL="mysql \
      --defaults-file=/etc/mysql/admin_user.cnf \
      --connect-timeout 10"
@@ -113,5 +114,471 @@ dump_databases_to_directory() {
   fi
 }
 
+# functions from  mariadb-verifier chart
+
+get_time_delta_secs () {
+  second_delta=0
+  input_date_second=$( date --date="$1" +%s )
+  if [ -n "$input_date_second" ]; then
+    current_date=$( date +"%Y-%m-%dT%H:%M:%SZ" )
+    current_date_second=$( date --date="$current_date" +%s )
+    ((second_delta=current_date_second-input_date_second))
+    if [ "$second_delta" -lt 0 ]; then
+      second_delta=0
+    fi
+  fi
+  echo $second_delta
+}
+
+
+check_data_freshness () {
+  archive_file=$(basename "$1")
+  archive_date=$(echo "$archive_file" | cut -d'.' -f 4)
+  SCOPE=$2
+
+  if [[ "${SCOPE}" != "all" ]]; then
+    log "Data freshness check is skipped for individual database."
+    return 0
+  fi
+
+  log "Checking for data freshness in the backups..."
+  # Get some idea of which database.table has changed in the last 30m
+  # Excluding the system DBs and aqua_test_database
+  #
+  changed_tables=$(${MYSQL_LIVE} -e "select TABLE_SCHEMA,TABLE_NAME from \
+information_schema.tables where UPDATE_TIME >= SUBTIME(now(),'00:30:00') AND TABLE_SCHEMA \
+NOT IN('information_schema', 'mysql', 'performance_schema', 'sys', 'aqua_test_database');" | \
+awk '{print $1 "." $2}')
+
+  if [ -n "${changed_tables}" ]; then
+    delta_secs=$(get_time_delta_secs "$archive_date")
+    age_offset={{ .Values.conf.backup.validateData.ageOffset }}
+    ((age_threshold=delta_secs+age_offset))
+
+    data_freshness=false
+    skipped_freshness=false
+
+    for table in ${changed_tables}; do
+      tab_schema=$(echo "$table" | awk -F. '{print $1}')
+      tab_name=$(echo "$table" | awk -F. '{print $2}')
+
+      local_table_existed=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select TABLE_SCHEMA,TABLE_NAME from \
+INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA=\"${tab_schema}\" AND TABLE_NAME=\"${tab_name}\";")
+
+      if [ -n "$local_table_existed" ]; then
+        # TODO: If last updated field of a table structure has different
+        # patterns (updated/timstamp), it may be worth to parameterize the patterns.
+        datetime=$(${MYSQL_LOCAL_SHORT_SILENT} -e "describe ${table};" | \
+                   awk '(/updated/ || /timestamp/) && /datetime/ {print $1}')
+
+        if [ -n "${datetime}" ]; then
+          data_ages=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select \
+time_to_sec(timediff(now(),${datetime})) from ${table} where ${datetime} is not null order by 1 limit 10;")
+
+          for age in $data_ages; do
+            if [ "$age" -le $age_threshold ]; then
+              data_freshness=true
+              break
+            fi
+          done
+
+          # As long as there is an indication of data freshness, no need to check further
+          if [ "$data_freshness" = true ] ; then
+            break
+          fi
+        else
+          skipped_freshness=true
+          log "No indicator to determine data freshness for table $table. Skipped data freshness check."
+
+          # Dumping out table structure to determine if enhancement is needed to include this table
+          debug_info=$(${MYSQL_LOCAL} --skip-column-names -e "describe ${table};" | awk '{print $2 " " $1}')
+          log "$debug_info" "DEBUG"
+        fi
+      else
+        log "Table $table doesn't exist in local database"
+        skipped_freshness=true
+      fi
+    done
+
+    if [ "$data_freshness" = true ] ; then
+      log "Database passed integrity (data freshness) check."
+    else
+      if [ "$skipped_freshness" = false ] ; then
+        log "Local backup database restore failed integrity check." "ERROR"
+        log "The backup may not have captured the up-to-date data." "INFO"
+        return 1
+      fi
+    fi
+  else
+    log "No tables changed in this backup. Skipped data freshness check as the"
+    log "check should have been performed by previous validation runs."
+  fi
+
+  return 0
+}
+
+
+cleanup_local_databases () {
+  old_local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
+    grep -ivE 'information_schema|performance_schema|mysql|sys' || true)
+
+  for db in $old_local_dbs; do
+    ${MYSQL_LOCAL_SHORT_SILENT} -e "drop database $db;"
+  done
+}
+
+list_archive_dir () {
+  archive_dir_content=$(ls -1R "$ARCHIVE_DIR")
+  if [ -n "$archive_dir_content" ]; then
+    log "Content of $ARCHIVE_DIR"
+    log "${archive_dir_content}"
+  fi
+}
+
+remove_remote_archive_file () {
+  archive_file=$(basename "$1")
+  token_req_file=$(mktemp --suffix ".json")
+  header_file=$(mktemp)
+  resp_file=$(mktemp --suffix ".json")
+  http_resp="404"
+
+  HEADER_CONTENT_TYPE="Content-Type: application/json"
+  HEADER_ACCEPT="Accept: application/json"
+
+  cat << JSON_EOF > "$token_req_file"
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "domain": {
+                        "name": "${OS_USER_DOMAIN_NAME}"
+                    },
+                    "name": "${OS_USERNAME}",
+                    "password": "${OS_PASSWORD}"
+                }
+            }
+        },
+        "scope": {
+            "project": {
+                "domain": {
+                    "name": "${OS_PROJECT_DOMAIN_NAME}"
+                },
+                "name": "${OS_PROJECT_NAME}"
+            }
+        }
+    }
+}
+JSON_EOF
+
+  http_resp=$(curl -s -X POST "$OS_AUTH_URL/auth/tokens"  -H "${HEADER_CONTENT_TYPE}" \
+       -H "${HEADER_ACCEPT}" -d @"${token_req_file}" -D "$header_file" -o "$resp_file" -w "%{http_code}")
+
+  if [ "$http_resp" = "201" ]; then
+    OS_TOKEN=$(grep -i "x-subject-token" "$header_file" | cut -d' ' -f2 | tr -d "\r")
+
+    if [ -n "$OS_TOKEN" ]; then
+      OS_OBJ_URL=$(python3 -c "import json,sys;print([[ep['url'] for ep in obj['endpoints'] if ep['interface']=='public'] for obj in json.load(sys.stdin)['token']['catalog'] if obj['type']=='object-store'][0][0])" < "$resp_file")
+
+      if [ -n "$OS_OBJ_URL" ]; then
+        http_resp=$(curl -s -X DELETE "$OS_OBJ_URL/$CONTAINER_NAME/$archive_file" \
+                         -H "${HEADER_CONTENT_TYPE}" -H "${HEADER_ACCEPT}" \
+                         -H "X-Auth-Token: ${OS_TOKEN}" -D "$header_file" -o "$resp_file" -w "%{http_code}")
+      fi
+    fi
+  fi
+
+  if [ "$http_resp" == "404" ] ; then
+    log "Failed to cleanup remote backup. Container object $archive_file is not on RGW."
+    return 1
+  fi
+
+  if [ "$http_resp" != "204" ] ; then
+    log "Failed to cleanup remote backup. Cannot delete container object $archive_file" "ERROR"
+    cat "$header_file"
+    cat "$resp_file"
+  fi
+  return 0
+}
+
+handle_bad_archive_file () {
+  archive_file=$1
+
+  if [ ! -d "$BAD_ARCHIVE_DIR" ]; then
+    mkdir -p "$BAD_ARCHIVE_DIR"
+  fi
+
+  # Move the file to quarantine directory such that
+  # file won't be used for restore in case of recovery
+  #
+  log "Moving $i to $BAD_ARCHIVE_DIR..."
+  mv "$i" "$BAD_ARCHIVE_DIR"
+  log "Removing $i from remote RGW..."
+  if remove_remote_archive_file "$i"; then
+    log "File $i has been successfully removed from RGW."
+  else
+    log "FIle $i cannot be removed form RGW." "ERROR"
+    return 1
+  fi
+
+  # Atmost only three bad files are kept. Deleting the oldest if
+  # number of files exceeded the threshold.
+  #
+  bad_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l)
+  if [ "$bad_files" -gt 3 ]; then
+    ((bad_files=bad_files-3))
+    delete_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | sort | head --lines=$bad_files)
+    for b in $delete_files; do
+      log "Deleting $b..."
+      rm -f "${b}"
+    done
+  fi
+  return 0
+}
+
+cleanup_old_validation_result_file () {
+  clean_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.passed" 2>/dev/null)
+  for d in $clean_files; do
+    archive_file=${d/.passed}
+    if [ ! -f "$archive_file" ]; then
+      log "Deleting $d as its associated archive file $archive_file nolonger existed."
+      rm -f "${d}"
+    fi
+  done
+}
+
+validate_databases_backup () {
+  archive_file=$1
+  SCOPE=${2:-"all"}
+
+  restore_log='/tmp/restore_error.log'
+  tmp_dir=$(mktemp -d)
+
+  rm -f $restore_log
+  cd "$tmp_dir"
+  log "Decompressing archive $archive_file..."
+  if ! tar zxvf - < "$archive_file" 1>/dev/null; then
+    log "Database restore from local backup failed. Archive decompression failed." "ERROR"
+    return 1
+  fi
+
+  db_list_file="$tmp_dir/db.list"
+  if [[ -e "$db_list_file" ]]; then
+    dbs=$(sort < "$db_list_file" | grep -ivE sys | tr '\n' ' ')
+  else
+    dbs=" "
+  fi
+
+  sql_file="${tmp_dir}/mariadb.${MARIADB_POD_NAMESPACE}.${SCOPE}.sql"
+
+  if [[ "${SCOPE}" == "all" ]]; then
+    grant_file="${tmp_dir}/grants.sql"
+  else
+    grant_file="${tmp_dir}/${SCOPE}_grant.sql"
+  fi
+
+  if [[ -f $sql_file ]]; then
+    if $MYSQL_LOCAL < "$sql_file" 2>$restore_log; then
+      local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
+        grep -ivE 'information_schema|performance_schema|mysql|sys' | sort | tr '\n' ' ')
+
+      if [ "$dbs" = "$local_dbs" ]; then
+        log "Databases restored successful."
+      else
+        log "Database restore from local backup failed. Database mismatched between local backup and local server" "ERROR"
+        log "Databases restored on local server: $local_dbs" "DEBUG"
+        log "Databases in the local backup: $dbs" "DEBUG"
+        return 1
+      fi
+    else
+      log "Database restore from local backup failed. $dbs" "ERROR"
+      cat $restore_log
+      return 1
+    fi
+
+    if [[ -f $grant_file ]]; then
+      if $MYSQL_LOCAL < "$grant_file" 2>$restore_log; then
+        if ! $MYSQL_LOCAL -e 'flush privileges;'; then
+          log "Database restore from local backup failed. Failed to flush privileges." "ERROR"
+          return 1
+        fi
+        log "Databases permission restored successful."
+      else
+        log "Database restore from local backup failed. Databases permission failed to restore." "ERROR"
+        cat "$restore_log"
+        cat "$grant_file"
+        log "Local DBs: $local_dbs" "DEBUG"
+        return 1
+      fi
+    else
+      log "Database restore from local backup failed. There is no permission file available" "ERROR"
+      return 1
+    fi
+
+    if ! check_data_freshness "$archive_file" ${SCOPE}; then
+      # Log has already generated during check data freshness
+      return 1
+    fi
+  else
+    log "Database restore from local backup failed. There is no database file available to restore from" "ERROR"
+    return 1
+  fi
+
+  return 0
+}
+
+# end of functions form mariadb verifier chart
+
+# Verify all the databases backup archives
+verify_databases_backup_archives() {
+  SCOPE=${1:-"all"}
+
+  # verification code
+  export DB_NAME="mariadb"
+  export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
+  export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
+  export MYSQL_OPTS="--silent --skip-column-names"
+  export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
+  export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
+  export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
+  export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
+  export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"
+
+  max_wait={{ .Values.conf.mariadb_server.setup_wait.iteration }}
+  duration={{ .Values.conf.mariadb_server.setup_wait.duration }}
+  counter=0
+  dbisup=false
+
+  log "Waiting for Mariadb backup verification server to start..."
+
+  # During Mariadb init/startup process, a temporary server is startup
+  # and shutdown prior to starting up the normal server.
+  # To avoid prematurely determine server availability, lets snooze
+  # a bit to give time for the process to complete prior to issue
+  # mysql commands.
+  #
+
+
+  while [ $counter -lt $max_wait ]; do
+    if ! $MYSQL_LOCAL_SHORT -e 'select 1' > /dev/null 2>&1 ; then
+      sleep $duration
+      ((counter=counter+1))
+    else
+      # Lets sleep for an additional duration just in case async
+      # init takes a bit more time to complete.
+      #
+      sleep $duration
+      dbisup=true
+      counter=$max_wait
+    fi
+  done
+
+  if ! $dbisup; then
+    log "Mariadb backup verification server is not running" "ERROR"
+    return 1
+  fi
+
+  # During Mariadb init process, a test database will be briefly
+  # created and deleted. Adding to the exclusion list for some
+  # edge cases
+  #
+  clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
+    grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true)
+
+  if [[ -z "${clean_db// }" ]]; then
+    log "Clean Server is up and running"
+  else
+    cleanup_local_databases
+    log "Old databases found on the Mariadb backup verification server were cleaned."
+    clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
+      grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true)
+
+    if [[ -z "${clean_db// }" ]]; then
+      log "Clean Server is up and running"
+    else
+      log "Cannot clean old databases on verification server." "ERROR"
+      return 1
+    fi
+    log "The server is ready for verification."
+  fi
+
+  # Starting with 10.4.13, new definer mariadb.sys was added. However, mariadb.sys was deleted
+  # during init mariadb as it was not on the exclusion list. This corrupted the view of mysql.user.
+  # Insert the tuple back to avoid other similar issues with error i.e
+  #   The user specified as a definer ('mariadb.sys'@'localhost') does not exist
+  #
+  # Before insert the tuple mentioned above, we should make sure that the MariaDB version is 10.4.+
+  mariadb_version=$($MYSQL_LOCAL_SHORT -e "status" | grep -E '^Server\s+version:')
+  log "Current database ${mariadb_version}"
+  if [[ ! -z ${mariadb_version} && -z $(grep '10.2' <<< ${mariadb_version}}) ]]; then
+    if [[ -z $(grep 'mariadb.sys' <<< $($MYSQL_LOCAL_SHORT mysql  -e "select * from global_priv where user='mariadb.sys'")) ]]; then
+      $MYSQL_LOCAL_SHORT -e "insert into mysql.global_priv values ('localhost','mariadb.sys',\
+    '{\"access\":0,\"plugin\":\"mysql_native_password\",\"authentication_string\":\"\",\"account_locked\":true,\"password_last_changed\":0}');"
+      $MYSQL_LOCAL_SHORT -e 'flush privileges;'
+    fi
+  fi
+
+  # Ensure archive dir existed
+  if [ -d "$ARCHIVE_DIR" ]; then
+    # List archive dir before
+    list_archive_dir
+
+      # Ensure the local databases are clean for each restore validation
+      #
+      cleanup_local_databases
+
+      if [[ "${SCOPE}" == "all" ]]; then
+        archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | sort)
+        for i in $archive_files; do
+          archive_file_passed=$i.passed
+          if [ ! -f "$archive_file_passed" ]; then
+            log "Validating archive file $i..."
+            if validate_databases_backup "$i"; then
+              touch "$archive_file_passed"
+            else
+              if handle_bad_archive_file "$i"; then
+                log "File $i has been removed from RGW."
+              else
+                log "File $i cannot be removed from RGW." "ERROR"
+                return 1
+              fi
+            fi
+          fi
+        done
+      else
+        archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | grep "${SCOPE}" | sort)
+        for i in $archive_files; do
+          archive_file_passed=$i.passed
+          if [ ! -f "$archive_file_passed" ]; then
+            log "Validating archive file $i..."
+            if validate_databases_backup "${i}" "${SCOPE}"; then
+              touch "$archive_file_passed"
+            else
+              if handle_bad_archive_file "$i"; then
+                log "File $i has been removed from RGW."
+              else
+                log "File $i cannot be removed from RGW." "ERROR"
+                return 1
+              fi
+            fi
+          fi
+        done
+      fi
+
+
+    # Cleanup passed files if its archive file nolonger existed
+    cleanup_old_validation_result_file
+
+    # List archive dir after
+    list_archive_dir
+  fi
+
+
+  return 0
+}
+
 # Call main program to start the database backup
 backup_databases ${SCOPE}
diff --git a/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl b/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl
new file mode 100644
index 000000000..dce67fa15
--- /dev/null
+++ b/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl
@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+log () {
+  msg_default="Need some text to log"
+  level_default="INFO"
+  component_default="Mariadb Backup Verifier"
+
+  msg=${1:-$msg_default}
+  level=${2:-$level_default}
+  component=${3:-"$component_default"}
+
+  echo "$(date +'%Y-%m-%d %H:%M:%S,%3N') - ${component} - ${level} - ${msg}"
+}
+
+log "Starting Mariadb server for backup verification..."
+MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml
index d0abd08e3..a1e3657ec 100644
--- a/mariadb/templates/configmap-bin.yaml
+++ b/mariadb/templates/configmap-bin.yaml
@@ -38,6 +38,8 @@ data:
 {{- if .Values.conf.backup.enabled }}
   backup_mariadb.sh: |
 {{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  start_verification_server.sh: |
+{{ tuple "bin/_start_mariadb_verify_server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   restore_mariadb.sh: |
 {{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   backup_main.sh: |
diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml
index ef9db9bc6..db8c06639 100644
--- a/mariadb/templates/cron-job-backup-mariadb.yaml
+++ b/mariadb/templates/cron-job-backup-mariadb.yaml
@@ -52,6 +52,7 @@ spec:
 {{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
           serviceAccountName: {{ $serviceAccountName }}
           restartPolicy: OnFailure
+          shareProcessNamespace: true
 {{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
 {{ end }}
@@ -76,10 +77,29 @@ spec:
                   name: pod-tmp
                 - mountPath: {{ .Values.conf.backup.base_path }}
                   name: mariadb-backup-dir
+            - name: verify-perms
+{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
+              command:
+                - chown
+                - -R
+                - "65534:65534"
+                - /var/lib/mysql
+              volumeMounts:
+                - mountPath: /tmp
+                  name: pod-tmp
+                - mountPath: /var/lib/mysql
+                  name: mysql-data
           containers:
             - name: mariadb-backup
               command:
-                - /tmp/backup_mariadb.sh
+                - /bin/sh
+              args:
+                - -c
+                - >-
+                    /tmp/backup_mariadb.sh;
+                    /usr/bin/pkill mysqld
               env:
                 - name: MARIADB_BACKUP_BASE_DIR
                   value: {{ .Values.conf.backup.base_path | quote }}
@@ -131,12 +151,62 @@ spec:
                   subPath: admin_user.cnf
                   readOnly: true
 {{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
+            - name: mariadb-verify-server
+{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              env:
+                {{- if $envAll.Values.manifests.certificates }}
+                - name: MARIADB_X509
+                  value: "REQUIRE X509"
+                {{- end }}
+                - name: MYSQL_HISTFILE
+                  value: /dev/null
+                - name: MARIADB_BACKUP_BASE_DIR
+                  value: {{ .Values.conf.backup.base_path | quote }}
+              ports:
+                - name: mysql
+                  protocol: TCP
+                  containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              command:
+                - /tmp/start_verification_server.sh
+              volumeMounts:
+                - name: pod-tmp
+                  mountPath: /tmp
+                - name: var-run
+                  mountPath: /var/run/mysqld
+                - name: mycnfd
+                  mountPath: /etc/mysql/conf.d
+                - name: mariadb-etc
+                  mountPath: /etc/mysql/my.cnf
+                  subPath: my.cnf
+                  readOnly: true
+                - name: mariadb-secrets
+                  mountPath: /etc/mysql/admin_user.cnf
+                  subPath: admin_user.cnf
+                  readOnly: true
+                - name: mysql-data
+                  mountPath: /var/lib/mysql
+                - name: mariadb-bin
+                  mountPath: /tmp/start_verification_server.sh
+                  readOnly: true
+                  subPath: start_verification_server.sh
           restartPolicy: OnFailure
           serviceAccount: {{ $serviceAccountName }}
           serviceAccountName: {{ $serviceAccountName }}
           volumes:
             - name: pod-tmp
               emptyDir: {}
+            - name: mycnfd
+              emptyDir: {}
+            - name: var-run
+              emptyDir: {}
+            - name: mariadb-etc
+              configMap:
+                name: mariadb-etc
+                defaultMode: 0444
+            - name: mysql-data
+              emptyDir: {}
             - name: mariadb-secrets
               secret:
                 secretName: mariadb-secrets
diff --git a/mariadb/values.yaml b/mariadb/values.yaml
index b2393eb3d..f67e54855 100644
--- a/mariadb/values.yaml
+++ b/mariadb/values.yaml
@@ -122,10 +122,17 @@ pod:
         backup_perms:
           runAsUser: 0
           readOnlyRootFilesystem: true
+        verify_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
         mariadb_backup:
           runAsUser: 65534
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
+        mariadb_verify_server:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
     tests:
       pod:
         runAsUser: 999
@@ -328,9 +335,15 @@ conf:
   ingress_conf:
     worker-processes: "auto"
     log-format-stream: "\"$remote_addr [$time_local] $protocol $status $bytes_received $bytes_sent $upstream_addr $upstream_connect_time $upstream_first_byte_time $upstream_session_time $session_time\""
+  mariadb_server:
+    setup_wait:
+      iteration: 30
+      duration: 5
   backup:
     enabled: false
     base_path: /var/backup
+    validateData:
+      ageOffset: 120
     mysqldump_options: >
       --single-transaction --quick --add-drop-database
       --add-drop-table --add-locks --databases
diff --git a/mariadb/values_overrides/apparmor.yaml b/mariadb/values_overrides/apparmor.yaml
index f2f16c6cf..ffde96e81 100644
--- a/mariadb/values_overrides/apparmor.yaml
+++ b/mariadb/values_overrides/apparmor.yaml
@@ -15,6 +15,7 @@ pod:
     mariadb-backup:
       init: runtime/default
       mariadb-backup: runtime/default
+      mariadb-verify-server: runtime/default
     mariadb-test:
       init: runtime/default
       mariadb-test: runtime/default
diff --git a/postgresql/Chart.yaml b/postgresql/Chart.yaml
index 206ce9641..6d52ec027 100644
--- a/postgresql/Chart.yaml
+++ b/postgresql/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v9.6
 description: OpenStack-Helm PostgreSQL
 name: postgresql
-version: 0.1.16
+version: 0.1.17
 home: https://www.postgresql.org
 sources:
   - https://github.com/postgres/postgres
diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl
index 7d85b9eb4..a9ea35c35 100755
--- a/postgresql/templates/bin/_backup_postgresql.sh.tpl
+++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl
@@ -82,5 +82,13 @@ dump_databases_to_directory() {
   fi
 }
 
+# Verify all the databases backup archives
+verify_databases_backup_archives() {
+  ####################################
+  # TODO: add implementation of local backup verification
+  ####################################
+  return 0
+}
+
 # Call main program to start the database backup
 backup_databases ${SCOPE}
diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml
index acca41601..6dcc3fae9 100644
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@@ -54,4 +54,5 @@ helm-toolkit:
   - 0.2.45 Modify use_external_ingress_controller place in openstack-helm values.yaml
   - 0.2.46 Fixed for getting kibana ingress value parameters
   - 0.2.47 Adjusting of kibana ingress value parameters
+  - 0.2.48 Added verify_databases_backup_archives function call to backup process and added remote backup sha256 hash verification
 ...
diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml
index 0b34f257c..fd1ed9928 100644
--- a/releasenotes/notes/mariadb.yaml
+++ b/releasenotes/notes/mariadb.yaml
@@ -43,4 +43,5 @@ mariadb:
   - 0.2.25 Add liveness probe to restart a pod that got stuck in a transfer wsrep_local_state_comment
   - 0.2.26 Added OCI registry authentication
   - 0.2.27 Fix broken helmrelease for helmv3
+  - 0.2.28 Added verify_databases_backup_in_directory function implementation
 ...
diff --git a/releasenotes/notes/postgresql.yaml b/releasenotes/notes/postgresql.yaml
index 0ea3f7898..c0110677a 100644
--- a/releasenotes/notes/postgresql.yaml
+++ b/releasenotes/notes/postgresql.yaml
@@ -17,4 +17,5 @@ postgresql:
   - 0.1.14 Fix invalid fields in values
   - 0.1.15 Migrated CronJob resource to batch/v1 API version
   - 0.1.16 Added OCI registry authentication
+  - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit
 ...