From 5ca69113fd2c070401fb7ab2f3b58e9f109c06c5 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 16 Feb 2021 14:42:36 +1100
Subject: [PATCH] borg-backup: send explicit email on backup failure
This sets a global BORG_UNDER_CRON=1 environment variable for
production hosts and makes the borg-backup script send an email if any
part of the backup job appears to fail (this avoids spamming ourselves
if we're testing backups, etc).
We should ideally never get this email, but if we do it's something we
want to investigate quickly. There's nothing worse than thinking
backups are working when they aren't.
Change-Id: Ibb63f19817782c25a5929781b0f6342fe4c82cf0
---
playbooks/roles/borg-backup/tasks/main.yaml | 6 ++++++
playbooks/roles/borg-backup/templates/borg-backup.j2 | 5 +++++
2 files changed, 11 insertions(+)
diff --git a/playbooks/roles/borg-backup/tasks/main.yaml b/playbooks/roles/borg-backup/tasks/main.yaml
index b8418ab178..4ccc7f867b 100644
--- a/playbooks/roles/borg-backup/tasks/main.yaml
+++ b/playbooks/roles/borg-backup/tasks/main.yaml
@@ -53,6 +53,12 @@
name: '{{ item }}'
with_inventory_hostnames: borg-backup-server
+- name: Set cron flag to enable error reports
+ cron:
+ name: BORG_UNDER_CRON
+ env: yes
+ job: '1'
+
- name: Install backup cron job
cron:
name: "Run borg backup to {{ item }}"
diff --git a/playbooks/roles/borg-backup/templates/borg-backup.j2 b/playbooks/roles/borg-backup/templates/borg-backup.j2
index d645575135..f5a24df369 100644
--- a/playbooks/roles/borg-backup/templates/borg-backup.j2
+++ b/playbooks/roles/borg-backup/templates/borg-backup.j2
@@ -60,7 +60,12 @@ if [ ${backup_exit} -eq 0 ]; then
info "Backup finished successfully"
else
info "Backup finished with errors"
+ if [ ${BORG_UNDER_CRON:-0} -eq 1 ]; then
+ echo "Backups failed on host $(hostname) at $(date)." | \
+ mail -s "ACTION REQUIRED: Backup failed on $(hostname)" infra-root@openstack.org
+ fi
fi
+
exit ${backup_exit}