From 5ca69113fd2c070401fb7ab2f3b58e9f109c06c5 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 16 Feb 2021 14:42:36 +1100 Subject: [PATCH] borg-backup: send explicit email on backup failure This sets a global BORG_UNDER_CRON=1 environment variable for production hosts and makes the borg-backup script send an email if any part of the backup job appears to fail (this avoids spamming ourselves if we're testing backups, etc). We should ideally never get this email, but if we do it's something we want to investigate quickly. There's nothing worse than thinking backups are working when they aren't. Change-Id: Ibb63f19817782c25a5929781b0f6342fe4c82cf0 --- playbooks/roles/borg-backup/tasks/main.yaml | 6 ++++++ playbooks/roles/borg-backup/templates/borg-backup.j2 | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/playbooks/roles/borg-backup/tasks/main.yaml b/playbooks/roles/borg-backup/tasks/main.yaml index b8418ab178..4ccc7f867b 100644 --- a/playbooks/roles/borg-backup/tasks/main.yaml +++ b/playbooks/roles/borg-backup/tasks/main.yaml @@ -53,6 +53,12 @@ name: '{{ item }}' with_inventory_hostnames: borg-backup-server +- name: Set cron flag to enable error reports + cron: + name: BORG_UNDER_CRON + env: yes + job: '1' + - name: Install backup cron job cron: name: "Run borg backup to {{ item }}" diff --git a/playbooks/roles/borg-backup/templates/borg-backup.j2 b/playbooks/roles/borg-backup/templates/borg-backup.j2 index d645575135..f5a24df369 100644 --- a/playbooks/roles/borg-backup/templates/borg-backup.j2 +++ b/playbooks/roles/borg-backup/templates/borg-backup.j2 @@ -60,7 +60,12 @@ if [ ${backup_exit} -eq 0 ]; then info "Backup finished successfully" else info "Backup finished with errors" + if [ ${BORG_UNDER_CRON:-0} -eq 1 ]; then + echo "Backups failed on host $(hostname) at $(date)." | \ + mail -s "ACTION REQUIRED: Backup failed on $(hostname)" infra-root@openstack.org + fi fi + exit ${backup_exit}