system-config/playbooks/roles/zuul-status-backup/tasks/main.yaml

28 lines
1.2 KiB
YAML

# Minutes, hours, days, etc are not specified here because we are
# interested in running this *every minute*.
# This is a mean of backing up status.json periodically in order to provide
# a mean of restoring lost scheduler queues if need be.
# If the status.json is unavailable for download, no new files are created.
- name: Install cron for status backup
cron:
name: 'zuul-scheduler-status-{{ tenant }}'
state: present
user: root
job: |
timeout -k 5 10 curl https://zuul.opendev.org/api/tenant/{{ tenant }}/status --fail -o /var/lib/zuul/backup/{{ tenant }}_status_$(date +\%s).json 2>/dev/null
# Rotate backups and keep no more than 120 files -- or 2 hours worth of
# backup if Zuul has 100% uptime.
# We're not basing the rotation on time because the scheduler/web service
# could be down for an extended period of time.
# This is run hourly so technically up to ~3 hours worth of backups will
# be kept.
- name: Clean up old status backups
cron:
name: 'zuul-scheduler-status-prune-{{ tenant }}'
state: present
user: root
minute: '0'
job: |
flock -n /var/run/{{ tenant }}_status_prune.lock ls -dt -1 /var/lib/zuul/backup/{{ tenant }}_* |sed -e '1,120d' |xargs rm -f