diff --git a/inventory/service/host_vars/etherpad01.opendev.org.yaml b/inventory/service/host_vars/etherpad01.opendev.org.yaml index c7bd142b12..e3e4112a23 100644 --- a/inventory/service/host_vars/etherpad01.opendev.org.yaml +++ b/inventory/service/host_vars/etherpad01.opendev.org.yaml @@ -8,3 +8,5 @@ etherpad_redirect_vhost: etherpad.openstack.org borg_backup_excludes_extra: # live db; we store daily dumps - /var/etherpad/* + # local db backups, we store stream + - /var/backups/etherpad-mariadb diff --git a/inventory/service/host_vars/gitea01.opendev.org.yaml b/inventory/service/host_vars/gitea01.opendev.org.yaml index 9587736b73..5bc0b20d63 100644 --- a/inventory/service/host_vars/gitea01.opendev.org.yaml +++ b/inventory/service/host_vars/gitea01.opendev.org.yaml @@ -7,3 +7,5 @@ borg_backup_excludes_extra: - /var/gitea/data/ # db is backed up in dumps, don't capture live files - /var/gitea/db + # backed up by streaming backup + - /var/backups/gitea-mariadb diff --git a/inventory/service/host_vars/review01.openstack.org.yaml b/inventory/service/host_vars/review01.openstack.org.yaml index 39992072a8..cec2216945 100644 --- a/inventory/service/host_vars/review01.openstack.org.yaml +++ b/inventory/service/host_vars/review01.openstack.org.yaml @@ -76,3 +76,5 @@ borg_backup_excludes_extra: - /home/gerrit2/review_site/cache/* - /home/gerrit2/review_site/tmp/* - /home/gerrit2/review_site/index/* + # dump directly via stream + - /home/gerrit2/mysql_backups/* diff --git a/playbooks/roles/borg-backup/README.rst b/playbooks/roles/borg-backup/README.rst index c97c88161f..b751d9f477 100644 --- a/playbooks/roles/borg-backup/README.rst +++ b/playbooks/roles/borg-backup/README.rst @@ -15,6 +15,12 @@ correctly on the backup server. This role sets a tuple ``borg_user`` with the username and public key; the ``borg-backup-server`` role uses this variable for each host in the ``borg-backup`` group to initalise users. +Hosts can place into ``/etc/borg-streams`` which should be a script +that outputs to stdout data to be fed into a backup archive on each +run. This will be saved to an archive with the name of the file. +This is useful for raw database dumps which allow ``borg`` to +deduplicate as much as possible. + **Role Variables** .. zuul:rolevar:: borg_username diff --git a/playbooks/roles/borg-backup/templates/borg-backup.j2 b/playbooks/roles/borg-backup/templates/borg-backup.j2 index 4ac8225e52..5a05b6e3f9 100644 --- a/playbooks/roles/borg-backup/templates/borg-backup.j2 +++ b/playbooks/roles/borg-backup/templates/borg-backup.j2 @@ -9,6 +9,7 @@ if [ -z "$1" ]; then fi BORG="/opt/borg/bin/borg" +BORG_CREATE="${BORG} create --verbose --filter AME --list --stats --show-rc --compression lz4 --exclude-caches " # Setting this, so the repo does not need to be given on the commandline: export BORG_REPO="ssh://{{ borg_username}}@${1}/opt/backups/{{ borg_username }}/backup" @@ -24,31 +25,35 @@ export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=1 # Backup the most important directories into an archive named after # the machine this script is currently running on: -${BORG} create \ - --verbose \ - --filter AME \ - --list \ - --stats \ - --show-rc \ - --compression lz4 \ - --exclude-caches \ +${BORG_CREATE} \ {% for item in borg_backup_excludes + borg_backup_excludes_extra -%} --exclude '{{ item }}' \ {% endfor -%} - \ - ::'{hostname}-{now}' \ + ::'{hostname}-filesystem-{now}' \ {% for item in borg_backup_dirs + borg_backup_dirs_extra -%} {{ item }} {{ '\\' if not loop.last }} {% endfor -%} backup_exit=$? -if [ ${backup_exit} -eq 0 ]; then - info "Running prune" - ${BORG} prune --verbose --list --prefix '{hostname}-' \ - --show-rc --keep-daily 7 --keep-weekly 4 --keep-monthly 12 - backup_exit=$? -fi +for f in $(shopt -s nullglob; echo /etc/borg-streams/*) +do + stream_name=$(basename $f) + info "Backing up stream archive $stream_name" + bash $f | ${BORG_CREATE} --stdin-name ${stream_name} \ + ::"{hostname}-${stream_name}-{now}" - + + if [[ ${PIPESTATUS[0]} -ne 0 ]]; then + info "Streaming script ${f} failed!" + stream_exit=${PIPESTATUS[0]} + elif [[ ${PIPESTATUS[1]} -ne 1 ]]; then + info "Borg failed!" + stream_exit=${PIPESTATUS[1]} + else + stream_exit=0 + fi + (( backup_exit = backup_exit || stream_exit )) +done if [ ${backup_exit} -eq 0 ]; then info "Backup finished successfully" diff --git a/playbooks/roles/etherpad/tasks/main.yaml b/playbooks/roles/etherpad/tasks/main.yaml index 4d5967aee3..7b118328f9 100644 --- a/playbooks/roles/etherpad/tasks/main.yaml +++ b/playbooks/roles/etherpad/tasks/main.yaml @@ -123,7 +123,7 @@ owner: root group: root -- name: Set up cron job to backup the database +- name: Set up cron job for local database backup cron: name: etherpad-db-backup state: present @@ -142,3 +142,17 @@ logrotate_rotate: 2 logrotate_file_name: /var/backups/etherpad-mariadb/etherpad-mariadb.sql.gz logrotate_compress: false + +- name: Setup db backup streaming job + block: + - name: Create backup streaming config dir + file: + path: /etc/borg-streams + state: directory + + - name: Create db streaming file + copy: + content: >- + /usr/local/bin/docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec -T mariadb + bash -c '/usr/bin/mysqldump --skip-extended-insert --databases etherpad-lite --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' + dest: /etc/borg-streams/mysql diff --git a/playbooks/roles/gerrit/tasks/main.yaml b/playbooks/roles/gerrit/tasks/main.yaml index 1c4f244a60..edad89e89a 100644 --- a/playbooks/roles/gerrit/tasks/main.yaml +++ b/playbooks/roles/gerrit/tasks/main.yaml @@ -338,3 +338,16 @@ job: 'find /home/gerrit2/review_site/logs/*.gz -mtime +30 -exec rm -f {} \;' minute: 1 hour: 6 + +- name: Setup db backup streaming job + block: + - name: Create backup streaming config dir + file: + path: /etc/borg-streams + state: directory + + - name: Create db streaming file + copy: + content: >- + /usr/bin/mysqldump --defaults-file=/root/.gerrit_db.cnf --skip-extended-insert --ignore-table mysql.event --all-databases --single-transaction + dest: /etc/borg-streams/mysql diff --git a/playbooks/roles/gitea/tasks/main.yaml b/playbooks/roles/gitea/tasks/main.yaml index 60164f0be3..6221e3d242 100644 --- a/playbooks/roles/gitea/tasks/main.yaml +++ b/playbooks/roles/gitea/tasks/main.yaml @@ -186,3 +186,16 @@ vars: logrotate_file_name: /var/backups/gitea-mariadb/gitea-mariadb.sql.gz logrotate_compress: false +- name: Setup db backup streaming job + block: + - name: Create backup streaming config dir + file: + path: /etc/borg-streams + state: directory + + - name: Create db streaming file + copy: + content: >- + /usr/local/bin/docker-compose -f /etc/gitea-docker/docker-compose.yaml exec -T mariadb + bash -c '/usr/bin/mysqldump --skip-extended-insert --databases gitea --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'T_PASSWORD"' + dest: /etc/borg-streams/mysql diff --git a/playbooks/test-borg-backup.yaml b/playbooks/test-borg-backup.yaml new file mode 100644 index 0000000000..3b449a3e07 --- /dev/null +++ b/playbooks/test-borg-backup.yaml @@ -0,0 +1,14 @@ +- hosts: "borg-backup" + tasks: + - name: Setup db backup streaming job + block: + - name: Create backup streaming config dir + file: + path: /etc/borg-streams + state: directory + + - name: Create sample streaming file + copy: + content: >- + dd if=/dev/urandom bs=1M count=5 + dest: /etc/borg-streams/random diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 45413988b0..f89c7b696d 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -347,7 +347,10 @@ vars: run_playbooks: - playbooks/service-borg-backup.yaml + run_test_playbook: playbooks/test-borg-backup.yaml files: + - playbooks/service-borg-backup.yaml + - playbooks/test-borg-bcakup.yaml - playbooks/install-ansible.yaml - playbooks/roles/install-borg - playbooks/roles/borg-backup