Add backups for the new Keycloak server

We should really be backing this up before it begins to get used by
additional services. Also, since our newer deployment uses a
separate RDBMS, back that up safely.

Change-Id: I4510dd05204f4b0f450d1925ed7be148d7d73e6e

inventory/service/group_vars/keycloak.yaml

@@ -4,3 +4,8 @@ letsencrypt_certs:
     # and is referenced in the apache config.
     - keycloak.opendev.org
     - "{{ inventory_hostname }}"
+borg_backup_excludes_extra:
+  # db is backed up in dumps, don't capture live files
+  - /var/lib/keycloak/db
+  # backed up by streaming backup
+  - /var/backups/keycloak-mariadb

inventory/service/groups.yaml

@@ -31,12 +31,14 @@ groups:
     - eavesdrop01.opendev.org
     - paste01.opendev.org
     - lists01.opendev.org
+    - keycloak03.opendev.org
     # These are test specific hosts that we add to the backup
     # group to mimic as much as possible what their prod version
     # end up doing.
     - gitea99.opendev.org
     - review99.opendev.org
     - lists99.opendev.org
+    - keycloak99.opendev.org
     # All these servers are "special-cased" in specifically
     # as they are puppet and should be replaced "soon"
     - storyboard01.opendev.org

playbooks/roles/keycloak/tasks/main.yaml

@@ -78,3 +78,46 @@
 - name: Run docker prune to cleanup unneeded images
   shell:
     cmd: docker image prune -f
+
+#### Database Backups ####
+
+- name: Create db backup dest
+  file:
+    state: directory
+    path: /var/backups/keycloak-mariadb
+    mode: 0700
+    owner: root
+    group: root
+
+- name: Set up cron job to backup the database
+  cron:
+    name: keycloak-db-backup
+    state: present
+    user: root
+    job: >
+      /usr/local/bin/docker-compose -f /etc/keycloak-docker/docker-compose.yaml exec -T mariadb
+      bash -c '/usr/bin/mysqldump --opt --databases keycloak --single-transaction -uroot -p"$MARIADB_ROOT_PASSWORD"' |
+      gzip -9 > /var/backups/keycloak-mariadb/keycloak-mariadb.sql.gz
+    minute: 14
+    hour: 5
+
+- name: Rotate db backups
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/backups/keycloak-mariadb/keycloak-mariadb.sql.gz
+    logrotate_compress: false
+
+- name: Setup db backup streaming job
+  block:
+    - name: Create backup streaming config dir
+      file:
+        path: /etc/borg-streams
+        state: directory
+
+    - name: Create db streaming file
+      copy:
+        content: >-
+            /usr/local/bin/docker-compose -f /etc/keycloak-docker/docker-compose.yaml exec -T mariadb
+            bash -c '/usr/bin/mysqldump --skip-extended-insert --databases keycloak --single-transaction -uroot -p"$MARIADB_ROOT_PASSWORD"'
+        dest: /etc/borg-streams/mysql