diff --git a/playbooks/roles/backup/tasks/main.yaml b/playbooks/roles/backup/tasks/main.yaml
index e07f9cbcf9..88abd05e30 100644
--- a/playbooks/roles/backup/tasks/main.yaml
+++ b/playbooks/roles/backup/tasks/main.yaml
@@ -10,22 +10,28 @@
   package:
     name:
       - bup
-    state: present
+    state: absent
 
-- name: Generate keypair for backups
-  openssh_keypair:
+- name: Remove old keypair
+  file:
     path: /root/.ssh/id_backup_ed25519
-    type: ed25519
-  register: bup_keypair
+    state: absent
 
-- name: Initalise bup
-  command: bup init
-  when: bup_keypair.changed
+- name: Remove old keypair
+  file:
+    path: /root/.ssh/id_backup_ed25519.pub
+    state: absent
 
-- name: Configure ssh for backup server
+- name: Remove old config directory
+  file:
+    path: /root/.bup
+    state: absent
+
+- name: Remove ssh config
   blockinfile:
     path: /root/.ssh/config
-    create: true
+    state: absent
+    create: false
     block: |
       Host {{ item }}
       HostName {{ item }}
@@ -33,29 +39,19 @@
       User {{ bup_username }}
     mode: 0600
   with_inventory_hostnames: backup-server
+  ignore_errors: True
 
-- name: Generate bup_user info tuple
-  set_fact:
-    bup_user: '{{ [ bup_username, bup_keypair["public_key"] ] }}'
+- name: Remove /etc/bup-excludes
+  file:
+    path: /etc/bup-excludes
+    state: absent
 
-- name: Accept hostkey of backup server
-  known_hosts:
-    state: present
-    key: '{{ item }} ssh-ed25519 {{ hostvars[item]["ansible_ssh_host_key_ed25519_public"] }}'
-    name: '{{ item }}'
-  with_inventory_hostnames: backup-server
-
-- name: Write /etc/bup-excludes
-  copy:
-    src: bup-excludes
-    dest: /etc/bup-excludes
-    mode: 0444
-
-- name: Install backup cron job
+- name: Remove backup cronjob
   cron:
     name: "Run bup backup"
     job: "tar -X /etc/bup-excludes -cPF - / | bup split -r {{ bup_username }}@{{ item }}: -n root -q"
     user: root
     hour: '5'
     minute: '{{ 59|random(seed=item) }}'
+    state: absent
   with_inventory_hostnames: backup-server
diff --git a/playbooks/service-backup.yaml b/playbooks/service-backup.yaml
index fc66a03a48..9866cb65e8 100644
--- a/playbooks/service-backup.yaml
+++ b/playbooks/service-backup.yaml
@@ -1,12 +1,8 @@
-# This needs to happen in order.  Backup hosts export their username/key
-# combos which are installed onto the backup server
+# NOTE(ianw) : we are removing bup for borg.  This just needs to run
+# once to remove bup parts from the backup clients, then we will
+# remove it completely.
 - hosts: "backup:!disabled"
   name: "Base: Generate backup users and keys"
   roles:
     - iptables
     - backup
-- hosts: "backup-server:!disabled"
-  name: "Generate bup configuration"
-  roles:
-    - iptables
-    - backup-server
diff --git a/testinfra/test_backups.py b/testinfra/test_backups.py
deleted file mode 100644
index ed0daebab7..0000000000
--- a/testinfra/test_backups.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright 2019 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os.path
-import pytest
-
-testinfra_hosts = ['backup01.region.provider.opendev.org',
-                   'backup-test01.opendev.org',
-                   'backup-test02.opendev.org']
-
-
-def test_bup_installed(host):
-    package = host.package("bup")
-    assert package.is_installed
-
-def test_server_users(host):
-    hostname = host.backend.get_hostname()
-    if hostname.startswith('backup-test'):
-        pytest.skip()
-
-    for username in 'bup-backup01', 'bup-backup-test02':
-        homedir = os.path.join('/opt/backups/', username)
-        bup_config = os.path.join(homedir, '.bup', 'config')
-        authorized_keys = os.path.join(homedir, '.ssh', 'authorized_keys')
-
-        user = host.user(username)
-        assert user.exists
-        assert user.home == homedir
-
-        f = host.file(authorized_keys)
-        assert f.exists
-        assert f.contains("ssh-ed25519")
-
-        f = host.file(bup_config)
-        assert f.exists
-
-def test_backup_host_config(host):
-    hostname = host.backend.get_hostname()
-    if hostname == 'backup01.region.provider.opendev.org':
-        pytest.skip()
-
-    f = host.file('/root/.ssh/id_backup_ed25519')
-    assert f.exists
-
-    f = host.file('/root/.ssh/config')
-    assert f.exists
-    assert f.contains('Host backup01.region.provider.opendev.org')
-
-    f = host.file('/root/.bup/config')
-    assert f.exists