diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py
index 7806db347d..f907cb8b46 100644
--- a/tests/unit/test_merger_repo.py
+++ b/tests/unit/test_merger_repo.py
@@ -163,6 +163,64 @@ class TestMergerRepo(ZuulTestCase):
         work_repo.reset()
         work_repo.checkout("foobar")
 
+    def test_rebase_merge_conflict_abort(self):
+        """Test that a failed rebase is properly aborted and related
+        directories are cleaned up."""
+        parent_path = os.path.join(self.upstream_root, 'org/project1')
+        parent_repo = git.Repo(parent_path)
+        parent_repo.create_head("feature")
+
+        files = {"test.txt": "master"}
+        self.create_commit("org/project1", files=files, head="master",
+                           message="Add master file")
+
+        files = {"test.txt": "feature"}
+        self.create_commit("org/project1", files=files, head="feature",
+                           message="Add feature file")
+
+        work_repo = Repo(parent_path, self.workspace_root,
+                         "none@example.org", "User Name", "0", "0")
+
+        item = {"ref": "refs/heads/feature"}
+        # We expect the rebase to fail because of a conflict, but the
+        # rebase will be aborted.
+        with testtools.ExpectedException(git.exc.GitCommandError):
+            work_repo.rebaseMerge(item, "master")
+
+        # Assert that the failed rebase doesn't leave any temporary
+        # directories behind.
+        self.assertFalse(
+            os.path.exists(f"{work_repo.local_path}/.git/rebase-merge"))
+        self.assertFalse(
+            os.path.exists(f"{work_repo.local_path}/.git/rebase-apply"))
+
+    def test_rebase_merge_conflict_reset_cleanup(self):
+        """Test temporary directories of a failed rebase merge are
+        removed on repo reset."""
+        parent_path = os.path.join(self.upstream_root, 'org/project1')
+        parent_repo = git.Repo(parent_path)
+        parent_repo.create_head("feature")
+
+        files = {"master.txt": "master"}
+        self.create_commit("org/project1", files=files, head="master",
+                           message="Add master file")
+
+        files = {"feature.txt": "feature"}
+        self.create_commit("org/project1", files=files, head="feature",
+                           message="Add feature file")
+
+        work_repo = Repo(parent_path, self.workspace_root,
+                         "none@example.org", "User Name", "0", "0")
+
+        # Simulate leftovers from a failed rebase
+        os.mkdir(f"{work_repo.local_path}/.git/rebase-merge")
+        os.mkdir(f"{work_repo.local_path}/.git/rebase-apply")
+
+        # Resetting the repo should clean up any leaked directories
+        work_repo.reset()
+        item = {"ref": "refs/heads/feature"}
+        work_repo.rebaseMerge(item, "master")
+
     def test_set_refs(self):
         parent_path = os.path.join(self.upstream_root, 'org/project1')
         remote_sha = self.create_commit('org/project1')
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 1e2de27b8d..e4688a1b7c 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -332,6 +332,26 @@ class Repo(object):
                     messages.append("Cleaning empty ref dir %s" % root)
                 os.rmdir(root)
 
+    @staticmethod
+    def _cleanup_leaked_rebase_dirs(local_path, log, messages):
+        for rebase_dir in [".git/rebase-merge", ".git/rebase-apply"]:
+            leaked_dir = os.path.join(local_path, rebase_dir)
+            if not os.path.exists(leaked_dir):
+                continue
+            if log:
+                log.debug("Cleaning leaked %s dir", leaked_dir)
+            else:
+                messages.append(
+                    f"Cleaning leaked {leaked_dir} dir")
+            try:
+                shutil.rmtree(leaked_dir)
+            except Exception as exc:
+                msg = f"Failed to remove leaked {leaked_dir} dir:"
+                if log:
+                    log.exception(msg)
+                else:
+                    messages.append(f"{msg}\n{exc}")
+
     @staticmethod
     def refNameToZuulRef(ref_name: str) -> str:
         return "refs/zuul/{}".format(
@@ -384,6 +404,8 @@ class Repo(object):
                 messages.append("Delete stale Zuul ref {}".format(ref))
             Repo._deleteRef(ref.path, repo)
 
+        Repo._cleanup_leaked_rebase_dirs(local_path, log, messages)
+
         # Note: Before git 2.13 deleting a a ref foo/bar leaves an empty
         # directory foo behind that will block creating the reference foo
         # in the future. As a workaround we must clean up empty directories
@@ -615,7 +637,11 @@ class Repo(object):
         self.fetch(ref, zuul_event_id=zuul_event_id)
         log.debug("Rebasing %s with args %s", ref, args)
         repo.git.checkout('FETCH_HEAD')
-        repo.git.rebase(*args)
+        try:
+            repo.git.rebase(*args)
+        except Exception:
+            repo.git.rebase(abort=True)
+            raise
         return repo.head.commit
 
     def fetch(self, ref, zuul_event_id=None):