diff --git a/octavia/amphorae/backends/agent/api_server/util.py b/octavia/amphorae/backends/agent/api_server/util.py
index 90922a896c..1a6114ea64 100644
--- a/octavia/amphorae/backends/agent/api_server/util.py
+++ b/octavia/amphorae/backends/agent/api_server/util.py
@@ -347,6 +347,13 @@ def vrrp_check_script_update(lb_id, action):
         if not udp_ids:
             with open(haproxy_check_script_path(), 'w') as text_file:
                 text_file.write('exit 1')
+        else:
+            try:
+                LOG.debug("Attempting to remove old haproxy check script...")
+                os.remove(haproxy_check_script_path())
+                LOG.debug("Finished removing old haproxy check script.")
+            except FileNotFoundError:
+                LOG.debug("No haproxy check script to remove.")
         return
     if action == consts.AMP_ACTION_STOP:
         lb_ids.remove(lb_id)
diff --git a/releasenotes/notes/fix-udp-only-lbs-c4ca42106fc1e2bb.yaml b/releasenotes/notes/fix-udp-only-lbs-c4ca42106fc1e2bb.yaml
new file mode 100644
index 0000000000..90e330895b
--- /dev/null
+++ b/releasenotes/notes/fix-udp-only-lbs-c4ca42106fc1e2bb.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed an issue where UDP only load balancers would not bring up the VIP
+    address.