diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py
index c93eb330e28..e34c1ddb172 100644
--- a/neutron/agent/l3/ha_router.py
+++ b/neutron/agent/l3/ha_router.py
@@ -199,7 +199,10 @@ class HaRouter(router.RouterInfo):
             return
         self.keepalived_manager.disable()
         conf_dir = self.keepalived_manager.get_conf_dir()
-        shutil.rmtree(conf_dir)
+        try:
+            shutil.rmtree(conf_dir)
+        except FileNotFoundError:
+            pass
 
     def _get_keepalived_instance(self):
         return self.keepalived_manager.config.get_instance(self.ha_vr_id)