diff --git a/kuryr_kubernetes/controller/drivers/lbaasv2.py b/kuryr_kubernetes/controller/drivers/lbaasv2.py
index 84773d3ff..ec74a4f76 100644
--- a/kuryr_kubernetes/controller/drivers/lbaasv2.py
+++ b/kuryr_kubernetes/controller/drivers/lbaasv2.py
@@ -170,11 +170,16 @@ class LBaaSv2Driver(base.LBaaSDriver):
         all_pod_rules = []
         add_default_rules = False
         os_net = clients.get_network_client()
+        sgs = []
 
         if new_sgs:
             sgs = new_sgs
-        else:
+        elif loadbalancer['security_groups']:
             sgs = loadbalancer['security_groups']
+        else:
+            # NOTE(gryf): in case there is no new SG rules and loadbalancer
+            # has the SG removed, just add default ones.
+            add_default_rules = True
 
         # Check if Network Policy allows listener on the pods
         for sg in sgs:
diff --git a/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py b/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py
index bd5b9d63c..3ea3a8149 100644
--- a/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py
+++ b/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py
@@ -298,6 +298,8 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
                 # Probably the network got removed already, we can ignore it.
                 pass
 
+            self._drv_policy.delete_np_sg(crd_sg)
+
             if (CONF.octavia_defaults.enforce_sg_rules and policy and
                     not self._is_egress_only_policy(policy)):
                 services = driver_utils.get_services(
@@ -306,7 +308,14 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
                     if (not svc['spec'].get('selector') or not
                             self._is_service_affected(svc, pods_to_update)):
                         continue
+
                     sgs = self._drv_svc_sg.get_security_groups(svc, project_id)
+
+                    if crd_sg in sgs:
+                        # Remove our crd_sg out of service groups since we
+                        # don't have it anymore
+                        sgs.remove(crd_sg)
+
                     try:
                         self._drv_lbaas.update_lbaas_sg(svc, sgs)
                     except exceptions.ResourceNotReady:
@@ -314,8 +323,6 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
                         # get handled when members will be getting created.
                         pass
 
-            self._drv_policy.delete_np_sg(crd_sg)
-
         LOG.debug("Removing finalizers from KuryrNetworkPolicy and "
                   "NetworkPolicy.")
         if policy: