diff --git a/zun/compute/manager.py b/zun/compute/manager.py
index 105edf275..3d826cd4c 100644
--- a/zun/compute/manager.py
+++ b/zun/compute/manager.py
@@ -233,8 +233,8 @@ class Manager(object):
         LOG.debug('Adding security_group to container: %s', container.uuid)
         try:
             sandbox_id = self.driver.get_sandbox_id(container)
-            self.driver.add_security_group(context, sandbox_id,
-                                           security_group)
+            self.driver.add_security_group(context, container, security_group,
+                                           sandbox_id=sandbox_id)
             container.security_groups += [security_group]
             container.save(context)
         except Exception as e:
diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py
index e296b86b6..c48bad35a 100644
--- a/zun/container/docker/driver.py
+++ b/zun/container/docker/driver.py
@@ -666,7 +666,8 @@ class DockerDriver(driver.ContainerDriver):
                         cpu_used += float(nanocpus) / 1e9
             return cpu_used
 
-    def add_security_group(self, context, sandbox_id, security_group):
+    def add_security_group(self, context, container, security_group,
+                           sandbox_id=None):
         security_group_ids = self._get_security_group_ids(
             context, [security_group])
         with docker_utils.docker_client() as docker:
@@ -674,14 +675,14 @@ class DockerDriver(driver.ContainerDriver):
             sandbox = docker.inspect_container(sandbox_id)
             for network in sandbox["NetworkSettings"]["Networks"]:
                 network_api.add_security_groups_to_ports(
-                    sandbox, network, security_group_ids)
+                    container, security_group_ids, sandbox_id)
 
     def get_available_nodes(self):
         return [self._host.get_hostname()]
 
 
 class NovaDockerDriver(DockerDriver):
-    def add_security_group(self, context, sandbox_id, security_group):
+    def add_security_group(self, context, container, security_group, **kwargs):
         msg = "NovaDockerDriver does not support security_groups"
         raise exception.ZunException(msg)
 
diff --git a/zun/container/driver.py b/zun/container/driver.py
index c2f534650..e470d7d21 100644
--- a/zun/container/driver.py
+++ b/zun/container/driver.py
@@ -200,7 +200,7 @@ class ContainerDriver(object):
     def get_cpu_used(self):
         raise NotImplementedError()
 
-    def add_security_group(self, context, sandbox_id, security_group):
+    def add_security_group(self, context, container, security_group, **kwargs):
         raise NotImplementedError()
 
     def get_available_resources(self, node):
diff --git a/zun/network/kuryr_network.py b/zun/network/kuryr_network.py
index fd640e065..801fbbb6d 100644
--- a/zun/network/kuryr_network.py
+++ b/zun/network/kuryr_network.py
@@ -182,32 +182,28 @@ class KuryrNetwork(network.Network):
                             'or neutron tag extension does not supported or'
                             ' not enabled.')
 
-    def add_security_groups_to_ports(self, container, network_name,
-                                     security_group_ids):
-        container_id = container['Id']
-        neutron_ports = None
-        if "NetworkSettings" in container:
-            network = container["NetworkSettings"]["Networks"][network_name]
-            endpoint_id = network["EndpointID"]
-            # Kuryr set the port's device_id as endpoint_id so we leverge it
-            neutron_ports = self.neutron.list_ports(device_id=endpoint_id)
-            neutron_ports = neutron_ports.get('ports', [])
-            if not neutron_ports:
-                raise exceptions.ZunException(
-                    "Cannot find the neutron port that bind container "
-                    "%s to network %s", container_id, network_name)
-            for port in neutron_ports:
-                if 'security_groups' not in port:
-                    port['security_groups'] = []
-                port['security_groups'].extend(security_group_ids)
-                updated_port = {'security_groups': port['security_groups']}
-                try:
-                    LOG.info("Adding security group %(security_group_ids)s "
-                             "to port %(port_id)s",
-                             {'security_group_ids': security_group_ids,
-                              'port_id': port['id']})
-                    self.neutron.update_port(port['id'],
-                                             {'port': updated_port})
-                except Exception:
-                    with excutils.save_and_reraise_exception():
-                        LOG.exception("Neutron Error:")
+    def add_security_groups_to_ports(self, container, security_group_ids,
+                                     sandbox_id=None):
+        port_ids = set()
+        for addrs_list in container.addresses.values():
+            for addr in addrs_list:
+                port_id = addr['port']
+                port_ids.add(port_id)
+
+        neutron_ports = self.neutron.list_ports().get('ports', [])
+        neutron_ports = [p for p in neutron_ports if p['id'] in port_ids]
+        for port in neutron_ports:
+            if 'security_groups' not in port:
+                port['security_groups'] = []
+            port['security_groups'].extend(security_group_ids)
+            updated_port = {'security_groups': port['security_groups']}
+            try:
+                LOG.info("Adding security group %(security_group_ids)s "
+                         "to port %(port_id)s" %
+                         {'security_group_ids': security_group_ids,
+                          'port_id': port['id']})
+                self.neutron.update_port(port['id'],
+                                         {'port': updated_port})
+            except Exception:
+                with excutils.save_and_reraise_exception():
+                    LOG.exception("Neutron Error:")