diff --git a/neutron/agent/securitygroups_rpc.py b/neutron/agent/securitygroups_rpc.py
index 9b0d5797fb..ffdc21fd22 100644
--- a/neutron/agent/securitygroups_rpc.py
+++ b/neutron/agent/securitygroups_rpc.py
@@ -125,13 +125,14 @@ class SecurityGroupAgentRpcMixin(object):
             'security_group_source_groups')
 
     def _security_group_updated(self, security_groups, attribute):
-        #check need update or not
+        devices = []
+        sec_grp_set = set(security_groups)
         for device in self.firewall.ports.values():
-            if set(device.get(attribute,
-                              [])).intersection(
-                    set(security_groups)):
-                    self.refresh_firewall()
-                    return
+            if sec_grp_set & set(device.get(attribute, [])):
+                devices.append(device)
+
+        if devices:
+            self.refresh_firewall(devices)
 
     def security_groups_provider_updated(self):
         LOG.info(_("Provider rule updated"))
@@ -148,10 +149,15 @@ class SecurityGroupAgentRpcMixin(object):
                     continue
                 self.firewall.remove_port_filter(device)
 
-    def refresh_firewall(self):
+    def refresh_firewall(self, devices=None):
         LOG.info(_("Refresh firewall rules"))
-        device_ids = self.firewall.ports.keys()
+
+        if devices:
+            device_ids = [d['device'] for d in devices]
+        else:
+            device_ids = self.firewall.ports.keys()
         if not device_ids:
+            LOG.info(_("No ports here to refresh firewall"))
             return
         devices = self.plugin_rpc.security_group_rules_for_devices(
             self.context, device_ids)
diff --git a/neutron/tests/unit/test_security_groups_rpc.py b/neutron/tests/unit/test_security_groups_rpc.py
index 235100d252..280b269ef7 100644
--- a/neutron/tests/unit/test_security_groups_rpc.py
+++ b/neutron/tests/unit/test_security_groups_rpc.py
@@ -465,7 +465,7 @@ class SecurityGroupAgentRpcTestCase(base.BaseTestCase):
         self.agent.prepare_devices_filter(['fake_port_id'])
         self.agent.security_groups_rule_updated(['fake_sgid1', 'fake_sgid3'])
         self.agent.refresh_firewall.assert_has_calls(
-            [call.refresh_firewall()])
+            [call.refresh_firewall([self.fake_device])])
 
     def test_security_groups_rule_not_updated(self):
         self.agent.refresh_firewall = mock.Mock()
@@ -478,7 +478,7 @@ class SecurityGroupAgentRpcTestCase(base.BaseTestCase):
         self.agent.prepare_devices_filter(['fake_port_id'])
         self.agent.security_groups_member_updated(['fake_sgid2', 'fake_sgid3'])
         self.agent.refresh_firewall.assert_has_calls(
-            [call.refresh_firewall()])
+            [call.refresh_firewall([self.fake_device])])
 
     def test_security_groups_member_not_updated(self):
         self.agent.refresh_firewall = mock.Mock()
@@ -501,6 +501,19 @@ class SecurityGroupAgentRpcTestCase(base.BaseTestCase):
                  call.update_port_filter(self.fake_device)]
         self.firewall.assert_has_calls(calls)
 
+    def test_refresh_firewall_devices(self):
+        self.agent.prepare_devices_filter(['fake_port_id'])
+        self.agent.refresh_firewall([self.fake_device])
+        calls = [call.defer_apply(),
+                 call.prepare_port_filter(self.fake_device),
+                 call.defer_apply(),
+                 call.update_port_filter(self.fake_device)]
+        self.firewall.assert_has_calls(calls)
+
+    def test_refresh_firewall_none(self):
+        self.agent.refresh_firewall([])
+        self.firewall.assert_has_calls([])
+
 
 class FakeSGRpcApi(agent_rpc.PluginApi,
                    sg_rpc.SecurityGroupServerRpcApiMixin):