diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
index 41ba0e8bf54..c4b3d542766 100644
--- a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
@@ -2272,8 +2272,9 @@ class OVSNeutronAgent(l2population_rpc.L2populationRpcCallBackTunnelMixin,
                        binding_no_activated_devices - migrating_devices)
         self.process_install_ports_egress_flows(need_binding_devices)
         added_to_datapath = added_ports - devices_not_in_datapath
-        self.sg_agent.setup_port_filters(added_to_datapath,
-                                         port_info.get('updated', set()))
+        self.sg_agent.setup_port_filters(
+            added_to_datapath,
+            port_info.get('updated', set()) - binding_no_activated_devices)
 
         LOG.info("process_network_ports - iteration:%(iter_num)d - "
                  "agent port security group processed in %(elapsed).3f",
diff --git a/neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py b/neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py
index d8dd778e906..da918476295 100644
--- a/neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py
+++ b/neutron/tests/unit/plugins/ml2/drivers/openvswitch/agent/test_ovs_neutron_agent.py
@@ -1212,7 +1212,7 @@ class TestOvsNeutronAgent(object):
             setup_port_filters.assert_called_once_with(
                 (added_devices - set(skipped_devices) -
                     binding_no_activated_devices),
-                port_info.get('updated', set()))
+                port_info.get('updated', set()) - binding_no_activated_devices)
             devices_added_updated = (added_devices |
                                      port_info.get('updated', set()))
             if devices_added_updated: