diff --git a/neutron/agent/linux/openvswitch_firewall/firewall.py b/neutron/agent/linux/openvswitch_firewall/firewall.py
index f0969c5b0bb..70f7e8b10b5 100644
--- a/neutron/agent/linux/openvswitch_firewall/firewall.py
+++ b/neutron/agent/linux/openvswitch_firewall/firewall.py
@@ -293,7 +293,7 @@ class ConjIPFlowManager(object):
         addr_to_conj = collections.defaultdict(list)
         for remote_id, conj_id_set in sg_conj_id_map.items():
             remote_group = self.driver.sg_port_map.get_sg(remote_id)
-            if not remote_group or not remote_group.ports:
+            if not remote_group or not remote_group.members:
                 LOG.debug('No member for SG %s', remote_id)
                 continue
             for addr in remote_group.get_ethertype_filtered_addresses(
diff --git a/neutron/tests/unit/agent/linux/openvswitch_firewall/test_firewall.py b/neutron/tests/unit/agent/linux/openvswitch_firewall/test_firewall.py
index a1de1f7de56..aa6992a5da5 100644
--- a/neutron/tests/unit/agent/linux/openvswitch_firewall/test_firewall.py
+++ b/neutron/tests/unit/agent/linux/openvswitch_firewall/test_firewall.py
@@ -303,9 +303,9 @@ class TestConjIPFlowManager(base.BaseTestCase):
         self.vlan_tag = 100
         self.conj_id = 16
 
-    def test_update_flows_for_vlan_no_ports(self):
+    def test_update_flows_for_vlan_no_members(self):
         remote_group = self.driver.sg_port_map.get_sg.return_value
-        remote_group.ports = {}
+        remote_group.members = {}
         with mock.patch.object(self.manager.conj_id_map,
                                'get_conj_id') as get_conj_id_mock:
             get_conj_id_mock.return_value = self.conj_id
@@ -315,6 +315,21 @@ class TestConjIPFlowManager(base.BaseTestCase):
         self.assertFalse(remote_group.get_ethertype_filtered_addresses.called)
         self.assertFalse(self.driver._add_flow.called)
 
+    def test_update_flows_for_vlan_no_ports_but_members(self):
+        remote_group = self.driver.sg_port_map.get_sg.return_value
+        remote_group.ports = set()
+        remote_group.members = {constants.IPv4: ['10.22.3.4']}
+        remote_group.get_ethertype_filtered_addresses.return_value = [
+            '10.22.3.4']
+        with mock.patch.object(self.manager.conj_id_map,
+                               'get_conj_id') as get_conj_id_mock:
+            get_conj_id_mock.return_value = self.conj_id
+            self.manager.add(self.vlan_tag, 'sg', 'remote_id',
+                             constants.INGRESS_DIRECTION, constants.IPv4, 0)
+            self.manager.update_flows_for_vlan(self.vlan_tag)
+        self.assertTrue(remote_group.get_ethertype_filtered_addresses.called)
+        self.assertTrue(self.driver._add_flow.called)
+
     def test_update_flows_for_vlan(self):
         remote_group = self.driver.sg_port_map.get_sg.return_value
         remote_group.get_ethertype_filtered_addresses.return_value = [
diff --git a/releasenotes/notes/fix-remote-security-group-no-port-on-host-9177e66d4b16e90c.yaml b/releasenotes/notes/fix-remote-security-group-no-port-on-host-9177e66d4b16e90c.yaml
new file mode 100644
index 00000000000..f75035d7bd5
--- /dev/null
+++ b/releasenotes/notes/fix-remote-security-group-no-port-on-host-9177e66d4b16e90c.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Fixes an issue that the OVS firewall driver does not configure security
+    group rules using remote group properly when a corresponding remote group
+    has no port on a local hypervisor. For more information
+    see bugs: `1862703 <https://bugs.launchpad.net/neutron/+bug/1862703>`_
+    and `1854131 <https://bugs.launchpad.net/neutron/+bug/1854131>`__.