diff --git a/etc/policy.json b/etc/policy.json
index fc1268c28ff..3d9830d610a 100644
--- a/etc/policy.json
+++ b/etc/policy.json
@@ -86,7 +86,6 @@
     "get_port:binding:vif_details": "rule:admin_only",
     "get_port:binding:host_id": "rule:admin_only",
     "get_port:binding:profile": "rule:admin_only",
-    "get_port:ipam_segment_id": "rule:admin_only",
     "update_port": "rule:admin_or_owner or rule:context_is_advsvc",
     "update_port:device_owner": "not rule:network_device or rule:context_is_advsvc or rule:admin_or_network_owner",
     "update_port:mac_address": "rule:admin_only or rule:context_is_advsvc",
diff --git a/neutron/extensions/segment.py b/neutron/extensions/segment.py
index 0b9be9394a1..0911726d806 100644
--- a/neutron/extensions/segment.py
+++ b/neutron/extensions/segment.py
@@ -31,7 +31,6 @@ SEGMENT_ID = 'segment_id'
 NETWORK_TYPE = 'network_type'
 PHYSICAL_NETWORK = 'physical_network'
 SEGMENTATION_ID = 'segmentation_id'
-IPAM_SEGMENT_ID = 'ipam_segment_id'
 
 # Attribute Map
 RESOURCE_ATTRIBUTE_MAP = {
@@ -74,10 +73,6 @@ RESOURCE_ATTRIBUTE_MAP = {
                      'validate': {'type:uuid_or_none': None},
                      'is_visible': True, },
     },
-    attributes.PORTS: {
-        IPAM_SEGMENT_ID: {'allow_post': False, 'allow_put': False,
-                          'is_visible': True},
-    },
 }
 
 
diff --git a/neutron/services/segments/plugin.py b/neutron/services/segments/plugin.py
index e30e742f210..7dcc77c12f6 100644
--- a/neutron/services/segments/plugin.py
+++ b/neutron/services/segments/plugin.py
@@ -45,23 +45,16 @@ def _extend_port_dict_binding(plugin, port_res, port_db):
         return
 
     value = ip_allocation.IP_ALLOCATION_IMMEDIATE
-    segment_id = None
-    # TODO(Carl) eliminate this query entirely and use optimistic joins
-    object_session = session.Session.object_session(port_db)
-    query = object_session.query(models_v2.Subnet)
-    query = query.filter_by(network_id=port_db.network_id)
-    query = query.filter(models_v2.Subnet.segment_id.isnot(None))
-    ips = port_res.get('fixed_ips')
-    if not ips:
+    if not port_res.get('fixed_ips'):
+        # NOTE Only routed network ports have deferred allocation. Check if it
+        # is routed by looking for subnets associated with segments.
+        object_session = session.Session.object_session(port_db)
+        query = object_session.query(models_v2.Subnet)
+        query = query.filter_by(network_id=port_db.network_id)
+        query = query.filter(models_v2.Subnet.segment_id.isnot(None))
         if query.count():
             value = ip_allocation.IP_ALLOCATION_DEFERRED
-    else:
-        query = query.filter_by(id=ips[0]['subnet_id'])
-        routed_subnet = query.one_or_none()
-        if routed_subnet:
-            segment_id = routed_subnet[segment.SEGMENT_ID]
     port_res[ip_allocation.IP_ALLOCATION] = value
-    port_res[segment.IPAM_SEGMENT_ID] = segment_id
 
 
 class Plugin(db.SegmentDbMixin, segment.SegmentPluginBase):
diff --git a/neutron/tests/etc/policy.json b/neutron/tests/etc/policy.json
index fc1268c28ff..3d9830d610a 100644
--- a/neutron/tests/etc/policy.json
+++ b/neutron/tests/etc/policy.json
@@ -86,7 +86,6 @@
     "get_port:binding:vif_details": "rule:admin_only",
     "get_port:binding:host_id": "rule:admin_only",
     "get_port:binding:profile": "rule:admin_only",
-    "get_port:ipam_segment_id": "rule:admin_only",
     "update_port": "rule:admin_or_owner or rule:context_is_advsvc",
     "update_port:device_owner": "not rule:network_device or rule:context_is_advsvc or rule:admin_or_network_owner",
     "update_port:mac_address": "rule:admin_only or rule:context_is_advsvc",
diff --git a/neutron/tests/unit/extensions/test_segment.py b/neutron/tests/unit/extensions/test_segment.py
index 57032a0ea00..93b14215340 100644
--- a/neutron/tests/unit/extensions/test_segment.py
+++ b/neutron/tests/unit/extensions/test_segment.py
@@ -720,8 +720,7 @@ class TestSegmentAwareIpam(SegmentTestCase):
                                      arg_list=(portbindings.HOST_ID,),
                                      **{portbindings.HOST_ID: 'fakehost'})
         res = self.deserialize(self.fmt, response)
-        self._validate_immediate_ip_allocation(
-            res['port']['id'], segment_id=segments[1]['segment']['id'])
+        self._validate_immediate_ip_allocation(res['port']['id'])
 
         # Since host mapped to middle segment, IP must come from middle subnet
         self._assert_one_ip_in_subnet(response, subnets[1]['subnet']['cidr'])
@@ -910,7 +909,7 @@ class TestSegmentAwareIpam(SegmentTestCase):
         ips = response['port']['fixed_ips']
         self.assertEqual(0, len(ips))
 
-    def _validate_immediate_ip_allocation(self, port_id, segment_id=None):
+    def _validate_immediate_ip_allocation(self, port_id):
         request = self.new_show_request('ports', port_id)
         response = self.deserialize(self.fmt, request.get_response(self.api))
 
@@ -918,8 +917,6 @@ class TestSegmentAwareIpam(SegmentTestCase):
                          response['port'][ip_allocation.IP_ALLOCATION])
         ips = response['port']['fixed_ips']
         self.assertNotEqual(0, len(ips))
-        self.assertEqual(segment_id,
-                         response['port'][ext_segment.IPAM_SEGMENT_ID])
 
     def _create_deferred_ip_port(self, network):
         response = self._create_port(self.fmt,