Add internal metadata network on demand

Previously, an internal metadata network is created
for a router whenever a subnet is attached to this router.
The purpose of this internal network is to help processing
metadata requests from instances on DHCP-disabled networks.

This commit adds a config option to create internal metadata
networks only when a DHCP-disabled subnet is attached to a router.
This will help saving system resources because each metadata
network consumes one DHCP name space.

Change-Id: Ia56050b3f431dbd65bb39da29ba6dbf8e62e36ea

etc/nsx.ini

@@ -391,3 +391,15 @@
 # The default is 8 nested groups, which allows a maximum of 4k security-groups,
 # to allow creation of more security-groups, modify this figure.
 # number_of_nested_groups =
+
+# Acceptable values for 'metadata_mode' are:
+#   - 'access_network': this enables a dedicated connection to the metadata
+#     proxy for metadata server access via Neutron router.
+#   - 'dhcp_host_route': this enables host route injection via the dhcp agent.
+# This option is only useful if running on a host that does not support
+# namespaces otherwise access_network should be used.
+# metadata_mode = access_network
+
+# If True, an internal metadata network will be created for a router only when
+# the router is attached to a DHCP-disabled subnet.
+# metadata_on_demand = True

vmware_nsx/common/config.py

@@ -243,6 +243,20 @@ nsx_v3_opts = [
     cfg.IntOpt('number_of_nested_groups',
                default=8,
                help=_("The number of nested NSGroups to use.")),
+    cfg.StrOpt('metadata_mode',
+               default=MetadataModes.DIRECT,
+               help=_("If set to access_network this enables a dedicated "
+                      "connection to the metadata proxy for metadata server "
+                      "access via Neutron router. If set to dhcp_host_route "
+                      "this enables host route injection via the dhcp agent. "
+                      "This option is only useful if running on a host that "
+                      "does not support namespaces otherwise access_network "
+                      "should be used.")),
+    cfg.BoolOpt('metadata_on_demand',
+                default=True,
+                help=_("If true, an internal metadata network will be created "
+                       "for a router only when the router is attached to a "
+                       "DHCP-disabled subnet.")),
 ]
 
 DEFAULT_STATUS_CHECK_INTERVAL = 2000

vmware_nsx/dhcp_meta/rpc.py

@@ -51,7 +51,8 @@ def handle_port_metadata_access(plugin, context, port, is_delete=False):
     # For instances supporting DHCP option 121 and created in a
     # DHCP-enabled but isolated network. This method is useful
     # only when no network namespace support.
-    if (cfg.CONF.NSX.metadata_mode == config.MetadataModes.INDIRECT and
+    plugin_cfg = getattr(cfg.CONF, plugin.cfg_group)
+    if (plugin_cfg.metadata_mode == config.MetadataModes.INDIRECT and
         port.get('device_owner') == const.DEVICE_OWNER_DHCP):
         if not port.get('fixed_ips'):
             # If port does not have an IP, the associated subnet is in
@@ -93,7 +94,10 @@ def handle_port_metadata_access(plugin, context, port, is_delete=False):
 def handle_router_metadata_access(plugin, context, router_id, interface=None):
     # For instances created in a DHCP-disabled network but connected to
     # a router.
-    if cfg.CONF.NSX.metadata_mode != config.MetadataModes.DIRECT:
+    # The parameter "interface" is only used as a Boolean flag to indicate
+    # whether to add (True) or delete (False) an internal metadata network.
+    plugin_cfg = getattr(cfg.CONF, plugin.cfg_group)
+    if plugin_cfg.metadata_mode != config.MetadataModes.DIRECT:
         LOG.debug("Metadata access network is disabled")
         return
     if not cfg.CONF.allow_overlapping_ips:
@@ -108,12 +112,19 @@ def handle_router_metadata_access(plugin, context, router_id, interface=None):
         plugin, ctx_elevated, filters=device_filter)
     try:
         if ports:
-            if (interface and
-                not _find_metadata_port(plugin, ctx_elevated, ports)):
-                _create_metadata_access_network(
-                    plugin, ctx_elevated, router_id)
-            elif len(ports) == 1:
-                # The only port left might be the metadata port
+            on_demand = getattr(plugin_cfg, 'metadata_on_demand', False)
+            if interface:
+                if (not on_demand or _find_dhcp_disabled_subnet(
+                    plugin, ctx_elevated, ports)) and (
+                    not _find_metadata_port(plugin, ctx_elevated, ports)):
+                    _create_metadata_access_network(
+                        plugin, ctx_elevated, router_id)
+            elif (len(ports) == 1 and _find_metadata_port(
+                plugin, ctx_elevated, ports)) or (on_demand and
+                not _find_dhcp_disabled_subnet(plugin, ctx_elevated, ports)):
+                # Delete the internal metadata network if the router port
+                # is the last port left or no more DHCP-disabled subnet
+                # attached to the router.
                 _destroy_metadata_access_network(
                     plugin, ctx_elevated, router_id, ports)
         else:
@@ -139,6 +150,14 @@ def _find_metadata_port(plugin, context, ports):
                 return port
 
 
+def _find_dhcp_disabled_subnet(plugin, context, ports):
+    for port in ports:
+        for fixed_ip in port['fixed_ips']:
+            subnet = plugin.get_subnet(context, fixed_ip['subnet_id'])
+            if not subnet['enable_dhcp']:
+                return subnet
+
+
 def _create_metadata_access_network(plugin, context, router_id):
     # Add network
     # Network name is likely to be truncated on NSX

vmware_nsx/plugins/nsx_mh/plugin.py

@@ -168,6 +168,7 @@ class NsxPluginV2(addr_pair_db.AllowedAddressPairsMixin,
 
         neutron_extensions.append_api_extensions_path(
             [vmware_nsx.NSX_EXT_PATH])
+        self.cfg_group = 'NSX'  # group name for nsx section in nsx.ini
         self.nsx_opts = cfg.CONF.NSX
         self.nsx_sync_opts = cfg.CONF.NSX_SYNC
         self.cluster = nsx_utils.create_nsx_cluster(
@@ -1750,7 +1751,7 @@ class NsxPluginV2(addr_pair_db.AllowedAddressPairsMixin,
             # is removed  (with the network) if this the last subnet
             # on the router
             self.handle_router_metadata_access(
-                context, router_id, interface=info)
+                context, router_id, interface=None)
             if not subnet:
                 subnet = self._get_subnet(context, subnet_id)
             router = self._get_router(context, router_id)

vmware_nsx/plugins/nsx_v3/plugin.py

@@ -123,6 +123,7 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin,
         self._nsx_client = nsx_client.NSX3Client(self._api_cluster)
         nsx_client._set_default_api_cluster(self._api_cluster)
 
+        self.cfg_group = 'nsx_v3'  # group name for nsx_v3 section in nsx.ini
         self.base_binding_dict = {
             pbin.VIF_TYPE: pbin.VIF_TYPE_OVS,
             pbin.VIF_DETAILS: {
@@ -563,6 +564,22 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin,
         # TODO(berlin): cancel public external subnet announcement
         return super(NsxV3Plugin, self).delete_subnet(context, subnet_id)
 
+    def update_subnet(self, context, subnet_id, subnet):
+        updated_subnet = super(NsxV3Plugin, self).update_subnet(
+            context, subnet_id, subnet)
+        if cfg.CONF.nsx_v3.metadata_on_demand:
+            # If enable_dhcp is changed on a subnet attached to a router,
+            # update internal metadata network accordingly.
+            if 'enable_dhcp' in subnet['subnet']:
+                port_filters = {'device_owner': const.ROUTER_INTERFACE_OWNERS,
+                                'fixed_ips': {'subnet_id': [subnet_id]}}
+                ports = self.get_ports(context, filters=port_filters)
+                for port in ports:
+                    nsx_rpc.handle_router_metadata_access(
+                        self, context, port['device_id'],
+                        interface=not updated_subnet['enable_dhcp'])
+        return updated_subnet
+
     def _build_address_bindings(self, port):
         address_bindings = []
         for fixed_ip in port['fixed_ips']:
@@ -1493,7 +1510,8 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin,
         info = super(NsxV3Plugin, self).remove_router_interface(
             context, router_id, interface_info)
         # Ensure the connection to the 'metadata access network' is removed
-        # (with the network) if this the last subnet on the router.
+        # (with the network) if this is the last DHCP-disabled subnet on the
+        # router.
         nsx_rpc.handle_router_metadata_access(self, context, router_id)
         return info
 

vmware_nsx/tests/unit/extensions/test_metadata.py

@@ -28,11 +28,38 @@ from vmware_nsx.common import config
 
 class MetaDataTestCase(object):
 
-    def _metadata_setup(self, mode=config.MetadataModes.DIRECT):
-        cfg.CONF.set_override('metadata_mode', mode, 'NSX')
+    def _metadata_setup(self, mode=config.MetadataModes.DIRECT,
+                        on_demand=False):
+        cfg.CONF.set_override('metadata_mode', mode, self.plugin.cfg_group)
+        if hasattr(getattr(cfg.CONF, self.plugin.cfg_group),
+                   'metadata_on_demand'):
+            cfg.CONF.set_override('metadata_on_demand', on_demand,
+                                  self.plugin.cfg_group)
 
     def _metadata_teardown(self):
-        cfg.CONF.set_override('metadata_mode', None, 'NSX')
+        cfg.CONF.set_override('metadata_mode', None, self.plugin.cfg_group)
+        if hasattr(getattr(cfg.CONF, self.plugin.cfg_group),
+                   'metadata_on_demand'):
+            cfg.CONF.set_override('metadata_on_demand', False,
+                                  self.plugin.cfg_group)
+
+    def _check_metadata(self, expected_subnets, expected_ports):
+        subnets = self._list('subnets')['subnets']
+        self.assertEqual(len(subnets), expected_subnets)
+        meta_net_id, meta_sub_id = None, None
+        meta_cidr = netaddr.IPNetwork('169.254.0.0/16')
+        for subnet in subnets:
+            cidr = netaddr.IPNetwork(subnet['cidr'])
+            if meta_cidr == cidr or meta_cidr in cidr.supernet(16):
+                meta_sub_id = subnet['id']
+                meta_net_id = subnet['network_id']
+                break
+        ports = self._list(
+            'ports',
+            query_params='network_id=%s' % meta_net_id)['ports']
+        self.assertEqual(len(ports), expected_ports)
+        meta_port_id = ports[0]['id'] if ports else None
+        return meta_net_id, meta_sub_id, meta_port_id
 
     def test_router_add_interface_subnet_with_metadata_access(self):
         self._metadata_setup()
@@ -178,19 +205,8 @@ class MetaDataTestCase(object):
             with self.subnet() as s:
                 self._router_interface_action('add', r['router']['id'],
                                               s['subnet']['id'], None)
-                subnets = self._list('subnets')['subnets']
-                self.assertEqual(len(subnets), 2)
-                meta_cidr = netaddr.IPNetwork('169.254.0.0/16')
-                for subnet in subnets:
-                    cidr = netaddr.IPNetwork(subnet['cidr'])
-                    if meta_cidr == cidr or meta_cidr in cidr.supernet(16):
-                        meta_sub_id = subnet['id']
-                        meta_net_id = subnet['network_id']
-                ports = self._list(
-                    'ports',
-                    query_params='network_id=%s' % meta_net_id)['ports']
-                self.assertEqual(len(ports), 1)
-                meta_port_id = ports[0]['id']
+                meta_net_id, meta_sub_id, meta_port_id = self._check_metadata(
+                    expected_subnets=2, expected_ports=1)
                 self._router_interface_action('remove', r['router']['id'],
                                               s['subnet']['id'], None)
                 self._show('networks', meta_net_id,
@@ -240,6 +256,54 @@ class MetaDataTestCase(object):
                            webob.exc.HTTPOk.code)
         self._metadata_teardown()
 
+    def test_metadata_network_with_update_subnet_dhcp_enable(self):
+        self._metadata_setup(on_demand=True)
+        with self.router() as r:
+            # Create a DHCP-disabled subnet.
+            with self.subnet(enable_dhcp=False) as s:
+                self._router_interface_action('add', r['router']['id'],
+                                              s['subnet']['id'], None)
+                meta_net_id, meta_sub_id, meta_port_id = self._check_metadata(
+                    expected_subnets=2, expected_ports=1)
+                # Update subnet to DHCP-enabled.
+                data = {'subnet': {'enable_dhcp': True}}
+                req = self.new_update_request('subnets', data,
+                                              s['subnet']['id'])
+                res = self.deserialize(self.fmt, req.get_response(self.api))
+                self.assertEqual(True, res['subnet']['enable_dhcp'])
+                self._check_metadata(expected_subnets=1, expected_ports=0)
+                self._show('networks', meta_net_id,
+                           webob.exc.HTTPNotFound.code)
+                self._show('ports', meta_port_id,
+                           webob.exc.HTTPNotFound.code)
+                self._show('subnets', meta_sub_id,
+                           webob.exc.HTTPNotFound.code)
+        self._metadata_teardown()
+
+    def test_metadata_network_with_update_subnet_dhcp_disable(self):
+        self._metadata_setup(on_demand=True)
+        with self.router() as r:
+            # Create a DHCP-enabled subnet.
+            with self.subnet(enable_dhcp=True) as s:
+                self._router_interface_action('add', r['router']['id'],
+                                              s['subnet']['id'], None)
+                self._check_metadata(expected_subnets=1, expected_ports=0)
+                # Update subnet to DHCP-disabled.
+                data = {'subnet': {'enable_dhcp': False}}
+                req = self.new_update_request('subnets', data,
+                                              s['subnet']['id'])
+                res = self.deserialize(self.fmt, req.get_response(self.api))
+                self.assertEqual(False, res['subnet']['enable_dhcp'])
+                meta_net_id, meta_sub_id, meta_port_id = self._check_metadata(
+                    expected_subnets=2, expected_ports=1)
+                self._show('networks', meta_net_id,
+                           webob.exc.HTTPOk.code)
+                self._show('ports', meta_port_id,
+                           webob.exc.HTTPOk.code)
+                self._show('subnets', meta_sub_id,
+                           webob.exc.HTTPOk.code)
+        self._metadata_teardown()
+
     def test_metadata_dhcp_host_route(self):
         self._metadata_setup(config.MetadataModes.INDIRECT)
         subnets = self._list('subnets')['subnets']

vmware_nsx/tests/unit/nsx_mh/test_plugin.py

@@ -889,6 +889,12 @@ class TestL3NatTestCase(L3NatTest,
     def test_floatingip_disassociate_notification(self):
         self.skipTest('not supported')
 
+    def test_metadata_network_with_update_subnet_dhcp_enable(self):
+        self.skipTest('not supported')
+
+    def test_metadata_network_with_update_subnet_dhcp_disable(self):
+        self.skipTest('not supported')
+
 
 class ExtGwModeTestCase(NsxPluginV2TestCase,
                         test_ext_gw_mode.ExtGwModeIntTestCase):