Merge "Schedule gateway on chassis from ovn-cms-options"

devstack/computenode-local.conf.sample

@@ -70,3 +70,9 @@ VNCSERVER_PROXYCLIENT_ADDRESS=$VNCSERVER_LISTEN
 #PHYSICAL_NETWORK=providernet
 #OVS_PHYSICAL_BRIDGE=br-provider
 #PUBLIC_INTERFACE=<public interface>
+
+# If the admin wants to enable this chassis to host gateway routers for
+# external connectivity, then set ENABLE_CHASSIS_AS_GW to True.
+# Then devstack will set ovn-cms-options with enable-chassis-as-gw
+# in Open_vSwitch table's external_ids column.
+#ENABLE_CHASSIS_AS_GW=False

devstack/devstackgaterc

@@ -38,6 +38,9 @@ else
     echo "No ovs branch specified, using the default from the devstack plugin"
 fi
 
+# Enable controller to host gateway routers
+export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_CHASSIS_AS_GW=True"
+
 if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]] ; then
     # NOTE(rtheis): Multinode does not require creating an OVN L3 public network.
     export DEVSTACK_LOCAL_CONFIG+=$'\n'"OVN_L3_CREATE_PUBLIC_NETWORK=False"

devstack/lib/networking-ovn

@@ -363,6 +363,10 @@ function start_ovs {
         ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int"
         ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve,vxlan"
         ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$HOST_IP"
+        # Select this chassis to host gateway routers
+        if [[ "$ENABLE_CHASSIS_AS_GW" == "True" ]]; then
+            ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-cms-options="enable-chassis-as-gw"
+        fi
 
         ovn_base_setup_bridge br-int
         ovs-vsctl --no-wait set bridge br-int fail-mode=secure other-config:disable-in-band=true

devstack/local.conf.sample

@@ -96,6 +96,13 @@ disable_service cinder c-sch c-api c-vol
 #PUBLIC_INTERFACE=<public interface>
 #OVS_PHYSICAL_BRIDGE=br-provider
 #PROVIDER_SUBNET_NAME=provider-subnet
+
+# If the admin wants to enable this chassis to host gateway routers for
+# external connectivity, then set ENABLE_CHASSIS_AS_GW to True.
+# Then devstack will set ovn-cms-options with enable-chassis-as-gw
+# in Open_vSwitch table's external_ids column
+#ENABLE_CHASSIS_AS_GW=True
+
 # use the following for IPv4
 #IP_VERSION=4
 #FIXED_RANGE=<CIDR for the Provider Network>

doc/source/admin/refarch/provider-networks.rst

@@ -44,6 +44,17 @@ Create a provider network
 
 #. On the controller node, source the administrative project credentials.
 
+#. On the controller node, to enable this chassis to host gateway routers
+   for external connectivity, set ovn-cms-options to enable-chassis-as-gw.
+
+   .. code-block:: console
+
+      # ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw"
+
+   .. note::
+
+      This command provide no output if successful.
+
 #. On the controller node, create the provider network in the Networking
    service. In this case, instances and routers in other projects can use
    the network.

doc/source/admin/refarch/refarch.rst

@@ -119,6 +119,10 @@ The compute nodes don't need connectivity to the external network,
 although it could be provided if we wanted to have direct connectivity
 to such network from some instances.
 
+For external connectivity, gateway nodes have to set ovn-cms-options
+with enable-chassis-as-gw in Open_vSwitch table's external_ids column.
+$ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw"
+
 Distributed Floating IPs (DVR)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/contributor/testing.rst

@@ -513,6 +513,14 @@ to the bridge 'br-provider".
     $ ovs-vsctl set open . \
     external-ids:ovn-bridge-mappings=providernet:br-provider
 
+If you want to enable this chassis to host a gateway router for
+external connectivity, then set ovn-cms-options to enable-chassis-as-gw.
+
+::
+
+    $ ovs-vsctl set open . \
+    external-ids:ovn-cms-options="enable-chassis-as-gw"
+
 Now create a Neutron provider network.
 
 ::

doc/source/install/index.rst

@@ -243,6 +243,17 @@ primary node. See the :ref:`faq` for more information.
 
 #. Start the ``neutron-server`` service.
 
+#. Configure the chassis to host gateway routers for external connectivity.
+
+   * Set ovn-cms-options with enable-chassis-as-gw in Open_vSwitch table's
+     external_ids column. Then if this chassis has proper bridge mappings,
+     it will be selected for scheduling gateway routers.
+
+     .. code-block:: console
+
+        # ovs-vsctl set open . external-ids:ovn-cms-options=enable-chassis-as-gw
+
+
 Network nodes
 -------------
 

migration/migrate-to-ovn.yml

@@ -161,6 +161,8 @@
   - name: Schedule gateway routers by running the sync util.
     when: ovn_central is defined
     command: neutron-ovn-db-sync-util --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
+  - name: Configure node for hosting gateway routers for external connectivity.
+    command: "ovs-vsctl set open . external_ids:ovn-cms-options=enable-chassis-as-gw"
 
 - hosts: overcloud
   remote_user: "{{ remote_user }}"

networking_ovn/common/ovn_client.py

@@ -982,16 +982,40 @@ class OVNClient(object):
             txn.add(self._nb_idl.delete_lrouter(lrouter_name))
         db_rev.delete_revision(router_id, ovn_const.TYPE_ROUTERS)
 
-    def get_candidates_for_scheduling(self, extnet):
+    def get_candidates_for_scheduling(self, physnet, cms=None,
+                                      chassis_physnets=None):
+        """Return chassis for scheduling gateway router.
+
+        Criteria for selecting chassis as candidates
+        1) chassis from cms with proper bridge mappings
+        2) if no chassis is available from 1) then,
+           select chassis with proper bridge mappings
+        """
+        cms = cms or self._sb_idl.get_gateway_chassis_from_cms_options()
+        chassis_physnets = (chassis_physnets or
+                            self._sb_idl.get_chassis_and_physnets())
+        cms_bmaps = []
+        bmaps = []
+        for chassis, physnets in chassis_physnets.items():
+            if physnet and physnet in physnets:
+                if chassis in cms:
+                    cms_bmaps.append(chassis)
+                else:
+                    bmaps.append(chassis)
+        candidates = cms_bmaps or bmaps
+        if not cms_bmaps:
+            LOG.debug("No eligible chassis with external connectivity"
+                      " through ovn-cms-options.")
+        LOG.debug("Chassis candidates with external connectivity: %s",
+                  candidates)
+        return candidates
+
+    def _get_physnet(self, net_id):
+        extnet = self._plugin.get_network(n_context.get_admin_context(),
+                                          net_id)
         if extnet.get(pnet.NETWORK_TYPE) in [const.TYPE_FLAT,
                                              const.TYPE_VLAN]:
-            physnet = extnet.get(pnet.PHYSICAL_NETWORK)
-            if not physnet:
-                return []
-            chassis_physnets = self._sb_idl.get_chassis_and_physnets()
-            return [chassis for chassis, physnets in chassis_physnets.items()
-                    if physnet in physnets]
-        return []
+            return extnet.get(pnet.PHYSICAL_NETWORK)
 
     def _gen_router_port_ext_ids(self, port):
         return {
@@ -1009,9 +1033,8 @@ class OVNClient(object):
             'device_owner')
         columns = {}
         if is_gw_port:
-            context = n_context.get_admin_context()
-            candidates = self.get_candidates_for_scheduling(
-                self._plugin.get_network(context, port['network_id']))
+            physnet = self._get_physnet(port['network_id'])
+            candidates = self.get_candidates_for_scheduling(physnet)
             selected_chassis = self._ovn_scheduler.select(
                 self._nb_idl, self._sb_idl, lrouter_port_name,
                 candidates=candidates)

networking_ovn/l3/l3_ovn.py

@@ -352,14 +352,14 @@ class OVNL3RouterPlugin(service_base.ServicePluginBase,
     def schedule_unhosted_gateways(self):
         port_physnet_dict = self._get_gateway_port_physnet_mapping()
         chassis_physnets = self._sb_ovn.get_chassis_and_physnets()
+        cms = self._sb_ovn.get_gateway_chassis_from_cms_options()
         unhosted_gateways = self._ovn.get_unhosted_gateways(
-            port_physnet_dict, chassis_physnets)
+            port_physnet_dict, chassis_physnets, cms)
         with self._ovn.transaction(check_error=True) as txn:
             for g_name in unhosted_gateways:
                 physnet = port_physnet_dict.get(g_name[len('lrp-'):])
-                candidates = [chassis
-                              for chassis, physnets in chassis_physnets.items()
-                              if physnet and physnet in physnets]
+                candidates = self._ovn_client.get_candidates_for_scheduling(
+                    physnet, cms=cms, chassis_physnets=chassis_physnets)
                 chassis = self.scheduler.select(
                     self._ovn, self._sb_ovn, g_name, candidates=candidates)
                 txn.add(self._ovn.update_lrouter_port(

networking_ovn/ovsdb/impl_idl_ovn.py

@@ -408,7 +408,8 @@ class OvsdbNbOvnIdl(nb_impl_idl.OvnNbApiIdlImpl, Backend):
         except idlutils.RowNotFound:
             return []
 
-    def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets):
+    def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets,
+                              gw_chassis):
         unhosted_gateways = []
         valid_chassis_list = list(chassis_physnets)
         for lrp in self._tables['Logical_Router_Port'].rows.values():
@@ -423,7 +424,8 @@ class OvsdbNbOvnIdl(nb_impl_idl.OvnNbApiIdlImpl, Backend):
                 if (chassis_name == ovn_const.OVN_GATEWAY_INVALID_CHASSIS or
                         chassis_name not in valid_chassis_list or
                         (physnet and
-                         physnet not in chassis_physnets.get(chassis_name))):
+                         physnet not in chassis_physnets.get(chassis_name)) or
+                        (gw_chassis and chassis_name not in gw_chassis)):
                     unhosted_gateways.append(lrp.name)
         return unhosted_gateways
 
@@ -672,6 +674,14 @@ class OvsdbSbOvnIdl(sb_impl_idl.OvnSbApiIdlImpl, Backend):
             chassis_info_dict[ch.hostname] = self._get_chassis_physnets(ch)
         return chassis_info_dict
 
+    def get_gateway_chassis_from_cms_options(self):
+        gw_chassis = []
+        for ch in self.chassis_list().execute(check_error=True):
+            cms_options = ch.external_ids.get('ovn-cms-options', '')
+            if 'enable-chassis-as-gw' in cms_options.split(','):
+                gw_chassis.append(ch.name)
+        return gw_chassis
+
     def get_chassis_and_physnets(self):
         chassis_info_dict = {}
         for ch in self.chassis_list().execute(check_error=True):

networking_ovn/ovsdb/ovn_api.py

@@ -337,11 +337,14 @@ class API(api.API):
         """
 
     @abc.abstractmethod
-    def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets):
+    def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets,
+                              gw_chassis):
         """Return a list of gateways not hosted on chassis
 
         :param port_physnet_dict: Dictionary of gateway ports and their physnet
         :param chassis_physnets:  Dictionary of chassis and physnets
+        :param gw_chassis:        List of gateway chassis provided by admin
+                                  through ovn-cms-options
         :returns:                 List of gateways not hosted on a valid
                                   chassis
         """
@@ -623,6 +626,18 @@ class SbAPI(api.API):
         value. And hostname and physnets are related to the same host.
         """
 
+    def get_gateway_chassis_from_cms_options(self):
+        """Get chassis eligible for external connectivity from CMS options.
+
+        When admin wants to enable router gateway on few chassis,
+        he would set the external_ids as
+
+        ovs-vsctl set open .
+           external_ids:ovn-cms-options="enable-chassis-as-gw"
+        In this function, we parse ovn-cms-options and return these chassis
+        :returns:              List with chassis names.
+        """
+
     @abc.abstractmethod
     def get_chassis_and_physnets(self):
         """Return a dict contains chassis name and physnets mapping.

networking_ovn/tests/functional/test_router.py

@@ -82,6 +82,63 @@ class TestRouter(base.TestOVNFunctionalBase):
                 rc = row.options.get(ovn_const.OVN_GATEWAY_CHASSIS_KEY)
                 self.assertIn(rc, expected)
 
+    def _check_gateway_chassis_candidates(self, candidates):
+        # In this test, fake_select() is called once from _create_router()
+        # and later from schedule_unhosted_gateways()
+        ovn_client = self.l3_plugin._ovn_client
+        ext1 = self._create_ext_network(
+            'ext1', 'vlan', 'physnet1', 1, "10.0.0.1", "10.0.0.0/24")
+        # mock select function and check if it is called with expected
+        # candidates.
+
+        def fake_select(*args, **kwargs):
+            self.assertItemsEqual(candidates, kwargs['candidates'])
+            # We are not interested in further processing, let us return
+            # INVALID_CHASSIS to avoid erros
+            return [ovn_const.OVN_GATEWAY_INVALID_CHASSIS]
+
+        with mock.patch.object(ovn_client._ovn_scheduler, 'select',
+                               side_effect=fake_select) as client_select,\
+            mock.patch.object(self.l3_plugin.scheduler, 'select',
+                              side_effect=fake_select) as plugin_select:
+            gw_info = {'network_id': ext1['network']['id']}
+            self._create_router('router1', gw_info=gw_info)
+            self.assertFalse(plugin_select.called)
+            self.assertTrue(client_select.called)
+            client_select.reset_mock()
+            plugin_select.reset_mock()
+
+            # set redirect-chassis to neutron-ovn-invalid-chassis, so
+            # that schedule_unhosted_gateways will try to schedule it
+            self._set_redirect_chassis_to_invalid_chassis(ovn_client)
+            self.l3_plugin.schedule_unhosted_gateways()
+            self.assertFalse(client_select.called)
+            self.assertTrue(plugin_select.called)
+
+    def test_gateway_chassis_with_cms_and_bridge_mappings(self):
+        # Both chassis1 and chassis3 are having proper bridge mappings,
+        # but only chassis3 is having enable-chassis-as-gw.
+        # Test if chassis3 is selected as candidate or not.
+        self.chassis3 = self.add_fake_chassis(
+            'ovs-host3', physical_nets=['physnet1'],
+            external_ids={'ovn-cms-options': 'enable-chassis-as-gw'})
+        self._check_gateway_chassis_candidates([self.chassis3])
+
+    def test_gateway_chassis_with_cms_and_no_bridge_mappings(self):
+        # chassis1 is having proper bridge mappings.
+        # chassis3 is having enable-chassis-as-gw, but no bridge mappings.
+        # Test if chassis1 is selected as candidate or not.
+        self.chassis3 = self.add_fake_chassis(
+            'ovs-host3',
+            external_ids={'ovn-cms-options': 'enable-chassis-as-gw'})
+        self._check_gateway_chassis_candidates([self.chassis1])
+
+    def test_gateway_chassis_with_bridge_mappings_and_no_cms(self):
+        # chassis1 is configured with proper bridge mappings,
+        # but none of the chassis having enable-chassis-as-gw.
+        # Test if chassis1 is selected as candidate or not.
+        self._check_gateway_chassis_candidates([self.chassis1])
+
     def test_gateway_chassis_with_bridge_mappings(self):
         ovn_client = self.l3_plugin._ovn_client
         # Create external networks with vlan, flat and geneve network types

networking_ovn/tests/unit/l3/test_l3_ovn.py

@@ -1065,6 +1065,10 @@ class OVNL3ExtrarouteTests(test_l3_gw.ExtGwModeIntTestCase,
             'networking_ovn.l3.l3_ovn_scheduler.'
             'OVNGatewayScheduler._schedule_gateway',
             return_value='hv1')
+        self._start_mock(
+            'networking_ovn.common.ovn_client.'
+            'OVNClient.get_candidates_for_scheduling',
+            return_value=[])
         self._start_mock(
             'networking_ovn.common.ovn_client.OVNClient.'
             '_get_v4_network_of_all_router_ports',

networking_ovn/tests/unit/ovsdb/test_impl_idl_ovn.py

@@ -580,7 +580,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn):
         self._load_nb_db()
         # Test only host-1 in the valid list
         unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways(
-            {}, {'host-1': 'physnet1'})
+            {}, {'host-1': 'physnet1'}, [])
         expected = {
             utils.ovn_lrouter_port_name('orp-id-b2'): {
                 ovn_const.OVN_GATEWAY_CHASSIS_KEY: 'host-2'},
@@ -590,7 +590,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn):
         self.assertItemsEqual(unhosted_gateways, expected)
         # Test both host-1, host-2 in valid list
         unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways(
-            {}, {'host-1': 'physnet1', 'host-2': 'physnet2'})
+            {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}, [])
         expected = {utils.ovn_lrouter_port_name('orp-id-a3'): {
             ovn_const.OVN_GATEWAY_CHASSIS_KEY:
                 ovn_const.OVN_GATEWAY_INVALID_CHASSIS}}
@@ -602,7 +602,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn):
             setattr(router_row, 'options', {
                 ovn_const.OVN_GATEWAY_CHASSIS_KEY: 'host-2'})
         unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways(
-            {}, {'host-1': 'physnet1', 'host-2': 'physnet2'})
+            {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}, [])
         self.assertItemsEqual(unhosted_gateways, {})
 
     def test_get_subnet_dhcp_options(self):

releasenotes/notes/ovn-cms-options-enable-chassis-as-gw-3adc7024478e3efa.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    New option "enable-chassis-as-gw" to select gateway router.
+    For external connectivity, gateway nodes have to set ovn-cms-options
+    with enable-chassis-as-gw in Open_vSwitch table's external_ids column.
+
+    $ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw"
+
+    Networking-ovn will parse ovn-cms-options and select this chassis
+    if it has proper bridge mappings. This helps admin to exclude compute
+    nodes to host gateway routers as they are more likely to be restarted
+    for maintenance operations. If no chassis with enable-chassis-as-gw and
+    proper bridge mappings available, then chassis with only bridge mappings
+    are selected for scheduling router gateway.
+
+    This is not a config option enabled through conf files. Instead admin
+    has to set it through openstack installer or manually in Open_vSwitch
+    table.

vagrant/provisioning/setup-compute.sh

@@ -44,6 +44,8 @@ PHYSICAL_NETWORK=provider
 # as necessary for your environment.
 NETWORK_GATEWAY=172.16.1.1
 FIXED_RANGE=172.16.1.0/24
+
+ENABLE_CHASSIS_AS_GW=False
 DEVSTACKEOF
 
 # Add unique post-config for DevStack here using a separate 'cat' with

vagrant/provisioning/setup-controller.sh

@@ -71,6 +71,12 @@ PUBLIC_SUBNET_NAME=provider-v4
 IPV6_PUBLIC_SUBNET_NAME=provider-v6
 Q_FLOATING_ALLOCATION_POOL="start=$start_ip,end=$end_ip"
 FLOATING_RANGE="$network"
+
+# If the admin wants to enable this chassis to host gateway routers for
+# external connectivity, then set ENABLE_CHASSIS_AS_GW to True.
+# Then devstack will set ovn-cms-options with enable-chassis-as-gw
+# in Open_vSwitch table's external_ids column
+ENABLE_CHASSIS_AS_GW=True
 DEVSTACKEOF
 
 # Add unique post-config for DevStack here using a separate 'cat' with