diff --git a/devstack/computenode-local.conf.sample b/devstack/computenode-local.conf.sample index 509bf65ca..def70ec6e 100644 --- a/devstack/computenode-local.conf.sample +++ b/devstack/computenode-local.conf.sample @@ -70,3 +70,9 @@ VNCSERVER_PROXYCLIENT_ADDRESS=$VNCSERVER_LISTEN #PHYSICAL_NETWORK=providernet #OVS_PHYSICAL_BRIDGE=br-provider #PUBLIC_INTERFACE= + +# If the admin wants to enable this chassis to host gateway routers for +# external connectivity, then set ENABLE_CHASSIS_AS_GW to True. +# Then devstack will set ovn-cms-options with enable-chassis-as-gw +# in Open_vSwitch table's external_ids column. +#ENABLE_CHASSIS_AS_GW=False diff --git a/devstack/devstackgaterc b/devstack/devstackgaterc index 4132070d1..883fb3edf 100644 --- a/devstack/devstackgaterc +++ b/devstack/devstackgaterc @@ -38,6 +38,9 @@ else echo "No ovs branch specified, using the default from the devstack plugin" fi +# Enable controller to host gateway routers +export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_CHASSIS_AS_GW=True" + if [[ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]] ; then # NOTE(rtheis): Multinode does not require creating an OVN L3 public network. export DEVSTACK_LOCAL_CONFIG+=$'\n'"OVN_L3_CREATE_PUBLIC_NETWORK=False" diff --git a/devstack/lib/networking-ovn b/devstack/lib/networking-ovn index 4ddfe13f1..6a1621681 100644 --- a/devstack/lib/networking-ovn +++ b/devstack/lib/networking-ovn @@ -363,6 +363,10 @@ function start_ovs { ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int" ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve,vxlan" ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$HOST_IP" + # Select this chassis to host gateway routers + if [[ "$ENABLE_CHASSIS_AS_GW" == "True" ]]; then + ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-cms-options="enable-chassis-as-gw" + fi ovn_base_setup_bridge br-int ovs-vsctl --no-wait set bridge br-int fail-mode=secure other-config:disable-in-band=true diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample index ff03c0559..37be5852f 100644 --- a/devstack/local.conf.sample +++ b/devstack/local.conf.sample @@ -96,6 +96,13 @@ disable_service cinder c-sch c-api c-vol #PUBLIC_INTERFACE= #OVS_PHYSICAL_BRIDGE=br-provider #PROVIDER_SUBNET_NAME=provider-subnet + +# If the admin wants to enable this chassis to host gateway routers for +# external connectivity, then set ENABLE_CHASSIS_AS_GW to True. +# Then devstack will set ovn-cms-options with enable-chassis-as-gw +# in Open_vSwitch table's external_ids column +#ENABLE_CHASSIS_AS_GW=True + # use the following for IPv4 #IP_VERSION=4 #FIXED_RANGE= diff --git a/doc/source/admin/refarch/provider-networks.rst b/doc/source/admin/refarch/provider-networks.rst index 526e7bd41..af54a448d 100644 --- a/doc/source/admin/refarch/provider-networks.rst +++ b/doc/source/admin/refarch/provider-networks.rst @@ -44,6 +44,17 @@ Create a provider network #. On the controller node, source the administrative project credentials. +#. On the controller node, to enable this chassis to host gateway routers + for external connectivity, set ovn-cms-options to enable-chassis-as-gw. + + .. code-block:: console + + # ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw" + + .. note:: + + This command provide no output if successful. + #. On the controller node, create the provider network in the Networking service. In this case, instances and routers in other projects can use the network. diff --git a/doc/source/admin/refarch/refarch.rst b/doc/source/admin/refarch/refarch.rst index 581f80c3e..6e3938a9b 100644 --- a/doc/source/admin/refarch/refarch.rst +++ b/doc/source/admin/refarch/refarch.rst @@ -119,6 +119,10 @@ The compute nodes don't need connectivity to the external network, although it could be provided if we wanted to have direct connectivity to such network from some instances. +For external connectivity, gateway nodes have to set ovn-cms-options +with enable-chassis-as-gw in Open_vSwitch table's external_ids column. +$ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw" + Distributed Floating IPs (DVR) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/contributor/testing.rst b/doc/source/contributor/testing.rst index c60d11990..a97999221 100644 --- a/doc/source/contributor/testing.rst +++ b/doc/source/contributor/testing.rst @@ -513,6 +513,14 @@ to the bridge 'br-provider". $ ovs-vsctl set open . \ external-ids:ovn-bridge-mappings=providernet:br-provider +If you want to enable this chassis to host a gateway router for +external connectivity, then set ovn-cms-options to enable-chassis-as-gw. + +:: + + $ ovs-vsctl set open . \ + external-ids:ovn-cms-options="enable-chassis-as-gw" + Now create a Neutron provider network. :: diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst index f9f41cc61..039435620 100644 --- a/doc/source/install/index.rst +++ b/doc/source/install/index.rst @@ -243,6 +243,17 @@ primary node. See the :ref:`faq` for more information. #. Start the ``neutron-server`` service. +#. Configure the chassis to host gateway routers for external connectivity. + + * Set ovn-cms-options with enable-chassis-as-gw in Open_vSwitch table's + external_ids column. Then if this chassis has proper bridge mappings, + it will be selected for scheduling gateway routers. + + .. code-block:: console + + # ovs-vsctl set open . external-ids:ovn-cms-options=enable-chassis-as-gw + + Network nodes ------------- diff --git a/migration/migrate-to-ovn.yml b/migration/migrate-to-ovn.yml index cb2a2bb76..61fe06555 100644 --- a/migration/migrate-to-ovn.yml +++ b/migration/migrate-to-ovn.yml @@ -161,6 +161,8 @@ - name: Schedule gateway routers by running the sync util. when: ovn_central is defined command: neutron-ovn-db-sync-util --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini + - name: Configure node for hosting gateway routers for external connectivity. + command: "ovs-vsctl set open . external_ids:ovn-cms-options=enable-chassis-as-gw" - hosts: overcloud remote_user: "{{ remote_user }}" diff --git a/networking_ovn/common/ovn_client.py b/networking_ovn/common/ovn_client.py index d574afc21..6da3fdeb4 100644 --- a/networking_ovn/common/ovn_client.py +++ b/networking_ovn/common/ovn_client.py @@ -982,16 +982,40 @@ class OVNClient(object): txn.add(self._nb_idl.delete_lrouter(lrouter_name)) db_rev.delete_revision(router_id, ovn_const.TYPE_ROUTERS) - def get_candidates_for_scheduling(self, extnet): + def get_candidates_for_scheduling(self, physnet, cms=None, + chassis_physnets=None): + """Return chassis for scheduling gateway router. + + Criteria for selecting chassis as candidates + 1) chassis from cms with proper bridge mappings + 2) if no chassis is available from 1) then, + select chassis with proper bridge mappings + """ + cms = cms or self._sb_idl.get_gateway_chassis_from_cms_options() + chassis_physnets = (chassis_physnets or + self._sb_idl.get_chassis_and_physnets()) + cms_bmaps = [] + bmaps = [] + for chassis, physnets in chassis_physnets.items(): + if physnet and physnet in physnets: + if chassis in cms: + cms_bmaps.append(chassis) + else: + bmaps.append(chassis) + candidates = cms_bmaps or bmaps + if not cms_bmaps: + LOG.debug("No eligible chassis with external connectivity" + " through ovn-cms-options.") + LOG.debug("Chassis candidates with external connectivity: %s", + candidates) + return candidates + + def _get_physnet(self, net_id): + extnet = self._plugin.get_network(n_context.get_admin_context(), + net_id) if extnet.get(pnet.NETWORK_TYPE) in [const.TYPE_FLAT, const.TYPE_VLAN]: - physnet = extnet.get(pnet.PHYSICAL_NETWORK) - if not physnet: - return [] - chassis_physnets = self._sb_idl.get_chassis_and_physnets() - return [chassis for chassis, physnets in chassis_physnets.items() - if physnet in physnets] - return [] + return extnet.get(pnet.PHYSICAL_NETWORK) def _gen_router_port_ext_ids(self, port): return { @@ -1009,9 +1033,8 @@ class OVNClient(object): 'device_owner') columns = {} if is_gw_port: - context = n_context.get_admin_context() - candidates = self.get_candidates_for_scheduling( - self._plugin.get_network(context, port['network_id'])) + physnet = self._get_physnet(port['network_id']) + candidates = self.get_candidates_for_scheduling(physnet) selected_chassis = self._ovn_scheduler.select( self._nb_idl, self._sb_idl, lrouter_port_name, candidates=candidates) diff --git a/networking_ovn/l3/l3_ovn.py b/networking_ovn/l3/l3_ovn.py index 740b35ee6..9215b00b1 100644 --- a/networking_ovn/l3/l3_ovn.py +++ b/networking_ovn/l3/l3_ovn.py @@ -352,14 +352,14 @@ class OVNL3RouterPlugin(service_base.ServicePluginBase, def schedule_unhosted_gateways(self): port_physnet_dict = self._get_gateway_port_physnet_mapping() chassis_physnets = self._sb_ovn.get_chassis_and_physnets() + cms = self._sb_ovn.get_gateway_chassis_from_cms_options() unhosted_gateways = self._ovn.get_unhosted_gateways( - port_physnet_dict, chassis_physnets) + port_physnet_dict, chassis_physnets, cms) with self._ovn.transaction(check_error=True) as txn: for g_name in unhosted_gateways: physnet = port_physnet_dict.get(g_name[len('lrp-'):]) - candidates = [chassis - for chassis, physnets in chassis_physnets.items() - if physnet and physnet in physnets] + candidates = self._ovn_client.get_candidates_for_scheduling( + physnet, cms=cms, chassis_physnets=chassis_physnets) chassis = self.scheduler.select( self._ovn, self._sb_ovn, g_name, candidates=candidates) txn.add(self._ovn.update_lrouter_port( diff --git a/networking_ovn/ovsdb/impl_idl_ovn.py b/networking_ovn/ovsdb/impl_idl_ovn.py index b0f930675..93f3e40eb 100644 --- a/networking_ovn/ovsdb/impl_idl_ovn.py +++ b/networking_ovn/ovsdb/impl_idl_ovn.py @@ -408,7 +408,8 @@ class OvsdbNbOvnIdl(nb_impl_idl.OvnNbApiIdlImpl, Backend): except idlutils.RowNotFound: return [] - def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets): + def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets, + gw_chassis): unhosted_gateways = [] valid_chassis_list = list(chassis_physnets) for lrp in self._tables['Logical_Router_Port'].rows.values(): @@ -423,7 +424,8 @@ class OvsdbNbOvnIdl(nb_impl_idl.OvnNbApiIdlImpl, Backend): if (chassis_name == ovn_const.OVN_GATEWAY_INVALID_CHASSIS or chassis_name not in valid_chassis_list or (physnet and - physnet not in chassis_physnets.get(chassis_name))): + physnet not in chassis_physnets.get(chassis_name)) or + (gw_chassis and chassis_name not in gw_chassis)): unhosted_gateways.append(lrp.name) return unhosted_gateways @@ -672,6 +674,14 @@ class OvsdbSbOvnIdl(sb_impl_idl.OvnSbApiIdlImpl, Backend): chassis_info_dict[ch.hostname] = self._get_chassis_physnets(ch) return chassis_info_dict + def get_gateway_chassis_from_cms_options(self): + gw_chassis = [] + for ch in self.chassis_list().execute(check_error=True): + cms_options = ch.external_ids.get('ovn-cms-options', '') + if 'enable-chassis-as-gw' in cms_options.split(','): + gw_chassis.append(ch.name) + return gw_chassis + def get_chassis_and_physnets(self): chassis_info_dict = {} for ch in self.chassis_list().execute(check_error=True): diff --git a/networking_ovn/ovsdb/ovn_api.py b/networking_ovn/ovsdb/ovn_api.py index 08b66dba2..541a011df 100644 --- a/networking_ovn/ovsdb/ovn_api.py +++ b/networking_ovn/ovsdb/ovn_api.py @@ -337,11 +337,14 @@ class API(api.API): """ @abc.abstractmethod - def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets): + def get_unhosted_gateways(self, port_physnet_dict, chassis_physnets, + gw_chassis): """Return a list of gateways not hosted on chassis :param port_physnet_dict: Dictionary of gateway ports and their physnet :param chassis_physnets: Dictionary of chassis and physnets + :param gw_chassis: List of gateway chassis provided by admin + through ovn-cms-options :returns: List of gateways not hosted on a valid chassis """ @@ -623,6 +626,18 @@ class SbAPI(api.API): value. And hostname and physnets are related to the same host. """ + def get_gateway_chassis_from_cms_options(self): + """Get chassis eligible for external connectivity from CMS options. + + When admin wants to enable router gateway on few chassis, + he would set the external_ids as + + ovs-vsctl set open . + external_ids:ovn-cms-options="enable-chassis-as-gw" + In this function, we parse ovn-cms-options and return these chassis + :returns: List with chassis names. + """ + @abc.abstractmethod def get_chassis_and_physnets(self): """Return a dict contains chassis name and physnets mapping. diff --git a/networking_ovn/tests/functional/test_router.py b/networking_ovn/tests/functional/test_router.py index f36653bab..e4fa0888b 100644 --- a/networking_ovn/tests/functional/test_router.py +++ b/networking_ovn/tests/functional/test_router.py @@ -82,6 +82,63 @@ class TestRouter(base.TestOVNFunctionalBase): rc = row.options.get(ovn_const.OVN_GATEWAY_CHASSIS_KEY) self.assertIn(rc, expected) + def _check_gateway_chassis_candidates(self, candidates): + # In this test, fake_select() is called once from _create_router() + # and later from schedule_unhosted_gateways() + ovn_client = self.l3_plugin._ovn_client + ext1 = self._create_ext_network( + 'ext1', 'vlan', 'physnet1', 1, "10.0.0.1", "10.0.0.0/24") + # mock select function and check if it is called with expected + # candidates. + + def fake_select(*args, **kwargs): + self.assertItemsEqual(candidates, kwargs['candidates']) + # We are not interested in further processing, let us return + # INVALID_CHASSIS to avoid erros + return [ovn_const.OVN_GATEWAY_INVALID_CHASSIS] + + with mock.patch.object(ovn_client._ovn_scheduler, 'select', + side_effect=fake_select) as client_select,\ + mock.patch.object(self.l3_plugin.scheduler, 'select', + side_effect=fake_select) as plugin_select: + gw_info = {'network_id': ext1['network']['id']} + self._create_router('router1', gw_info=gw_info) + self.assertFalse(plugin_select.called) + self.assertTrue(client_select.called) + client_select.reset_mock() + plugin_select.reset_mock() + + # set redirect-chassis to neutron-ovn-invalid-chassis, so + # that schedule_unhosted_gateways will try to schedule it + self._set_redirect_chassis_to_invalid_chassis(ovn_client) + self.l3_plugin.schedule_unhosted_gateways() + self.assertFalse(client_select.called) + self.assertTrue(plugin_select.called) + + def test_gateway_chassis_with_cms_and_bridge_mappings(self): + # Both chassis1 and chassis3 are having proper bridge mappings, + # but only chassis3 is having enable-chassis-as-gw. + # Test if chassis3 is selected as candidate or not. + self.chassis3 = self.add_fake_chassis( + 'ovs-host3', physical_nets=['physnet1'], + external_ids={'ovn-cms-options': 'enable-chassis-as-gw'}) + self._check_gateway_chassis_candidates([self.chassis3]) + + def test_gateway_chassis_with_cms_and_no_bridge_mappings(self): + # chassis1 is having proper bridge mappings. + # chassis3 is having enable-chassis-as-gw, but no bridge mappings. + # Test if chassis1 is selected as candidate or not. + self.chassis3 = self.add_fake_chassis( + 'ovs-host3', + external_ids={'ovn-cms-options': 'enable-chassis-as-gw'}) + self._check_gateway_chassis_candidates([self.chassis1]) + + def test_gateway_chassis_with_bridge_mappings_and_no_cms(self): + # chassis1 is configured with proper bridge mappings, + # but none of the chassis having enable-chassis-as-gw. + # Test if chassis1 is selected as candidate or not. + self._check_gateway_chassis_candidates([self.chassis1]) + def test_gateway_chassis_with_bridge_mappings(self): ovn_client = self.l3_plugin._ovn_client # Create external networks with vlan, flat and geneve network types diff --git a/networking_ovn/tests/unit/l3/test_l3_ovn.py b/networking_ovn/tests/unit/l3/test_l3_ovn.py index 326f479b4..48af1a700 100644 --- a/networking_ovn/tests/unit/l3/test_l3_ovn.py +++ b/networking_ovn/tests/unit/l3/test_l3_ovn.py @@ -1065,6 +1065,10 @@ class OVNL3ExtrarouteTests(test_l3_gw.ExtGwModeIntTestCase, 'networking_ovn.l3.l3_ovn_scheduler.' 'OVNGatewayScheduler._schedule_gateway', return_value='hv1') + self._start_mock( + 'networking_ovn.common.ovn_client.' + 'OVNClient.get_candidates_for_scheduling', + return_value=[]) self._start_mock( 'networking_ovn.common.ovn_client.OVNClient.' '_get_v4_network_of_all_router_ports', diff --git a/networking_ovn/tests/unit/ovsdb/test_impl_idl_ovn.py b/networking_ovn/tests/unit/ovsdb/test_impl_idl_ovn.py index 051e31817..23a14cde2 100644 --- a/networking_ovn/tests/unit/ovsdb/test_impl_idl_ovn.py +++ b/networking_ovn/tests/unit/ovsdb/test_impl_idl_ovn.py @@ -580,7 +580,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn): self._load_nb_db() # Test only host-1 in the valid list unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways( - {}, {'host-1': 'physnet1'}) + {}, {'host-1': 'physnet1'}, []) expected = { utils.ovn_lrouter_port_name('orp-id-b2'): { ovn_const.OVN_GATEWAY_CHASSIS_KEY: 'host-2'}, @@ -590,7 +590,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn): self.assertItemsEqual(unhosted_gateways, expected) # Test both host-1, host-2 in valid list unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways( - {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}) + {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}, []) expected = {utils.ovn_lrouter_port_name('orp-id-a3'): { ovn_const.OVN_GATEWAY_CHASSIS_KEY: ovn_const.OVN_GATEWAY_INVALID_CHASSIS}} @@ -602,7 +602,7 @@ class TestNBImplIdlOvn(TestDBImplIdlOvn): setattr(router_row, 'options', { ovn_const.OVN_GATEWAY_CHASSIS_KEY: 'host-2'}) unhosted_gateways = self.nb_ovn_idl.get_unhosted_gateways( - {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}) + {}, {'host-1': 'physnet1', 'host-2': 'physnet2'}, []) self.assertItemsEqual(unhosted_gateways, {}) def test_get_subnet_dhcp_options(self): diff --git a/releasenotes/notes/ovn-cms-options-enable-chassis-as-gw-3adc7024478e3efa.yaml b/releasenotes/notes/ovn-cms-options-enable-chassis-as-gw-3adc7024478e3efa.yaml new file mode 100644 index 000000000..6fb8e98ba --- /dev/null +++ b/releasenotes/notes/ovn-cms-options-enable-chassis-as-gw-3adc7024478e3efa.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + New option "enable-chassis-as-gw" to select gateway router. + For external connectivity, gateway nodes have to set ovn-cms-options + with enable-chassis-as-gw in Open_vSwitch table's external_ids column. + + $ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw" + + Networking-ovn will parse ovn-cms-options and select this chassis + if it has proper bridge mappings. This helps admin to exclude compute + nodes to host gateway routers as they are more likely to be restarted + for maintenance operations. If no chassis with enable-chassis-as-gw and + proper bridge mappings available, then chassis with only bridge mappings + are selected for scheduling router gateway. + + This is not a config option enabled through conf files. Instead admin + has to set it through openstack installer or manually in Open_vSwitch + table. diff --git a/vagrant/provisioning/setup-compute.sh b/vagrant/provisioning/setup-compute.sh index 2713cfaac..db7ac7ed5 100644 --- a/vagrant/provisioning/setup-compute.sh +++ b/vagrant/provisioning/setup-compute.sh @@ -44,6 +44,8 @@ PHYSICAL_NETWORK=provider # as necessary for your environment. NETWORK_GATEWAY=172.16.1.1 FIXED_RANGE=172.16.1.0/24 + +ENABLE_CHASSIS_AS_GW=False DEVSTACKEOF # Add unique post-config for DevStack here using a separate 'cat' with diff --git a/vagrant/provisioning/setup-controller.sh b/vagrant/provisioning/setup-controller.sh index 18034e06c..0e450cb2a 100644 --- a/vagrant/provisioning/setup-controller.sh +++ b/vagrant/provisioning/setup-controller.sh @@ -71,6 +71,12 @@ PUBLIC_SUBNET_NAME=provider-v4 IPV6_PUBLIC_SUBNET_NAME=provider-v6 Q_FLOATING_ALLOCATION_POOL="start=$start_ip,end=$end_ip" FLOATING_RANGE="$network" + +# If the admin wants to enable this chassis to host gateway routers for +# external connectivity, then set ENABLE_CHASSIS_AS_GW to True. +# Then devstack will set ovn-cms-options with enable-chassis-as-gw +# in Open_vSwitch table's external_ids column +ENABLE_CHASSIS_AS_GW=True DEVSTACKEOF # Add unique post-config for DevStack here using a separate 'cat' with