[L3][QoS] Neutron server side router gateway IP QoS

This patch enables to bind a QoS policy to the router gateway,
then in L3 agent side SNAT traffic for the VMs without floating
IPs can be limited under the policy bandwidth rules. This is
suit for all kinds of L3 routers: DVR, DVR with SNAT HA, L3 HA
and Legacy.

API update router gateway json:
{
router": {
  "external_gateway_info": {
    ...
    "qos_policy_id": "policy-uuid"
    }
  }
}

Depends-On: https://review.openstack.org/#/c/567497/

Partially-Implements blueprint: router-gateway-ip-qos
Closes-Bug: #1757044
Related-Bug: #1596611
Change-Id: I26e22bce7edd1f93b2ac0048b61b14f858938537

neutron/common/exceptions.py

@@ -119,3 +119,13 @@ class ProcessExecutionError(RuntimeError):
     def __init__(self, message, returncode):
         super(ProcessExecutionError, self).__init__(message)
         self.returncode = returncode
+
+
+class RouterQosBindingNotFound(exceptions.NotFound):
+    message = _("QoS binding for router %(router_id)s gateway and policy "
+                "%(policy_id)s could not be found.")
+
+
+class RouterQosBindingError(exceptions.NeutronException):
+    message = _("QoS binding for router %(router_id)s gateway and policy "
+                "%(policy_id)s could not be created: %(db_error)s.")

neutron/db/l3_gateway_ip_qos.py

@@ -0,0 +1,129 @@
+# Copyright 2018 OpenStack Foundation
+# Copyright 2017 Letv Cloud Computing
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+from neutron_lib.api.definitions import l3 as l3_apidef
+from neutron_lib.api.definitions import qos_gateway_ip
+from neutron_lib.api import extensions
+from neutron_lib.services.qos import constants as qos_consts
+from oslo_log import log as logging
+
+from neutron.db import _resource_extend as resource_extend
+from neutron.db import l3_db
+from neutron.db import l3_gwmode_db
+from neutron.objects.qos import policy as policy_object
+
+LOG = logging.getLogger(__name__)
+
+
+@resource_extend.has_resource_extenders
+class L3_gw_ip_qos_dbonly_mixin(l3_gwmode_db.L3_NAT_dbonly_mixin):
+    """Mixin class to add router gateway IP's QoS extra attributes."""
+
+    _gw_ip_qos = None
+
+    @staticmethod
+    @resource_extend.extends([l3_apidef.ROUTERS])
+    def _extend_router_dict_gw_qos(router_res, router_db):
+        if router_db.gw_port_id and router_db.get('qos_policy_binding'):
+            policy_id = router_db.qos_policy_binding.policy_id
+            router_res[l3_apidef.EXTERNAL_GW_INFO].update(
+                {qos_consts.QOS_POLICY_ID: policy_id})
+
+    @property
+    def _is_gw_ip_qos_supported(self):
+        if self._gw_ip_qos is None:
+            # Check L3 service plugin
+            self._gw_ip_qos = extensions.is_extension_supported(
+                self, qos_gateway_ip.ALIAS)
+        return self._gw_ip_qos
+
+    def _create_gw_ip_qos_db(self, context, router_id, policy_id):
+        policy = policy_object.QosPolicy.get_policy_obj(context, policy_id)
+        policy.attach_router(router_id)
+
+    def _delete_gw_ip_qos_db(self, context, router_id, policy_id):
+        policy = policy_object.QosPolicy.get_policy_obj(context, policy_id)
+        policy.detach_router(router_id)
+
+    def _update_router_gw_info(self, context, router_id, info, router=None):
+        # Calls superclass, pass router db object for avoiding re-loading
+        router = super(L3_gw_ip_qos_dbonly_mixin,
+                       self)._update_router_gw_info(
+            context, router_id, info, router)
+
+        if self._is_gw_ip_qos_supported and router.gw_port:
+            self._update_router_gw_qos_policy(context, router_id,
+                                              info, router)
+
+        return router
+
+    def _get_router_gateway_policy_binding(self, context, router_id):
+        router = self._get_router(context, router_id)
+        return router.qos_policy_binding
+
+    def _update_router_gw_qos_policy(self, context, router_id, info, router):
+        if not info or qos_consts.QOS_POLICY_ID not in info:
+            # An explicit 'None' for `qos_polcy_id` indicates to clear
+            # the router gateway IP policy. So if info does not have
+            # the key `qos_polcy_id`, we can not decide what behavior
+            # to be done, then directly return here.
+            return
+
+        new_qos_policy_id = info[qos_consts.QOS_POLICY_ID]
+        if router.qos_policy_binding:
+            old_qos_policy_id = router.qos_policy_binding.policy_id
+
+            if old_qos_policy_id == new_qos_policy_id:
+                return
+            if old_qos_policy_id:
+                self._delete_gw_ip_qos_db(context,
+                                          router_id,
+                                          old_qos_policy_id)
+
+        with context.session.begin(subtransactions=True):
+            context.session.refresh(router)
+
+        if new_qos_policy_id:
+            self._create_gw_ip_qos_db(
+                context, router_id, new_qos_policy_id)
+
+    def _build_routers_list(self, context, routers, gw_ports):
+        routers = super(L3_gw_ip_qos_dbonly_mixin,
+                        self)._build_routers_list(
+                            context, routers, gw_ports)
+
+        if not self._is_gw_ip_qos_supported:
+            return routers
+
+        for rtr in routers:
+            gw_port_id = rtr['gw_port_id']
+            # Collect gw ports only if available
+            if gw_port_id and gw_ports.get(gw_port_id):
+                rtr['gw_port'] = gw_ports[gw_port_id]
+                router_gateway_policy_binding = (
+                    self._get_router_gateway_policy_binding(
+                        context, rtr['id']))
+                qos_policy_id = None
+                if router_gateway_policy_binding:
+                    qos_policy_id = router_gateway_policy_binding.policy_id
+                rtr['gw_port'][qos_consts.QOS_POLICY_ID] = qos_policy_id
+        return routers
+
+
+class L3_gw_ip_qos_db_mixin(L3_gw_ip_qos_dbonly_mixin,
+                            l3_db.L3_NAT_db_mixin):
+    pass

neutron/db/l3_gwmode_db.py

@@ -45,7 +45,7 @@ class L3_NAT_dbonly_mixin(l3_db.L3_NAT_dbonly_mixin):
     def _extend_router_dict_gw_mode(router_res, router_db):
         if router_db.gw_port_id:
             nw_id = router_db.gw_port['network_id']
-            router_res[l3_apidef.EXTERNAL_GW_INFO] = {
+            router_res[l3_apidef.EXTERNAL_GW_INFO].update({
                 'network_id': nw_id,
                 'enable_snat': router_db.enable_snat,
                 'external_fixed_ips': [
@@ -53,7 +53,7 @@ class L3_NAT_dbonly_mixin(l3_db.L3_NAT_dbonly_mixin):
                      'ip_address': ip["ip_address"]}
                     for ip in router_db.gw_port['fixed_ips']
                 ]
-            }
+            })
 
     def _update_router_gw_info(self, context, router_id, info, router=None):
         # Load the router only if necessary

neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -1 +1 @@
-cada2437bf41
+195176fb410d

neutron/db/migration/alembic_migrations/versions/stein/expand/195176fb410d_router_gateway_ip_qos.py

@@ -0,0 +1,45 @@
+# Copyright 2018 OpenStack Foundation
+# Copyright 2017 Letv Cloud Computing
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""router gateway IP QoS
+
+Revision ID: 195176fb410d
+Revises: cada2437bf41
+Create Date: 2016-04-28 12:38:09.872706
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from neutron_lib.db import constants as db_const
+
+# revision identifiers, used by Alembic.
+revision = '195176fb410d'
+down_revision = 'cada2437bf41'
+
+
+def upgrade():
+    op.create_table(
+        'qos_router_gw_policy_bindings',
+        sa.Column('policy_id',
+                  sa.String(length=db_const.UUID_FIELD_SIZE),
+                  sa.ForeignKey('qos_policies.id', ondelete='CASCADE'),
+                  nullable=False, primary_key=True),
+        sa.Column('router_id',
+                  sa.String(length=db_const.UUID_FIELD_SIZE),
+                  sa.ForeignKey('routers.id', ondelete='CASCADE'),
+                  nullable=False, unique=True, primary_key=True))

neutron/db/qos/models.py

@@ -76,6 +76,26 @@ class QosFIPPolicyBinding(model_base.BASEV2):
                                cascade='delete', lazy='joined'))
 
 
+class QosRouterGatewayIPPolicyBinding(model_base.BASEV2):
+    __tablename__ = 'qos_router_gw_policy_bindings'
+    policy_id = sa.Column(sa.String(db_const.UUID_FIELD_SIZE),
+                          sa.ForeignKey('qos_policies.id',
+                                        ondelete='CASCADE'),
+                          nullable=False,
+                          primary_key=True)
+    router_id = sa.Column(sa.String(db_const.UUID_FIELD_SIZE),
+                          sa.ForeignKey('routers.id',
+                                        ondelete='CASCADE'),
+                          nullable=False,
+                          unique=True,
+                          primary_key=True)
+    revises_on_change = ('router', )
+    router = sa.orm.relationship(
+        l3.Router, load_on_pending=True,
+        backref=sa.orm.backref("qos_policy_binding", uselist=False,
+                               cascade='delete', lazy='joined'))
+
+
 class QosPortPolicyBinding(model_base.BASEV2):
     __tablename__ = 'qos_port_policy_bindings'
     policy_id = sa.Column(sa.String(36),

neutron/extensions/qos_gateway_ip.py

@@ -0,0 +1,24 @@
+# Copyright 2018 OpenStack Foundation
+# Copyright 2017 Letv Cloud Computing
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from neutron_lib.api.definitions import qos_gateway_ip as apidef
+from neutron_lib.api import extensions
+
+
+class Qos_gateway_ip(extensions.APIExtensionDescriptor):
+    """Extension class supporting gateway IP rate limit in all router."""
+
+    api_definition = apidef

neutron/objects/qos/binding.py

@@ -65,3 +65,19 @@ class QosPolicyFloatingIPBinding(base.NeutronDbObject):
 
     primary_keys = ['policy_id', 'fip_id']
     fields_no_update = ['policy_id', 'fip_id']
+
+
+@base.NeutronObjectRegistry.register
+class QosPolicyRouterGatewayIPBinding(base.NeutronDbObject):
+    # Version 1.0: Initial version
+    VERSION = '1.0'
+
+    db_model = qos_db_model.QosRouterGatewayIPPolicyBinding
+
+    fields = {
+        'policy_id': common_types.UUIDField(),
+        'router_id': common_types.UUIDField()
+    }
+
+    primary_keys = ['policy_id', 'router_id']
+    fields_no_update = ['policy_id', 'router_id']

neutron/objects/qos/policy.py

@@ -58,7 +58,8 @@ class QosPolicy(rbac_db.NeutronRbacObject):
     # Version 1.5: Direction for bandwidth limit rule added
     # Version 1.6: Added "is_default" field
     # Version 1.7: Added floating IP bindings
-    VERSION = '1.7'
+    # Version 1.8: Added router gateway QoS policy bindings
+    VERSION = '1.8'
 
     # required by RbacNeutronMetaclass
     rbac_db_cls = QosPolicyRBAC
@@ -81,7 +82,8 @@ class QosPolicy(rbac_db.NeutronRbacObject):
 
     binding_models = {'port': binding.QosPolicyPortBinding,
                       'network': binding.QosPolicyNetworkBinding,
-                      'fip': binding.QosPolicyFloatingIPBinding}
+                      'fip': binding.QosPolicyFloatingIPBinding,
+                      'router': binding.QosPolicyRouterGatewayIPBinding}
 
     def obj_load_attr(self, attrname):
         if attrname == 'rules':
@@ -201,6 +203,12 @@ class QosPolicy(rbac_db.NeutronRbacObject):
         return cls._get_object_policy(
             context, binding.QosPolicyFloatingIPBinding, fip_id=fip_id)
 
+    @classmethod
+    def get_router_policy(cls, context, router_id):
+        return cls._get_object_policy(
+            context, binding.QosPolicyRouterGatewayIPBinding,
+            router_id=router_id)
+
     # TODO(QoS): Consider extending base to trigger registered methods for us
     def create(self):
         with self.db_context_writer(self.obj_context):
@@ -265,6 +273,16 @@ class QosPolicy(rbac_db.NeutronRbacObject):
                                                        fip_id=fip_id,
                                                        db_error=e)
 
+    def attach_router(self, router_id):
+        router_binding_obj = binding.QosPolicyRouterGatewayIPBinding(
+            self.obj_context, policy_id=self.id, router_id=router_id)
+        try:
+            router_binding_obj.create()
+        except db_exc.DBReferenceError as e:
+            raise exceptions.RouterQosBindingError(policy_id=self.id,
+                                                   router_id=router_id,
+                                                   db_error=e)
+
     def detach_network(self, network_id):
         deleted = binding.QosPolicyNetworkBinding.delete_objects(
             self.obj_context, network_id=network_id)
@@ -286,6 +304,13 @@ class QosPolicy(rbac_db.NeutronRbacObject):
             raise exceptions.FloatingIPQosBindingNotFound(fip_id=fip_id,
                                                           policy_id=self.id)
 
+    def detach_router(self, router_id):
+        deleted = binding.QosPolicyRouterGatewayIPBinding.delete_objects(
+            self.obj_context, router_id=router_id)
+        if not deleted:
+            raise exceptions.RouterQosBindingNotFound(router_id=router_id,
+                                                      policy_id=self.id)
+
     def set_default(self):
         if not self.get_default():
             qos_default_policy = QosPolicyDefault(self.obj_context,
@@ -329,6 +354,13 @@ class QosPolicy(rbac_db.NeutronRbacObject):
                 self.obj_context, policy_id=self.id)
         ]
 
+    def get_bound_routers(self):
+        return [
+            rb.router_id
+            for rb in binding.QosPolicyRouterGatewayIPBinding.get_objects(
+                self.obj_context, policy_id=self.id)
+        ]
+
     @classmethod
     def _get_bound_tenant_ids(cls, session, binding_db, bound_db,
                               binding_db_id_column, policy_id):
@@ -349,6 +381,8 @@ class QosPolicy(rbac_db.NeutronRbacObject):
         qosport = qos_db_model.QosPortPolicyBinding
         fip = l3.FloatingIP
         qosfip = qos_db_model.QosFIPPolicyBinding
+        router = l3.Router
+        qosrouter = qos_db_model.QosRouterGatewayIPPolicyBinding
         bound_tenants = []
         with cls.db_context_reader(context):
             bound_tenants.extend(cls._get_bound_tenant_ids(
@@ -359,6 +393,9 @@ class QosPolicy(rbac_db.NeutronRbacObject):
             bound_tenants.extend(
                 cls._get_bound_tenant_ids(context.session, qosfip, fip,
                                           qosfip.fip_id, policy_id))
+            bound_tenants.extend(
+                cls._get_bound_tenant_ids(context.session, qosrouter, router,
+                                          qosrouter.router_id, policy_id))
         return set(bound_tenants)
 
     def obj_make_compatible(self, primitive, target_version):

neutron/services/l3_router/l3_router_plugin.py

@@ -35,7 +35,7 @@ from neutron.db import l3_dvrscheduler_db
 from neutron.db import l3_fip_pools_db
 from neutron.db import l3_fip_port_details
 from neutron.db import l3_fip_qos
-from neutron.db import l3_gwmode_db
+from neutron.db import l3_gateway_ip_qos
 from neutron.db import l3_hamode_db
 from neutron.db import l3_hascheduler_db
 from neutron.db.models import l3 as l3_models
@@ -54,11 +54,11 @@ def disable_dvr_extension_by_config(aliases):
             aliases.remove('dvr')
 
 
-def disable_qos_fip_extension_by_plugins(aliases):
+def disable_l3_qos_extension_by_plugins(ext, aliases):
     qos_class = 'neutron.services.qos.qos_plugin.QoSPlugin'
     if all(p not in cfg.CONF.service_plugins for p in ['qos', qos_class]):
-        if 'qos-fip' in aliases:
+        if ext in aliases:
-            aliases.remove('qos-fip')
+            aliases.remove(ext)
 
 
 @resource_extend.has_resource_extenders
@@ -66,7 +66,7 @@ class L3RouterPlugin(service_base.ServicePluginBase,
                      common_db_mixin.CommonDbMixin,
                      extraroute_db.ExtraRoute_db_mixin,
                      l3_hamode_db.L3_HA_NAT_db_mixin,
-                     l3_gwmode_db.L3_NAT_db_mixin,
+                     l3_gateway_ip_qos.L3_gw_ip_qos_db_mixin,
                      l3_dvr_ha_scheduler_db.L3_DVR_HA_scheduler_db_mixin,
                      dns_db.DNSDbMixin,
                      l3_fip_qos.FloatingQoSDbMixin,
@@ -86,7 +86,8 @@ class L3RouterPlugin(service_base.ServicePluginBase,
                                     "extraroute", "l3_agent_scheduler",
                                     "l3-ha", "router_availability_zone",
                                     "l3-flavors", "qos-fip",
-                                    "fip-port-details", "floatingip-pools"]
+                                    "fip-port-details", "floatingip-pools",
+                                    "qos-gateway-ip"]
 
     __native_pagination_support = True
     __native_sorting_support = True
@@ -116,7 +117,8 @@ class L3RouterPlugin(service_base.ServicePluginBase,
         if not hasattr(self, '_aliases'):
             aliases = self._supported_extension_aliases[:]
             disable_dvr_extension_by_config(aliases)
-            disable_qos_fip_extension_by_plugins(aliases)
+            disable_l3_qos_extension_by_plugins('qos-fip', aliases)
+            disable_l3_qos_extension_by_plugins('qos-gateway-ip', aliases)
             self._aliases = aliases
         return self._aliases
 

neutron/tests/common/l3_test_common.py

@@ -105,6 +105,8 @@ def prepare_router_data(ip_version=lib_constants.IP_VERSION_4,
                       'subnets': subnets,
                       'extra_subnets': extra_subnets}
 
+    external_gateway_info = {"qos_policy_id": kwargs.get('qos_policy_id')}
+
     routes = []
     if extra_routes:
         routes = [{'destination': '8.8.8.0/24', 'nexthop': '19.4.4.4'}]
@@ -114,7 +116,8 @@ def prepare_router_data(ip_version=lib_constants.IP_VERSION_4,
         'distributed': False,
         lib_constants.INTERFACE_KEY: [],
         'routes': routes,
-        'gw_port': ex_gw_port}
+        'gw_port': ex_gw_port,
+        'external_gateway_info': external_gateway_info}
 
     router_fips = router.get(lib_constants.FLOATINGIP_KEY, [])
     if enable_floating_ip:

neutron/tests/contrib/hooks/api_all_extensions

@@ -38,6 +38,7 @@ NETWORK_API_EXTENSIONS+=",project-id"
 NETWORK_API_EXTENSIONS+=",provider"
 NETWORK_API_EXTENSIONS+=",qos"
 NETWORK_API_EXTENSIONS+=",qos-fip"
+NETWORK_API_EXTENSIONS+=",qos-gateway-ip"
 NETWORK_API_EXTENSIONS+=",quotas"
 NETWORK_API_EXTENSIONS+=",quota_details"
 NETWORK_API_EXTENSIONS+=",rbac-policies"

neutron/tests/unit/extensions/test_l3.py

@@ -395,13 +395,17 @@ class L3NatTestCaseMixin(object):
 
     def _add_external_gateway_to_router(self, router_id, network_id,
                                         expected_code=exc.HTTPOk.code,
-                                        neutron_context=None, ext_ips=None):
+                                        neutron_context=None, ext_ips=None,
+                                        **kwargs):
         ext_ips = ext_ips or []
         body = {'router':
                 {'external_gateway_info': {'network_id': network_id}}}
         if ext_ips:
             body['router']['external_gateway_info'][
                 'external_fixed_ips'] = ext_ips
+        if 'policy_id' in kwargs:
+            body['router']['external_gateway_info'][
+                'qos_policy_id'] = kwargs.get('policy_id')
         return self._update('routers', router_id, body,
                             expected_code=expected_code,
                             neutron_context=neutron_context)

neutron/tests/unit/extensions/test_qos_gateway_ip.py

@@ -0,0 +1,233 @@
+# Copyright 2018 OpenStack Foundation
+# Copyright 2017 Letv Cloud Computing
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from neutron_lib.api.definitions import l3 as l3_apidef
+from neutron_lib.api.definitions import qos_gateway_ip
+from neutron_lib import context
+from neutron_lib.services.qos import constants as qos_consts
+from oslo_config import cfg
+from oslo_utils import uuidutils
+
+from neutron.conf.db import extraroute_db
+from neutron.db import l3_gateway_ip_qos
+from neutron.extensions import l3
+from neutron.objects.qos import policy
+from neutron.tests.unit.extensions import test_l3
+
+
+class GatewayIPQoSTestExtensionManager(object):
+
+    def get_resources(self):
+        l3_apidef.RESOURCE_ATTRIBUTE_MAP['routers'].update(
+            qos_gateway_ip.RESOURCE_ATTRIBUTE_MAP['routers'])
+        return l3.L3.get_resources()
+
+    def get_actions(self):
+        return []
+
+    def get_request_extensions(self):
+        return []
+
+
+class TestGatewayIPQoSIntPlugin(
+        test_l3.TestL3NatIntPlugin,
+        l3_gateway_ip_qos.L3_gw_ip_qos_db_mixin):
+    supported_extension_aliases = ["external-net",
+                                   "router",
+                                   "ext-gw-mode",
+                                   qos_gateway_ip.ALIAS]
+
+
+class TestGatewayIPQoSL3NatServicePlugin(
+        test_l3.TestL3NatServicePlugin,
+        l3_gateway_ip_qos.L3_gw_ip_qos_db_mixin):
+    supported_extension_aliases = ["router",
+                                   "ext-gw-mode",
+                                   qos_gateway_ip.ALIAS]
+
+
+class GatewayIPQoSDBTestCaseBase(object):
+
+    def test_create_router_gateway_with_qos_policy(self):
+        ctx = context.get_admin_context()
+        policy_obj = policy.QosPolicy(ctx,
+                                      id=uuidutils.generate_uuid(),
+                                      project_id='tenant', name='pol1',
+                                      rules=[])
+        policy_obj.create()
+        with self.subnet(cidr='11.0.0.0/24') as public_sub,\
+                self.router() as r:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                policy_id=policy_obj.id)
+            self.assertEqual(
+                policy_obj.id,
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+    def test_update_router_gateway_with_qos_policy(self):
+        ctx = context.get_admin_context()
+        policy_obj = policy.QosPolicy(ctx,
+                                      id=uuidutils.generate_uuid(),
+                                      project_id='tenant', name='pol1',
+                                      rules=[])
+        policy_obj.create()
+        with self.subnet(cidr='11.0.0.0/24') as public_sub,\
+                self.router() as r:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'])
+            self.assertIsNone(
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+            # update router gateway
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                policy_id=policy_obj.id)
+            self.assertEqual(
+                policy_obj.id,
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+    def test_clear_router_gateway_and_create_with_old_qos_policy_implicitly(
+            self):
+        ctx = context.get_admin_context()
+        policy_obj = policy.QosPolicy(ctx,
+                                      id=uuidutils.generate_uuid(),
+                                      project_id='tenant', name='pol1',
+                                      rules=[])
+        policy_obj.create()
+        with self.subnet(cidr='11.0.0.0/24') as public_sub,\
+                self.router() as r:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                policy_id=policy_obj.id)
+            self.assertEqual(
+                policy_obj.id,
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+            # Clear router gateway
+            self._remove_external_gateway_from_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                external_gw_info={})
+
+            # Create router gateway again, then the qos policy binding will be
+            # reused here.
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'])
+            self.assertEqual(
+                policy_obj.id,
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+    def test_clear_router_gateway_qos_policy(self):
+        ctx = context.get_admin_context()
+        policy_obj = policy.QosPolicy(ctx,
+                                      id=uuidutils.generate_uuid(),
+                                      project_id='tenant', name='pol1',
+                                      rules=[])
+        policy_obj.create()
+        with self.subnet(cidr='11.0.0.0/24') as public_sub,\
+                self.router() as r:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'])
+            self.assertIsNone(
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+            # update router gateway
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                policy_id=policy_obj.id)
+            self.assertEqual(
+                policy_obj.id,
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+            # Explicitly clear router gateway qos policy binding
+            res = self._add_external_gateway_to_router(
+                r['router']['id'],
+                public_sub['subnet']['network_id'],
+                policy_id=None,
+                is_remove=True)
+            self.assertIsNone(
+                res['router']['external_gateway_info'].get(
+                    qos_consts.QOS_POLICY_ID))
+
+
+class GatewayIPQoSDBIntTestCase(test_l3.L3BaseForIntTests,
+                                test_l3.L3NatTestCaseMixin,
+                                GatewayIPQoSDBTestCaseBase):
+
+    def setUp(self, plugin=None):
+        if not plugin:
+            plugin = ('neutron.tests.unit.extensions.test_qos_gateway_ip.'
+                      'TestGatewayIPQoSIntPlugin')
+        service_plugins = {'qos': 'neutron.services.qos.qos_plugin.QoSPlugin'}
+
+        extraroute_db.register_db_extraroute_opts()
+        # for these tests we need to enable overlapping ips
+        cfg.CONF.set_default('allow_overlapping_ips', True)
+        cfg.CONF.set_default('max_routes', 3)
+
+        ext_mgr = GatewayIPQoSTestExtensionManager()
+        super(test_l3.L3BaseForIntTests, self).setUp(
+            plugin=plugin,
+            ext_mgr=ext_mgr,
+            service_plugins=service_plugins)
+
+        self.setup_notification_driver()
+
+
+class GatewayIPQoSDBSepTestCase(test_l3.L3BaseForSepTests,
+                                test_l3.L3NatTestCaseMixin,
+                                GatewayIPQoSDBTestCaseBase):
+
+    def setUp(self):
+        # the plugin without L3 support
+        plugin = 'neutron.tests.unit.extensions.test_l3.TestNoL3NatPlugin'
+        # the L3 service plugin
+        l3_plugin = ('neutron.tests.unit.extensions.test_qos_gateway_ip.'
+                     'TestGatewayIPQoSL3NatServicePlugin')
+        service_plugins = {'l3_plugin_name': l3_plugin,
+                           'qos': 'neutron.services.qos.qos_plugin.QoSPlugin'}
+
+        extraroute_db.register_db_extraroute_opts()
+        # for these tests we need to enable overlapping ips
+        cfg.CONF.set_default('allow_overlapping_ips', True)
+        cfg.CONF.set_default('max_routes', 3)
+
+        ext_mgr = GatewayIPQoSTestExtensionManager()
+        super(test_l3.L3BaseForSepTests, self).setUp(
+            plugin=plugin,
+            ext_mgr=ext_mgr,
+            service_plugins=service_plugins)
+
+        self.setup_notification_driver()

neutron/tests/unit/objects/qos/test_binding.py

@@ -68,3 +68,22 @@ class QosPolicyFloatingIPBindingDbObjectTestCase(
         for db_obj in self.db_objs:
             self._create_test_qos_policy(id=db_obj['policy_id'])
             self._create_test_fip_id(fip_id=db_obj['fip_id'])
+
+
+class QosPolicyRouterGatewayIPBindingObjectTestCase(
+        test_base.BaseObjectIfaceTestCase):
+
+    _test_class = binding.QosPolicyRouterGatewayIPBinding
+
+
+class QosPolicyRouterGatewayIPBindingDbObjectTestCase(
+        test_base.BaseDbObjectTestCase,
+        testlib_api.SqlTestCase):
+
+    _test_class = binding.QosPolicyRouterGatewayIPBinding
+
+    def setUp(self):
+        super(QosPolicyRouterGatewayIPBindingDbObjectTestCase, self).setUp()
+        for db_obj in self.db_objs:
+            self._create_test_qos_policy(id=db_obj['policy_id'])
+            self._create_test_router_id(router_id=db_obj['router_id'])

neutron/tests/unit/objects/test_base.py

@@ -1542,10 +1542,12 @@ class BaseDbObjectTestCase(_BaseObjectTestCase,
         segment.create()
         return segment.id
 
-    def _create_test_router_id(self):
+    def _create_test_router_id(self, router_id=None):
         attrs = {
             'name': 'test_router',
         }
+        if router_id:
+            attrs['id'] = router_id
         self._router = router.Router(self.context, **attrs)
         self._router.create()
         return self._router['id']

neutron/tests/unit/objects/test_objects.py

@@ -78,9 +78,10 @@ object_data = {
     'QosPolicyRBAC': '1.0-c8a67f39809c5a3c8c7f26f2f2c620b2',
     'QosRuleType': '1.3-7286188edeb3a0386f9cf7979b9700fc',
     'QosRuleTypeDriver': '1.0-7d8cb9f0ef661ac03700eae97118e3db',
-    'QosPolicy': '1.7-4adb0cde3102c10d8970ec9487fd7fe7',
+    'QosPolicy': '1.8-4adb0cde3102c10d8970ec9487fd7fe7',
     'QosPolicyDefault': '1.0-59e5060eedb1f06dd0935a244d27d11c',
     'QosPolicyFloatingIPBinding': '1.0-5625df4205a18778cd6aa40f99be024e',
+    'QosPolicyRouterGatewayIPBinding': '1.0-da064fbfe5ee18c950b905b483bf59e3',
     'QosPolicyNetworkBinding': '1.0-df53a1e0f675aab8d27a1ccfed38dc42',
     'QosPolicyPortBinding': '1.0-66cb364ac99aa64523ade07f9f868ea6',
     'Quota': '1.0-6bb6a0f1bd5d66a2134ffa1a61873097',

releasenotes/notes/gateway-rate-limit-905bee1ed60c6b8e.yaml

@@ -0,0 +1,15 @@
+---
+features:
+- |
+   A new attribute ``qos_policy_id`` is added to the L3 router
+   gateway.
+
+   * It enables users to associate QoS policies to L3 router gateways
+     to control the rate of transmission of the associated SNAT traffic.
+   * At the moment, only bandwidth limit rules are supported in the
+     QoS polices.
+   * To enable this feature, the ``qos`` service plugin has to be
+     configured in the Neutron server and the ``gateway_ip_qos``
+     extension has to be configured in the L3 agents. Please refer to
+     the ``QoS`` section of the ``OpenStack Networking Guide`` for more
+     specific details.

setup.cfg

@@ -206,6 +206,7 @@ neutron.objects =
     QosPolicyNetworkBinding = neutron.objects.qos.binding:QosPolicyNetworkBinding
     QosPolicyPortBinding = neutron.objects.qos.binding:QosPolicyPortBinding
     QosPolicyRBAC = neutron.objects.qos.policy:QosPolicyRBAC
+    QosPolicyRouterGatewayIPBinding = neutron.objects.qos.binding:QosPolicyRouterGatewayIPBinding
     QosRule = neutron.objects.qos.rule:QosRule
     QosRuleType = neutron.objects.qos.rule_type:QosRuleType
     QosRuleTypeDriver = neutron.objects.qos.rule_type:QosRuleTypeDriver