Test QoS DSCP for router gateways

Adds testing for feature [1], according to the following bigger priority
order findings from LP [2].

QoS enforced on router gateway in manner that allows testing precedence
with port and network, while testing functionality works with all rest
of QoS types when used before/after in test scenario.

Added missing steps documentation into bandwidth limit validation
method.

[1]
https://bugs.launchpad.net/neutron/+bug/1893625
https://issues.redhat.com/browse/OSPRH-2909

[2]
https://bugs.launchpad.net/neutron/+bug/2116317

Change-Id: I43fc3969392bd38111bdda287fc4c9767f6979c3

whitebox_neutron_tempest_plugin/tests/scenario/test_qos.py

@@ -55,6 +55,8 @@ class QosBaseTest(test_qos.QoSTestMixin, base.TrafficFlowTest):
     dscp_mark_port = 48
     dscp_mark_fip = 36
     dscp_mark_fip_new = 38
+    dscp_mark_gw = 10
+    dscp_mark_gw_new = 12
     MIN_KBPS_NO_BWLIMIT = 10000
     IPERF_PORT = 4321
     bwlimit_kbps_net = 1000
@@ -638,8 +640,9 @@ class QosBaseTest(test_qos.QoSTestMixin, base.TrafficFlowTest):
             self.assertFalse(remote_capture.is_empty(), msg)
 
     def _validate_traffic_dscp_marking(
-            self, src_server, dst_server, ipv6=False, fip_qos=False):
+            self, src_server, dst_server, ipv6=False, fip_qos=False,
-        """Validate that traffic between servers has a dscp mark
+            gw_qos=False):
+        """Validate that traffic between servers has a dscp mark.
 
         Scenario:
             1. First make sure that traffic between servers is not marked.
@@ -655,8 +658,17 @@ class QosBaseTest(test_qos.QoSTestMixin, base.TrafficFlowTest):
             7. Send traffic between 2 servers and make sure that now packets
             are marked with a new DSCP mark (since rule for port has higher
             priority).
-            8. Delete DSCP marking rule from the port QoS policy and make sure
+            8. Create another QoS policy applied to router gateway routing to
+            src_server, verify it takes precedence over port policy.
+            9. Delete DSCP marking rule from the router gateway QoS policy
+            and make sure that traffic is marked for port value again.
+            10. Create another QoS policy applied to FIP of src_server, verify
+            it takes precedence over port policy.
+            11. Delete DSCP marking rule from the FIP QoS policy
+            and make sure that traffic is marked for port value again.
+            12. Delete DSCP marking rule from the port QoS policy and make sure
             that traffic is not marked.
+            13. Verify a QoS policy attached to a port cannot be deleted.
         """
         # First, let's make sure that traffic is not marked
         self._validate_traffic_marked(0, src_server, dst_server)
@@ -699,6 +711,41 @@ class QosBaseTest(test_qos.QoSTestMixin, base.TrafficFlowTest):
         self._validate_traffic_marked(
             self.dscp_mark_port, src_server, dst_server)
 
+        # Test QoS policy enforced on router gateway (north-south topology)
+        if gw_qos:
+            gw_dscp_policy_id = self._create_qos_policy()
+            self.router['external_gateway_info'][
+                'qos_policy_id'] = gw_dscp_policy_id
+            self.router = self.admin_client.update_router(
+                self.router['id'],
+                external_gateway_info=self.router['external_gateway_info']
+            )['router']
+            del self.router['external_gateway_info']['qos_policy_id']
+            self.addCleanup(
+                self.admin_client.update_router,
+                self.router['id'],
+                external_gateway_info=self.router['external_gateway_info'])
+            gw_rule_id = self.admin_client.create_dscp_marking_rule(
+                gw_dscp_policy_id, self.dscp_mark_gw)[
+                    'dscp_marking_rule']['id']
+            # Verify that traffic is marked with a value from gw qos policy
+            self._validate_traffic_marked(
+                self.dscp_mark_gw, src_server, dst_server)
+
+            # update dscp mark associated with gw qos policy
+            self.admin_client.update_dscp_marking_rule(
+                gw_dscp_policy_id, gw_rule_id,
+                dscp_mark=self.dscp_mark_gw_new)
+            self._validate_traffic_marked(
+                self.dscp_mark_gw_new, src_server, dst_server)
+
+            # delete dscp mark associated gw qos policy
+            # port dscp rule applies
+            self.admin_client.delete_dscp_marking_rule(
+                gw_dscp_policy_id, gw_rule_id)
+            self._validate_traffic_marked(
+                self.dscp_mark_port, src_server, dst_server)
+
         # Create a new QoS Policy and attach to the FIP of src server
         # This only applies to south-north tests because the traffic from the
         # src server to the dst server goes through the src FIP
@@ -875,7 +922,7 @@ class QosTestCommon(QosBaseTest):
             topology='north-south')
         if self.has_ovn_support:
             self._validate_traffic_dscp_marking(
-                src_server, dst_server, fip_qos=True)
+                src_server, dst_server, fip_qos=True, gw_qos=True)
         else:
             self._validate_traffic_dscp_marking(src_server, dst_server)