Iptables metering driver

this is a part of the blueprint bandwidth-router-measurement

Change-Id: I37e4dc5abeaca4e13b32155bb7e2f07883ef9b2d

etc/metering_agent.ini

@@ -10,3 +10,6 @@
 # Interval between two metering reports
 # report_interval = 300
 
+# interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
+
+# use_namespaces = True

neutron/services/metering/drivers/iptables/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Sylvain Afchain <sylvain.afchain@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

neutron/services/metering/drivers/iptables/iptables_driver.py

@@ -0,0 +1,293 @@
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Sylvain Afchain <sylvain.afchain@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo.config import cfg
+
+from neutron.agent.common import config
+from neutron.agent.linux import interface
+from neutron.agent.linux import iptables_manager
+from neutron.common import constants as constants
+from neutron.common import log
+from neutron.openstack.common import importutils
+from neutron.openstack.common import log as logging
+from neutron.services.metering.drivers import abstract_driver
+
+
+LOG = logging.getLogger(__name__)
+NS_PREFIX = 'qrouter-'
+WRAP_NAME = 'neutron-meter'
+EXTERNAL_DEV_PREFIX = 'qg-'
+TOP_CHAIN = WRAP_NAME + "-FORWARD"
+RULE = '-r-'
+LABEL = '-l-'
+
+IptablesDriverOpts = [
+    cfg.StrOpt('interface_driver',
+               help=_("The driver used to manage the virtual "
+                      "interface.")),
+    cfg.BoolOpt('use_namespaces', default=True,
+                help=_("Allow overlapping IP."))
+]
+config.register_root_helper(cfg.CONF)
+cfg.CONF.register_opts(interface.OPTS)
+cfg.CONF.register_opts(IptablesDriverOpts)
+
+
+class IptablesManagerTransaction(object):
+    __transactions = {}
+
+    def __init__(self, im):
+        self.im = im
+
+        transaction = self.__transactions.get(im, 0)
+        transaction += 1
+        self.__transactions[im] = transaction
+
+    def __enter__(self):
+        return self.im
+
+    def __exit__(self, type, value, traceback):
+        transaction = self.__transactions.get(self.im)
+        if transaction == 1:
+            self.im.apply()
+            del self.__transactions[self.im]
+        else:
+            transaction -= 1
+            self.__transactions[self.im] = transaction
+
+
+class RouterWithMetering(object):
+
+    def __init__(self, conf, router):
+        self.conf = conf
+        self.id = router['id']
+        self.router = router
+        self.root_helper = config.get_root_helper(self.conf)
+        self.iptables_manager = iptables_manager.IptablesManager(
+            root_helper=self.conf.root_helper,
+            namespace=self.ns_name(),
+            binary_name=WRAP_NAME)
+        self.metering_labels = {}
+
+    def ns_name(self):
+        if self.conf.use_namespaces:
+            return NS_PREFIX + self.router['id']
+
+
+class IptablesMeteringDriver(abstract_driver.MeteringAbstractDriver):
+
+    def __init__(self, plugin, conf):
+        self.plugin = plugin
+        self.conf = conf or cfg.CONF
+        self.routers = {}
+
+        if not self.conf.interface_driver:
+            raise SystemExit(_('An interface driver must be specified'))
+        LOG.info(_("Loading interface driver %s") % self.conf.interface_driver)
+        self.driver = importutils.import_object(self.conf.interface_driver,
+                                                self.conf)
+
+    def _update_router(self, router):
+        r = self.routers.get(router['id'],
+                             RouterWithMetering(self.conf, router))
+        r.router = router
+        self.routers[r.id] = r
+
+        return r
+
+    @log.log
+    def update_routers(self, context, routers):
+        # disassociate removed routers
+        router_ids = [router['id'] for router in routers]
+        for router_id in self.routers:
+            if router_id not in router_ids:
+                self._process_disassociate_metering_label(router)
+
+        for router in routers:
+            old_gw_port_id = None
+            old_rm = self.routers.get(router['id'])
+            if old_rm:
+                old_gw_port_id = old_rm.router['gw_port_id']
+            gw_port_id = router['gw_port_id']
+
+            if gw_port_id != old_gw_port_id:
+                if old_rm:
+                    with IptablesManagerTransaction(old_rm.iptables_manager):
+                        self._process_disassociate_metering_label(router)
+                        if gw_port_id:
+                            self._process_associate_metering_label(router)
+                elif gw_port_id:
+                    self._process_associate_metering_label(router)
+
+    @log.log
+    def remove_router(self, context, router_id):
+        if router_id in self.routers:
+            del self.routers[router_id]
+
+    def get_external_device_name(self, port_id):
+        return (EXTERNAL_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN]
+
+    def _process_metering_label_rules(self, rm, rules, label_chain,
+                                      rules_chain):
+        im = rm.iptables_manager
+        ext_dev = self.get_external_device_name(rm.router['gw_port_id'])
+        if not ext_dev:
+            return
+
+        for rule in rules:
+            remote_ip = rule['remote_ip_prefix']
+
+            dir = '-i ' + ext_dev
+            if rule['direction'] == 'egress':
+                dir = '-o ' + ext_dev
+
+            if rule['excluded'] == 'true':
+                ipt_rule = dir + ' -d ' + remote_ip + ' -j RETURN'
+                im.ipv4['filter'].add_rule(rules_chain, ipt_rule, wrap=False,
+                                           top=True)
+            else:
+                ipt_rule = dir + ' -d ' + remote_ip + ' -j ' + label_chain
+                im.ipv4['filter'].add_rule(rules_chain, ipt_rule,
+                                           wrap=False, top=False)
+
+    def _process_associate_metering_label(self, router):
+        self._update_router(router)
+        rm = self.routers.get(router['id'])
+
+        with IptablesManagerTransaction(rm.iptables_manager):
+            labels = router.get(constants.METERING_LABEL_KEY, [])
+            for label in labels:
+                label_id = label['id']
+
+                label_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              LABEL + label_id,
+                                                              wrap=False)
+                rm.iptables_manager.ipv4['filter'].add_chain(label_chain,
+                                                             wrap=False)
+
+                rules_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              RULE + label_id,
+                                                              wrap=False)
+                rm.iptables_manager.ipv4['filter'].add_chain(rules_chain,
+                                                             wrap=False)
+                rm.iptables_manager.ipv4['filter'].add_rule(TOP_CHAIN, '-j ' +
+                                                            rules_chain,
+                                                            wrap=False)
+
+                rm.iptables_manager.ipv4['filter'].add_rule(label_chain,
+                                                            '',
+                                                            wrap=False)
+
+                rules = label.get('rules')
+                if rules:
+                    self._process_metering_label_rules(rm, rules,
+                                                       label_chain,
+                                                       rules_chain)
+
+                rm.metering_labels[label_id] = label
+
+    def _process_disassociate_metering_label(self, router):
+        rm = self.routers.get(router['id'])
+        if not rm:
+            return
+
+        with IptablesManagerTransaction(rm.iptables_manager):
+            labels = router.get(constants.METERING_LABEL_KEY, [])
+            for label in labels:
+                label_id = label['id']
+                if label_id not in rm.metering_labels:
+                    continue
+
+                label_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              LABEL + label_id,
+                                                              wrap=False)
+                rules_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              RULE + label_id,
+                                                              wrap=False)
+
+                rm.iptables_manager.ipv4['filter'].remove_chain(label_chain,
+                                                                wrap=False)
+                rm.iptables_manager.ipv4['filter'].remove_chain(rules_chain,
+                                                                wrap=False)
+
+                del rm.metering_labels[label_id]
+
+    @log.log
+    def add_metering_label(self, context, routers):
+        for router in routers:
+            self._process_associate_metering_label(router)
+
+    @log.log
+    def update_metering_label_rules(self, context, routers):
+        for router in routers:
+            self._update_metering_label_rules(router)
+
+    def _update_metering_label_rules(self, router):
+        rm = self.routers.get(router['id'])
+        if not rm:
+            return
+
+        with IptablesManagerTransaction(rm.iptables_manager):
+            labels = router.get(constants.METERING_LABEL_KEY, [])
+            for label in labels:
+                label_id = label['id']
+
+                label_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              LABEL + label_id,
+                                                              wrap=False)
+                rules_chain = iptables_manager.get_chain_name(WRAP_NAME +
+                                                              RULE + label_id,
+                                                              wrap=False)
+                rm.iptables_manager.ipv4['filter'].empty_chain(rules_chain,
+                                                               wrap=False)
+
+                rules = label.get('rules')
+                if rules:
+                    self._process_metering_label_rules(rm, rules,
+                                                       label_chain,
+                                                       rules_chain)
+
+    @log.log
+    def remove_metering_label(self, context, routers):
+        for router in routers:
+            self._process_disassociate_metering_label(router)
+
+    @log.log
+    def get_traffic_counters(self, context, routers):
+        accs = {}
+        for router in routers:
+            rm = self.routers.get(router['id'])
+            if not rm:
+                continue
+
+            for label_id, label in rm.metering_labels.items():
+                chain = iptables_manager.get_chain_name(WRAP_NAME + LABEL +
+                                                        label_id, wrap=False)
+
+                chain_acc = rm.iptables_manager.get_traffic_counters(
+                    chain, wrap=False, zero=True)
+
+                if not chain_acc:
+                    continue
+
+                acc = accs.get(label_id, {'pkts': 0, 'bytes': 0})
+
+                acc['pkts'] += chain_acc['pkts']
+                acc['bytes'] += chain_acc['bytes']
+
+                accs[label_id] = acc
+
+        return accs

neutron/tests/unit/services/metering/drivers/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Sylvain Afchain <sylvain.afchain@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

neutron/tests/unit/services/metering/drivers/test_iptables_driver.py

@@ -0,0 +1,362 @@
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Sylvain Afchain <sylvain.afchain@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+
+import mock
+from mock import call
+from oslo.config import cfg
+
+from neutron.services.metering.drivers.iptables import iptables_driver
+from neutron.tests import base
+from neutron.tests.unit import test_api_v2
+
+_uuid = test_api_v2._uuid
+
+
+class IptablesDriverTestCase(base.BaseTestCase):
+    def setUp(self):
+        super(IptablesDriverTestCase, self).setUp()
+        self.utils_exec_p = mock.patch(
+            'neutron.agent.linux.utils.execute')
+        self.utils_exec = self.utils_exec_p.start()
+        self.addCleanup(self.utils_exec_p.stop)
+        self.iptables_cls_p = mock.patch(
+            'neutron.agent.linux.iptables_manager.IptablesManager')
+        iptables_cls = self.iptables_cls_p.start()
+        self.addCleanup(self.iptables_cls_p.stop)
+        self.iptables_inst = mock.Mock()
+        self.v4filter_inst = mock.Mock()
+        self.v6filter_inst = mock.Mock()
+        self.v4filter_inst.chains = []
+        self.v6filter_inst.chains = []
+        self.iptables_inst.ipv4 = {'filter': self.v4filter_inst}
+        self.iptables_inst.ipv6 = {'filter': self.v6filter_inst}
+        iptables_cls.return_value = self.iptables_inst
+        cfg.CONF.set_override('interface_driver',
+                              'neutron.agent.linux.interface.NullDriver')
+        self.router_info_inst = mock.Mock()
+        self.router_info_inst.iptables_manager = self.iptables_inst
+
+        self.metering = iptables_driver.IptablesMeteringDriver('metering',
+                                                               cfg.CONF)
+
+    def test_add_metering_label(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': []}],
+            'admin_state_up': True,
+            'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)
+
+    def test_add_metering_label_with_rules(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'}]}],
+            'admin_state_up': True,
+            'gw_port_id': '6d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'},
+            {'_metering_labels': [
+             {'id': 'eeef45da-c600-4a2a-b2f4-c0fb6df73c83',
+              'rules': [{
+                  'direction': 'ingress',
+                  'excluded': True,
+                  'id': 'fa2441e8-2489-4ed1-870c-a62754501379',
+                  'metering_label_id': 'eeef45da-c600-4a2a-b2f4-c0fb6df73c83',
+                  'remote_ip_prefix': '20.0.0.0/24'}]}],
+             'admin_state_up': True,
+             'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+             'id': '373ec392-1711-44e3-b008-3251ccfc5099',
+             'name': 'router2',
+             'status': 'ACTIVE',
+             'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-6d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.add_chain('neutron-meter-l-eeef45da-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-eeef45da-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-eeef45da-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-eeef45da-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-eeef45da-c60',
+                               '-i qg-7d411f48-ec -d 20.0.0.0/24 -j '
+                               'neutron-meter-l-eeef45da-c60',
+                               wrap=False, top=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)
+
+    def test_update_metering_label_rules(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'}]}],
+            'admin_state_up': True,
+            'gw_port_id': '6d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+
+        updates = copy.deepcopy(routers)
+        updates[0]['_metering_labels'][0]['rules'] = [{
+            'direction': 'egress',
+            'excluded': True,
+            'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+            'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+            'remote_ip_prefix': '10.0.0.0/24'},
+            {'direction': 'ingress',
+             'excluded': False,
+             'id': '6f1a261f-2489-4ed1-870c-a62754501379',
+             'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'remote_ip_prefix': '20.0.0.0/24'}]
+
+        self.metering.update_metering_label_rules(None, updates)
+
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-6d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.empty_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-o qg-6d411f48-ec -d 10.0.0.0/24 -j '
+                               'neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-6d411f48-ec -d 20.0.0.0/24 -j '
+                               'neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)
+
+    def test_remove_metering_label_rule(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'},
+                 {'direction': 'ingress',
+                  'excluded': False,
+                  'id': 'aaaa261f-2489-4ed1-870c-a62754501379',
+                  'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                  'remote_ip_prefix': '20.0.0.0/24'}]
+             }],
+            'admin_state_up': True,
+            'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'}]
+             }],
+            'admin_state_up': True,
+            'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.update_metering_label_rules(None, routers)
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-7d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-7d411f48-ec -d 20.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.empty_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-7d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)
+
+    def test_remove_metering_label(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'}]
+             }],
+            'admin_state_up': True,
+            'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+        self.metering.remove_metering_label(None, routers)
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-7d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.remove_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.remove_chain('neutron-meter-r-c5df2fe5-c60', wrap=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)
+
+    def test_update_routers(self):
+        routers = [{'_metering_labels': [
+            {'id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+             'rules': [{
+                 'direction': 'ingress',
+                 'excluded': False,
+                 'id': '7f1a261f-2489-4ed1-870c-a62754501379',
+                 'metering_label_id': 'c5df2fe5-c600-4a2a-b2f4-c0fb6df73c83',
+                 'remote_ip_prefix': '10.0.0.0/24'}]}],
+            'admin_state_up': True,
+            'gw_port_id': '6d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+            'id': '473ec392-1711-44e3-b008-3251ccfc5099',
+            'name': 'router1',
+            'status': 'ACTIVE',
+            'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'},
+            {'_metering_labels': [
+             {'id': 'eeef45da-c600-4a2a-b2f4-c0fb6df73c83',
+              'rules': [{
+                  'direction': 'ingress',
+                  'excluded': True,
+                  'id': 'fa2441e8-2489-4ed1-870c-a62754501379',
+                  'metering_label_id': 'eeef45da-c600-4a2a-b2f4-c0fb6df73c83',
+                  'remote_ip_prefix': '20.0.0.0/24'}]}],
+             'admin_state_up': True,
+             'gw_port_id': '7d411f48-ecc7-45e0-9ece-3b5bdb54fcee',
+             'id': '373ec392-1711-44e3-b008-3251ccfc5099',
+             'name': 'router2',
+             'status': 'ACTIVE',
+             'tenant_id': '6c5f5d2a1fa2441e88e35422926f48e8'}]
+
+        self.metering.add_metering_label(None, routers)
+
+        updates = copy.deepcopy(routers)
+        updates[0]['gw_port_id'] = '587b63c1-22a3-40b3-9834-486d1fb215a5'
+
+        self.metering.update_routers(None, updates)
+        calls = [call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-6d411f48-ec -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False),
+                 call.add_chain('neutron-meter-l-eeef45da-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-eeef45da-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-eeef45da-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-eeef45da-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-eeef45da-c60',
+                               '-i qg-7d411f48-ec -d 20.0.0.0/24 -j '
+                               'neutron-meter-l-eeef45da-c60',
+                               wrap=False, top=False),
+                 call.remove_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.remove_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-l-c5df2fe5-c60', wrap=False),
+                 call.add_chain('neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-FORWARD', '-j '
+                               'neutron-meter-r-c5df2fe5-c60', wrap=False),
+                 call.add_rule('neutron-meter-l-c5df2fe5-c60',
+                               '',
+                               wrap=False),
+                 call.add_rule('neutron-meter-r-c5df2fe5-c60',
+                               '-i qg-587b63c1-22 -d 10.0.0.0/24'
+                               ' -j neutron-meter-l-c5df2fe5-c60',
+                               wrap=False, top=False)]
+
+        self.v4filter_inst.assert_has_calls(calls)