openstack
/
neutron
Code Issues Proposed changes

Add segments to hosts mappings 

This patchset adds code to update a segments to hosts mapping in the
DB from L2 agents state reports to the Neutron server

Change-Id: If3e0b4723f11e52a520a969f45c82e32f3980fd9
Partially-Implements: blueprint routed-networks
This commit is contained in:
Miguel Lavalle 2016-02-26 15:12:01 -06:00
parent c455bf79f6
commit 3d3f0595eb
7 changed files with 437 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
neutron/db/agents_db.py
View File

@ -43,6 +43,7 @@ from neutron.db import model_base
from neutron.extensions import agent as ext_agent
from neutron.extensions import availability_zone as az_ext
from neutron import manager
from neutron.services.segments import db as segments_db
LOG = logging.getLogger(__name__)
@ -396,6 +397,8 @@ class AgentDbMixin(ext_agent.AgentPluginBase, AgentAvailabilityZoneMixin):
self._log_heartbeat(agent_state, agent_db, configurations_dict)
status = n_const.AGENT_NEW
greenthread.sleep(0)
segments_db.update_segment_host_mapping_for_agent(
context, agent_state['host'], self, agent_state)
return status
def create_or_update_agent(self, context, agent):

2
neutron/db/migration/alembic_migrations/versions/CONTRACT_HEAD
View File

@ -1 +1 @@
c879c5e1ee90
8fd3918ef6f4

58
neutron/db/migration/alembic_migrations/versions/newton/contract/8fd3918ef6f4_add_segment_host_mapping.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2016 IBM
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Add segment_host_mapping table.
Revision ID: 8fd3918ef6f4
Revises: c879c5e1ee90
Create Date: 2016-02-25 00:22:47.618593
"""
# revision identifiers, used by Alembic.
revision = '8fd3918ef6f4'
down_revision = 'c879c5e1ee90'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table('segmenthostmappings',
sa.Column('segment_id',
sa.String(length=36),
index=True,
nullable=False),
sa.Column('host',
sa.String(255),
index=True,
nullable=False),
sa.PrimaryKeyConstraint('segment_id', 'host'),
sa.ForeignKeyConstraint(['segment_id'],
['networksegments.id'],
ondelete='CASCADE'))
def contract_creation_exceptions():
"""
Return create exceptions.
These elements depend on the networksegments table which was renamed
in the contract branch.
"""
return {
sa.Table: ['segmenthostmappings'],
sa.Index: ['segmenthostmappings']
}

8
neutron/plugins/ml2/plugin.py
View File

@ -1685,3 +1685,11 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
segments = db.get_network_segments(context.session, network_id)
return self.mechanism_manager.filter_hosts_with_segment_access(
context, segments, candidate_hosts, self.get_agents)
def check_segment_for_agent(self, segment, agent):
for mech_driver in self.mechanism_manager.ordered_mech_drivers:
driver_agent_type = getattr(mech_driver.obj, 'agent_type', None)
if driver_agent_type and driver_agent_type == agent['agent_type']:
if mech_driver.obj.check_segment_for_agent(segment, agent):
return True
return False

79
neutron/services/segments/db.py
View File

@ -19,15 +19,44 @@ import functools
from neutron_lib import constants
from oslo_log import helpers as log_helpers
from oslo_log import log as logging
from oslo_utils import uuidutils
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.orm import exc
from neutron._i18n import _LI
from neutron.api.v2 import attributes
from neutron.db import common_db_mixin
from neutron.db import model_base
from neutron.db import segments_db as db
from neutron.services.segments import exceptions
LOG = logging.getLogger(__name__)
class SegmentHostMapping(model_base.BASEV2):
segment_id = sa.Column(sa.String(36),
sa.ForeignKey('networksegments.id',
ondelete="CASCADE"),
primary_key=True,
index=True,
nullable=False)
host = sa.Column(sa.String(255),
primary_key=True,
index=True,
nullable=False)
# Add a relationship to the NetworkSegment model in order to instruct
# SQLAlchemy to eagerly load this association
network_segment = orm.relationship(
db.NetworkSegment, backref=orm.backref("segment_host_mapping",
lazy='joined',
cascade='delete'))
def _extend_subnet_dict_binding(plugin, subnet_res, subnet_db):
subnet_res['segment_id'] = subnet_db.get('segment_id')
@ -124,3 +153,53 @@ class SegmentDbMixin(common_db_mixin.CommonDbMixin):
query = query.filter(db.NetworkSegment.id == uuid)
if 0 == query.delete():
raise exceptions.SegmentNotFound(segment_id=uuid)
def update_segment_host_mapping(context, host, current_segment_ids):
with context.session.begin(subtransactions=True):
segments_host_query = context.session.query(
SegmentHostMapping).filter_by(host=host)
previous_segment_ids = {
seg_host['segment_id'] for seg_host in segments_host_query}
for segment_id in current_segment_ids - previous_segment_ids:
context.session.add(SegmentHostMapping(segment_id=segment_id,
host=host))
stale_segment_ids = previous_segment_ids - current_segment_ids
if stale_segment_ids:
context.session.query(SegmentHostMapping).filter(
SegmentHostMapping.segment_id.in_(
stale_segment_ids)).delete(synchronize_session=False)
def _get_phys_nets(agent):
configurations_dict = agent.get('configurations', {})
mappings = configurations_dict.get('bridge_mappings', {})
mappings.update(configurations_dict.get('interface_mappings', {}))
mappings.update(configurations_dict.get('device_mappings', {}))
return mappings.keys()
reported_hosts = set()
def update_segment_host_mapping_for_agent(context, host, plugin, agent):
check_segment_for_agent = getattr(plugin, 'check_segment_for_agent', None)
if not check_segment_for_agent:
LOG.info(_LI("Core plug-in does not implement "
"'check_segment_for_agent'. It is not possible to "
"build a hosts segments mapping"))
return
phys_nets = _get_phys_nets(agent)
if not phys_nets:
return
start_flag = agent.get('start_flag', None)
if host in reported_hosts and not start_flag:
return
reported_hosts.add(host)
with context.session.begin(subtransactions=True):
segments = context.session.query(db.NetworkSegment).filter(
db.NetworkSegment.physical_network.in_(phys_nets))
current_segment_ids = {
segment['id'] for segment in segments
if check_segment_for_agent(segment, agent)}
update_segment_host_mapping(context, host, current_segment_ids)

64
neutron/tests/common/helpers.py
View File

@ -70,8 +70,9 @@ def _get_l3_agent_dict(host, agent_mode, internal_only=True,
'router_id': router_id}}
def _register_agent(agent):
plugin = FakePlugin()
def _register_agent(agent, plugin=None):
if not plugin:
plugin = FakePlugin()
admin_context = context.get_admin_context()
plugin.create_or_update_agent(admin_context, agent)
return plugin._get_agent_by_type_and_host(
@ -136,25 +137,30 @@ def set_agent_admin_state(agent_id, admin_state_up=False):
{'agent': {'admin_state_up': admin_state_up}})
def _get_ovs_agent_dict(host, agent_type, binary, tunnel_types,
tunneling_ip='20.0.0.1', interface_mappings=None,
bridge_mappings=None, l2pop_network_types=None):
def _get_l2_agent_dict(host, agent_type, binary, tunnel_types=None,
tunneling_ip='20.0.0.1', interface_mappings=None,
bridge_mappings=None, l2pop_network_types=None,
device_mappings=None, start_flag=True):
agent = {
'binary': binary,
'host': host,
'topic': constants.L2_AGENT_TOPIC,
'configurations': {'tunneling_ip': tunneling_ip,
'tunnel_types': tunnel_types},
'configurations': {},
'agent_type': agent_type,
'tunnel_type': [],
'start_flag': True}
'start_flag': start_flag}
if tunnel_types is not None:
agent['configurations']['tunneling_ip'] = tunneling_ip
agent['configurations']['tunnel_types'] = tunnel_types
if bridge_mappings is not None:
agent['configurations']['bridge_mappings'] = bridge_mappings
if interface_mappings is not None:
agent['configurations']['interface_mappings'] = interface_mappings
if l2pop_network_types is not None:
agent['configurations']['l2pop_network_types'] = l2pop_network_types
if device_mappings is not None:
agent['configurations']['device_mappings'] = device_mappings
return agent
@ -162,11 +168,43 @@ def register_ovs_agent(host=HOST, agent_type=constants.AGENT_TYPE_OVS,
binary='neutron-openvswitch-agent',
tunnel_types=['vxlan'], tunneling_ip='20.0.0.1',
interface_mappings=None, bridge_mappings=None,
l2pop_network_types=None):
agent = _get_ovs_agent_dict(host, agent_type, binary, tunnel_types,
tunneling_ip, interface_mappings,
bridge_mappings, l2pop_network_types)
return _register_agent(agent)
l2pop_network_types=None, plugin=None, start_flag=True):
agent = _get_l2_agent_dict(host, agent_type, binary, tunnel_types,
tunneling_ip, interface_mappings,
bridge_mappings, l2pop_network_types,
start_flag=start_flag)
return _register_agent(agent, plugin)
def register_linuxbridge_agent(host=HOST,
agent_type=constants.AGENT_TYPE_LINUXBRIDGE,
binary='neutron-linuxbridge-agent',
tunnel_types=['vxlan'], tunneling_ip='20.0.0.1',
interface_mappings=None, bridge_mappings=None,
plugin=None):
agent = _get_l2_agent_dict(host, agent_type, binary, tunnel_types,
tunneling_ip=tunneling_ip,
interface_mappings=interface_mappings,
bridge_mappings=bridge_mappings)
return _register_agent(agent, plugin)
def register_macvtap_agent(host=HOST,
agent_type=constants.AGENT_TYPE_MACVTAP,
binary='neutron-macvtap-agent',
interface_mappings=None, plugin=None):
agent = _get_l2_agent_dict(host, agent_type, binary,
interface_mappings=interface_mappings)
return _register_agent(agent, plugin)
def register_sriovnicswitch_agent(host=HOST,
agent_type=constants.AGENT_TYPE_NIC_SWITCH,
binary='neutron-sriov-nic-agent',
device_mappings=None, plugin=None):
agent = _get_l2_agent_dict(host, agent_type, binary,
device_mappings=device_mappings)
return _register_agent(agent, plugin)
def requires_py2(testcase):

240
neutron/tests/unit/extensions/test_segment.py
View File

@ -12,16 +12,21 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
from neutron_lib import constants
from oslo_utils import uuidutils
import webob.exc
from neutron.api.v2 import attributes
from neutron import context
from neutron.db import agents_db
from neutron.db import db_base_plugin_v2
from neutron.db import segments_db
from neutron.extensions import segment as ext_segment
from neutron.plugins.common import constants as p_constants
from neutron.plugins.ml2 import config
from neutron.services.segments import db
from neutron.tests.common import helpers
from neutron.tests.unit.db import test_db_base_plugin_v2
DB_PLUGIN_KLASS = ('neutron.tests.unit.extensions.test_segment.'
@ -48,10 +53,12 @@ class SegmentTestExtensionManager(object):
class SegmentTestCase(test_db_base_plugin_v2.NeutronDbPluginV2TestCase):
def setUp(self):
plugin = DB_PLUGIN_KLASS
def setUp(self, plugin=None, service_plugins=None):
if not plugin:
plugin = DB_PLUGIN_KLASS
ext_mgr = SegmentTestExtensionManager()
super(SegmentTestCase, self).setUp(plugin=plugin, ext_mgr=ext_mgr)
super(SegmentTestCase, self).setUp(plugin=plugin, ext_mgr=ext_mgr,
service_plugins=service_plugins)
def _create_segment(self, fmt, expected_res_status=None, **kwargs):
segment = {'segment': {}}
@ -94,6 +101,12 @@ class SegmentTestPlugin(db_base_plugin_v2.NeutronDbPluginV2,
supported_extension_aliases = ["segment"]
def get_plugin_description(self):
return "Network Segments"
def get_plugin_type(self):
return "segments"
class TestSegment(SegmentTestCase):
@ -261,3 +274,224 @@ class TestSegmentSubnetAssociation(SegmentTestCase):
res = self.deserialize(self.fmt, response)
# Don't allocate IPs in this case because it doesn't have binding info
self.assertEqual(0, len(res['port']['fixed_ips']))
class HostSegmentMappingTestCase(SegmentTestCase):
_mechanism_drivers = ['logger']
def setUp(self, plugin=None):
config.cfg.CONF.set_override('mechanism_drivers',
self._mechanism_drivers,
group='ml2')
if not plugin:
plugin = 'neutron.plugins.ml2.plugin.Ml2Plugin'
segments_plugin = ('neutron.tests.unit.extensions.test_segment.'
'SegmentTestPlugin')
service_plugins = {'segments_plugin_name': segments_plugin}
super(HostSegmentMappingTestCase, self).setUp(
plugin=plugin, service_plugins=service_plugins)
def _get_segments_for_host(self, host):
ctx = context.get_admin_context()
segments_host_list = ctx.session.query(
db.SegmentHostMapping).filter_by(host=host)
return {seg_host['segment_id']: seg_host
for seg_host in segments_host_list}
def _register_agent(self, host, mappings=None, plugin=None,
start_flag=True):
helpers.register_ovs_agent(host=host, bridge_mappings=mappings,
plugin=self.plugin, start_flag=start_flag)
def _test_one_segment_one_host(self, host):
physical_network = 'phys_net1'
with self.network() as network:
network = network['network']
segment = self._test_create_segment(
network_id=network['id'], physical_network=physical_network,
segmentation_id=200, network_type=p_constants.TYPE_VLAN)['segment']
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host)
self.assertEqual(1, len(segments_host_db))
self.assertEqual(segment['id'],
segments_host_db[segment['id']]['segment_id'])
self.assertEqual(host, segments_host_db[segment['id']]['host'])
return segment
class TestMl2HostSegmentMappingOVS(HostSegmentMappingTestCase):
_mechanism_drivers = ['openvswitch', 'logger']
mock_path = 'neutron.services.segments.db.update_segment_host_mapping'
def test_new_agent(self):
host = 'host1'
self._test_one_segment_one_host(host)
def test_updated_agent_changed_physical_networks(self):
host = 'host1'
physical_networks = ['phys_net1', 'phys_net2']
networks = []
segments = []
for i in range(len(physical_networks)):
with self.network() as network:
networks.append(network['network'])
segments.append(self._test_create_segment(
network_id=networks[i]['id'],
physical_network=physical_networks[i],
segmentation_id=200,
network_type=p_constants.TYPE_VLAN)['segment'])
self._register_agent(host, mappings={physical_networks[0]: 'br-eth-1',
physical_networks[1]: 'br-eth-2'},
plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host)
self.assertEqual(len(physical_networks), len(segments_host_db))
for segment in segments:
self.assertEqual(segment['id'],
segments_host_db[segment['id']]['segment_id'])
self.assertEqual(host, segments_host_db[segment['id']]['host'])
self._register_agent(host, mappings={physical_networks[0]: 'br-eth-1'},
plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host)
self.assertEqual(1, len(segments_host_db))
self.assertEqual(segments[0]['id'],
segments_host_db[segments[0]['id']]['segment_id'])
self.assertEqual(host, segments_host_db[segments[0]['id']]['host'])
def test_same_segment_two_hosts(self):
host1 = 'host1'
host2 = 'host2'
physical_network = 'phys_net1'
segment = self._test_one_segment_one_host(host1)
self._register_agent(host2, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host2)
self.assertEqual(1, len(segments_host_db))
self.assertEqual(segment['id'],
segments_host_db[segment['id']]['segment_id'])
self.assertEqual(host2, segments_host_db[segment['id']]['host'])
def test_segment_deletion_removes_host_mapping(self):
host = 'host1'
segment = self._test_one_segment_one_host(host)
self._delete('segments', segment['id'])
segments_host_db = self._get_segments_for_host(host)
self.assertFalse(segments_host_db)
@mock.patch(mock_path)
def test_agent_with_no_mappings(self, mock):
host = 'host1'
physical_network = 'phys_net1'
with self.network() as network:
network = network['network']
self._test_create_segment(
network_id=network['id'], physical_network=physical_network,
segmentation_id=200, network_type=p_constants.TYPE_VLAN)
self._register_agent(host, plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host)
self.assertFalse(segments_host_db)
self.assertFalse(mock.mock_calls)
class TestMl2HostSegmentMappingLinuxBridge(TestMl2HostSegmentMappingOVS):
_mechanism_drivers = ['linuxbridge', 'logger']
def _register_agent(self, host, mappings=None, plugin=None):
helpers.register_linuxbridge_agent(host=host,
bridge_mappings=mappings,
plugin=self.plugin)
class TestMl2HostSegmentMappingMacvtap(TestMl2HostSegmentMappingOVS):
_mechanism_drivers = ['macvtap', 'logger']
def _register_agent(self, host, mappings=None, plugin=None):
helpers.register_macvtap_agent(host=host, interface_mappings=mappings,
plugin=self.plugin)
class TestMl2HostSegmentMappingSriovNicSwitch(TestMl2HostSegmentMappingOVS):
_mechanism_drivers = ['sriovnicswitch', 'logger']
def _register_agent(self, host, mappings=None, plugin=None):
helpers.register_sriovnicswitch_agent(host=host,
device_mappings=mappings,
plugin=self.plugin)
class NoSupportHostSegmentMappingPlugin(db_base_plugin_v2.NeutronDbPluginV2,
db.SegmentDbMixin,
agents_db.AgentDbMixin):
__native_pagination_support = True
__native_sorting_support = True
supported_extension_aliases = []
class TestHostSegmentMappingNoSupportFromPlugin(HostSegmentMappingTestCase):
mock_path = 'neutron.services.segments.db.update_segment_host_mapping'
def setUp(self):
plugin = ('neutron.tests.unit.extensions.test_segment.'
'NoSupportHostSegmentMappingPlugin')
super(TestHostSegmentMappingNoSupportFromPlugin, self).setUp(
plugin=plugin)
@mock.patch(mock_path)
def test_host_segments_not_updated(self, mock):
host = 'host1'
physical_network = 'phys_net1'
with self.network() as network:
network = network['network']
self._test_create_segment(network_id=network['id'],
physical_network=physical_network,
segmentation_id=200,
network_type=p_constants.TYPE_VLAN)
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin)
segments_host_db = self._get_segments_for_host(host)
self.assertFalse(segments_host_db)
self.assertFalse(mock.mock_calls)
class TestMl2HostSegmentMappingAgentServerSynch(HostSegmentMappingTestCase):
_mechanism_drivers = ['openvswitch', 'logger']
mock_path = 'neutron.services.segments.db.update_segment_host_mapping'
@mock.patch(mock_path)
def test_starting_server_processes_agents(self, mock_function):
host = 'agent_updating_starting_server'
physical_network = 'phys_net1'
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin, start_flag=False)
self.assertTrue(host in db.reported_hosts)
self.assertEqual(1, mock_function.call_count)
expected_call = mock.call(mock.ANY, host, set())
mock_function.assert_has_calls([expected_call])
@mock.patch(mock_path)
def test_starting_agent_is_processed(self, mock_function):
host = 'starting_agent'
physical_network = 'phys_net1'
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin, start_flag=False)
self.assertTrue(host in db.reported_hosts)
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin, start_flag=True)
self.assertTrue(host in db.reported_hosts)
self.assertEqual(2, mock_function.call_count)
expected_call = mock.call(mock.ANY, host, set())
mock_function.assert_has_calls([expected_call, expected_call])
@mock.patch(mock_path)
def test_no_starting_agent_is_not_processed(self, mock_function):
host = 'agent_with_no_start_update'
physical_network = 'phys_net1'
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin, start_flag=False)
self.assertTrue(host in db.reported_hosts)
mock_function.reset_mock()
self._register_agent(host, mappings={physical_network: 'br-eth-1'},
plugin=self.plugin, start_flag=False)
self.assertTrue(host in db.reported_hosts)
mock_function.assert_not_called()