NSX-v3: Initial framework for api-replay-mode

This patch includes the initial framework to allow existing
neutron deployments running different backends to be migrated
over to the nsx-v3 plugin. The main logic that is required to
do this is to allow the ability of an id to be specified for
a given resource. This patch makes this possible with the addition
of a new extension api-replay.

The reason why a new extension is needed is because the RESOURCE_MAP
is loaded after the plugin is loaded. Therefore, there is no way
for me to change the mapping directly in the plugin without creating
an extension to do so.

This patch also adds support for migrating the router-uplink and
floatingips which was missing in the previous patchset.

Here's an example output of the migration tool
running: http://codepad.org/I7x6Rq3u

Change-Id: I2ee9778374a8d137e06125f2732524c7c662c002
This commit is contained in:
Aaron Rosen 2016-05-24 17:12:57 -04:00
parent a8443957cb
commit b004985c24
9 changed files with 273 additions and 35 deletions

View File

View File

@ -12,7 +12,7 @@
import argparse import argparse
from vmware_nsx.plugins.nsx_v3.api_replay import client from vmware_nsx.api_replay import client
class ApiReplayCli(object): class ApiReplayCli(object):

View File

@ -47,6 +47,7 @@ class ApiReplayClient(object):
self.migrate_security_groups() self.migrate_security_groups()
self.migrate_routers() self.migrate_routers()
self.migrate_networks_subnets_ports() self.migrate_networks_subnets_ports()
self.migrate_floatingips()
def find_subnet_by_id(self, subnet_id, subnets): def find_subnet_by_id(self, subnet_id, subnets):
for subnet in subnets: for subnet in subnets:
@ -104,7 +105,7 @@ class ApiReplayClient(object):
dest_sec_group = self.have_id(sg['id'], dest_sec_groups) dest_sec_group = self.have_id(sg['id'], dest_sec_groups)
# If the security group already exists on the the dest_neutron # If the security group already exists on the the dest_neutron
if dest_sec_group: if dest_sec_group:
# make sure all the security group rules are theree and # make sure all the security group rules are there and
# create them if not # create them if not
for sg_rule in sg['security_group_rules']: for sg_rule in sg['security_group_rules']:
if(self.have_id(sg_rule['id'], if(self.have_id(sg_rule['id'],
@ -126,18 +127,19 @@ class ApiReplayClient(object):
else: else:
sg_rules = sg.pop('security_group_rules') sg_rules = sg.pop('security_group_rules')
try: try:
print(self.dest_neutron.create_security_group( new_sg = self.dest_neutron.create_security_group(
{'security_group': sg})) {'security_group': sg})
print ("Created security-group %s" % new_sg)
except Exception as e: except Exception as e:
# TODO(arosen): improve exception handing here. # TODO(arosen): improve exception handing here.
print (e) print (e)
pass
for sg_rule in sg_rules: for sg_rule in sg_rules:
try: try:
print (self.dest_neutron.create_security_group_rule( rule = self.dest_neutron.create_security_group_rule(
{'security_group_rule': sg_rule})) {'security_group_rule': sg_rule})
except n_exc.Conflict: print ("created security group rule %s " % rule['id'])
except Exception:
# NOTE(arosen): when you create a default # NOTE(arosen): when you create a default
# security group it is automatically populated # security group it is automatically populated
# with some rules. When we go to create the rules # with some rules. When we go to create the rules
@ -155,13 +157,15 @@ class ApiReplayClient(object):
if dest_router is False: if dest_router is False:
drop_router_fields = ['status', drop_router_fields = ['status',
'routes', 'routes',
'ha',
'external_gateway_info'] 'external_gateway_info']
body = self.drop_fields(router, drop_router_fields) body = self.drop_fields(router, drop_router_fields)
print (self.dest_neutron.create_router( new_router = (self.dest_neutron.create_router(
{'router': body})) {'router': body}))
print ("created router %s" % new_router)
def migrate_networks_subnets_ports(self): def migrate_networks_subnets_ports(self):
"""Migrates routers from source to dest neutron.""" """Migrates networks/ports/router-uplinks from src to dest neutron."""
source_ports = self.source_neutron.list_ports()['ports'] source_ports = self.source_neutron.list_ports()['ports']
source_subnets = self.source_neutron.list_subnets()['subnets'] source_subnets = self.source_neutron.list_subnets()['subnets']
source_networks = self.source_neutron.list_networks()['networks'] source_networks = self.source_neutron.list_networks()['networks']
@ -185,10 +189,12 @@ class ApiReplayClient(object):
'port_security_enabled', 'port_security_enabled',
'binding:vif_details', 'binding:vif_details',
'binding:vif_type', 'binding:vif_type',
'binding:host_id'] 'binding:host_id', 'qos_policy_id']
drop_network_fields = ['status', 'subnets', 'availability_zones', drop_network_fields = ['status', 'subnets', 'availability_zones',
'created_at', 'updated_at', 'tags'] 'created_at', 'updated_at', 'tags',
'qos_policy_id', 'ipv4_address_scope',
'ipv6_address_scope', 'mtu']
for network in source_networks: for network in source_networks:
body = self.drop_fields(network, drop_network_fields) body = self.drop_fields(network, drop_network_fields)
@ -202,7 +208,7 @@ class ApiReplayClient(object):
if self.have_id(network['id'], dest_networks) is False: if self.have_id(network['id'], dest_networks) is False:
created_net = self.dest_neutron.create_network( created_net = self.dest_neutron.create_network(
{'network': body})['network'] {'network': body})['network']
print ("Created network: " + created_net['id']) print ("Created network: %s " % created_net)
for subnet_id in network['subnets']: for subnet_id in network['subnets']:
subnet = self.find_subnet_by_id(subnet_id, source_subnets) subnet = self.find_subnet_by_id(subnet_id, source_subnets)
@ -240,24 +246,49 @@ class ApiReplayClient(object):
# only create port if the dest server doesn't have it # only create port if the dest server doesn't have it
if self.have_id(port['id'], dest_ports) is False: if self.have_id(port['id'], dest_ports) is False:
if port['device_owner'] == 'network:router_gateway':
body = {
"external_gateway_info":
{"network_id": port['network_id']}}
router_uplink = self.dest_neutron.update_router(
port['device_id'], # router_id
{'router': body})
print ("Uplinked router %s" % router_uplink)
continue
if port['device_owner'] in ['network:router_interface', # Let the neutron dhcp-agent recreate this on it's own
'network:router_gateway']: if port['device_owner'] == 'network:dhcp':
if port['allowed_address_pairs'] == []: continue
del body['allowed_address_pairs']
created_port = self.dest_neutron.create_port( # ignore these as we create them ourselves later
{'port': body})['port'] if port['device_owner'] == 'network:floatingip':
print ("Created port: " + created_port['id']) continue
if port['device_owner'] == 'network:router_interface': if port['device_owner'] == 'network:router_interface':
try: try:
print (self.dest_neutron.add_interface_router( # uplink router_interface ports
self.dest_neutron.add_interface_router(
port['device_id'], port['device_id'],
{'port_id': port['id']})) {'subnet_id': created_subnet['id']})
print ("Uplinked router %s to subnet %s" %
(port['device_id'], created_subnet['id']))
continue
except n_exc.BadRequest as e: except n_exc.BadRequest as e:
# NOTE(arosen): this occurs here if you run the # NOTE(arosen): this occurs here if you run the
# script multiple times as we don't track this. # script multiple times as we don't track this.
print (e) print (e)
raise
# TODO(arosen): handle 'network:router_gateway' uplinking created_port = self.dest_neutron.create_port(
{'port': body})['port']
print ("Created port: " + created_port['id'])
def migrate_floatingips(self):
"""Migrates floatingips from source to dest neutron."""
source_fips = self.source_neutron.list_floatingips()['floatingips']
drop_fip_fields = ['status', 'router_id', 'id']
for source_fip in source_fips:
body = self.drop_fields(source_fip, drop_fip_fields)
fip = self.dest_neutron.create_floatingip({'floatingip': body})
print ("Created floatingip %s" % fip)

View File

@ -0,0 +1,41 @@
# Copyright 2016 VMware, Inc.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.api.v2 import attributes
from oslo_config import cfg
from oslo_utils import uuidutils
import webob.exc
def _fixup_res_dict(context, attr_name, res_dict, check_allow_post=True):
# This method is a replacement of _fixup_res_dict which is used in
# neutron.plugin.common.utils. All this mock does is insert a uuid
# for the id field if one is not found ONLY if running in api_replay_mode.
if cfg.CONF.api_replay_mode and 'id' not in res_dict:
res_dict['id'] = uuidutils.generate_uuid()
attr_info = attributes.RESOURCE_ATTRIBUTE_MAP[attr_name]
try:
attributes.populate_tenant_id(context, res_dict, attr_info, True)
attributes.verify_attributes(res_dict, attr_info)
except webob.exc.HTTPBadRequest as e:
# convert webob exception into ValueError as these functions are
# for internal use. webob exception doesn't make sense.
raise ValueError(e.detail)
attributes.fill_default_value(attr_info, res_dict,
check_allow_post=check_allow_post)
attributes.convert_value(attr_info, res_dict)
return res_dict

View File

@ -230,6 +230,12 @@ nsx_common_opts = [
"parameter to tooz coordinator. By default, value is " "parameter to tooz coordinator. By default, value is "
"None and oslo_concurrency is used for single-node " "None and oslo_concurrency is used for single-node "
"lock management.")), "lock management.")),
cfg.BoolOpt('api_replay_mode',
default=False,
help=_("If true, the server then allows the caller to "
"specify the id of resources. This should only "
"be enabled in order to allow one to migrate an "
"existing install of neutron to the nsx-v3 plugin.")),
] ]
nsx_v3_opts = [ nsx_v3_opts = [

View File

@ -0,0 +1,73 @@
# Copyright 2016 VMware, Inc.
#
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from neutron.api import extensions
RESOURCE_ATTRIBUTE_MAP = {
'ports': {
'id': {'allow_post': True, 'allow_put': False,
'validate': {'type:uuid': None},
'is_visible': True},
},
'networks': {
'id': {'allow_post': True, 'allow_put': False,
'validate': {'type:uuid': None},
'is_visible': True},
},
'security_groups': {
'id': {'allow_post': True, 'allow_put': False,
'validate': {'type:uuid': None},
'is_visible': True},
},
'security_group_rules': {
'id': {'allow_post': True, 'allow_put': False,
'validate': {'type:uuid': None},
'is_visible': True},
},
'routers': {
'id': {'allow_post': True, 'allow_put': False,
'validate': {'type:uuid': None},
'is_visible': True},
},
}
class Api_replay(extensions.ExtensionDescriptor):
"""Extension for api replay which allows us to specify ids of resources."""
@classmethod
def get_name(cls):
return "Api Replay"
@classmethod
def get_alias(cls):
return 'api-replay'
@classmethod
def get_description(cls):
return "Enables mode to allow api to be replayed"
@classmethod
def get_updated(cls):
return "2016-05-05T10:00:00-00:00"
def get_extended_resources(self, version):
if version == "2.0":
return RESOURCE_ATTRIBUTE_MAP
else:
return {}

View File

@ -12,6 +12,8 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import mock
import netaddr import netaddr
import six import six
@ -66,6 +68,7 @@ from oslo_utils import importutils
from oslo_utils import uuidutils from oslo_utils import uuidutils
from vmware_nsx._i18n import _, _LE, _LI, _LW from vmware_nsx._i18n import _, _LE, _LI, _LW
from vmware_nsx.api_replay import utils as api_replay_utils
from vmware_nsx.common import config # noqa from vmware_nsx.common import config # noqa
from vmware_nsx.common import exceptions as nsx_exc from vmware_nsx.common import exceptions as nsx_exc
from vmware_nsx.common import locking from vmware_nsx.common import locking
@ -196,6 +199,8 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
"switching profile: %s") % NSX_V3_DHCP_PROFILE_NAME "switching profile: %s") % NSX_V3_DHCP_PROFILE_NAME
raise nsx_exc.NsxPluginException(msg) raise nsx_exc.NsxPluginException(msg)
self._unsubscribe_callback_events() self._unsubscribe_callback_events()
if cfg.CONF.api_replay_mode:
self.supported_extension_aliases.append('api-replay')
# translate configured transport zones/rotuers names to uuid # translate configured transport zones/rotuers names to uuid
self._translate_configured_names_2_uuids() self._translate_configured_names_2_uuids()
@ -1658,8 +1663,12 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
name = utils.get_name_and_uuid( name = utils.get_name_and_uuid(
router_name, port['id'], tag='port') router_name, port['id'], tag='port')
self._port_client.update(nsx_port_id, None, name=name) self._port_client.update(nsx_port_id, None, name=name)
return super(NsxV3Plugin, self).update_router(
context, router_id, router) # NOTE(arosen): the mock.patch here is needed for api_replay_mode
with mock.patch("neutron.plugins.common.utils._fixup_res_dict",
side_effect=api_replay_utils._fixup_res_dict):
return super(NsxV3Plugin, self).update_router(
context, router_id, router)
except nsx_exc.ResourceNotFound: except nsx_exc.ResourceNotFound:
with context.session.begin(subtransactions=True): with context.session.begin(subtransactions=True):
router_db = self._get_router(context, router_id) router_db = self._get_router(context, router_id)
@ -1742,9 +1751,11 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
# to routers # to routers
self._validate_multiple_subnets_routers(context, self._validate_multiple_subnets_routers(context,
router_id, interface_info) router_id, interface_info)
# NOTE(arosen): the mock.patch here is needed for api_replay_mode
info = super(NsxV3Plugin, self).add_router_interface( with mock.patch("neutron.plugins.common.utils._fixup_res_dict",
context, router_id, interface_info) side_effect=api_replay_utils._fixup_res_dict):
info = super(NsxV3Plugin, self).add_router_interface(
context, router_id, interface_info)
try: try:
subnet = self.get_subnet(context, info['subnet_ids'][0]) subnet = self.get_subnet(context, info['subnet_ids'][0])
port = self.get_port(context, info['port_id']) port = self.get_port(context, info['port_id'])
@ -1860,11 +1871,15 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
return info return info
def create_floatingip(self, context, floatingip): def create_floatingip(self, context, floatingip):
new_fip = super(NsxV3Plugin, self).create_floatingip( # NOTE(arosen): the mock.patch here is needed for api_replay_mode
context, floatingip, initial_status=( with mock.patch("neutron.plugins.common.utils._fixup_res_dict",
const.FLOATINGIP_STATUS_ACTIVE side_effect=api_replay_utils._fixup_res_dict):
if floatingip['floatingip']['port_id']
else const.FLOATINGIP_STATUS_DOWN)) new_fip = super(NsxV3Plugin, self).create_floatingip(
context, floatingip, initial_status=(
const.FLOATINGIP_STATUS_ACTIVE
if floatingip['floatingip']['port_id']
else const.FLOATINGIP_STATUS_DOWN))
router_id = new_fip['router_id'] router_id = new_fip['router_id']
if not router_id: if not router_id:
return new_fip return new_fip
@ -1973,6 +1988,33 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
super(NsxV3Plugin, self).disassociate_floatingips( super(NsxV3Plugin, self).disassociate_floatingips(
context, port_id, do_notify=False) context, port_id, do_notify=False)
def _ensure_default_security_group(self, context, tenant_id):
# NOTE(arosen): if in replay mode we'll create all the default
# security groups for the user with their data so we don't
# want this to be called.
if (cfg.CONF.api_replay_mode is False):
return super(NsxV3Plugin, self)._ensure_default_security_group(
context, tenant_id)
def _stub__validate_name_not_default(self):
# NOTE(arosen): if in replay mode we need stub out this validator to
# all default security groups to be created via the api
if cfg.CONF.api_replay_mode:
def _pass(data, foo=None):
pass
ext_sg.validators.validators['type:name_not_default'] = _pass
def get_security_groups(self, context, filters=None, fields=None,
sorts=None, limit=None,
marker=None, page_reverse=False, default_sg=False):
self._stub__validate_name_not_default()
return super(NsxV3Plugin, self).get_security_groups(
context, filters=filters, fields=fields,
sorts=sorts, limit=limit,
marker=marker, page_reverse=page_reverse,
default_sg=default_sg)
def create_security_group(self, context, security_group, default_sg=False): def create_security_group(self, context, security_group, default_sg=False):
secgroup = security_group['security_group'] secgroup = security_group['security_group']
secgroup['id'] = secgroup.get('id') or uuidutils.generate_uuid() secgroup['id'] = secgroup.get('id') or uuidutils.generate_uuid()

View File

@ -0,0 +1,45 @@
# Copyright (c) 2015 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from vmware_nsx.tests.unit.nsx_v3 import test_plugin
# FIXME(arosen): - these tests pass but seem to break the other tests
# as the attribute map doesn't get reset after each test class. I tried
# backing it up and restoring it here though that doesn't seem to be doing
# the trick either...
class TestApiReplay(test_plugin.NsxV3PluginTestCaseMixin):
def setUp(self, plugin=None, ext_mgr=None, service_plugins=None):
# enables api_replay_mode for these tests
super(TestApiReplay, self).setUp()
def test_create_port_specify_id(self):
self.skipTest("...fixme...")
specified_network_id = '555e762b-d7a1-4b44-b09b-2a34ada56c9f'
specified_port_id = 'e55e762b-d7a1-4b44-b09b-2a34ada56c9f'
network_res = self._create_network(self.fmt,
'test-network',
True,
arg_list=('id',),
id=specified_network_id)
network = self.deserialize(self.fmt, network_res)
self.assertEqual(specified_network_id, network['network']['id'])
port_res = self._create_port(self.fmt,
network['network']['id'],
arg_list=('id',),
id=specified_port_id)
port = self.deserialize(self.fmt, port_res)
self.assertEqual(specified_port_id, port['port']['id'])