From 7c8d9d6108d66a79dece84aff6d8bd304feffdf8 Mon Sep 17 00:00:00 2001 From: Sayaji Date: Mon, 11 Aug 2014 18:16:12 -0700 Subject: [PATCH] Implements sync mechanism between Neutron and Nuage VSD This will sync resources between Neutron and VSD based on the configuration parameters. "enable_sync" will enable/disable the sync and "sync_interval" will control the time interval between consecutive sync cycles. DocImpact Change-Id: I6730bf0166dfd1e35795b679293a2831b5afbc75 Implements: blueprint nuage-neutron-sync --- etc/neutron/plugins/nuage/nuage_plugin.ini | 31 ++ neutron/plugins/nuage/common/config.py | 12 + neutron/plugins/nuage/nuagedb.py | 179 +++++++- neutron/plugins/nuage/plugin.py | 22 +- neutron/plugins/nuage/syncmanager.py | 423 ++++++++++++++++++ neutron/tests/unit/nuage/fake_nuageclient.py | 123 +++++ neutron/tests/unit/nuage/test_nuage_plugin.py | 32 +- neutron/tests/unit/nuage/test_syncmanager.py | 343 ++++++++++++++ 8 files changed, 1147 insertions(+), 18 deletions(-) create mode 100644 neutron/plugins/nuage/syncmanager.py create mode 100644 neutron/tests/unit/nuage/test_syncmanager.py diff --git a/etc/neutron/plugins/nuage/nuage_plugin.ini b/etc/neutron/plugins/nuage/nuage_plugin.ini index 994d1206ce2..aad37bd52b0 100644 --- a/etc/neutron/plugins/nuage/nuage_plugin.ini +++ b/etc/neutron/plugins/nuage/nuage_plugin.ini @@ -1,10 +1,41 @@ # Please fill in the correct data for all the keys below and uncomment key-value pairs [restproxy] +# (StrOpt) Default Network partition in which VSD will +# orchestrate network resources using openstack +# #default_net_partition_name = + +# (StrOpt) Nuage provided uri for initial authorization to +# access VSD +# #auth_resource = /auth + +# (StrOpt) IP Address and Port of VSD +# #server = ip:port + +# (StrOpt) Organization name in which VSD will orchestrate +# network resources using openstack +# #organization = org + +# (StrOpt) Username and password of VSD for authentication +# #serverauth = uname:pass + +# (BoolOpt) Boolean for SSL connection with VSD server +# #serverssl = True + +# (StrOpt) Nuage provided base uri to reach out to VSD +# #base_uri = /base +[syncmanager] +# (BoolOpt) Boolean to enable sync between openstack and VSD +# +#enable_sync = False + +# (IntOpt) Sync interval in seconds between openstack and VSD +# +#sync_interval = 0 \ No newline at end of file diff --git a/neutron/plugins/nuage/common/config.py b/neutron/plugins/nuage/common/config.py index cd5a8a80a3b..6ba6d5d3f3a 100644 --- a/neutron/plugins/nuage/common/config.py +++ b/neutron/plugins/nuage/common/config.py @@ -42,6 +42,18 @@ restproxy_opts = [ help=_("Per Net Partition quota of floating ips")), ] +syncmanager_opts = [ + cfg.BoolOpt('enable_sync', default=False, + help=_("Nuage plugin will sync resources between openstack " + "and VSD")), + cfg.IntOpt('sync_interval', default=0, + help=_("Sync interval in seconds between openstack and VSD. " + "It defines how often the synchronization is done. " + "If not set, value of 0 is assumed and sync will be " + "performed only once, at the Neutron startup time.")), +] + def nuage_register_cfg_opts(): cfg.CONF.register_opts(restproxy_opts, "RESTPROXY") + cfg.CONF.register_opts(syncmanager_opts, "SYNCMANAGER") \ No newline at end of file diff --git a/neutron/plugins/nuage/nuagedb.py b/neutron/plugins/nuage/nuagedb.py index d7ef52f0e73..2a4c0eafb2f 100644 --- a/neutron/plugins/nuage/nuagedb.py +++ b/neutron/plugins/nuage/nuagedb.py @@ -15,6 +15,10 @@ # @author: Ronak Shah, Nuage Networks, Alcatel-Lucent USA Inc. from neutron.db import common_db_mixin +from neutron.db import extraroute_db +from neutron.db import l3_db +from neutron.db import models_v2 +from neutron.db import securitygroups_db from neutron.plugins.nuage import nuage_models @@ -33,6 +37,11 @@ def delete_net_partition(session, net_partition): session.delete(net_partition) +def delete_net_partition_by_id(session, netpart_id): + query = session.query(nuage_models.NetPartition) + query.filter_by(id=netpart_id).delete() + + def get_net_partition_by_name(session, name): query = session.query(nuage_models.NetPartition) return query.filter_by(name=name).first() @@ -52,6 +61,74 @@ def get_net_partitions(session, filters=None, fields=None): return query +def get_net_partition_ids(session): + query = session.query(nuage_models.NetPartition.id) + return [netpart[0] for netpart in query] + + +def get_net_partition_with_lock(session, netpart_id): + query = session.query(nuage_models.NetPartition) + netpart_db = query.filter_by(id=netpart_id).with_lockmode('update').one() + return make_net_partition_dict(netpart_db) + + +def get_subnet_ids(session): + query = session.query(models_v2.Subnet.id) + return [subn[0] for subn in query] + + +def get_subnet_with_lock(session, sub_id): + query = session.query(models_v2.Subnet) + subnet_db = query.filter_by(id=sub_id).with_lockmode('update').one() + return subnet_db + + +def get_router_ids(session): + query = session.query(l3_db.Router.id) + return [router[0] for router in query] + + +def get_router_with_lock(session, router_id): + query = session.query(l3_db.Router) + router_db = query.filter_by(id=router_id).with_lockmode('update').one() + return router_db + + +def get_secgrp_ids(session): + query = session.query(securitygroups_db.SecurityGroup.id) + return [secgrp[0] for secgrp in query] + + +def get_secgrp_with_lock(session, secgrp_id): + query = session.query(securitygroups_db.SecurityGroup) + secgrp_db = query.filter_by(id=secgrp_id).with_lockmode('update').one() + return secgrp_db + + +def get_secgrprule_ids(session): + query = session.query(securitygroups_db.SecurityGroupRule.id) + return [secgrprule[0] for secgrprule in query] + + +def get_secgrprule_with_lock(session, secgrprule_id): + query = session.query(securitygroups_db.SecurityGroupRule) + secgrprule_db = (query.filter_by(id=secgrprule_id).with_lockmode( + 'update').one()) + return secgrprule_db + + +def get_port_with_lock(session, port_id): + query = session.query(models_v2.Port) + port_db = query.filter_by(id=port_id).with_lockmode('update').one() + return port_db + + +def get_fip_with_lock(session, fip_id): + query = session.query(l3_db.FloatingIP) + fip_db = query.filter_by(id=fip_id).with_lockmode('update').one() + return fip_db + + def add_entrouter_mapping(session, np_id, router_id, n_l3id): @@ -81,6 +158,20 @@ def update_subnetl2dom_mapping(subnet_l2dom, subnet_l2dom.update(new_dict) +def get_update_subnetl2dom_mapping(session, new_dict): + subnet_l2dom = get_subnet_l2dom_with_lock(session, new_dict['subnet_id']) + subnet_l2dom.update(new_dict) + + +def update_entrtr_mapping(ent_rtr, new_dict): + ent_rtr.update(new_dict) + + +def get_update_entrtr_mapping(session, new_dict): + ent_rtr = get_ent_rtr_mapping_with_lock(session, new_dict['router_id']) + ent_rtr.update(new_dict) + + def delete_subnetl2dom_mapping(session, subnet_l2dom): session.delete(subnet_l2dom) @@ -90,8 +181,13 @@ def get_subnet_l2dom_by_id(session, id): return query.filter_by(subnet_id=id).first() -def get_ent_rtr_mapping_by_entid(session, - entid): +def get_subnet_l2dom_with_lock(session, id): + query = session.query(nuage_models.SubnetL2Domain) + subl2dom = query.filter_by(subnet_id=id).with_lockmode('update').one() + return subl2dom + + +def get_ent_rtr_mapping_by_entid(session, entid): query = session.query(nuage_models.NetPartitionRouter) return query.filter_by(net_partition_id=entid).all() @@ -115,3 +211,82 @@ def get_network_binding(session, network_id): return (session.query(nuage_models.ProviderNetBinding). filter_by(network_id=network_id). first()) + + +def get_ent_rtr_mapping_with_lock(session, rtrid): + query = session.query(nuage_models.NetPartitionRouter) + entrtr = query.filter_by(router_id=rtrid).with_lockmode('update').one() + return entrtr + + +def get_ipalloc_for_fip(session, network_id, ip, lock=False): + query = session.query(models_v2.IPAllocation) + if lock: + # Lock is required when the resource is synced + ipalloc_db = (query.filter_by(network_id=network_id).filter_by( + ip_address=ip).with_lockmode('update').one()) + else: + ipalloc_db = (query.filter_by(network_id=network_id).filter_by( + ip_address=ip).one()) + return make_ipalloc_dict(ipalloc_db) + + +def get_all_net_partitions(session): + net_partitions = get_net_partitions(session) + return make_net_partition_list(net_partitions) + + +def get_all_routes(session): + routes = session.query(extraroute_db.RouterRoute) + return make_route_list(routes) + + +def get_route_with_lock(session, dest, nhop): + query = session.query(extraroute_db.RouterRoute) + route_db = (query.filter_by(destination=dest).filter_by(nexthop=nhop) + .with_lockmode('update').one()) + return make_route_dict(route_db) + + +def make_ipalloc_dict(subnet_db): + return {'port_id': subnet_db['port_id'], + 'subnet_id': subnet_db['subnet_id'], + 'network_id': subnet_db['network_id'], + 'ip_address': subnet_db['ip_address']} + + +def make_net_partition_dict(net_partition): + return {'id': net_partition['id'], + 'name': net_partition['name'], + 'l3dom_tmplt_id': net_partition['l3dom_tmplt_id'], + 'l2dom_tmplt_id': net_partition['l2dom_tmplt_id']} + + +def make_net_partition_list(net_partitions): + return [make_net_partition_dict(net_partition) for net_partition in + net_partitions] + + +def make_route_dict(route): + return {'destination': route['destination'], + 'nexthop': route['nexthop'], + 'router_id': route['router_id']} + + +def make_route_list(routes): + return [make_route_dict(route) for route in routes] + + +def make_subnl2dom_dict(subl2dom): + return {'subnet_id': subl2dom['subnet_id'], + 'net_partition_id': subl2dom['net_partition_id'], + 'nuage_subnet_id': subl2dom['nuage_subnet_id'], + 'nuage_l2dom_tmplt_id': subl2dom['nuage_l2dom_tmplt_id'], + 'nuage_user_id': subl2dom['nuage_user_id'], + 'nuage_group_id': subl2dom['nuage_group_id']} + + +def make_entrtr_dict(entrtr): + return {'net_partition_id': entrtr['net_partition_id'], + 'router_id': entrtr['router_id'], + 'nuage_router_id': entrtr['nuage_router_id']} \ No newline at end of file diff --git a/neutron/plugins/nuage/plugin.py b/neutron/plugins/nuage/plugin.py index ea639153311..f0041e18ee9 100644 --- a/neutron/plugins/nuage/plugin.py +++ b/neutron/plugins/nuage/plugin.py @@ -41,12 +41,14 @@ from neutron.extensions import securitygroup as ext_sg from neutron.openstack.common import excutils from neutron.openstack.common import importutils from neutron.openstack.common import lockutils +from neutron.openstack.common import loopingcall from neutron.plugins.nuage.common import config from neutron.plugins.nuage.common import constants from neutron.plugins.nuage.common import exceptions as nuage_exc from neutron.plugins.nuage import extensions from neutron.plugins.nuage.extensions import netpartition from neutron.plugins.nuage import nuagedb +from neutron.plugins.nuage import syncmanager from neutron import policy @@ -71,6 +73,9 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2, self.nuageclient_init() net_partition = cfg.CONF.RESTPROXY.default_net_partition_name self._create_default_net_partition(net_partition) + if cfg.CONF.SYNCMANAGER.enable_sync: + self.syncmanager = syncmanager.SyncManager(self.nuageclient) + self._synchronization_thread() def nuageclient_init(self): server = cfg.CONF.RESTPROXY.server @@ -85,6 +90,16 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2, auth_resource, organization) + def _synchronization_thread(self): + sync_interval = cfg.CONF.SYNCMANAGER.sync_interval + fip_quota = str(cfg.CONF.RESTPROXY.default_floatingip_quota) + if sync_interval > 0: + sync_loop = loopingcall.FixedIntervalLoopingCall( + self.syncmanager.synchronize, fip_quota) + sync_loop.start(interval=sync_interval) + else: + self.syncmanager.synchronize(fip_quota) + def _resource_finder(self, context, for_resource, resource, user_req): match = re.match(attributes.UUID_PATTERN, user_req[resource]) if match: @@ -1084,10 +1099,13 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2, neutron_fip, port_id): rtr_id = neutron_fip['router_id'] net_id = neutron_fip['floating_network_id'] + subn = nuagedb.get_ipalloc_for_fip(context.session, + net_id, + neutron_fip['floating_ip_address']) - fip_pool = self.nuageclient.get_nuage_fip_pool_by_id(net_id) + fip_pool = self.nuageclient.get_nuage_fip_pool_by_id(subn['subnet_id']) if not fip_pool: - msg = _('sharedresource %s not found on VSD') % net_id + msg = _('sharedresource %s not found on VSD') % subn['subnet_id'] raise n_exc.BadRequest(resource='floatingip', msg=msg) diff --git a/neutron/plugins/nuage/syncmanager.py b/neutron/plugins/nuage/syncmanager.py new file mode 100644 index 00000000000..a719015b6da --- /dev/null +++ b/neutron/plugins/nuage/syncmanager.py @@ -0,0 +1,423 @@ +# Copyright 2014 Alcatel-Lucent USA Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Sayaji Patil, Nuage Networks, Alcatel-Lucent USA Inc. + +from oslo.config import cfg +import sqlalchemy.orm.exc as db_exc + +from neutron import context as ncontext +from neutron.db import db_base_plugin_v2 +from neutron.db import extraroute_db +from neutron.db import securitygroups_db +from neutron.openstack.common import importutils +from neutron.openstack.common import log +from neutron.openstack.common.gettextutils import _LE, _LI, _LW +from neutron.plugins.nuage.common import config +from neutron.plugins.nuage import nuagedb + + +LOG = log.getLogger(__name__) +NUAGE_CONFIG_FILE = '/etc/neutron/plugins/nuage/nuage_plugin.ini' + + +class SyncManager(db_base_plugin_v2.NeutronDbPluginV2, + extraroute_db.ExtraRoute_db_mixin, + securitygroups_db.SecurityGroupDbMixin): + """ + This class provides functionality to sync data between OpenStack and VSD. + """ + + def __init__(self, nuageclient): + self.context = ncontext.get_admin_context() + self.nuageclient = nuageclient + + def synchronize(self, fipquota): + LOG.info(_LI("Starting the sync between Neutron and VSD")) + try: + # Get all data to determine the resources to sync + data = self._get_all_data() + resources = self.nuageclient.get_resources_to_sync(data) + + # Sync all resources + self._sync(resources, fipquota) + except Exception as e: + LOG.error(_LE("Cannot complete the sync between Neutron and VSD " + "because of error:%s"), str(e)) + return + + LOG.info(_LI("Sync between Neutron and VSD completed successfully")) + + def _get_all_data(self): + # Get all net-partitions + net_partition_list = nuagedb.get_all_net_partitions( + self.context.session) + + # Get all subnet ids + subnet_id_list = nuagedb.get_subnet_ids(self.context.session) + + # Get all router ids + router_id_list = nuagedb.get_router_ids(self.context.session) + + # Get all ports + port_list = self.get_ports(self.context) + + # Get all routes + route_list = nuagedb.get_all_routes(self.context.session) + + # Get all floatingips + fip_list = self.get_floatingips(self.context) + + # Get all securitygrp ids + secgrp_id_list = nuagedb.get_secgrp_ids(self.context.session) + + # Get all securitygrprules + secgrprule_id_list = self.get_security_group_rules(self.context) + + # Get all portbindings + portbinding_list = self._get_port_security_group_bindings(self.context) + + data = { + 'netpartition': net_partition_list, + 'subnet': subnet_id_list, + 'router': router_id_list, + 'port': port_list, + 'route': route_list, + 'fip': fip_list, + 'secgroup': secgrp_id_list, + 'secgrouprule': secgrprule_id_list, + 'portbinding': portbinding_list, + } + return data + + def _sync(self, resources, fip_quota): + # Sync net-partitions + net_partition_id_dict = self.sync_net_partitions(fip_quota, resources) + + # Sync sharednetworks + self.sync_sharednetworks(resources) + + # Sync l2domains + self.sync_l2domains(net_partition_id_dict, resources) + + # Sync domains + self.sync_domains(net_partition_id_dict, resources) + + # Sync domainsubnets + self.sync_domainsubnets(resources) + + # Sync routes + self.sync_routes(resources) + + # Sync vms + self.sync_vms(resources) + + # Sync secgrps + self.sync_secgrps(resources) + + # Sync secgrprules + self.sync_secgrp_rules(resources) + + # Sync fips + self._sync_fips(resources) + + # Delete the old net-partitions + for net_id in net_partition_id_dict: + nuagedb.delete_net_partition_by_id(self.context.session, + net_id) + + def sync_net_partitions(self, fip_quota, resources): + net_partition_id_dict = {} + for netpart_id in resources['netpartition']['add']: + with self.context.session.begin(subtransactions=True): + netpart = self._get_netpart_data(netpart_id) + if netpart: + result = self.nuageclient.create_netpart(netpart, + fip_quota) + netpart = result.get(netpart_id) + if netpart: + net_partition_id_dict[netpart_id] = netpart['id'] + nuagedb.add_net_partition( + self.context.session, + netpart['id'], + netpart['l3dom_tmplt_id'], + netpart['l2dom_tmplt_id'], + netpart['name']) + + return net_partition_id_dict + + def sync_sharednetworks(self, resources): + for sharednet_id in resources['sharednetwork']['add']: + with self.context.session.begin(subtransactions=True): + subnet, subl2dom = self._get_subnet_data( + sharednet_id, + get_mapping=False) + if subnet: + self.nuageclient.create_sharednetwork(subnet) + + def sync_l2domains(self, net_partition_id_dict, resources): + for l2dom_id in resources['l2domain']['add']: + with self.context.session.begin(subtransactions=True): + subnet, subl2dom = self._get_subnet_data(l2dom_id) + if subnet: + # if subnet exists, subl2dom will exist + netpart_id = subl2dom['net_partition_id'] + if netpart_id in net_partition_id_dict.keys(): + # Use the id of the newly created net_partition + netpart_id = net_partition_id_dict[netpart_id] + + result = self.nuageclient.create_l2domain(netpart_id, + subnet) + if result: + nuagedb.get_update_subnetl2dom_mapping( + self.context.session, + result) + + def sync_domains(self, net_partition_id_dict, resources): + for domain_id in resources['domain']['add']: + with self.context.session.begin(subtransactions=True): + router, entrtr = self._get_router_data(domain_id) + if router: + # if router exists, entrtr will exist + netpart_id = entrtr['net_partition_id'] + if netpart_id in net_partition_id_dict.keys(): + # Use the id of the newly created net_partition + netpart_id = net_partition_id_dict[netpart_id] + + netpart = nuagedb.get_net_partition_by_id( + self.context.session, + netpart_id) + result = self.nuageclient.create_domain(netpart, router) + if result: + nuagedb.get_update_entrtr_mapping(self.context.session, + result) + + def sync_domainsubnets(self, resources): + for domsubn_id in resources['domainsubnet']['add']: + # This is a dict of subn_id and the router interface port + subn_rtr_intf_port_dict = ( + resources['port']['sub_rtr_intf_port_dict']) + port_id = subn_rtr_intf_port_dict[domsubn_id] + port = self._get_port_data(port_id) + if port: + with self.context.session.begin(subtransactions=True): + subnet, subl2dom = self._get_subnet_data(domsubn_id) + if subnet: + result = self.nuageclient.create_domainsubnet(subnet, + port) + if result: + nuagedb.get_update_subnetl2dom_mapping( + self.context.session, + result) + + def sync_routes(self, resources): + for rt in resources['route']['add']: + with self.context.session.begin(subtransactions=True): + route = self._get_route_data(rt) + if route: + self.nuageclient.create_route(route) + + def sync_vms(self, resources): + for port_id in resources['port']['vm']: + port = self._get_port_data(port_id) + if port: + self.nuageclient.create_vm(port) + + def sync_secgrps(self, resources): + secgrp_dict = resources['security']['secgroup'] + for secgrp_id, ports in secgrp_dict['l2domain']['add'].iteritems(): + with self.context.session.begin(subtransactions=True): + secgrp = self._get_sec_grp_data(secgrp_id) + if secgrp: + self.nuageclient.create_security_group(secgrp, ports) + + for secgrp_id, ports in secgrp_dict['domain']['add'].iteritems(): + with self.context.session.begin(subtransactions=True): + secgrp = self._get_sec_grp_data(secgrp_id) + if secgrp: + self.nuageclient.create_security_group(secgrp, ports) + + def sync_secgrp_rules(self, resources): + secrule_list = resources['security']['secgrouprule'] + for secrule_id in secrule_list['l2domain']['add']: + with self.context.session.begin(subtransactions=True): + secgrprule = self._get_sec_grp_rule_data(secrule_id) + if secgrprule: + self.nuageclient.create_security_group_rule(secgrprule) + + for secrule_id in secrule_list['domain']['add']: + with self.context.session.begin(subtransactions=True): + secgrprule = self._get_sec_grp_rule_data(secrule_id) + if secgrprule: + self.nuageclient.create_security_group_rule(secgrprule) + + def _sync_fips(self, resources): + for fip_id in resources['fip']['add']: + with self.context.session.begin(subtransactions=True): + fip = self._get_fip_data(fip_id) + if fip: + ipalloc = self._get_ipalloc_for_fip(fip) + self.nuageclient.create_fip(fip, ipalloc) + + for fip_id in resources['fip']['disassociate']: + with self.context.session.begin(subtransactions=True): + fip = self._get_fip_data(fip_id) + if fip: + self.nuageclient.disassociate_fip(fip) + + for fip_id in resources['fip']['associate']: + with self.context.session.begin(subtransactions=True): + fip = self._get_fip_data(fip_id) + if fip: + self.nuageclient.associate_fip(fip) + + def _get_subnet_data(self, subnet_id, get_mapping=True): + subnet = None + subl2dom = None + try: + if get_mapping: + subl2dom_db = nuagedb.get_subnet_l2dom_with_lock( + self.context.session, + subnet_id) + subl2dom = nuagedb.make_subnl2dom_dict(subl2dom_db) + + subnet_db = nuagedb.get_subnet_with_lock(self.context.session, + subnet_id) + subnet = self._make_subnet_dict(subnet_db) + except db_exc.NoResultFound: + LOG.warning(_LW("Subnet %s not found in neutron for sync"), + subnet_id) + + return subnet, subl2dom + + def _get_router_data(self, router_id): + router = None + entrtr = None + try: + entrtr_db = nuagedb.get_ent_rtr_mapping_with_lock( + self.context.session, + router_id) + entrtr = nuagedb.make_entrtr_dict(entrtr_db) + + router_db = nuagedb.get_router_with_lock(self.context.session, + router_id) + router = self._make_router_dict(router_db) + except db_exc.NoResultFound: + LOG.warning(_LW("Router %s not found in neutron for sync"), + router_id) + + return router, entrtr + + def _get_route_data(self, rt): + route = None + try: + route = nuagedb.get_route_with_lock(self.context.session, + rt['destination'], + rt['nexthop']) + except db_exc.NoResultFound: + LOG.warning(_LW("Route with destination %(dest)s and nexthop " + "%(hop)s not found in neutron for sync"), + {'dest': rt['destination'], + 'hop': rt['nexthop']}) + + return route + + def _get_sec_grp_data(self, secgrp_id): + secgrp = None + try: + secgrp_db = nuagedb.get_secgrp_with_lock(self.context.session, + secgrp_id) + secgrp = self._make_security_group_dict(secgrp_db) + except db_exc.NoResultFound: + LOG.warning(_LW("Security group %s not found in neutron for sync"), + secgrp_id) + return secgrp + + def _get_sec_grp_rule_data(self, secgrprule_id): + secgrprule = None + try: + secrule_db = nuagedb.get_secgrprule_with_lock(self.context.session, + secgrprule_id) + secgrprule = self._make_security_group_rule_dict(secrule_db) + except db_exc.NoResultFound: + LOG.warning(_LW("Security group rule %s not found in neutron for " + "sync"), secgrprule_id) + return secgrprule + + def _get_fip_data(self, fip_id): + fip = None + try: + fip_db = nuagedb.get_fip_with_lock(self.context.session, fip_id) + fip = self._make_floatingip_dict(fip_db) + except db_exc.NoResultFound: + LOG.warning(_LW("Floating ip %s not found in neutron for sync"), + fip_id) + return fip + + def _get_ipalloc_for_fip(self, fip): + ipalloc = None + try: + ipalloc = nuagedb.get_ipalloc_for_fip(self.context.session, + fip['floating_network_id'], + fip['floating_ip_address'], + lock=True) + except db_exc.NoResultFound: + LOG.warning(_LW("IP allocation for floating ip %s not found in " + "neutron for sync"), fip['id']) + return ipalloc + + def _get_netpart_data(self, netpart_id): + netpart = None + try: + netpart = nuagedb.get_net_partition_with_lock( + self.context.session, + netpart_id) + except db_exc.NoResultFound: + LOG.warning(_LW("Net-partition %s not found in neutron for sync"), + netpart_id) + return netpart + + def _get_port_data(self, port_id): + port = None + try: + port_db = nuagedb.get_port_with_lock(self.context.session, port_id) + port = self._make_port_dict(port_db) + except db_exc.NoResultFound: + LOG.warning(_LW("VM port %s not found in neutron for sync"), + port_id) + return port + + +def main(): + cfg.CONF(default_config_files=( + [NUAGE_CONFIG_FILE])) + config.nuage_register_cfg_opts() + server = cfg.CONF.RESTPROXY.server + serverauth = cfg.CONF.RESTPROXY.serverauth + serverssl = cfg.CONF.RESTPROXY.serverssl + base_uri = cfg.CONF.RESTPROXY.base_uri + auth_resource = cfg.CONF.RESTPROXY.auth_resource + organization = cfg.CONF.RESTPROXY.organization + fipquota = str(cfg.CONF.RESTPROXY.default_floatingip_quota) + logging = importutils.import_module('logging') + nuageclientinst = importutils.import_module('nuagenetlib.nuageclient') + nuageclient = nuageclientinst.NuageClient(server, base_uri, + serverssl, serverauth, + auth_resource, + organization) + logging.basicConfig(level=logging.DEBUG) + SyncManager(nuageclient).synchronize(fipquota) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/neutron/tests/unit/nuage/fake_nuageclient.py b/neutron/tests/unit/nuage/fake_nuageclient.py index 332b385e9ae..37995ba3247 100644 --- a/neutron/tests/unit/nuage/fake_nuageclient.py +++ b/neutron/tests/unit/nuage/fake_nuageclient.py @@ -197,3 +197,126 @@ class FakeNuageClient(object): def remove_router_interface(self, params): pass + + def get_resources_to_sync(self, data): + netpart_id_list = [] + for netpart in data['netpartition']: + netpart_id_list.append(netpart['id']) + + netpart_dict = { + 'add': netpart_id_list, + 'sync': [] + } + + subn_id_list = [] + if data['subnet']: + subn_id_list.append(data['subnet'][0]) + + l2domain_dict = { + 'add': subn_id_list + } + + rtr_id_list = [] + if data['router']: + rtr_id_list.append(data['router'][0]) + + domain_dict = { + 'add': rtr_id_list + } + + domain_subn_id = uuidutils.generate_uuid() + + result = { + 'netpartition': netpart_dict, + 'l2domain': l2domain_dict, + 'domain': domain_dict, + 'domainsubnet': {'add': [domain_subn_id]}, + 'sharednetwork': {'add': [uuidutils.generate_uuid()]}, + 'route': {'add': []}, + 'security': { + 'secgroup': { + 'l2domain': {'add': { + uuidutils.generate_uuid(): [uuidutils.generate_uuid()] + }}, + 'domain': {'add': { + uuidutils.generate_uuid(): [uuidutils.generate_uuid()] + }} + }, + 'secgrouprule': { + 'l2domain': {'add': [uuidutils.generate_uuid()]}, + 'domain': {'add': [uuidutils.generate_uuid()]} + }, + }, + 'port': { + 'vm': [uuidutils.generate_uuid()], + 'sub_rtr_intf_port_dict': { + domain_subn_id: uuidutils.generate_uuid() + }, + 'secgroup': [uuidutils.generate_uuid()] + }, + 'subl2dommapping': [uuidutils.generate_uuid()], + 'fip': { + 'add': [uuidutils.generate_uuid()], + 'associate': [uuidutils.generate_uuid()], + 'disassociate': [uuidutils.generate_uuid()] + } + } + return result + + def create_netpart(self, netpart, fip_quota): + if netpart['name'] == 'sync-new-netpartition': + oldid = netpart['id'] + netpart['id'] = 'a917924f-3139-4bdb-a4c3-ea7c8011582f' + netpart = { + oldid: netpart + } + return netpart + return {} + + def create_sharednetwork(self, subnet): + pass + + def create_l2domain(self, netpart_id, subnet): + subl2dom = { + 'subnet_id': subnet['id'], + 'nuage_subnet_id': '52daa465-cf33-4efd-91d3-f5bc2aebd', + 'net_partition_id': netpart_id, + 'nuage_l2dom_tmplt_id': uuidutils.generate_uuid(), + 'nuage_user_id': uuidutils.generate_uuid(), + 'nuage_group_id': uuidutils.generate_uuid(), + } + + return subl2dom + + def create_domain(self, netpart, router): + entrtr = { + 'router_id': router['id'], + 'nuage_router_id': '2d782c02-b88e-44ad-a79b-4bdf11f7df3d', + 'net_partition_id': netpart['id'] + } + + return entrtr + + def create_domainsubnet(self, subnet, ports): + pass + + def create_route(self, route): + pass + + def create_vm(self, port): + pass + + def create_security_group(self, secgrp, ports): + pass + + def create_security_group_rule(self, secgrprule): + pass + + def create_fip(self, fip, ipalloc): + pass + + def associate_fip(self, fip): + pass + + def disassociate_fip(self, fip): + pass diff --git a/neutron/tests/unit/nuage/test_nuage_plugin.py b/neutron/tests/unit/nuage/test_nuage_plugin.py index d28f5d7991d..16c049a703a 100644 --- a/neutron/tests/unit/nuage/test_nuage_plugin.py +++ b/neutron/tests/unit/nuage/test_nuage_plugin.py @@ -59,6 +59,22 @@ FAKE_ORGANIZATION = 'fake_org' _plugin_name = ('%s.NuagePlugin' % NUAGE_PLUGIN_PATH) +def getNuageClient(): + server = FAKE_SERVER + serverauth = FAKE_SERVER_AUTH + serverssl = FAKE_SERVER_SSL + base_uri = FAKE_BASE_URI + auth_resource = FAKE_AUTH_RESOURCE + organization = FAKE_ORGANIZATION + nuageclient = fake_nuageclient.FakeNuageClient(server, + base_uri, + serverssl, + serverauth, + auth_resource, + organization) + return nuageclient + + class NuagePluginV2TestCase(test_db_plugin.NeutronDbPluginV2TestCase): def setUp(self, plugin=_plugin_name, ext_mgr=None, service_plugins=None): @@ -67,19 +83,7 @@ class NuagePluginV2TestCase(test_db_plugin.NeutronDbPluginV2TestCase): self.skipTest("Nuage Plugin does not support IPV6.") def mock_nuageClient_init(self): - server = FAKE_SERVER - serverauth = FAKE_SERVER_AUTH - serverssl = FAKE_SERVER_SSL - base_uri = FAKE_BASE_URI - auth_resource = FAKE_AUTH_RESOURCE - organization = FAKE_ORGANIZATION - self.nuageclient = None - self.nuageclient = fake_nuageclient.FakeNuageClient(server, - base_uri, - serverssl, - serverauth, - auth_resource, - organization) + self.nuageclient = getNuageClient() with mock.patch.object(nuage_plugin.NuagePlugin, 'nuageclient_init', new=mock_nuageClient_init): @@ -541,4 +545,4 @@ class TestNuageSecurityGroupTestCase(NuagePluginV2TestCase, # The Nuage plugin reserve the first port port = ports['ports'][1] self.assertEqual(1, len(port[ext_sg.SECURITYGROUPS])) - self._delete('ports', port['id']) \ No newline at end of file + self._delete('ports', port['id']) diff --git a/neutron/tests/unit/nuage/test_syncmanager.py b/neutron/tests/unit/nuage/test_syncmanager.py new file mode 100644 index 00000000000..e72d235f6aa --- /dev/null +++ b/neutron/tests/unit/nuage/test_syncmanager.py @@ -0,0 +1,343 @@ +# Copyright 2014 Alcatel-Lucent USA Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Sayaji Patil, Nuage Networks, Alcatel-Lucent USA Inc. + +import contextlib + +from neutron import context +from neutron.openstack.common import uuidutils +from neutron.plugins.nuage import nuage_models +from neutron.plugins.nuage import syncmanager as sync +from neutron.tests.unit.nuage import test_netpartition +from neutron.tests.unit.nuage import test_nuage_plugin +from neutron.tests.unit import test_extension_extraroute as extraroute_test +from neutron.tests.unit import test_extension_security_group as test_sg +from neutron.tests.unit import test_l3_plugin + +_uuid = uuidutils.generate_uuid + + +class TestL3Sync(test_nuage_plugin.NuagePluginV2TestCase, + test_l3_plugin.L3NatDBIntTestCase): + + def setUp(self): + self.session = context.get_admin_context().session + self.syncmanager = sync.SyncManager( + test_nuage_plugin.getNuageClient()) + super(TestL3Sync, self).setUp() + + def _make_floatingip_for_tenant_port(self, net_id, port_id, tenant_id): + data = {'floatingip': {'floating_network_id': net_id, + 'tenant_id': tenant_id, + 'port_id': port_id}} + floatingip_req = self.new_create_request('floatingips', data, self.fmt) + res = floatingip_req.get_response(self.ext_api) + return self.deserialize(self.fmt, res) + + def test_router_sync(self): + # If the router exists in neutron and not in VSD, + # sync will create it in VSD. But the nuage_router_id + # will now change and will be updated in neutron + # accordingly + rtr_res = self._create_router('json', 'foo', 'test-router', True) + router = self.deserialize('json', rtr_res) + + self.syncmanager.synchronize('250') + + # Check that the nuage_router_id is updated in entrtrmapping table + router_db = self.session.query( + nuage_models.NetPartitionRouter).filter_by( + router_id=router['router']['id']).first() + + self.assertEqual('2d782c02-b88e-44ad-a79b-4bdf11f7df3d', + router_db['nuage_router_id']) + + self._delete('routers', router['router']['id']) + + def test_router_deleted_get(self): + data = self.syncmanager._get_router_data(_uuid()) + self.assertIsNone(data[0]) + self.assertIsNone(data[1]) + + def test_fip_sync(self): + with self.subnet(cidr='200.0.0.0/24') as public_sub: + self._set_net_external(public_sub['subnet']['network_id']) + with contextlib.nested(self.port(), self.port(), self.port()) as ( + p1, p2, p3): + p1_id = p1['port']['id'] + p2_id = p2['port']['id'] + p3_id = p3['port']['id'] + with contextlib.nested(self.floatingip_with_assoc( + port_id=p1_id), self.floatingip_with_assoc( + port_id=p2_id), self.floatingip_with_assoc( + port_id=p3_id)) as (fip1, fip2, fip3): + fip_dict = {'fip': { + 'add': [fip1['floatingip']['id']], + 'associate': [fip2['floatingip']['id']], + 'disassociate': [fip3['floatingip']['id']] + }} + self.syncmanager._sync_fips(fip_dict) + + def test_deleted_fip_sync(self): + fip_dict = {'fip': { + 'add': [_uuid()], + 'associate': [_uuid()], + 'disassociate': [_uuid()] + }} + self.syncmanager._sync_fips(fip_dict) + + def test_fip_and_ipalloc_get(self): + with self.subnet(cidr='200.0.0.0/24') as public_sub: + self._set_net_external(public_sub['subnet']['network_id']) + with self.port() as port: + p_id = port['port']['id'] + with self.floatingip_with_assoc(port_id=p_id) as fip: + + data = self.syncmanager._get_fip_data( + fip['floatingip']['id']) + + self.assertEqual(fip['floatingip']['id'], data['id']) + + data = self.syncmanager._get_ipalloc_for_fip( + fip['floatingip']) + self.assertEqual(fip['floatingip']['floating_ip_address'], + data['ip_address']) + + def test_fip_and_ipalloc_deleted_get(self): + data = self.syncmanager._get_fip_data(_uuid()) + self.assertIsNone(data) + + fip = { + 'id': _uuid(), + 'floating_network_id': _uuid(), + 'floating_ip_address': '176.176.10.10' + } + data = self.syncmanager._get_ipalloc_for_fip(fip) + self.assertIsNone(data) + + def test_domainsubnet_sync(self): + with self.subnet() as s1: + with contextlib.nested( + self.router(), + self.port()) as (r1, p1): + self._router_interface_action( + 'add', r1['router']['id'], + s1['subnet']['id'], p1['port']['id']) + domainsubn_dict = { + 'domainsubnet': {'add': [s1['subnet']['id']]}, + 'port': {'sub_rtr_intf_port_dict': {s1['subnet']['id']: + p1['port']['id']}}} + self.syncmanager.sync_domainsubnets(domainsubn_dict) + self._router_interface_action('remove', r1['router']['id'], + s1['subnet']['id'], None) + + def test_floatingip_update_different_router(self): + self._test_floatingip_update_different_router() + + def test_floatingip_update_different_fixed_ip_same_port(self): + self._test_floatingip_update_different_fixed_ip_same_port() + + def test_floatingip_create_different_fixed_ip_same_port(self): + self._test_floatingip_create_different_fixed_ip_same_port() + + def test_network_update_external_failure(self): + self._test_network_update_external_failure() + + +class TestExtraRouteSync(extraroute_test.ExtraRouteDBIntTestCase): + + def setUp(self): + self.session = context.get_admin_context().session + self.syncmanager = sync.SyncManager( + test_nuage_plugin.getNuageClient()) + super(TestExtraRouteSync, self).setUp() + + def test_route_sync(self): + route = {'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'} + with self.router() as r: + with self.subnet(cidr='10.0.1.0/24') as s: + net_id = s['subnet']['network_id'] + res = self._create_port('json', net_id) + p = self.deserialize(self.fmt, res) + self._routes_update_prepare(r['router']['id'], + None, p['port']['id'], [route]) + + route_dict = {'route': {'add': [route]}} + self.syncmanager.sync_routes(route_dict) + + self._routes_update_cleanup(p['port']['id'], + None, r['router']['id'], []) + + def test_route_get(self): + routes = [{'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'}] + with self.router() as r: + with self.subnet(cidr='10.0.1.0/24') as s: + net_id = s['subnet']['network_id'] + res = self._create_port('json', net_id) + p = self.deserialize(self.fmt, res) + self._routes_update_prepare(r['router']['id'], + None, p['port']['id'], routes) + + data = self.syncmanager._get_route_data(routes[0]) + self.assertEqual(routes[0]['destination'], data['destination']) + self.assertEqual(routes[0]['nexthop'], data['nexthop']) + self._routes_update_cleanup(p['port']['id'], + None, r['router']['id'], []) + + def test_route_deleted_get(self): + route = {'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'} + data = self.syncmanager._get_route_data(route) + self.assertIsNone(data) + + +class TestNetPartSync(test_netpartition.NetPartitionTestCase): + + def setUp(self): + self.session = context.get_admin_context().session + self.syncmanager = sync.SyncManager( + test_nuage_plugin.getNuageClient()) + super(TestNetPartSync, self).setUp() + + def test_net_partition_sync(self): + # If the net-partition exists in neutron and not in VSD, + # sync will create it in VSD. But the net-partition + # id will now change and has to be updated in neutron + # accordingly + netpart = self._make_netpartition('json', 'sync-new-netpartition') + + self.syncmanager.synchronize('250') + + # Check that the net-partition id is updated in db + netpart_db = self.session.query( + nuage_models.NetPartition).filter_by(name=netpart['net_partition'][ + 'name']).first() + + self.assertEqual('a917924f-3139-4bdb-a4c3-ea7c8011582f', + netpart_db['id']) + self._del_netpartition(netpart_db['id']) + + def test_net_partition_deleted_get(self): + data = self.syncmanager._get_netpart_data(_uuid()) + self.assertIsNone(data) + + +class TestL2Sync(test_nuage_plugin.NuagePluginV2TestCase): + + def setUp(self): + self.session = context.get_admin_context().session + self.syncmanager = sync.SyncManager( + test_nuage_plugin.getNuageClient()) + super(TestL2Sync, self).setUp() + + def test_subnet_sync(self): + # If the subnet exists in neutron and not in VSD, + # sync will create it in VSD. But the nuage_subnet_id + # will now change and will be updated in neutron + # accordingly + net_res = self._create_network("json", "pub", True) + network = self.deserialize('json', net_res) + + sub_res = self._create_subnet("json", network['network']['id'], + '10.0.0.0/24') + subnet = self.deserialize('json', sub_res) + + self.syncmanager.synchronize('250') + + # Check that the nuage_subnet_id is updated in db + subl2dom_db = self.session.query( + nuage_models.SubnetL2Domain).filter_by(subnet_id=subnet[ + 'subnet']['id']).first() + self.assertEqual('52daa465-cf33-4efd-91d3-f5bc2aebd', + subl2dom_db['nuage_subnet_id']) + + self._delete('subnets', subnet['subnet']['id']) + self._delete('networks', network['network']['id']) + + def test_subnet_deleted_get(self): + data = self.syncmanager._get_subnet_data(_uuid()) + self.assertIsNone(data[0]) + self.assertIsNone(data[1]) + + def test_sharednetwork_sync(self): + with self.subnet(cidr='200.0.0.0/24') as public_sub: + sharednet_dict = {'sharednetwork': {'add': [public_sub['subnet'][ + 'id']]}} + self.syncmanager.sync_sharednetworks(sharednet_dict) + + def test_vm_sync(self): + with self.port() as p: + port_dict = {'port': {'vm': [p['port']['id']]}} + self.syncmanager.sync_vms(port_dict) + + +class TestSecurityGroupSync(test_sg.TestSecurityGroups): + + def setUp(self): + self.session = context.get_admin_context().session + self.syncmanager = sync.SyncManager( + test_nuage_plugin.getNuageClient()) + super(TestSecurityGroupSync, self).setUp() + + def test_sg_get(self): + with self.security_group() as sg: + data = self.syncmanager._get_sec_grp_data( + sg['security_group']['id']) + self.assertEqual(sg['security_group']['id'], data['id']) + + def test_sg_deleted_get(self): + data = self.syncmanager._get_sec_grp_data(_uuid()) + self.assertIsNone(data) + + def test_sg_rule_get(self): + with self.security_group() as sg: + sg_rule_id = sg['security_group']['security_group_rules'][0]['id'] + data = self.syncmanager._get_sec_grp_rule_data(sg_rule_id) + self.assertEqual(sg_rule_id, data['id']) + + def test_sg_rule_deleted_get(self): + data = self.syncmanager._get_sec_grp_rule_data(_uuid()) + self.assertIsNone(data) + + def test_sg_grp_sync(self): + with contextlib.nested(self.security_group(), + self.security_group()) as (sg1, sg2): + sg1_id = sg1['security_group']['id'] + sg2_id = sg2['security_group']['id'] + sg_dict = {'security': {'secgroup': {'l2domain': {'add': {sg1_id: [ + _uuid()]}}, 'domain': {'add': {sg2_id: [_uuid()]}}}}} + self.syncmanager.sync_secgrps(sg_dict) + + def test_deleted_sg_grp_sync(self): + sg_dict = {'security': {'secgroup': {'l2domain': {'add': {_uuid(): [ + _uuid()]}}, 'domain': {'add': {_uuid(): [_uuid()]}}}}} + self.syncmanager.sync_secgrps(sg_dict) + + def test_sg_rule_sync(self): + with contextlib.nested(self.security_group(), + self.security_group()) as (sg1, sg2): + sg1_rule_id = ( + sg1['security_group']['security_group_rules'][0]['id']) + sg2_rule_id = ( + sg2['security_group']['security_group_rules'][0]['id']) + + sg_dict = {'security': {'secgrouprule': {'l2domain': { + 'add': [sg1_rule_id]}, 'domain': {'add': [sg2_rule_id]}}}} + self.syncmanager.sync_secgrp_rules(sg_dict) + + def test_deleted_sg_grp_rule_sync(self): + sg_dict = {'security': {'secgrouprule': + {'l2domain': {'add': [_uuid()]}, + 'domain': {'add': [_uuid()]}}}} + self.syncmanager.sync_secgrp_rules(sg_dict) \ No newline at end of file