x
/
group-based-policy
Code Issues Proposed changes

Add support for ussuri 

Add support for the ussuri stable branch.
* Removed use of services in devstack (e.g. FWaaS and LBaas), which
  were only used by the deprecated legacy plugin.
* https://review.opendev.org/#/c/572767/ changed the return
  value of _get_security_groups_on_port from a list of security
  group IDs to a list of security group OVOs. The monkey patch
  of this method has been updated to be consistent with this
  upstream change.
* https://review.opendev.org/#/c/703143/ removed the upstream
  get_binding_levels, which is replaced by the corresponding
  OVO call, get_binding_level_objs.
* https://review.opendev.org/#/c/709122/ broke the __repr__
  method in the AddressScope model class. This patch works
  around this by using the dictionary representation instead.
* https://review.opendev.org/#/c/679399/ made the MTU field
  of networks non-nullable, and sets it to a constant if not
  set explicitly. This broke GBP APIs which create networks
  as part of their implementation. This patch adds a monkey
  patch to pass in a value of 0, if one wasn't specified.
* Fixed alias uncovered by PEP8 checks.

Change-Id: I219bc9a5c2034499e59788ab11ef0ae310e97e1e
This commit is contained in:
Thomas Bachman 2020-09-16 22:27:21 +00:00
parent dfa4cd7fbc
commit 5588d7217e
18 changed files with 93 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.zuul.yaml
View File

@ -1,7 +1,7 @@
- project:
name: x/group-based-policy
templates:
- openstack-python-jobs
- openstack-python3-ussuri-jobs
- publish-to-pypi
# REVISIT: In the jobs below, the required-projects clause is needed on
# the master branch to select the correct version of the requirements
@ -15,27 +15,17 @@
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
- openstack-tox-py27:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
- openstack-tox-py35:
nodeset: ubuntu-xenial
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri
- openstack-tox-py36:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri
- openstack-tox-py37:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri
- legacy-group-based-policy-dsvm-functional:
voting: false
- legacy-group-based-policy-dsvm-aim:
@ -48,24 +38,14 @@
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
- openstack-tox-py27:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
- openstack-tox-py35:
nodeset: ubuntu-xenial
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri
- openstack-tox-py36:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri
- openstack-tox-py37:
nodeset: ubuntu-bionic
required-projects:
- name: openstack/requirements
override-checkout: stable/train
override-checkout: stable/ussuri

2
devstack/lib/nfp
View File

@ -25,7 +25,7 @@ NEUTRON_CONF_DIR=/etc/neutron
NEUTRON_CONF=$NEUTRON_CONF_DIR/neutron.conf
NFP_CONF_DIR=/etc/nfp
DISKIMAGE_CREATE_DIR=$NFPSERVICE_DIR/gbpservice/contrib/nfp/tools/image_builder
NEUTRON_SRC_BRANCH_FOR_NFP_CONTROLLER=stable/train
NEUTRON_SRC_BRANCH_FOR_NFP_CONTROLLER=stable/ussuri
# Save trace setting
XTRACE=$(set +o | grep xtrace)

10
devstack/local.conf.nfp
View File

@ -43,11 +43,11 @@ if [[ $ENABLE_NFP = True ]]; then
# Make sure that your public interface is not attached to any bridge.
PUBLIC_INTERFACE=
enable_plugin neutron-fwaas http://opendev.org/openstack/neutron-fwaas.git stable/train
enable_plugin neutron-lbaas https://opendev.org/openstack/neutron-lbaas.git stable/train
enable_plugin neutron https://opendev.org/openstack/neutron.git stable/train
enable_plugin neutron-vpnaas https://opendev.org/openstack/neutron-vpnaas.git stable/train
enable_plugin octavia https://opendev.org/openstack/octavia.git stable/train
enable_plugin neutron-fwaas http://opendev.org/openstack/neutron-fwaas.git stable/ussuri
enable_plugin neutron-lbaas https://opendev.org/openstack/neutron-lbaas.git stable/ussuri
enable_plugin neutron https://opendev.org/openstack/neutron.git stable/ussuri
enable_plugin neutron-vpnaas https://opendev.org/openstack/neutron-vpnaas.git stable/ussuri
enable_plugin octavia https://opendev.org/openstack/octavia.git stable/ussuri
fi
fi

7
devstack/settings
View File

@ -25,13 +25,13 @@ GIT_BASE=${GIT_BASE:-https://opendev.org}
GBPSERVICE_REPO=${GBPSERVICE_REPO:-${GIT_BASE}/x/group-based-policy.git}
GBPSERVICE_BRANCH=${GBPSERVICE_BRANCH:-master}
GBPCLIENT_REPO=${GBPCLIENT_REPO:-${GIT_BASE}/x/python-group-based-policy-client.git}
GBPCLIENT_BRANCH=${GBPCLIENT_BRANCH:-stable/train}
GBPCLIENT_BRANCH=${GBPCLIENT_BRANCH:-stable/ussuri}
GBPUI_REPO=${GBPUI_REPO:-${GIT_BASE}/x/group-based-policy-ui.git}
GBPUI_BRANCH=${GBPUI_BRANCH:-master}
GBPHEAT_REPO=${GBPHEAT_REPO:-${GIT_BASE}/x/group-based-policy-automation.git}
GBPHEAT_BRANCH=${GBPHEAT_BRANCH:-master}
AIM_BRANCH=${AIM_BRANCH:-master}
OPFLEX_BRANCH=${OPFLEX_BRANCH:-stable/train}
OPFLEX_BRANCH=${OPFLEX_BRANCH:-stable/ussuri}
APICAPI_BRANCH=${APICAPI_BRANCH:-master}
ACITOOLKIT_BRANCH=${ACITOOLKIT_BRANCH:-noiro-lite}
@ -47,13 +47,10 @@ else
fi
if [[ $ENABLE_APIC_AIM_GATE = True ]]; then
disable_service q-dhcp
disable_service q-lbaas
disable_service q-meta
else
enable_service q-dhcp
fi
enable_service q-fwaas-v1
enable_service q-lbaasv2
enable_service q-meta
enable_service neutron
enable_service group-policy

9
gbpservice/neutron/extensions/patch.py
View File

@ -17,6 +17,7 @@ from neutron.api.v2 import resource as neutron_resource
from neutron.db import l3_db
from neutron.db import models_v2
from neutron.db import securitygroups_db
from neutron.objects import securitygroup as sg_obj
from neutron.plugins.ml2 import db as ml2_db
from neutron.quota import resource as quota_resource
from neutron_lib.api import attributes
@ -100,10 +101,8 @@ def _get_security_groups_on_port(self, context, port):
return
port_sg = p.get(securitygroups_db.ext_sg.SECURITYGROUPS, [])
filters = {'id': port_sg}
valid_groups = set(g['id'] for g in
self.get_security_groups(context, fields=['id'],
filters=filters))
sg_objs = sg_obj.SecurityGroup.get_objects(context, id=port_sg)
valid_groups = set(g['id'] for g in sg_objs)
requested_groups = set(port_sg)
port_sg_missing = requested_groups - valid_groups
@ -111,7 +110,7 @@ def _get_security_groups_on_port(self, context, port):
raise securitygroups_db.ext_sg.SecurityGroupNotFound(
id=', '.join(port_sg_missing))
return requested_groups
return sg_objs
securitygroups_db.SecurityGroupDbMixin._get_security_groups_on_port = (

13
gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/data_migrations.py
View File

@ -127,7 +127,18 @@ def do_apic_aim_persist_migration(session):
scope_dbs = (session.query(as_db.AddressScope)
.options(lazyload('*')).all())
for scope_db in scope_dbs:
alembic_util.msg("Migrating address scope: %s" % scope_db)
# REVISIT: commit eb6104c0ac61216234ea958f2fd322e70b8e4bec
# in upstream neutron breaks the __repr__ method of the model
# class. We work around this for now by using the dict members.
# This should be removed once the model class is fixed upstream.
scope_dict = {}
for k, v in scope_db.__dict__.items():
if k == '_sa_instance_state':
continue
if k == 'shared_':
k = 'shared'
scope_dict[k] = v
alembic_util.msg("Migrating address scope: %s" % scope_dict)
vrf = None
ext_db = (session.query(DefunctAddressScopeExtensionDb).
filter_by(address_scope_id=scope_db.id).

12
gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py
View File

@ -3990,10 +3990,10 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
LOG.debug("Adding networks %s to topology",
[net.id for net in new_networks])
added_ids = []
for net in new_networks:
if net.id not in visited_networks:
visited_networks[net.id] = net
added_ids.append(net.id)
for new_net in new_networks:
if new_net.id not in visited_networks:
visited_networks[new_net.id] = new_net
added_ids.append(new_net.id)
if added_ids:
LOG.debug("Querying for routers interfaced to networks %s",
added_ids)
@ -6005,8 +6005,8 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
plugin_context, port_db.network_id)
host = (port_db.port_bindings[0].host if port_db.port_bindings
else None)
levels = (n_db.get_binding_levels(plugin_context, port_id, host)
if host else None)
levels = (n_db.get_binding_level_objs(plugin_context,
port_id, host) if host else None)
return ml2_context.PortContext(
self.plugin, plugin_context, port, network,
port_db.port_bindings[0] if port_db.port_bindings else None,

20
gbpservice/neutron/plugins/ml2plus/patch_neutron.py
View File

@ -64,3 +64,23 @@ from neutron.common import _constants
DEVICE_OWNER_SVI_PORT = 'apic:svi'
_constants.AUTO_DELETE_PORT_OWNERS.append(DEVICE_OWNER_SVI_PORT)
from neutron.db import db_base_plugin_v2 as db_v2
patched_create_network_db = db_v2.NeutronDbPluginV2.create_network_db
# REVISIT: this is a monkey patch of the upstream DB layer call.
# https://review.opendev.org/#/c/679399/ set the default value
# for the network's MTU to the constant defined in neutron-lib, if
# one wasn't specified. This modifies that behavior by explicitly
# setting it to 0, if not set already, avoiding this behavior.
def create_network_db(self, context, network):
n = network['network']
n.update({'mtu': n.get('mtu', 0)})
return patched_create_network_db(self, context, network)
db_v2.NeutronDbPluginV2.create_network_db = create_network_db

2
gbpservice/neutron/tests/unit/plugins/ml2plus/test_l3_apic_aim.py
View File

@ -58,7 +58,7 @@ class TestCiscoApicAimL3Plugin(test_aim_mapping_driver.AIMBaseTestCase):
super(TestCiscoApicAimL3Plugin, self).setUp()
get_sec_group_port_patch = mock.patch(
'neutron.db.securitygroups_db.SecurityGroupDbMixin.'
'_get_security_groups_on_port')
'_get_security_groups_on_port', return_value=set())
get_sec_group_port_patch.start()
# Set up L2 objects for L3 test

5
gbpservice/neutron/tests/unit/plugins/ml2plus/test_tracked_resources.py
View File

@ -18,6 +18,7 @@ from neutron.tests.unit.plugins.ml2 import (
test_tracked_resources as n_tracked)
from neutron.tests.unit.plugins.ml2 import test_plugin
from gbpservice.neutron.db import all_models # noqa
import gbpservice.neutron.extensions
PLUGIN_NAME = 'ml2plus'
@ -39,7 +40,7 @@ class Ml2PlusTestTrackedResourcesEventHandler(
super(Ml2PlusTestTrackedResourcesEventHandler, self).setUp()
get_sec_group_port_patch = mock.patch(
'neutron.db.securitygroups_db.SecurityGroupDbMixin.'
'_get_security_groups_on_port')
'_get_security_groups_on_port', return_value=set())
get_sec_group_port_patch.start()
def setup_parent(self):
@ -66,7 +67,7 @@ class Ml2PlusTestTrackedResources(n_tracked.TestTrackedResources):
super(Ml2PlusTestTrackedResources, self).setUp()
get_sec_group_port_patch = mock.patch(
'neutron.db.securitygroups_db.SecurityGroupDbMixin.'
'_get_security_groups_on_port')
'_get_security_groups_on_port', return_value=set())
get_sec_group_port_patch.start()
def setup_parent(self):

4
gbpservice/tests/contrib/devstack/local-aim.conf
View File

@ -13,11 +13,11 @@ SKIP_EXERCISES=volumes,trove,swift,sahara,euca,bundle,boot_from_volume,aggregate
enable_plugin group-based-policy https://opendev.org/x/group-based-policy.git master
enable_plugin networking-sfc https://opendev.org/openstack/networking-sfc.git stable/train
enable_plugin networking-sfc https://opendev.org/openstack/networking-sfc.git stable/ussuri
ENABLE_APIC_AIM_GATE=True
AIM_BRANCH=master
OPFLEX_BRANCH=stable/train
OPFLEX_BRANCH=stable/ussuri
APICAPI_BRANCH=master
ACITOOLKIT_BRANCH=noiro-lite

10
gbpservice/tests/contrib/devstack/local-nfp.conf
View File

@ -20,7 +20,7 @@ GBPSERVICE_BRANCH=master
#GBPSERVICE_BRANCH=refs/changes/85/298385/154
GBPCLIENT_REPO=${GIT_BASE}/x/python-group-based-policy-client.git
GBPCLIENT_BRANCH=stable/train
GBPCLIENT_BRANCH=stable/ussuri
GBPUI_REPO=${GIT_BASE}/x/group-based-policy-ui.git
GBPUI_BRANCH=master
@ -43,13 +43,13 @@ enable_service q-svc
enable_service q-agt
enable_service q-dhcp
enable_service q-l3
enable_plugin neutron-fwaas https://opendev.org/openstack/neutron-fwaas.git stable/train
enable_plugin neutron-lbaas https://opendev.org/openstack/neutron-lbaas.git stable/train
enable_plugin neutron https://opendev.org/openstack/neutron.git stable/train
enable_plugin neutron-fwaas https://opendev.org/openstack/neutron-fwaas.git stable/ussuri
enable_plugin neutron-lbaas https://opendev.org/openstack/neutron-lbaas.git stable/ussuri
enable_plugin neutron https://opendev.org/openstack/neutron.git stable/ussuri
#ENBALE OCTAVIA
enable_plugin octavia https://opendev.org/openstack/octavia stable/train
enable_plugin octavia https://opendev.org/openstack/octavia stable/ussuri
#ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
enable_service q-fwaas-v1

2
gbpservice/tests/contrib/devstack/local.conf
View File

@ -20,7 +20,7 @@ GBPSERVICE_BRANCH=master
#GBPSERVICE_BRANCH=refs/changes/54/240954/47
GBPCLIENT_REPO=${GIT_BASE}/x/python-group-based-policy-client.git
GBPCLIENT_BRANCH=stable/train
GBPCLIENT_BRANCH=stable/ussuri
GBPUI_REPO=${GIT_BASE}/x/group-based-policy-ui.git
GBPUI_BRANCH=master

16
gbpservice/tests/contrib/functions-gbp
View File

@ -16,7 +16,7 @@ XTRACE=$(set +o | grep xtrace)
function prepare_gbp_devstack_pre {
cd $TOP_DIR
sudo git checkout stable/train
sudo git checkout stable/ussuri
sudo sed -i 's/DEST=\/opt\/stack/DEST=\/opt\/stack\/new/g' $TOP_DIR/stackrc
sudo sed -i 's/source $TOP_DIR\/lib\/neutron/source $TOP_DIR\/lib\/neutron\nsource $TOP_DIR\/lib\/neutron-legacy/g' $TOP_DIR/stack.sh
}
@ -25,15 +25,15 @@ function prepare_gbp_devstack_post {
# The following should updated when master moves to a new release
# We need to do the following since the infra job clones these repos and
# checks out the master branch (as this is the master branch) and later
# does not switch to the stable/train branch when installing devstack
# does not switch to the stable/ussuri branch when installing devstack
# since the repo is already present.
# This can be worked around by changing the job description in
# project-config to set BRANCH_OVERRIDE to use the stable/train branch
sudo git --git-dir=/opt/stack/new/neutron/.git --work-tree=/opt/stack/new/neutron checkout stable/train
sudo git --git-dir=/opt/stack/new/nova/.git --work-tree=/opt/stack/new/nova checkout stable/train
sudo git --git-dir=/opt/stack/new/keystone/.git --work-tree=/opt/stack/new/keystone checkout stable/train
sudo git --git-dir=/opt/stack/new/cinder/.git --work-tree=/opt/stack/new/cinder checkout stable/train
sudo git --git-dir=/opt/stack/new/requirements/.git --work-tree=/opt/stack/new/requirements checkout stable/train
# project-config to set BRANCH_OVERRIDE to use the stable/ussuri branch
sudo git --git-dir=/opt/stack/new/neutron/.git --work-tree=/opt/stack/new/neutron checkout stable/ussuri
sudo git --git-dir=/opt/stack/new/nova/.git --work-tree=/opt/stack/new/nova checkout stable/ussuri
sudo git --git-dir=/opt/stack/new/keystone/.git --work-tree=/opt/stack/new/keystone checkout stable/ussuri
sudo git --git-dir=/opt/stack/new/cinder/.git --work-tree=/opt/stack/new/cinder checkout stable/ussuri
sudo git --git-dir=/opt/stack/new/requirements/.git --work-tree=/opt/stack/new/requirements checkout stable/ussuri
source $TOP_DIR/functions
source $TOP_DIR/functions-common

1
requirements.txt
View File

@ -1,4 +1,5 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr>=4.0.0 # Apache-2.0
pyOpenSSL>=16.2.0

7
setup.cfg
View File

@ -13,11 +13,9 @@ classifier =
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
[files]
packages =
@ -47,7 +45,6 @@ neutron.service_plugins =
group_policy = gbpservice.neutron.services.grouppolicy.plugin:GroupPolicyPlugin
ncp = gbpservice.neutron.services.servicechain.plugins.ncp.plugin:NodeCompositionPlugin
apic_gbp_l3 = gbpservice.neutron.services.l3_router.l3_apic:ApicGBPL3ServicePlugin
nfp_fwaas = gbpservice.contrib.nfp.service_plugins.firewall.nfp_fwaas_plugin:NFPFirewallPlugin
apic_aim_l3 = gbpservice.neutron.services.apic_aim.l3_plugin:ApicL3Plugin
gbpservice.neutron.group_policy.extension_drivers =
test = gbpservice.neutron.tests.unit.services.grouppolicy.drivers.extension_test:TestExtensionDriver

10
test-requirements.txt
View File

@ -4,21 +4,19 @@
hacking>=1.1.0,<1.2.0 # Apache-2.0
# Since version numbers for these are specified in
# https://releases.openstack.org/constraints/upper/train, they cannot be
# https://releases.openstack.org/constraints/upper/ussuri, they cannot be
# referenced as GIT URLs.
neutron
python-heatclient
python-keystoneclient
-e git+https://opendev.org/openstack/neutron-vpnaas.git@stable/train#egg=neutron-vpnaas
-e git+https://opendev.org/openstack/neutron-fwaas.git@stable/train#egg=neutron-fwaas
-e git+https://opendev.org/openstack/networking-sfc.git@stable/train#egg=networking-sfc
-e git+https://opendev.org/openstack/networking-sfc.git@stable/ussuri#egg=networking-sfc
-e git+https://github.com/noironetworks/apicapi.git@master#egg=apicapi
-e git+https://github.com/noironetworks/python-opflex-agent.git@stable/train#egg=python-opflexagent-agent
-e git+https://github.com/noironetworks/python-opflex-agent.git@stable/ussuri#egg=python-opflexagent-agent
-e git+https://opendev.org/x/python-group-based-policy-client.git@stable/train#egg=gbpclient
-e git+https://opendev.org/x/python-group-based-policy-client.git@stable/ussuri#egg=gbpclient
coverage!=4.4,>=4.0 # Apache-2.0
flake8-import-order==0.12 # LGPLv3

16
tox.ini
View File

@ -1,9 +1,11 @@
[tox]
envlist = py35,py36,py37,py27,pep8
minversion = 2.3.2
envlist = py36,py37,pep8
minversion = 3.2.0
skipsdist = True
ignore_basepython_conflict = True
[testenv]
basepython = python3
setenv = VIRTUAL_ENV={envdir}
OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true}
OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true}
@ -14,7 +16,7 @@ usedevelop = True
install_command =
pip install {opts} {packages}
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri}
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals = sh
@ -27,12 +29,12 @@ commands = stestr run {posargs}
setenv = VIRTUAL_ENV={envdir}
[testenv:functional]
basepython = python2.7
basepython = python3
setenv = OS_TEST_PATH=./gbpservice/tests/functional
OS_SUDO_TESTING=1
[testenv:dsvm-functional]
basepython = python2.7
basepython = python3
setenv = OS_TEST_PATH=./gbpservice/tests/functional
OS_SUDO_TESTING=1
OS_ROOTWRAP_CMD=sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
@ -43,7 +45,7 @@ sitepackages=True
sitepackages = True
[testenv:pep8]
basepython = python2.7
basepython = python3
commands =
flake8
gbp-db-manage check_migration
@ -53,7 +55,7 @@ commands =
python ./tools/check_i18n.py ./neutron ./tools/i18n_cfg.py
[testenv:cover]
basepython = python2.7
basepython = python3
commands =
coverage erase
coverage run -m testtools.run