openstack/neutron
Code Issues Proposed changes

Pecan routing for agent schedulers

Browse Source 
For pecan to support existing agent scheduler controllers, a couple of shim
pecan controllers need to be added.  These shim controllers will be used if
there are extensions that have legacy controllers that have not been
registered.  The shim controller is just a passthrough to those legacy
controllers.  This may have the added benefit of support existing out of
tree extensions that have defined their legacy extension controllers the
same way.

Changes to how the router(s) controllers determines whether something
is a member action has been changed a bit to support this.

Closes-Bug: #1552978
Co-Authored-By: Kevin Benton <kevin@benton.pub>
Change-Id: Icec56676d83b604c3db3377838076d6429d61e48
This commit is contained in:
Brandon Logan
2016-01-15 01:29:22 -06:00
committed by Kevin Benton
parent 113e59b12e
commit 6659428669
11 changed files with 385 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
neutron/api/v2/resource.py
View File

@@ -148,6 +148,11 @@ def Resource(controller, faults=None, deserializers=None, serializers=None,
return webob.Response(request=request, status=status,
content_type=content_type,
body=body)
# NOTE(blogan): this is something that is needed for the transition to
# pecan. This will allow the pecan code to have a handle on the controller
# for an extension so it can reuse the code instead of forcing every
# extension to rewrite the code for use with pecan.
setattr(resource, 'controller', controller)
return resource

19
neutron/pecan_wsgi/controllers/resource.py
View File

@@ -12,13 +12,19 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
import pecan
from pecan import request
from neutron.api import api_common
from neutron.i18n import _LW
from neutron import manager
from neutron.pecan_wsgi.controllers import utils
LOG = logging.getLogger(__name__)
class ItemController(utils.NeutronPecanController):
def __init__(self, resource, item):
@@ -59,6 +65,17 @@ class ItemController(utils.NeutronPecanController):
deleter = getattr(self.plugin, 'delete_%s' % self.resource)
return deleter(neutron_context, self.item)
@utils.expose()
def _lookup(self, collection, *remainder):
request.context['collection'] = collection
controller = manager.NeutronManager.get_controller_for_resource(
collection)
if not controller:
LOG.warn(_LW("No controller found for: %s - returning response "
"code 404"), collection)
pecan.abort(404)
return controller, remainder
class CollectionsController(utils.NeutronPecanController):
@@ -68,6 +85,8 @@ class CollectionsController(utils.NeutronPecanController):
def _lookup(self, item, *remainder):
# Store resource identifier in request context
request.context['resource_id'] = item
uri_identifier = '%s_id' % self.resource
request.context['uri_identifiers'][uri_identifier] = item
return self.item_controller_class(self.resource, item), remainder
@utils.expose(generic=True)

7
neutron/pecan_wsgi/controllers/root.py
View File

@@ -102,6 +102,13 @@ class V2Controller(object):
# properly sanitized (eg: replacing dashes with underscores)
request.context['resource'] = controller.resource
request.context['collection'] = controller.collection
# NOTE(blogan): initialize a dict to store the ids of the items walked
# in the path for example: /networks/1234 would cause uri_identifiers
# to contain: {'network_id': '1234'}
# This is for backwards compatibility with legacy extensions that
# defined their own controllers and expected kwargs to be passed in
# with the uri_identifiers
request.context['uri_identifiers'] = {}
return controller, remainder

81
neutron/pecan_wsgi/controllers/router.py
View File

@@ -18,6 +18,7 @@ from oslo_log import log
import pecan
from pecan import request
from neutron import manager
from neutron.pecan_wsgi.controllers import resource
from neutron.pecan_wsgi.controllers import utils
@@ -41,38 +42,23 @@ class RouterController(resource.ItemController):
@utils.when(index, method='PUT')
def put(self, *args, **kwargs):
neutron_context = request.context['neutron_context']
if args:
# There is a member action to process
member_action = args[0]
LOG.debug("Processing member action %(action)s for resource "
"%(resource)s identified by %(item)s",
{'action': member_action,
'resource': self.resource,
'item': self.item})
# NOTE(salv-orlando): The following simply verify that the plugin
# has a method for a given action. It therefore enables plugins to
# implement actions which are not part of the API specification.
# Unfortunately the API extension descriptor does not do a good job
# of sanctioning which actions are available on a given resource.
# TODO(salv-orlando): prevent plugins from implementing actions
# which are not part of the Neutron API spec
try:
member_action_method = getattr(self.plugin, member_action)
return member_action_method(neutron_context, self.item,
request.context['request_data'])
except AttributeError:
LOG.error(_LE("Action %(action)s is not defined on resource "
"%(resource)s"),
{'action': member_action, 'resource': self.resource})
pecan.abort(404)
# Do standard PUT processing
return super(RouterController, self).put(*args, **kwargs)
@utils.when(index, method='DELETE')
def delete(self):
return super(RouterController, self).delete()
@utils.expose()
def _lookup(self, member_action, *remainder):
# This check is mainly for the l3-agents resource. If there isn't
# a controller for it then we'll just assume its a member action.
controller = manager.NeutronManager.get_controller_for_resource(
member_action)
if not controller:
controller = RouterMemberActionController(
self.resource, self.item, member_action)
return controller, remainder
class RoutersController(resource.CollectionsController):
@@ -80,3 +66,46 @@ class RoutersController(resource.CollectionsController):
def __init__(self):
super(RoutersController, self).__init__('routers', 'router')
class RouterMemberActionController(resource.ItemController):
def __init__(self, resource, item, member_action):
super(RouterMemberActionController, self).__init__(resource, item)
self.member_action = member_action
@utils.expose(generic=True)
def index(self, *args, **kwargs):
pecan.abort(405)
@utils.when(index, method='HEAD')
@utils.when(index, method='POST')
@utils.when(index, method='PATCH')
def not_supported(self):
return super(RouterMemberActionController, self).not_supported()
@utils.when(index, method='PUT')
def put(self, *args, **kwargs):
neutron_context = request.context['neutron_context']
LOG.debug("Processing member action %(action)s for resource "
"%(resource)s identified by %(item)s",
{'action': self.member_action,
'resource': self.resource,
'item': self.item})
# NOTE(salv-orlando): The following simply verify that the plugin
# has a method for a given action. It therefore enables plugins to
# implement actions which are not part of the API specification.
# Unfortunately the API extension descriptor does not do a good job
# of sanctioning which actions are available on a given resource.
# TODO(salv-orlando): prevent plugins from implementing actions
# which are not part of the Neutron API spec
try:
member_action_method = getattr(self.plugin, self.member_action)
return member_action_method(neutron_context, self.item,
request.context['request_data'])
except AttributeError:
LOG.error(_LE("Action %(action)s is not defined on resource "
"%(resource)s"),
{'action': self.member_action,
'resource': self.resource})
pecan.abort(404)

61
neutron/pecan_wsgi/controllers/utils.py
View File

@@ -14,6 +14,7 @@
# under the License.
import pecan
from pecan import request
from neutron.api.v2 import attributes as api_attributes
from neutron import manager
@@ -50,3 +51,63 @@ class NeutronPecanController(object):
self._plugin = manager.NeutronManager.get_plugin_for_resource(
self.resource)
return self._plugin
class ShimRequest(object):
def __init__(self, context):
self.context = context
class ShimItemController(NeutronPecanController):
def __init__(self, collection, resource, item, controller):
super(ShimItemController, self).__init__(collection, resource)
self.item = item
self.controller_delete = getattr(controller, 'delete', None)
@expose(generic=True)
def index(self):
pecan.abort(405)
@when(index, method='DELETE')
def delete(self):
if not self.controller_delete:
pecan.abort(405)
shim_request = ShimRequest(request.context['neutron_context'])
uri_identifiers = request.context['uri_identifiers']
return self.controller_delete(shim_request, self.item,
**uri_identifiers)
class ShimCollectionsController(NeutronPecanController):
def __init__(self, collection, resource, controller):
super(ShimCollectionsController, self).__init__(collection, resource)
self.controller = controller
self.controller_index = getattr(controller, 'index', None)
self.controller_create = getattr(controller, 'create', None)
@expose(generic=True)
def index(self):
if not self.controller_index:
pecan.abort(405)
shim_request = ShimRequest(request.context['neutron_context'])
uri_identifiers = request.context['uri_identifiers']
return self.controller_index(shim_request, **uri_identifiers)
@when(index, method='POST')
def create(self):
if not self.controller_create:
pecan.abort(405)
shim_request = ShimRequest(request.context['neutron_context'])
uri_identifiers = request.context['uri_identifiers']
return self.controller_create(shim_request, request.json,
**uri_identifiers)
@expose()
def _lookup(self, item, *remainder):
request.context['resource'] = self.resource
request.context['resource_id'] = item
return ShimItemController(self.collection, self.resource, item,
self.controller), remainder

2
neutron/pecan_wsgi/hooks/body_validation.py
View File

@@ -48,7 +48,7 @@ class BodyValidationHook(hooks.PecanHook):
state.request.context['request_data'] = json_data
if not (resource in json_data or collection in json_data):
# there is no resource in the request. This can happen when a
# member action is being processed.
# member action is being processed or on agent scheduler operations
return
# Prepare data to be passed to the plugin from request body
data = v2_base.Controller.prepare_request_body(

24
neutron/pecan_wsgi/hooks/policy_enforcement.py
View File

@@ -41,6 +41,12 @@ class PolicyHook(hooks.PecanHook):
def _fetch_resource(self, neutron_context, resource, resource_id):
attrs = v2_attributes.get_resource_info(resource)
if not attrs:
# this isn't a request for a normal resource. it could be
# an action like removing a network from a dhcp agent.
# return None and assume the custom controller for this will
# handle the necessary logic.
return
field_list = [name for (name, value) in attrs.items()
if (value.get('required_by_policy') or
value.get('primary_key') or 'default' not in value)]
@@ -89,14 +95,16 @@ class PolicyHook(hooks.PecanHook):
# Ops... this was a delete after all!
item = {}
resource_id = state.request.context.get('resource_id')
obj = copy.copy(self._fetch_resource(neutron_context,
resource,
resource_id))
obj.update(item)
merged_resources.append(obj.copy())
obj[const.ATTRIBUTES_TO_UPDATE] = item.keys()
# Put back the item in the list so that policies could be enforced
resources_copy.append(obj)
resource_obj = self._fetch_resource(neutron_context,
resource, resource_id)
if resource_obj:
obj = copy.copy(resource_obj)
obj.update(item)
merged_resources.append(obj.copy())
obj[const.ATTRIBUTES_TO_UPDATE] = item.keys()
# Put back the item in the list so that policies could be
# enforced
resources_copy.append(obj)
# TODO(salv-orlando): as other hooks might need to prefetch resources,
# store them in the request context. However, this should be done in a
# separate hook which is conventietly called before all other hooks

4
neutron/pecan_wsgi/hooks/quota_enforcement.py
View File

@@ -32,10 +32,10 @@ class QuotaEnforcementHook(hooks.PecanHook):
def before(self, state):
resource = state.request.context.get('resource')
if state.request.method != 'POST' or not resource:
items = state.request.context.get('resources')
if state.request.method != 'POST' or not resource or not items:
return
plugin = manager.NeutronManager.get_plugin_for_resource(resource)
items = state.request.context.get('resources')
# Store requested resource amounts grouping them by tenant
deltas = collections.Counter(map(lambda x: x['tenant_id'], items))
# Perform quota enforcement

38
neutron/pecan_wsgi/startup.py
View File

@@ -21,6 +21,7 @@ from neutron.api.v2 import attributes
from neutron.api.v2 import router
from neutron import manager
from neutron.pecan_wsgi.controllers import resource as res_ctrl
from neutron.pecan_wsgi.controllers import utils
from neutron import policy
from neutron.quota import resource_registry
@@ -65,6 +66,40 @@ def _handle_plurals(collection):
return resource
def initialize_legacy_extensions(legacy_extensions):
leftovers = []
for ext in legacy_extensions:
ext_resources = ext.get_resources()
for ext_resource in ext_resources:
controller = ext_resource.controller.controller
collection = ext_resource.collection
resource = _handle_plurals(collection)
if manager.NeutronManager.get_plugin_for_resource(resource):
continue
# NOTE(blogan): It is possible that a plugin is tied to the
# collection rather than the resource. An example of this is
# the auto_allocated_topology extension. All other extensions
# created their legacy resources with the collection/plural form
# except auto_allocated_topology. Making that extension
# conform with the rest of extensions could invalidate this, but
# it's possible out of tree extensions did the same thing. Since
# the auto_allocated_topology resources have already been loaded
# we definitely don't want to load them up with shim controllers,
# so this will prevent that.
if manager.NeutronManager.get_plugin_for_resource(collection):
continue
# NOTE(blogan): Since this does not have a plugin, we know this
# extension has not been loaded and controllers for its resources
# have not been created nor set.
leftovers.append((collection, resource, controller))
# NOTE(blogan): at this point we have leftover extensions that never
# had a controller set which will force us to use shim controllers.
for leftover in leftovers:
shim_controller = utils.ShimCollectionsController(*leftover)
manager.NeutronManager.set_controller_for_resource(
shim_controller.collection, shim_controller)
def initialize_all():
ext_mgr = extensions.PluginAwareExtensionManager.get_instance()
ext_mgr.extend_resources("2.0", attributes.RESOURCE_ATTRIBUTE_MAP)
@@ -73,6 +108,7 @@ def initialize_all():
# and extensions)
pecanized_exts = [ext for ext in ext_mgr.extensions.values() if
hasattr(ext, 'get_pecan_controllers')]
non_pecanized_exts = set(ext_mgr.extensions.values()) - set(pecanized_exts)
pecan_controllers = {}
for ext in pecanized_exts:
LOG.info(_LI("Extension %s is pecan-aware. Fetching resources "
@@ -109,6 +145,8 @@ def initialize_all():
{'resource': resource,
'collection': collection})
initialize_legacy_extensions(non_pecanized_exts)
# NOTE(salv-orlando): If you are care about code quality, please read below
# Hackiness is strong with the piece of code below. It is used for
# populating resource plurals and registering resources with the quota

169
neutron/tests/functional/pecan_wsgi/test_controllers.py
View File

@@ -14,17 +14,22 @@ from collections import namedtuple
import mock
from oslo_config import cfg
from oslo_policy import policy as oslo_policy
from oslo_serialization import jsonutils
import pecan
from pecan import request
from neutron.api import extensions
from neutron.api.v2 import attributes
from neutron.common import constants as n_const
from neutron import context
from neutron import manager
from neutron.pecan_wsgi.controllers import root as controllers
from neutron.plugins.common import constants
from neutron import policy
from neutron.tests.common import helpers
from neutron.tests.functional.pecan_wsgi import test_functional
from neutron.tests.functional.pecan_wsgi import utils as pecan_utils
_SERVICE_PLUGIN_RESOURCE = 'serviceplugin'
_SERVICE_PLUGIN_COLLECTION = _SERVICE_PLUGIN_RESOURCE + 's'
@@ -50,6 +55,8 @@ class TestRootController(test_functional.PecanFunctionalTest):
def setUp(self):
super(TestRootController, self).setUp()
self.setup_service_plugin()
self.plugin = manager.NeutronManager.get_plugin()
self.ctx = context.get_admin_context()
def setup_service_plugin(self):
manager.NeutronManager.set_controller_for_resource(
@@ -228,12 +235,11 @@ class TestResourceController(TestRootController):
self._gen_port()
def _gen_port(self):
pl = manager.NeutronManager.get_plugin()
network_id = pl.create_network(context.get_admin_context(), {
network_id = self.plugin.create_network(context.get_admin_context(), {
'network':
{'name': 'pecannet', 'tenant_id': 'tenid', 'shared': False,
'admin_state_up': True, 'status': 'ACTIVE'}})['id']
self.port = pl.create_port(context.get_admin_context(), {
self.port = self.plugin.create_port(context.get_admin_context(), {
'port':
{'tenant_id': 'tenid', 'network_id': network_id,
'fixed_ips': attributes.ATTR_NOT_SPECIFIED,
@@ -403,43 +409,14 @@ class TestRouterController(TestResourceController):
cfg.CONF.set_override(
'service_plugins',
['neutron.services.l3_router.l3_router_plugin.L3RouterPlugin'])
super(TestRouterController, self).setUp()
# Create a network, a subnet, and a router
pl = manager.NeutronManager.get_plugin()
plugin = manager.NeutronManager.get_plugin()
ctx = context.get_admin_context()
service_plugins = manager.NeutronManager.get_service_plugins()
l3_plugin = service_plugins[constants.L3_ROUTER_NAT]
ctx = context.get_admin_context()
network_id = pl.create_network(
ctx,
{'network':
{'name': 'pecannet',
'tenant_id': 'tenid',
'shared': False,
'admin_state_up': True,
'status': 'ACTIVE'}})['id']
self.subnet = pl.create_subnet(
ctx,
{'subnet':
{'tenant_id': 'tenid',
'network_id': network_id,
'name': 'pecansub',
'ip_version': 4,
'cidr': '10.20.30.0/24',
'gateway_ip': '10.20.30.1',
'enable_dhcp': True,
'allocation_pools': [
{'start': '10.20.30.2',
'end': '10.20.30.254'}],
'dns_nameservers': [],
'host_routes': []}})
self.router = l3_plugin.create_router(
ctx,
{'router':
{'name': 'pecanrtr',
'tenant_id': 'tenid',
'admin_state_up': True}})
network_id = pecan_utils.create_network(ctx, plugin)['id']
self.subnet = pecan_utils.create_subnet(ctx, plugin, network_id)
self.router = pecan_utils.create_router(ctx, l3_plugin)
def test_member_actions_processing(self):
response = self.app.put_json(
@@ -455,3 +432,121 @@ class TestRouterController(TestResourceController):
headers={'X-Project-Id': 'tenid'},
expect_errors=True)
self.assertEqual(404, response.status_int)
def test_unsupported_method_member_action(self):
response = self.app.post_json(
'/v2.0/routers/%s/add_router_interface.json' % self.router['id'],
params={'subnet_id': self.subnet['id']},
headers={'X-Project-Id': 'tenid'},
expect_errors=True)
self.assertEqual(405, response.status_int)
response = self.app.get(
'/v2.0/routers/%s/add_router_interface.json' % self.router['id'],
headers={'X-Project-Id': 'tenid'},
expect_errors=True)
self.assertEqual(405, response.status_int)
class TestDHCPAgentShimControllers(test_functional.PecanFunctionalTest):
def setUp(self):
super(TestDHCPAgentShimControllers, self).setUp()
policy.init()
policy._ENFORCER.set_rules(
oslo_policy.Rules.from_dict(
{'get_dhcp-agents': 'role:admin',
'get_dhcp-networks': 'role:admin',
'create_dhcp-networks': 'role:admin',
'delete_dhcp-networks': 'role:admin'}),
overwrite=False)
plugin = manager.NeutronManager.get_plugin()
ctx = context.get_admin_context()
self.network = pecan_utils.create_network(ctx, plugin)
self.agent = helpers.register_dhcp_agent()
# NOTE(blogan): Not sending notifications because this test is for
# testing the shim controllers
plugin.agent_notifiers[n_const.AGENT_TYPE_DHCP] = None
def test_list_dhcp_agents_hosting_network(self):
response = self.app.get(
'/v2.0/networks/%s/dhcp-agents.json' % self.network['id'],
headers={'X-Roles': 'admin'})
self.assertEqual(200, response.status_int)
def test_list_networks_on_dhcp_agent(self):
response = self.app.get(
'/v2.0/agents/%s/dhcp-networks.json' % self.agent.id,
headers={'X-Project-Id': 'tenid', 'X-Roles': 'admin'})
self.assertEqual(200, response.status_int)
def test_add_remove_dhcp_agent(self):
headers = {'X-Project-Id': 'tenid', 'X-Roles': 'admin'}
self.app.post_json(
'/v2.0/agents/%s/dhcp-networks.json' % self.agent.id,
headers=headers, params={'network_id': self.network['id']})
response = self.app.get(
'/v2.0/networks/%s/dhcp-agents.json' % self.network['id'],
headers=headers)
self.assertIn(self.agent.id,
[a['id'] for a in response.json['agents']])
self.app.delete('/v2.0/agents/%(a)s/dhcp-networks/%(n)s.json' % {
'a': self.agent.id, 'n': self.network['id']}, headers=headers)
response = self.app.get(
'/v2.0/networks/%s/dhcp-agents.json' % self.network['id'],
headers=headers)
self.assertNotIn(self.agent.id,
[a['id'] for a in response.json['agents']])
class TestL3AgentShimControllers(test_functional.PecanFunctionalTest):
def setUp(self):
cfg.CONF.set_override(
'service_plugins',
['neutron.services.l3_router.l3_router_plugin.L3RouterPlugin'])
super(TestL3AgentShimControllers, self).setUp()
policy.init()
policy._ENFORCER.set_rules(
oslo_policy.Rules.from_dict(
{'get_l3-agents': 'role:admin',
'get_l3-routers': 'role:admin'}),
overwrite=False)
ctx = context.get_admin_context()
service_plugins = manager.NeutronManager.get_service_plugins()
l3_plugin = service_plugins[constants.L3_ROUTER_NAT]
self.router = pecan_utils.create_router(ctx, l3_plugin)
self.agent = helpers.register_l3_agent()
# NOTE(blogan): Not sending notifications because this test is for
# testing the shim controllers
l3_plugin.agent_notifiers[n_const.AGENT_TYPE_L3] = None
def test_list_l3_agents_hosting_router(self):
response = self.app.get(
'/v2.0/routers/%s/l3-agents.json' % self.router['id'],
headers={'X-Roles': 'admin'})
self.assertEqual(200, response.status_int)
def test_list_routers_on_l3_agent(self):
response = self.app.get(
'/v2.0/agents/%s/l3-routers.json' % self.agent.id,
headers={'X-Roles': 'admin'})
self.assertEqual(200, response.status_int)
def test_add_remove_l3_agent(self):
headers = {'X-Project-Id': 'tenid', 'X-Roles': 'admin'}
self.app.post_json(
'/v2.0/agents/%s/l3-routers.json' % self.agent.id,
headers=headers, params={'router_id': self.router['id']})
response = self.app.get(
'/v2.0/routers/%s/l3-agents.json' % self.router['id'],
headers=headers)
self.assertIn(self.agent.id,
[a['id'] for a in response.json['agents']])
self.app.delete('/v2.0/agents/%(a)s/l3-routers/%(n)s.json' % {
'a': self.agent.id, 'n': self.router['id']}, headers=headers)
response = self.app.get(
'/v2.0/routers/%s/l3-agents.json' % self.router['id'],
headers=headers)
self.assertNotIn(self.agent.id,
[a['id'] for a in response.json['agents']])

49
neutron/tests/functional/pecan_wsgi/utils.py Normal file
View File

@@ -0,0 +1,49 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def create_network(context, plugin):
return plugin.create_network(
context,
{'network':
{'name': 'pecannet',
'tenant_id': 'tenid',
'shared': False,
'admin_state_up': True,
'status': 'ACTIVE'}})
def create_subnet(context, plugin, network_id):
return plugin.create_subnet(
context,
{'subnet':
{'tenant_id': 'tenid',
'network_id': network_id,
'name': 'pecansub',
'ip_version': 4,
'cidr': '10.20.30.0/24',
'gateway_ip': '10.20.30.1',
'enable_dhcp': True,
'allocation_pools': [
{'start': '10.20.30.2',
'end': '10.20.30.254'}],
'dns_nameservers': [],
'host_routes': []}})
def create_router(context, l3_plugin):
return l3_plugin.create_router(
context,
{'router':
{'name': 'pecanrtr',
'tenant_id': 'tenid',
'admin_state_up': True}})