Merge "Add common way to extend standard attribute models"

This commit is contained in:
Jenkins 2016-09-16 16:35:06 +00:00 committed by Gerrit Code Review
commit 4c4a55ce6b
22 changed files with 207 additions and 106 deletions

View File

@ -38,3 +38,41 @@ by studying an existing API extension and explaining the different layers.
:maxdepth: 1
security_group_api
Extensions for Resources with standard attributes
-------------------------------------------------
Resources that inherit from the HasStandardAttributes DB class can
automatically have the extensions written for standard attributes
(e.g. timestamps, revision number, etc) extend their resources
by defining the 'api_collections' on their model. These are used
by extensions for standard attr resources to generate the extended
resources map.
Any new addition of a resource to the standard attributes collection
must be accompanied with a new extension to ensure that it is discoverable
via the API. If it's a completely new resource, the extension describing
that resource will suffice. If it's an existing resource that was released
in a previous cycle having the standard attributes added for the first time,
then a dummy extension needs to be added indicating that the resource
now has standard attributes. This ensures that an API caller can always
discover if an attribute will be available.
For example, if Flavors were migrated to include standard attributes, we
we need a new 'flavor-standardattr' extension. Then as an API caller, I will
know that flavors will have timestamps by checking for 'flavor-standardattr'
and 'timestamps'.
Current API resources extended by standard attr extensions:
- subnets: neutron.db.models_v2.Subnet
- trunks: neutron.services.trunk.models.Trunk
- routers: neutron.db.l3_db.Router
- segments: neutron.db.segments_db.NetworkSegment
- security_group_rules: neutron.db.models.securitygroup.SecurityGroupRule
- networks: neutron.db.models_v2.Network
- policies: neutron.db.qos.models.QosPolicy
- subnetpools: neutron.db.models_v2.SubnetPool
- ports: neutron.db.models_v2.Port
- security_groups: neutron.db.models.securitygroup.SecurityGroup
- floatingips: neutron.db.l3_db.FloatingIP

View File

@ -81,6 +81,12 @@ column to the model with a foreign key relationship to the 'standardattribute'
table. The model will then be able to access any columns of the
'standardattribute' table and any tables related to it.
A model that inherits HasStandardAttributes must implement the property
'api_collections', which is a list of API resources that the new object
may appear under. In most cases, this will only be one (e.g. 'ports' for
the Port model). This is used by all of the service plugins that add standard
attribute fields to determine which API responses need to be populated.
The introduction of a new standard attribute only requires one column addition
to the 'standardattribute' table for one-to-one relationships or a new table
for one-to-many or one-to-zero relationships. Then all of the models using the

View File

@ -109,6 +109,7 @@ class Router(standard_attr.HasStandardAttributes, model_base.BASEV2,
l3_agents = orm.relationship(
'Agent', lazy='joined', viewonly=True,
secondary=l3_agt.RouterL3AgentBinding.__table__)
api_collections = [l3.ROUTERS]
class FloatingIP(standard_attr.HasStandardAttributes, model_base.BASEV2,
@ -148,6 +149,7 @@ class FloatingIP(standard_attr.HasStandardAttributes, model_base.BASEV2,
name=('uniq_floatingips0floatingnetworkid'
'0fixedportid0fixedipaddress')),
model_base.BASEV2.__table_args__,)
api_collections = [l3.FLOATINGIPS]
class L3_NAT_dbonly_mixin(l3.RouterPluginBase,

View File

@ -19,6 +19,7 @@ from sqlalchemy import orm
from neutron.api.v2 import attributes
from neutron.db import models_v2
from neutron.db import standard_attr
from neutron.extensions import securitygroup as sg
class SecurityGroup(standard_attr.HasStandardAttributes, model_base.BASEV2,
@ -26,6 +27,7 @@ class SecurityGroup(standard_attr.HasStandardAttributes, model_base.BASEV2,
"""Represents a v2 neutron security group."""
name = sa.Column(sa.String(attributes.NAME_MAX_LEN))
api_collections = [sg.SECURITYGROUPS]
class DefaultSecurityGroup(model_base.BASEV2, model_base.HasProjectPrimaryKey):
@ -90,3 +92,4 @@ class SecurityGroupRule(standard_attr.HasStandardAttributes, model_base.BASEV2,
SecurityGroup,
backref=orm.backref('source_rules', cascade='all,delete'),
primaryjoin="SecurityGroup.id==SecurityGroupRule.remote_group_id")
api_collections = [sg.SECURITYGROUPRULES]

View File

@ -108,6 +108,7 @@ class Port(standard_attr.HasStandardAttributes, model_base.BASEV2,
name='uniq_ports0network_id0mac_address'),
model_base.BASEV2.__table_args__
)
api_collections = [attr.PORTS]
def __init__(self, id=None, tenant_id=None, name=None, network_id=None,
mac_address=None, admin_state_up=None, status=None,
@ -194,6 +195,7 @@ class Subnet(standard_attr.HasStandardAttributes, model_base.BASEV2,
rbac_db_models.NetworkRBAC, lazy='joined', uselist=True,
foreign_keys='Subnet.network_id',
primaryjoin='Subnet.network_id==NetworkRBAC.object_id')
api_collections = [attr.SUBNETS]
class SubnetPoolPrefix(model_base.BASEV2):
@ -230,6 +232,7 @@ class SubnetPool(standard_attr.HasStandardAttributes, model_base.BASEV2,
backref='subnetpools',
cascade='all, delete, delete-orphan',
lazy='joined')
api_collections = [attr.SUBNETPOOLS]
class Network(standard_attr.HasStandardAttributes, model_base.BASEV2,
@ -251,6 +254,7 @@ class Network(standard_attr.HasStandardAttributes, model_base.BASEV2,
dhcp_agents = orm.relationship(
'Agent', lazy='joined', viewonly=True,
secondary=ndab_model.NetworkDhcpAgentBinding.__table__)
api_collections = [attr.NETWORKS]
_deprecate._MovedGlobals()

View File

@ -30,6 +30,7 @@ class QosPolicy(standard_attr.HasStandardAttributes, model_base.BASEV2,
rbac_entries = sa.orm.relationship(rbac_db_models.QosPolicyRBAC,
backref='qos_policy', lazy='joined',
cascade='all, delete, delete-orphan')
api_collections = ['policies']
class QosNetworkPolicyBinding(model_base.BASEV2):

View File

@ -22,6 +22,7 @@ from neutron.callbacks import events
from neutron.callbacks import registry
from neutron.callbacks import resources
from neutron.db import standard_attr
from neutron.extensions import segment
LOG = logging.getLogger(__name__)
@ -53,6 +54,7 @@ class NetworkSegment(standard_attr.HasStandardAttributes,
segment_index = sa.Column(sa.Integer, nullable=False, server_default='0')
name = sa.Column(sa.String(attributes.NAME_MAX_LEN),
nullable=True)
api_collections = [segment.SEGMENTS]
NETWORK_TYPE = NetworkSegment.network_type.name

View File

@ -18,6 +18,7 @@ import sqlalchemy as sa
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext import declarative
from neutron._i18n import _LE
from neutron.api.v2 import attributes as attr
from neutron.db import sqlalchemytypes
@ -68,6 +69,26 @@ class StandardAttribute(model_base.BASEV2):
class HasStandardAttributes(object):
@classmethod
def get_api_collections(cls):
"""Define the API collection this object will appear under.
This should return a list of API collections that the object
will be exposed under. Most should be exposed in just one
collection (e.g. the network model is just exposed under
'networks').
This is used by the standard attr extensions to discover which
resources need to be extended with the standard attr fields
(e.g. created_at/updated_at/etc).
"""
# NOTE(kevinbenton): can't use abc because the metaclass conflicts
# with the declarative base others inherit from.
if hasattr(cls, 'api_collections'):
return cls.api_collections
raise NotImplementedError("%s must define api_collections" % cls)
@declarative.declared_attr
def standard_attr_id(cls):
return sa.Column(
@ -132,3 +153,17 @@ class HasStandardAttributes(object):
# this is a brand new object uncommited so we don't bump now
return
self.standard_attr.revision_number += 1
def get_standard_attr_resource_model_map():
rs_map = {}
for subclass in HasStandardAttributes.__subclasses__():
for resource in subclass.get_api_collections():
if resource in rs_map:
raise RuntimeError(_LE("Model %(sub)s tried to register for "
"API resource %(res)s which conflicts "
"with model %(other)s.") %
dict(sub=subclass, other=rs_map[resource],
res=resource))
rs_map[resource] = subclass
return rs_map

View File

@ -12,10 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from neutron.api.v2 import attributes
from neutron.db import common_db_mixin
from neutron.extensions import l3
from neutron.extensions import securitygroup
from neutron.db import standard_attr
class StandardAttrDescriptionMixin(object):
@ -26,10 +24,9 @@ class StandardAttrDescriptionMixin(object):
return
res['description'] = db_object.description
for resource in [attributes.NETWORKS, attributes.PORTS,
attributes.SUBNETS, attributes.SUBNETPOOLS,
securitygroup.SECURITYGROUPS,
securitygroup.SECURITYGROUPRULES,
l3.ROUTERS, l3.FLOATINGIPS]:
common_db_mixin.CommonDbMixin.register_dict_extend_funcs(
resource, ['_extend_standard_attr_description'])
def __new__(cls, *args, **kwargs):
for resource in standard_attr.get_standard_attr_resource_model_map():
common_db_mixin.CommonDbMixin.register_dict_extend_funcs(
resource, ['_extend_standard_attr_description'])
return super(StandardAttrDescriptionMixin, cls).__new__(cls, *args,
**kwargs)

View File

@ -12,6 +12,7 @@
# under the License.
from neutron.api import extensions
from neutron.db import standard_attr
REVISION = 'revision_number'
@ -19,11 +20,6 @@ REVISION_BODY = {
REVISION: {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None},
}
RESOURCES = ('security_group_rules', 'security_groups', 'ports', 'subnets',
'networks', 'routers', 'floatingips', 'subnetpools')
EXTENDED_ATTRIBUTES_2_0 = {}
for resource in RESOURCES:
EXTENDED_ATTRIBUTES_2_0[resource] = REVISION_BODY
class Revisions(extensions.ExtensionDescriptor):
@ -35,7 +31,7 @@ class Revisions(extensions.ExtensionDescriptor):
@classmethod
def get_alias(cls):
return "revisions"
return "standard-attr-revisions"
@classmethod
def get_description(cls):
@ -47,7 +43,7 @@ class Revisions(extensions.ExtensionDescriptor):
return "2016-04-11T10:00:00-00:00"
def get_extended_resources(self, version):
if version == "2.0":
return EXTENDED_ATTRIBUTES_2_0
else:
if version != "2.0":
return {}
rs_map = standard_attr.get_standard_attr_resource_model_map()
return {resource: REVISION_BODY for resource in rs_map}

View File

@ -15,17 +15,14 @@
from neutron.api import extensions
from neutron.api.v2 import attributes as attr
from neutron.db import standard_attr
EXTENDED_ATTRIBUTES_2_0 = {}
for resource in ('security_group_rules', 'security_groups', 'ports', 'subnets',
'networks', 'routers', 'floatingips', 'subnetpools'):
EXTENDED_ATTRIBUTES_2_0[resource] = {
'description': {'allow_post': True, 'allow_put': True,
'validate': {'type:string': attr.DESCRIPTION_MAX_LEN},
'is_visible': True, 'default': ''},
}
DESCRIPTION_BODY = {
'description': {'allow_post': True, 'allow_put': True,
'validate': {'type:string': attr.DESCRIPTION_MAX_LEN},
'is_visible': True, 'default': ''}
}
class Standardattrdescription(extensions.ExtensionDescriptor):
@ -50,6 +47,7 @@ class Standardattrdescription(extensions.ExtensionDescriptor):
return ['security-group', 'router']
def get_extended_resources(self, version):
if version == "2.0":
return dict(EXTENDED_ATTRIBUTES_2_0.items())
return {}
if version != "2.0":
return {}
rs_map = standard_attr.get_standard_attr_resource_model_map()
return {resource: DESCRIPTION_BODY for resource in rs_map}

View File

@ -13,6 +13,8 @@
# under the License.
from neutron.api import extensions
from neutron.db import standard_attr
# Attribute Map
CREATED = 'created_at'
@ -25,12 +27,6 @@ TIMESTAMP_BODY = {
'is_visible': True, 'default': None
},
}
EXTENDED_ATTRIBUTES_2_0 = {
'networks': TIMESTAMP_BODY,
'subnets': TIMESTAMP_BODY,
'ports': TIMESTAMP_BODY,
'subnetpools': TIMESTAMP_BODY,
}
class Timestamp_core(extensions.ExtensionDescriptor):
@ -59,7 +55,7 @@ class Timestamp_core(extensions.ExtensionDescriptor):
return "2016-03-01T10:00:00-00:00"
def get_extended_resources(self, version):
if version == "2.0":
return EXTENDED_ATTRIBUTES_2_0
else:
if version != "2.0":
return {}
rs_map = standard_attr.get_standard_attr_resource_model_map()
return {resource: TIMESTAMP_BODY for resource in rs_map}

View File

@ -13,26 +13,6 @@
# under the License.
from neutron.api import extensions
from neutron.extensions import l3
from neutron.extensions import securitygroup as sg
# Attribute Map
CREATED = 'created_at'
UPDATED = 'updated_at'
TIMESTAMP_BODY = {
CREATED: {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None
},
UPDATED: {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None
},
}
EXTENDED_ATTRIBUTES_2_0 = {
l3.ROUTERS: TIMESTAMP_BODY,
l3.FLOATINGIPS: TIMESTAMP_BODY,
sg.SECURITYGROUPS: TIMESTAMP_BODY,
sg.SECURITYGROUPRULES: TIMESTAMP_BODY,
}
class Timestamp_ext(extensions.ExtensionDescriptor):
@ -52,17 +32,16 @@ class Timestamp_ext(extensions.ExtensionDescriptor):
@classmethod
def get_description(cls):
return ("This extension can be used for recording "
"create/update timestamps for ext resources "
"like router, floatingip, security_group, "
"security_group_rule.")
return ("This extension adds create/update timestamps for all "
"standard neutron resources not included by the "
"'timestamp_core' extension.")
@classmethod
def get_updated(cls):
return "2016-05-05T10:00:00-00:00"
def get_extended_resources(self, version):
if version == "2.0":
return EXTENDED_ATTRIBUTES_2_0
else:
return {}
# NOTE(kevinbenton): this extension is basically a no-op because
# the timestamp_core extension already defines all of the resources
# now.
return {}

View File

@ -32,11 +32,6 @@ RESOURCE_ATTRIBUTE_MAP = {
'name': {'allow_post': True, 'allow_put': True,
'validate': {'type:string': attr.NAME_MAX_LEN},
'default': '', 'is_visible': True},
# TODO(armax): consolidate use of standardattr attributes
'description': {'allow_post': True,
'allow_put': True,
'validate': {'type:string': attr.DESCRIPTION_MAX_LEN},
'default': '', 'is_visible': True},
'tenant_id': {'allow_post': True, 'allow_put': False,
'required_by_policy': True,
'validate':
@ -54,13 +49,6 @@ RESOURCE_ATTRIBUTE_MAP = {
'validate': {'type:subports': None},
'enforce_policy': True,
'is_visible': True},
# TODO(armax): consolidate use of standardattr attributes
'created_at': {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None},
'updated_at': {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None},
'revision_number': {'allow_post': False, 'allow_put': False,
'is_visible': True, 'default': None},
},
}

View File

@ -19,7 +19,6 @@ from sqlalchemy.orm import session as se
from neutron._i18n import _, _LW
from neutron.db import db_base_plugin_v2
from neutron.db import standard_attr
from neutron.extensions import revisions
from neutron.services import service_base
LOG = logging.getLogger(__name__)
@ -28,11 +27,11 @@ LOG = logging.getLogger(__name__)
class RevisionPlugin(service_base.ServicePluginBase):
"""Plugin to populate revision numbers into standard attr resources."""
supported_extension_aliases = ['revisions']
supported_extension_aliases = ['standard-attr-revisions']
def __init__(self):
super(RevisionPlugin, self).__init__()
for resource in revisions.RESOURCES:
for resource in standard_attr.get_standard_attr_resource_model_map():
db_base_plugin_v2.NeutronDbPluginV2.register_dict_extend_funcs(
resource, [self.extend_resource_dict_revision])
event.listen(se.Session, 'before_flush', self.bump_revisions)

View File

@ -12,13 +12,9 @@
# License for the specific language governing permissions and limitations
# under the License.
from neutron.api.v2 import attributes
from neutron.db import db_base_plugin_v2
from neutron.db import l3_db
from neutron.db.models import securitygroup as sg_db
from neutron.db import models_v2
from neutron.extensions import l3
from neutron.extensions import securitygroup as sg
from neutron.db import standard_attr
from neutron.objects import base as base_obj
from neutron.services import service_base
from neutron.services.timestamp import timestamp_db as ts_db
@ -33,17 +29,7 @@ class TimeStampPlugin(service_base.ServicePluginBase,
def __init__(self):
super(TimeStampPlugin, self).__init__()
self.register_db_events()
rs_model_maps = {
attributes.NETWORKS: models_v2.Network,
attributes.PORTS: models_v2.Port,
attributes.SUBNETS: models_v2.Subnet,
attributes.SUBNETPOOLS: models_v2.SubnetPool,
l3.ROUTERS: l3_db.Router,
l3.FLOATINGIPS: l3_db.FloatingIP,
sg.SECURITYGROUPS: sg_db.SecurityGroup,
sg.SECURITYGROUPRULES: sg_db.SecurityGroupRule
}
rs_model_maps = standard_attr.get_standard_attr_resource_model_map()
for rsmap, model in rs_model_maps.items():
db_base_plugin_v2.NeutronDbPluginV2.register_dict_extend_funcs(
rsmap, [self.extend_resource_dict_timestamp])

View File

@ -44,6 +44,7 @@ class Trunk(standard_attr.HasStandardAttributes, model_base.BASEV2,
sub_ports = sa.orm.relationship(
'SubPort', lazy='joined', uselist=True, cascade="all, delete-orphan")
api_collections = ['trunks']
class SubPort(model_base.BASEV2):

View File

@ -28,13 +28,13 @@ NETWORK_API_EXTENSIONS="
qos, \
quotas, \
rbac-policies, \
revisions, \
router, \
router_availability_zone, \
security-group, \
service-type, \
sorting, \
standard-attr-description, \
standard-attr-revisions, \
subnet_allocation, \
tag, \
timestamp_core, \

View File

@ -20,7 +20,7 @@ from neutron.tests.tempest import config
class TestRevisions(base.BaseAdminNetworkTest, bsg.BaseSecGroupTest):
@classmethod
@test.requires_ext(extension="revisions", service="network")
@test.requires_ext(extension="standard-attr-revisions", service="network")
def skip_checks(cls):
super(TestRevisions, cls).skip_checks()

View File

@ -0,0 +1,70 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sqlalchemy.ext import declarative
import testtools
from neutron.db import standard_attr
from neutron.tests import base
from neutron.tests.unit import testlib_api
class StandardAttrTestCase(base.BaseTestCase):
def _make_decl_base(self):
# construct a new base so we don't interfere with the main
# base used in the sql test fixtures
return declarative.declarative_base(
cls=standard_attr.model_base.NeutronBaseV2)
def test_standard_attr_resource_model_map(self):
rs_map = standard_attr.get_standard_attr_resource_model_map()
base = self._make_decl_base()
class MyModel(standard_attr.HasStandardAttributes,
standard_attr.model_base.HasId,
base):
api_collections = ['my_resource', 'my_resource2']
rs_map = standard_attr.get_standard_attr_resource_model_map()
self.assertEqual(MyModel, rs_map['my_resource'])
self.assertEqual(MyModel, rs_map['my_resource2'])
class Dup(standard_attr.HasStandardAttributes,
standard_attr.model_base.HasId,
base):
api_collections = ['my_resource']
with testtools.ExpectedException(RuntimeError):
standard_attr.get_standard_attr_resource_model_map()
class StandardAttrAPIImapctTestCase(testlib_api.SqlTestCase):
"""Test case to determine if a resource has had new fields exposed."""
def test_api_collections_are_expected(self):
# NOTE to reviewers. If this test is being modified, it means the
# resources being extended by standard attr extensions have changed.
# Ensure that the patch has made this discoverable to API users.
# This means a new extension for a new resource or a new extension
# indicating that an existing resource now has standard attributes.
# Ensure devref list of resources is updated at
# doc/source/devref/api_extensions.rst
expected = ['subnets', 'trunks', 'routers', 'segments',
'security_group_rules', 'networks', 'policies',
'subnetpools', 'ports', 'security_groups', 'floatingips']
self.assertEqual(
set(expected),
set(standard_attr.get_standard_attr_resource_model_map().keys())
)

View File

@ -45,13 +45,12 @@ class SecurityGroupTestExtensionManager(object):
# The description of security_group_rules will be added by extending
# standardattrdescription. But as API router will not be initialized
# in test code, manually add it.
if (ext_sg.SECURITYGROUPRULES in
standardattrdescription.EXTENDED_ATTRIBUTES_2_0):
ext_res = (standardattrdescription.Standardattrdescription().
get_extended_resources("2.0"))
if ext_sg.SECURITYGROUPRULES in ext_res:
existing_sg_rule_attr_map = (
ext_sg.RESOURCE_ATTRIBUTE_MAP[ext_sg.SECURITYGROUPRULES])
sg_rule_attr_desc = (
standardattrdescription.
EXTENDED_ATTRIBUTES_2_0[ext_sg.SECURITYGROUPRULES])
sg_rule_attr_desc = ext_res[ext_sg.SECURITYGROUPRULES]
existing_sg_rule_attr_map.update(sg_rule_attr_desc)
# Add the resources to the global attribute map
# This is done here as the setup process won't

View File

@ -27,6 +27,7 @@ class FakeDbModelWithStandardAttributes(
standard_attr.HasStandardAttributes, model_base.BASEV2):
id = sa.Column(sa.String(36), primary_key=True, nullable=False)
item = sa.Column(sa.String(64))
api_collections = []
@obj_base.VersionedObjectRegistry.register_if(False)