openstack/neutron
Code Issues Proposed changes

Use dynamic lazy mode for fetching security group rules

Browse Source 
In conjunction with the prior fix to only get a subset of fields
when needed, this makes the querying of non-rules SG objects
very very fast.

Before the two fixes, if you have about ten security groups with 2000 rules each:

list all: 14s
list all, just 'id' field: 14s
list one: 0.6s
list one, just 'id' field: 0.6s

With just the previous partial fix:

list all: 14s
list all, just 'id' field: 6s
list one: 0.6s
list one, just 'id' field: 0.2s

Now with this change:

list all: 14s
list all, just 'id' field: 0.04s
list one: 0.6s
list one, just 'id' field: 0.03s

Closes-Bug: #1810563
Change-Id: I15df276ba7dbcb3763ab20b63b26cddf2d594954
This commit is contained in:
Doug Wiegley 2019-03-03 10:42:16 -07:00
parent e0cfe41491
commit 1e9086f6e2
No known key found for this signature in database
GPG Key ID: 4D3C112B76BBDB5F
6 changed files with 55 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lower-constraints.txt
View File

@ -81,7 +81,7 @@ oslo.serialization==2.18.0
oslo.service==1.24.0
oslo.upgradecheck==0.1.0
oslo.utils==3.33.0
oslo.versionedobjects==1.31.2
oslo.versionedobjects==1.35.1
oslosphinx==4.7.0
oslotest==3.2.0
osprofiler==1.4.0

2
neutron/db/models/securitygroup.py
View File

@ -93,7 +93,7 @@ class SecurityGroupRule(standard_attr.HasStandardAttributes, model_base.BASEV2,
remote_ip_prefix = sa.Column(sa.String(255))
security_group = orm.relationship(
SecurityGroup, load_on_pending=True,
backref=orm.backref('rules', cascade='all,delete', lazy='subquery'),
backref=orm.backref('rules', cascade='all,delete', lazy='dynamic'),
primaryjoin="SecurityGroup.id==SecurityGroupRule.security_group_id")
source_group = orm.relationship(
SecurityGroup,

44
neutron/objects/base.py
View File

@ -55,6 +55,25 @@ def register_filter_hook_on_model(model, filter_name):
obj_class.add_extra_filter_name(filter_name)
class LazyQueryIterator(six.Iterator):
def __init__(self, obj_class, lazy_query):
self.obj_class = obj_class
self.context = None
self.query = lazy_query
def __iter__(self):
self.results = self.query.all()
self.i = 0
return self
def __next__(self):
if self.i >= len(self.results):
raise StopIteration()
item = self.obj_class._load_object(self.context, self.results[self.i])
self.i += 1
return item
class Pager(object):
'''Pager class
@ -131,6 +150,11 @@ class NeutronObject(obj_base.VersionedObject,
synthetic_fields = []
extra_filter_names = set()
# To use lazy queries for child objects, you must set the ORM
# relationship in the db model to 'dynamic'. By default, all
# children are eager loaded.
lazy_fields = set()
def __init__(self, context=None, **kwargs):
super(NeutronObject, self).__init__(context, **kwargs)
self._load_synthetic_fields = True
@ -428,8 +452,15 @@ class NeutronDbObject(NeutronObject):
'''Return a database model that persists object data.'''
return self._captured_db_model
def _set_lazy_contexts(self, fields, context):
for field in self.lazy_fields.intersection(fields):
if isinstance(fields[field], LazyQueryIterator):
fields[field].context = context
def from_db_object(self, db_obj):
fields = self.modify_fields_from_db(db_obj)
if self.lazy_fields:
self._set_lazy_contexts(fields, self.obj_context)
for field in self.fields:
if field in fields and not self.is_synthetic(field):
setattr(self, field, fields[field])
@ -459,12 +490,23 @@ class NeutronDbObject(NeutronObject):
:param fields: dict of fields from NeutronDbObject
:return: modified dict of fields
"""
for k, v in fields.items():
if isinstance(v, LazyQueryIterator):
fields[k] = list(v)
result = copy.deepcopy(dict(fields))
for field, field_db in cls.fields_need_translation.items():
if field in result:
result[field_db] = result.pop(field)
return result
@classmethod
def _get_lazy_iterator(cls, field, appender_query):
if field not in cls.lazy_fields:
raise KeyError(_('Field %s is not a lazy query field') % field)
n_obj_classes = NeutronObjectRegistry.obj_classes()
n_obj = n_obj_classes.get(cls.fields[field].objname)
return LazyQueryIterator(n_obj[0], appender_query)
@classmethod
def modify_fields_from_db(cls, db_obj):
"""Modify the fields after data were fetched from DB.
@ -490,6 +532,8 @@ class NeutronDbObject(NeutronObject):
# don't allow sqlalchemy lists to propagate outside
if isinstance(v, orm.collections.InstrumentedList):
result[k] = list(v)
if isinstance(v, orm.dynamic.AppenderQuery):
result[k] = cls._get_lazy_iterator(k, v)
return result
@classmethod

2
neutron/objects/securitygroup.py
View File

@ -60,6 +60,8 @@ class SecurityGroup(rbac_db.NeutronRbacObject):
extra_filter_names = {'is_default'}
lazy_fields = set(['rules'])
def create(self):
# save is_default before super() resets it to False
is_default = self.is_default

7
neutron/tests/unit/objects/test_base.py
View File

@ -32,6 +32,7 @@ from oslo_utils import uuidutils
from oslo_versionedobjects import base as obj_base
from oslo_versionedobjects import exception
from oslo_versionedobjects import fields as obj_fields
from sqlalchemy import orm
import testtools
from neutron import objects
@ -2126,7 +2127,11 @@ class BaseDbObjectTestCase(_BaseObjectTestCase,
obj.update()
self.assertIsNotNone(obj.db_obj)
for k, v in obj.modify_fields_to_db(fields_to_update).items():
self.assertEqual(v, obj.db_obj[k], '%s attribute differs' % k)
if isinstance(obj.db_obj[k], orm.dynamic.AppenderQuery):
self.assertIsInstance(v, list)
else:
self.assertEqual(v, obj.db_obj[k],
'%s attribute differs' % k)
obj.delete()
self.assertIsNone(obj.db_obj)

2
requirements.txt
View File

@ -41,7 +41,7 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
oslo.upgradecheck>=0.1.0 # Apache-2.0
oslo.utils>=3.33.0 # Apache-2.0
oslo.versionedobjects>=1.31.2 # Apache-2.0
oslo.versionedobjects>=1.35.1 # Apache-2.0
osprofiler>=1.4.0 # Apache-2.0
os-ken >= 0.3.0 # Apache-2.0
ovs>=2.8.0 # Apache-2.0