openstack/octavia
Code Issues Proposed changes

Merge "Fix the API list performance regression"

Browse Source
This commit is contained in:
Zuul 2018-09-26 08:32:38 +00:00 committed by Gerrit Code Review
commit 7fa83ef8e6
11 changed files with 265 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia/api/v2/controllers/amphora.py
View File

@ -62,7 +62,7 @@ class AmphoraController(base.BaseController):
self._auth_validate_action(context, context.project_id,
constants.RBAC_GET_ALL)
db_amp, links = self.repositories.amphora.get_all(
db_amp, links = self.repositories.amphora.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER))
result = self._convert_db_to_type(

2
octavia/api/v2/controllers/health_monitor.py
View File

@ -69,7 +69,7 @@ class HealthMonitorController(base.BaseController):
query_filter = self._auth_get_all(context, project_id)
db_hm, links = self.repositories.health_monitor.get_all(
db_hm, links = self.repositories.health_monitor.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(consts.PAGINATION_HELPER),
**query_filter)

2
octavia/api/v2/controllers/l7policy.py
View File

@ -71,7 +71,7 @@ class L7PolicyController(base.BaseController):
query_filter = self._auth_get_all(context, project_id)
db_l7policies, links = self.repositories.l7policy.get_all(
db_l7policies, links = self.repositories.l7policy.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
**query_filter)

2
octavia/api/v2/controllers/l7rule.py
View File

@ -72,7 +72,7 @@ class L7RuleController(base.BaseController):
self._auth_validate_action(context, l7policy.project_id,
constants.RBAC_GET_ALL)
db_l7rules, links = self.repositories.l7rule.get_all(
db_l7rules, links = self.repositories.l7rule.get_all_API_list(
context.session, show_deleted=False, l7policy_id=self.l7policy_id,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER))
result = self._convert_db_to_type(

2
octavia/api/v2/controllers/listener.py
View File

@ -82,7 +82,7 @@ class ListenersController(base.BaseController):
query_filter = self._auth_get_all(context, project_id)
db_listeners, links = self.repositories.listener.get_all(
db_listeners, links = self.repositories.listener.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
**query_filter)

5
octavia/api/v2/controllers/load_balancer.py
View File

@ -82,10 +82,11 @@ class LoadBalancersController(base.BaseController):
query_filter = self._auth_get_all(context, project_id)
load_balancers, links = self.repositories.load_balancer.get_all(
load_balancers, links = (
self.repositories.load_balancer.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
**query_filter)
**query_filter))
result = self._convert_db_to_type(
load_balancers, [lb_types.LoadBalancerResponse])
if fields is not None:

2
octavia/api/v2/controllers/member.py
View File

@ -73,7 +73,7 @@ class MemberController(base.BaseController):
self._auth_validate_action(context, pool.project_id,
constants.RBAC_GET_ALL)
db_members, links = self.repositories.member.get_all(
db_members, links = self.repositories.member.get_all_API_list(
context.session, show_deleted=False,
pool_id=self.pool_id,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER))

2
octavia/api/v2/controllers/pool.py
View File

@ -71,7 +71,7 @@ class PoolsController(base.BaseController):
query_filter = self._auth_get_all(context, project_id)
db_pools, links = self.repositories.pool.get_all(
db_pools, links = self.repositories.pool.get_all_API_list(
context.session, show_deleted=False,
pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
**query_filter)

81
octavia/db/models.py
View File

@ -133,9 +133,7 @@ class SessionPersistence(base_models.BASE):
persistence_timeout = sa.Column(sa.Integer(), nullable=True)
persistence_granularity = sa.Column(sa.String(64), nullable=True)
pool = orm.relationship("Pool", uselist=False,
backref=orm.backref("session_persistence",
uselist=False,
cascade="delete"))
back_populates="session_persistence")
class ListenerStatistics(base_models.BASE):
@ -206,9 +204,7 @@ class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
name="fk_member_operating_status_name"),
nullable=False)
enabled = sa.Column(sa.Boolean(), nullable=False)
pool = orm.relationship("Pool", backref=orm.backref("members",
uselist=True,
cascade="delete"))
pool = orm.relationship("Pool", back_populates="members")
class HealthMonitor(base_models.BASE, base_models.IdMixin,
@ -244,9 +240,8 @@ class HealthMonitor(base_models.BASE, base_models.IdMixin,
expected_codes = sa.Column(sa.String(64), nullable=True)
enabled = sa.Column(sa.Boolean, nullable=False)
pool = orm.relationship("Pool", uselist=False,
backref=orm.backref("health_monitor",
uselist=False,
cascade="delete"))
back_populates="health_monitor")
provisioning_status = sa.Column(
sa.String(16),
sa.ForeignKey("provisioning_status.name",
@ -292,10 +287,19 @@ class Pool(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
sa.String(36),
sa.ForeignKey("load_balancer.id", name="fk_pool_load_balancer_id"),
nullable=True)
health_monitor = orm.relationship("HealthMonitor", uselist=False,
cascade="delete", back_populates="pool")
load_balancer = orm.relationship("LoadBalancer", uselist=False,
backref=orm.backref("pools",
uselist=True,
cascade="delete"))
back_populates="pools")
members = orm.relationship("Member", uselist=True, cascade="delete",
back_populates="pool")
session_persistence = orm.relationship(
"SessionPersistence", uselist=False, cascade="delete",
back_populates="pool")
_default_listeners = orm.relationship("Listener", uselist=True,
back_populates="default_pool")
l7policies = orm.relationship("L7Policy", uselist=True,
back_populates="redirect_pool")
# This property should be a unique list of any listeners that reference
# this pool as its default_pool and any listeners referenced by enabled
@ -342,10 +346,15 @@ class LoadBalancer(base_models.BASE, base_models.IdMixin,
nullable=True)
enabled = sa.Column(sa.Boolean, nullable=False)
amphorae = orm.relationship("Amphora", uselist=True,
backref=orm.backref("load_balancer",
uselist=False))
back_populates="load_balancer")
server_group_id = sa.Column(sa.String(36), nullable=True)
provider = sa.Column(sa.String(64), nullable=True)
vip = orm.relationship('Vip', cascade='delete', uselist=False,
backref=orm.backref('load_balancer', uselist=False))
pools = orm.relationship('Pool', cascade='delete', uselist=True,
back_populates="load_balancer")
listeners = orm.relationship('Listener', cascade='delete', uselist=True,
back_populates='load_balancer')
class VRRPGroup(base_models.BASE):
@ -387,9 +396,6 @@ class Vip(base_models.BASE):
port_id = sa.Column(sa.String(36), nullable=True)
subnet_id = sa.Column(sa.String(36), nullable=True)
network_id = sa.Column(sa.String(36), nullable=True)
load_balancer = orm.relationship("LoadBalancer", uselist=False,
backref=orm.backref("vip", uselist=False,
cascade="delete"))
qos_policy_id = sa.Column(sa.String(36), nullable=True)
octavia_owned = sa.Column(sa.Boolean(), nullable=True)
@ -437,12 +443,17 @@ class Listener(base_models.BASE, base_models.IdMixin,
nullable=False)
enabled = sa.Column(sa.Boolean(), nullable=False)
load_balancer = orm.relationship("LoadBalancer", uselist=False,
backref=orm.backref("listeners",
uselist=True,
cascade="delete"))
back_populates="listeners")
default_pool = orm.relationship("Pool", uselist=False,
backref=orm.backref("_default_listeners",
uselist=True))
back_populates="_default_listeners")
sni_containers = orm.relationship(
'SNI', cascade='delete', uselist=True,
backref=orm.backref('listener', uselist=False))
l7policies = orm.relationship(
'L7Policy', uselist=True, order_by='L7Policy.position',
collection_class=orderinglist.ordering_list('position', count_from=1),
cascade='delete', back_populates='listener')
peer_port = sa.Column(sa.Integer(), nullable=True)
insert_headers = sa.Column(sa.PickleType())
@ -487,10 +498,6 @@ class SNI(base_models.BASE):
nullable=False)
tls_container_id = sa.Column(sa.String(128), nullable=False)
position = sa.Column(sa.Integer(), nullable=True)
listener = orm.relationship("Listener", uselist=False,
backref=orm.backref("sni_containers",
uselist=True,
cascade="delete"))
class Amphora(base_models.BASE, base_models.IdMixin, models.TimestampMixin):
@ -528,6 +535,8 @@ class Amphora(base_models.BASE, base_models.IdMixin, models.TimestampMixin):
vrrp_priority = sa.Column(sa.Integer(), nullable=True)
cached_zone = sa.Column(sa.String(255), nullable=True)
image_id = sa.Column(sa.String(36), nullable=True)
load_balancer = orm.relationship("LoadBalancer", uselist=False,
back_populates='amphorae')
class AmphoraHealth(base_models.BASE):
@ -572,9 +581,7 @@ class L7Rule(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
invert = sa.Column(sa.Boolean(), default=False, nullable=False)
enabled = sa.Column(sa.Boolean(), nullable=False)
l7policy = orm.relationship("L7Policy", uselist=False,
backref=orm.backref("l7rules",
uselist=True,
cascade="delete"))
back_populates="l7rules")
provisioning_status = sa.Column(
sa.String(16),
sa.ForeignKey("provisioning_status.name",
@ -616,18 +623,12 @@ class L7Policy(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
nullable=True)
position = sa.Column(sa.Integer, nullable=False)
enabled = sa.Column(sa.Boolean(), nullable=False)
listener = orm.relationship(
"Listener", uselist=False,
backref=orm.backref(
"l7policies",
uselist=True,
order_by="L7Policy.position",
collection_class=orderinglist.ordering_list('position',
count_from=1),
cascade="delete"))
listener = orm.relationship("Listener", uselist=False,
back_populates="l7policies")
redirect_pool = orm.relationship("Pool", uselist=False,
backref=orm.backref("l7policies",
uselist=True))
back_populates="l7policies")
l7rules = orm.relationship("L7Rule", uselist=True, cascade="delete",
back_populates="l7policy")
provisioning_status = sa.Column(
sa.String(16),
sa.ForeignKey("provisioning_status.name",

210
octavia/db/repositories.py
View File

@ -27,6 +27,8 @@ from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import uuidutils
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import noload
from sqlalchemy.orm import subqueryload
from octavia.common import constants as consts
from octavia.common import data_models
@ -118,19 +120,21 @@ class BaseRepository(object):
return model.to_data_model()
def get_all(self, session, pagination_helper=None, **filters):
def get_all(self, session, pagination_helper=None,
query_options=None, **filters):
"""Retrieves a list of entities from the database.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param query_options: Optional query options to apply.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
deleted = filters.pop('show_deleted', True)
query = session.query(self.model_class).filter_by(**filters)
# Only make one trip to the database
query = query.options(joinedload('*'))
if query_options:
query = query.options(query_options)
if not deleted:
if hasattr(self.model_class, 'status'):
@ -666,6 +670,32 @@ class Repositories(object):
class LoadBalancerRepository(BaseRepository):
model_class = models.LoadBalancer
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of load balancers for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API load
balancer list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.LoadBalancer.vip),
subqueryload(models.LoadBalancer.amphorae),
subqueryload(models.LoadBalancer.pools),
subqueryload(models.LoadBalancer.listeners),
noload('*'))
return super(LoadBalancerRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
def test_and_set_provisioning_status(self, session, id, status,
raise_exception=False):
"""Tests and sets a load balancer and provisioning status.
@ -763,6 +793,29 @@ class VipRepository(BaseRepository):
class HealthMonitorRepository(BaseRepository):
model_class = models.HealthMonitor
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of health monitors for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API health
monitor list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.HealthMonitor.pool),
noload('*'))
return super(HealthMonitorRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
class SessionPersistenceRepository(BaseRepository):
model_class = models.SessionPersistence
@ -782,10 +835,65 @@ class SessionPersistenceRepository(BaseRepository):
class PoolRepository(BaseRepository):
model_class = models.Pool
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of pools for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API pool
list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.Pool._default_listeners),
subqueryload(models.Pool.health_monitor),
subqueryload(models.Pool.l7policies),
(subqueryload(models.Pool.l7policies).
subqueryload(models.L7Policy.l7rules)),
(subqueryload(models.Pool.l7policies).
subqueryload(models.L7Policy.listener)),
subqueryload(models.Pool.load_balancer),
subqueryload(models.Pool.members),
subqueryload(models.Pool.session_persistence),
noload('*'))
return super(PoolRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
class MemberRepository(BaseRepository):
model_class = models.Member
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of members for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API member
list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.Member.pool),
noload('*'))
return super(MemberRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
def delete_members(self, session, member_ids):
"""Batch deletes members from a pool."""
self.delete_batch(session, member_ids)
@ -806,6 +914,31 @@ class MemberRepository(BaseRepository):
class ListenerRepository(BaseRepository):
model_class = models.Listener
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of listeners for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API listener
list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.Listener.l7policies),
subqueryload(models.Listener.load_balancer),
subqueryload(models.Listener.sni_containers),
noload('*'))
return super(ListenerRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
def _find_next_peer_port(self, session, lb_id):
"""Finds the next available peer port on the load balancer."""
max_peer_port = 0
@ -903,6 +1036,29 @@ class ListenerStatisticsRepository(BaseRepository):
class AmphoraRepository(BaseRepository):
model_class = models.Amphora
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of amphorae for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API amphora
list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.Amphora.load_balancer),
noload('*'))
return super(AmphoraRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
def associate(self, session, load_balancer_id, amphora_id):
"""Associates an amphora with a load balancer.
@ -1281,6 +1437,29 @@ class VRRPGroupRepository(BaseRepository):
class L7RuleRepository(BaseRepository):
model_class = models.L7Rule
def get_all_API_list(self, session, pagination_helper=None, **filters):
"""Get a list of L7 Rules for the API list call.
This get_all returns a data set that is only one level deep
in the data graph. This is an optimized query for the API L7 Rule
list method.
:param session: A Sql Alchemy database session.
:param pagination_helper: Helper to apply pagination and sorting.
:param filters: Filters to decide which entities should be retrieved.
:returns: [octavia.common.data_model]
"""
# sub-query load the tables we need
# no-load (blank) the tables we don't need
query_options = (
subqueryload(models.L7Rule.l7policy),
noload('*'))
return super(L7RuleRepository, self).get_all(
session, pagination_helper=pagination_helper,
query_options=query_options, **filters)
def update(self, session, id, **model_kwargs):
with session.begin(subtransactions=True):
l7rule_db = session.query(self.model_class).filter_by(
@ -1361,6 +1540,31 @@ class L7PolicyRepository(BaseRepository):
data_model_list = [model.to_data_model() for model in model_list]
return data_model_list, links
def get_all_API_list(self, session, pagination_helper=None, **filters):
deleted = filters.pop('show_deleted', True)
query = session.query(self.model_class).filter_by(
**filters)
query = query.options(
subqueryload(models.L7Policy.l7rules),
subqueryload(models.L7Policy.listener),
subqueryload(models.L7Policy.redirect_pool),
noload('*'))
if not deleted:
query = query.filter(
self.model_class.provisioning_status != consts.DELETED)
if pagination_helper:
model_list, links = pagination_helper.apply(
query, self.model_class)
else:
links = None
model_list = query.order_by(self.model_class.position).all()
data_model_list = [model.to_data_model() for model in model_list]
return data_model_list, links
def update(self, session, id, **model_kwargs):
with session.begin(subtransactions=True):
l7policy_db = session.query(self.model_class).filter_by(

5
releasenotes/notes/fix-API-list-performance-3b121deffbc3ce4a.yaml Normal file
View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixed a performance regression in the Octavia v2 API when using the
"list" APIs.