Merge "Fix the API list performance regression"

octavia/api/v2/controllers/amphora.py

@@ -62,7 +62,7 @@ class AmphoraController(base.BaseController):
         self._auth_validate_action(context, context.project_id,
                                    constants.RBAC_GET_ALL)
 
-        db_amp, links = self.repositories.amphora.get_all(
+        db_amp, links = self.repositories.amphora.get_all_API_list(
             context.session, show_deleted=False,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER))
         result = self._convert_db_to_type(

octavia/api/v2/controllers/health_monitor.py

@@ -69,7 +69,7 @@ class HealthMonitorController(base.BaseController):
 
         query_filter = self._auth_get_all(context, project_id)
 
-        db_hm, links = self.repositories.health_monitor.get_all(
+        db_hm, links = self.repositories.health_monitor.get_all_API_list(
             context.session, show_deleted=False,
             pagination_helper=pcontext.get(consts.PAGINATION_HELPER),
             **query_filter)

octavia/api/v2/controllers/l7policy.py

@@ -71,7 +71,7 @@ class L7PolicyController(base.BaseController):
 
         query_filter = self._auth_get_all(context, project_id)
 
-        db_l7policies, links = self.repositories.l7policy.get_all(
+        db_l7policies, links = self.repositories.l7policy.get_all_API_list(
             context.session, show_deleted=False,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
             **query_filter)

octavia/api/v2/controllers/l7rule.py

@@ -72,7 +72,7 @@ class L7RuleController(base.BaseController):
         self._auth_validate_action(context, l7policy.project_id,
                                    constants.RBAC_GET_ALL)
 
-        db_l7rules, links = self.repositories.l7rule.get_all(
+        db_l7rules, links = self.repositories.l7rule.get_all_API_list(
             context.session, show_deleted=False, l7policy_id=self.l7policy_id,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER))
         result = self._convert_db_to_type(

octavia/api/v2/controllers/listener.py

@@ -82,7 +82,7 @@ class ListenersController(base.BaseController):
 
         query_filter = self._auth_get_all(context, project_id)
 
-        db_listeners, links = self.repositories.listener.get_all(
+        db_listeners, links = self.repositories.listener.get_all_API_list(
             context.session, show_deleted=False,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
             **query_filter)

octavia/api/v2/controllers/load_balancer.py

@@ -82,10 +82,11 @@ class LoadBalancersController(base.BaseController):
 
         query_filter = self._auth_get_all(context, project_id)
 
-        load_balancers, links = self.repositories.load_balancer.get_all(
+        load_balancers, links = (
-            context.session, show_deleted=False,
+            self.repositories.load_balancer.get_all_API_list(
-            pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
+                context.session, show_deleted=False,
-            **query_filter)
+                pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
+                **query_filter))
         result = self._convert_db_to_type(
             load_balancers, [lb_types.LoadBalancerResponse])
         if fields is not None:

octavia/api/v2/controllers/member.py

@@ -73,7 +73,7 @@ class MemberController(base.BaseController):
         self._auth_validate_action(context, pool.project_id,
                                    constants.RBAC_GET_ALL)
 
-        db_members, links = self.repositories.member.get_all(
+        db_members, links = self.repositories.member.get_all_API_list(
             context.session, show_deleted=False,
             pool_id=self.pool_id,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER))

octavia/api/v2/controllers/pool.py

@@ -71,7 +71,7 @@ class PoolsController(base.BaseController):
 
         query_filter = self._auth_get_all(context, project_id)
 
-        db_pools, links = self.repositories.pool.get_all(
+        db_pools, links = self.repositories.pool.get_all_API_list(
             context.session, show_deleted=False,
             pagination_helper=pcontext.get(constants.PAGINATION_HELPER),
             **query_filter)

octavia/db/models.py

@@ -133,9 +133,7 @@ class SessionPersistence(base_models.BASE):
     persistence_timeout = sa.Column(sa.Integer(), nullable=True)
     persistence_granularity = sa.Column(sa.String(64), nullable=True)
     pool = orm.relationship("Pool", uselist=False,
-                            backref=orm.backref("session_persistence",
+                            back_populates="session_persistence")
-                                                uselist=False,
-                                                cascade="delete"))
 
 
 class ListenerStatistics(base_models.BASE):
@@ -206,9 +204,7 @@ class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
                       name="fk_member_operating_status_name"),
         nullable=False)
     enabled = sa.Column(sa.Boolean(), nullable=False)
-    pool = orm.relationship("Pool", backref=orm.backref("members",
+    pool = orm.relationship("Pool", back_populates="members")
-                                                        uselist=True,
-                                                        cascade="delete"))
 
 
 class HealthMonitor(base_models.BASE, base_models.IdMixin,
@@ -244,9 +240,8 @@ class HealthMonitor(base_models.BASE, base_models.IdMixin,
     expected_codes = sa.Column(sa.String(64), nullable=True)
     enabled = sa.Column(sa.Boolean, nullable=False)
     pool = orm.relationship("Pool", uselist=False,
-                            backref=orm.backref("health_monitor",
+                            back_populates="health_monitor")
-                                                uselist=False,
+
-                                                cascade="delete"))
     provisioning_status = sa.Column(
         sa.String(16),
         sa.ForeignKey("provisioning_status.name",
@@ -292,10 +287,19 @@ class Pool(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
         sa.String(36),
         sa.ForeignKey("load_balancer.id", name="fk_pool_load_balancer_id"),
         nullable=True)
+    health_monitor = orm.relationship("HealthMonitor", uselist=False,
+                                      cascade="delete", back_populates="pool")
     load_balancer = orm.relationship("LoadBalancer", uselist=False,
-                                     backref=orm.backref("pools",
+                                     back_populates="pools")
-                                                         uselist=True,
+    members = orm.relationship("Member", uselist=True, cascade="delete",
-                                                         cascade="delete"))
+                               back_populates="pool")
+    session_persistence = orm.relationship(
+        "SessionPersistence", uselist=False, cascade="delete",
+        back_populates="pool")
+    _default_listeners = orm.relationship("Listener", uselist=True,
+                                          back_populates="default_pool")
+    l7policies = orm.relationship("L7Policy", uselist=True,
+                                  back_populates="redirect_pool")
 
     # This property should be a unique list of any listeners that reference
     # this pool as its default_pool and any listeners referenced by enabled
@@ -342,10 +346,15 @@ class LoadBalancer(base_models.BASE, base_models.IdMixin,
         nullable=True)
     enabled = sa.Column(sa.Boolean, nullable=False)
     amphorae = orm.relationship("Amphora", uselist=True,
-                                backref=orm.backref("load_balancer",
+                                back_populates="load_balancer")
-                                                    uselist=False))
     server_group_id = sa.Column(sa.String(36), nullable=True)
     provider = sa.Column(sa.String(64), nullable=True)
+    vip = orm.relationship('Vip', cascade='delete', uselist=False,
+                           backref=orm.backref('load_balancer', uselist=False))
+    pools = orm.relationship('Pool', cascade='delete', uselist=True,
+                             back_populates="load_balancer")
+    listeners = orm.relationship('Listener', cascade='delete', uselist=True,
+                                 back_populates='load_balancer')
 
 
 class VRRPGroup(base_models.BASE):
@@ -387,9 +396,6 @@ class Vip(base_models.BASE):
     port_id = sa.Column(sa.String(36), nullable=True)
     subnet_id = sa.Column(sa.String(36), nullable=True)
     network_id = sa.Column(sa.String(36), nullable=True)
-    load_balancer = orm.relationship("LoadBalancer", uselist=False,
-                                     backref=orm.backref("vip", uselist=False,
-                                                         cascade="delete"))
     qos_policy_id = sa.Column(sa.String(36), nullable=True)
     octavia_owned = sa.Column(sa.Boolean(), nullable=True)
 
@@ -437,12 +443,17 @@ class Listener(base_models.BASE, base_models.IdMixin,
         nullable=False)
     enabled = sa.Column(sa.Boolean(), nullable=False)
     load_balancer = orm.relationship("LoadBalancer", uselist=False,
-                                     backref=orm.backref("listeners",
+                                     back_populates="listeners")
-                                                         uselist=True,
-                                                         cascade="delete"))
     default_pool = orm.relationship("Pool", uselist=False,
-                                    backref=orm.backref("_default_listeners",
+                                    back_populates="_default_listeners")
-                                                        uselist=True))
+    sni_containers = orm.relationship(
+        'SNI', cascade='delete', uselist=True,
+        backref=orm.backref('listener', uselist=False))
+
+    l7policies = orm.relationship(
+        'L7Policy', uselist=True, order_by='L7Policy.position',
+        collection_class=orderinglist.ordering_list('position', count_from=1),
+        cascade='delete', back_populates='listener')
 
     peer_port = sa.Column(sa.Integer(), nullable=True)
     insert_headers = sa.Column(sa.PickleType())
@@ -487,10 +498,6 @@ class SNI(base_models.BASE):
         nullable=False)
     tls_container_id = sa.Column(sa.String(128), nullable=False)
     position = sa.Column(sa.Integer(), nullable=True)
-    listener = orm.relationship("Listener", uselist=False,
-                                backref=orm.backref("sni_containers",
-                                                    uselist=True,
-                                                    cascade="delete"))
 
 
 class Amphora(base_models.BASE, base_models.IdMixin, models.TimestampMixin):
@@ -528,6 +535,8 @@ class Amphora(base_models.BASE, base_models.IdMixin, models.TimestampMixin):
     vrrp_priority = sa.Column(sa.Integer(), nullable=True)
     cached_zone = sa.Column(sa.String(255), nullable=True)
     image_id = sa.Column(sa.String(36), nullable=True)
+    load_balancer = orm.relationship("LoadBalancer", uselist=False,
+                                     back_populates='amphorae')
 
 
 class AmphoraHealth(base_models.BASE):
@@ -572,9 +581,7 @@ class L7Rule(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
     invert = sa.Column(sa.Boolean(), default=False, nullable=False)
     enabled = sa.Column(sa.Boolean(), nullable=False)
     l7policy = orm.relationship("L7Policy", uselist=False,
-                                backref=orm.backref("l7rules",
+                                back_populates="l7rules")
-                                                    uselist=True,
-                                                    cascade="delete"))
     provisioning_status = sa.Column(
         sa.String(16),
         sa.ForeignKey("provisioning_status.name",
@@ -616,18 +623,12 @@ class L7Policy(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
         nullable=True)
     position = sa.Column(sa.Integer, nullable=False)
     enabled = sa.Column(sa.Boolean(), nullable=False)
-    listener = orm.relationship(
+    listener = orm.relationship("Listener", uselist=False,
-        "Listener", uselist=False,
+                                back_populates="l7policies")
-        backref=orm.backref(
-            "l7policies",
-            uselist=True,
-            order_by="L7Policy.position",
-            collection_class=orderinglist.ordering_list('position',
-                                                        count_from=1),
-            cascade="delete"))
     redirect_pool = orm.relationship("Pool", uselist=False,
-                                     backref=orm.backref("l7policies",
+                                     back_populates="l7policies")
-                                                         uselist=True))
+    l7rules = orm.relationship("L7Rule", uselist=True, cascade="delete",
+                               back_populates="l7policy")
     provisioning_status = sa.Column(
         sa.String(16),
         sa.ForeignKey("provisioning_status.name",

octavia/db/repositories.py

@@ -27,6 +27,8 @@ from oslo_log import log as logging
 from oslo_utils import excutils
 from oslo_utils import uuidutils
 from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import noload
+from sqlalchemy.orm import subqueryload
 
 from octavia.common import constants as consts
 from octavia.common import data_models
@@ -118,19 +120,21 @@ class BaseRepository(object):
 
         return model.to_data_model()
 
-    def get_all(self, session, pagination_helper=None, **filters):
+    def get_all(self, session, pagination_helper=None,
+                query_options=None, **filters):
 
         """Retrieves a list of entities from the database.
 
         :param session: A Sql Alchemy database session.
         :param pagination_helper: Helper to apply pagination and sorting.
+        :param query_options: Optional query options to apply.
         :param filters: Filters to decide which entities should be retrieved.
         :returns: [octavia.common.data_model]
         """
         deleted = filters.pop('show_deleted', True)
         query = session.query(self.model_class).filter_by(**filters)
-        # Only make one trip to the database
+        if query_options:
-        query = query.options(joinedload('*'))
+            query = query.options(query_options)
 
         if not deleted:
             if hasattr(self.model_class, 'status'):
@@ -666,6 +670,32 @@ class Repositories(object):
 class LoadBalancerRepository(BaseRepository):
     model_class = models.LoadBalancer
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of load balancers for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API load
+        balancer list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.LoadBalancer.vip),
+            subqueryload(models.LoadBalancer.amphorae),
+            subqueryload(models.LoadBalancer.pools),
+            subqueryload(models.LoadBalancer.listeners),
+            noload('*'))
+
+        return super(LoadBalancerRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
     def test_and_set_provisioning_status(self, session, id, status,
                                          raise_exception=False):
         """Tests and sets a load balancer and provisioning status.
@@ -763,6 +793,29 @@ class VipRepository(BaseRepository):
 class HealthMonitorRepository(BaseRepository):
     model_class = models.HealthMonitor
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of health monitors for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API health
+        monitor list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.HealthMonitor.pool),
+            noload('*'))
+
+        return super(HealthMonitorRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
 
 class SessionPersistenceRepository(BaseRepository):
     model_class = models.SessionPersistence
@@ -782,10 +835,65 @@ class SessionPersistenceRepository(BaseRepository):
 class PoolRepository(BaseRepository):
     model_class = models.Pool
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of pools for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API pool
+        list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.Pool._default_listeners),
+            subqueryload(models.Pool.health_monitor),
+            subqueryload(models.Pool.l7policies),
+            (subqueryload(models.Pool.l7policies).
+             subqueryload(models.L7Policy.l7rules)),
+            (subqueryload(models.Pool.l7policies).
+             subqueryload(models.L7Policy.listener)),
+            subqueryload(models.Pool.load_balancer),
+            subqueryload(models.Pool.members),
+            subqueryload(models.Pool.session_persistence),
+            noload('*'))
+
+        return super(PoolRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
 
 class MemberRepository(BaseRepository):
     model_class = models.Member
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of members for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API member
+        list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.Member.pool),
+            noload('*'))
+
+        return super(MemberRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
     def delete_members(self, session, member_ids):
         """Batch deletes members from a pool."""
         self.delete_batch(session, member_ids)
@@ -806,6 +914,31 @@ class MemberRepository(BaseRepository):
 class ListenerRepository(BaseRepository):
     model_class = models.Listener
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of listeners for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API listener
+        list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.Listener.l7policies),
+            subqueryload(models.Listener.load_balancer),
+            subqueryload(models.Listener.sni_containers),
+            noload('*'))
+
+        return super(ListenerRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
     def _find_next_peer_port(self, session, lb_id):
         """Finds the next available peer port on the load balancer."""
         max_peer_port = 0
@@ -903,6 +1036,29 @@ class ListenerStatisticsRepository(BaseRepository):
 class AmphoraRepository(BaseRepository):
     model_class = models.Amphora
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of amphorae for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API amphora
+        list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.Amphora.load_balancer),
+            noload('*'))
+
+        return super(AmphoraRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
     def associate(self, session, load_balancer_id, amphora_id):
         """Associates an amphora with a load balancer.
 
@@ -1281,6 +1437,29 @@ class VRRPGroupRepository(BaseRepository):
 class L7RuleRepository(BaseRepository):
     model_class = models.L7Rule
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        """Get a list of L7 Rules for the API list call.
+
+        This get_all returns a data set that is only one level deep
+        in the data graph. This is an optimized query for the API L7 Rule
+        list method.
+
+        :param session: A Sql Alchemy database session.
+        :param pagination_helper: Helper to apply pagination and sorting.
+        :param filters: Filters to decide which entities should be retrieved.
+        :returns: [octavia.common.data_model]
+        """
+
+        # sub-query load the tables we need
+        # no-load (blank) the tables we don't need
+        query_options = (
+            subqueryload(models.L7Rule.l7policy),
+            noload('*'))
+
+        return super(L7RuleRepository, self).get_all(
+            session, pagination_helper=pagination_helper,
+            query_options=query_options, **filters)
+
     def update(self, session, id, **model_kwargs):
         with session.begin(subtransactions=True):
             l7rule_db = session.query(self.model_class).filter_by(
@@ -1361,6 +1540,31 @@ class L7PolicyRepository(BaseRepository):
         data_model_list = [model.to_data_model() for model in model_list]
         return data_model_list, links
 
+    def get_all_API_list(self, session, pagination_helper=None, **filters):
+        deleted = filters.pop('show_deleted', True)
+        query = session.query(self.model_class).filter_by(
+            **filters)
+
+        query = query.options(
+            subqueryload(models.L7Policy.l7rules),
+            subqueryload(models.L7Policy.listener),
+            subqueryload(models.L7Policy.redirect_pool),
+            noload('*'))
+
+        if not deleted:
+            query = query.filter(
+                self.model_class.provisioning_status != consts.DELETED)
+
+        if pagination_helper:
+            model_list, links = pagination_helper.apply(
+                query, self.model_class)
+        else:
+            links = None
+            model_list = query.order_by(self.model_class.position).all()
+
+        data_model_list = [model.to_data_model() for model in model_list]
+        return data_model_list, links
+
     def update(self, session, id, **model_kwargs):
         with session.begin(subtransactions=True):
             l7policy_db = session.query(self.model_class).filter_by(

releasenotes/notes/fix-API-list-performance-3b121deffbc3ce4a.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed a performance regression in the Octavia v2 API when using the
+    "list" APIs.