More optimizations for #75

Condenses many of the SQL queries down to a single query per call,
dramatically speeding up quark in several instances. Additionally, adds
the profiling decorators I've been using in testing to util.py for
future usage and optimization.

quark/db/api.py

@@ -173,6 +173,9 @@ def port_find(context, **filters):
     if filters.get("device_id"):
         model_filters.append(models.Port.device_id.in_(filters["device_id"]))
 
+    if "join_security_groups" in filters:
+        query = query.options(orm.joinedload(models.Port.security_groups))
+
     return query.filter(*model_filters).order_by(asc(models.Port.created_at))
 
 
@@ -235,7 +238,7 @@ def ip_address_find(context, lock_mode=False, **filters):
         if lock_mode:
             stmt = stmt.with_lockmode("update")
         stmt = stmt.outerjoin(models.port_ip_association_table)
-        stmt = stmt.group_by(models.IPAddress).subquery()
+        stmt = stmt.group_by(models.IPAddress.id).subquery()
 
         query = query.outerjoin(stmt, stmt.c.id == models.IPAddress.id)
 
@@ -345,6 +348,9 @@ def _network_find(context, fields, defaults=None, **filters):
     else:
         query = query.filter(*model_filters)
 
+    if "join_subnets" in filters:
+        query = query.options(orm.joinedload(models.Network.subnets))
+
     return query
 
 
@@ -381,7 +387,7 @@ def subnet_find_allocation_counts(context, net_id, **filters):
                                   label("count")).with_lockmode('update')
     query = query.filter_by(do_not_use=False)
     query = query.outerjoin(models.Subnet.generated_ips)
-    query = query.group_by(models.Subnet)
+    query = query.group_by(models.Subnet.id)
     query = query.order_by("count DESC")
 
     query = query.filter(models.Subnet.network_id == net_id)
@@ -399,9 +405,15 @@ def subnet_find_allocation_counts(context, net_id, **filters):
 def subnet_find(context, **filters):
     if "shared" in filters and True in filters["shared"]:
         return []
-    query = context.session.query(models.Subnet).\
+    query = context.session.query(models.Subnet)
-        options(orm.joinedload(models.Subnet.routes))
     model_filters = _model_query(context, models.Subnet, filters)
+
+    if "join_dns" in filters:
+        query = query.options(orm.joinedload(models.Subnet.dns_nameservers))
+
+    if "join_routes" in filters:
+        query = query.options(orm.joinedload(models.Subnet.routes))
+
     return query.filter(*model_filters)
 
 

quark/plugin_modules/networks.py

@@ -155,7 +155,8 @@ def get_network(context, id, fields=None):
     LOG.info("get_network %s for tenant %s fields %s" %
             (id, context.tenant_id, fields))
 
-    network = db_api.network_find(context, id=id, scope=db_api.ONE)
+    network = db_api.network_find(context, id=id, join_subnets=True,
+                                  scope=db_api.ONE)
     if not network:
         raise exceptions.NetworkNotFound(net_id=id)
     return v._make_network_dict(network)
@@ -182,7 +183,7 @@ def get_networks(context, filters=None, fields=None):
     """
     LOG.info("get_networks for tenant %s with filters %s, fields %s" %
             (context.tenant_id, filters, fields))
-    nets = db_api.network_find(context, **filters) or []
+    nets = db_api.network_find(context, join_subnets=True, **filters) or []
     nets = [v._make_network_dict(net) for net in nets]
     return nets
 

quark/plugin_modules/ports.py

@@ -383,7 +383,8 @@ def get_ports(context, filters=None, fields=None):
             (context.tenant_id, filters, fields))
     if filters is None:
         filters = {}
-    query = db_api.port_find(context, fields=fields, **filters)
+    query = db_api.port_find(context, fields=fields, join_security_groups=True,
+                             **filters)
     return v._make_ports_list(query, fields)
 
 
@@ -406,7 +407,7 @@ def get_ports_count(context, filters=None):
     """
     LOG.info("get_ports_count for tenant %s filters %s" %
             (context.tenant_id, filters))
-    return db_api.port_count_all(context, **filters)
+    return db_api.port_count_all(context, join_security_groups=True, **filters)
 
 
 def delete_port(context, id):

quark/plugin_modules/subnets.py

@@ -231,7 +231,8 @@ def get_subnet(context, id, fields=None):
     """
     LOG.info("get_subnet %s for tenant %s with fields %s" %
             (id, context.tenant_id, fields))
-    subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE)
+    subnet = db_api.subnet_find(context, id=id, join_dns=True,
+                                join_routes=True, scope=db_api.ONE)
     if not subnet:
         raise exceptions.SubnetNotFound(subnet_id=id)
 
@@ -264,7 +265,8 @@ def get_subnets(context, filters=None, fields=None):
     """
     LOG.info("get_subnets for tenant %s with filters %s fields %s" %
             (context.tenant_id, filters, fields))
-    subnets = db_api.subnet_find(context, **filters)
+    subnets = db_api.subnet_find(context, join_dns=True, join_routes=True,
+                                 **filters)
     return v._make_subnets_list(subnets, fields=fields,
                                 default_route=routes.DEFAULT_ROUTE)
 

quark/plugin_views.py

@@ -21,15 +21,26 @@ import netaddr
 
 from neutron.extensions import securitygroup as sg_ext
 from neutron.openstack.common import log as logging
+from oslo.config import cfg
 
 from quark.db import api as db_api
 from quark.db import models
 from quark import network_strategy
 from quark import utils
 
+CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 STRATEGY = network_strategy.STRATEGY
 
+quark_view_opts = [
+    cfg.BoolOpt('show_allocation_pools',
+                default=True,
+                help=_('Controls whether or not to calculate and display'
+                       'allocation pools or not'))
+]
+
+CONF.register_opts(quark_view_opts, "QUARK")
+
 
 def _make_network_dict(network, fields=None):
     shared_net = STRATEGY.is_parent_network(network["id"])
@@ -85,12 +96,16 @@ def _make_subnet_dict(subnet, default_route=None, fields=None):
            "tenant_id": subnet.get("tenant_id"),
            "network_id": net_id,
            "ip_version": subnet.get("ip_version"),
-           "allocation_pools": _allocation_pools(subnet),
            "dns_nameservers": dns_nameservers or [],
            "cidr": subnet.get("cidr"),
            "shared": STRATEGY.is_parent_network(net_id),
            "enable_dhcp": None}
 
+    if CONF.QUARK.show_allocation_pools:
+        res["allocation_pools"] = _allocation_pools(subnet)
+    else:
+        res["allocation_pools"] = []
+
     def _host_route(route):
         return {"destination": route["cidr"],
                 "nexthop": route["gateway"]}

quark/tests/plugin_modules/test_networks.py

@@ -121,6 +121,7 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin):
         with self._stubs(nets=[net]) as net_find:
             self.plugin.get_networks(self.context, {"shared": [True]})
             net_find.assert_called_with(self.context, None,
+                                        join_subnets=True,
                                         defaults=["public_network"])
 
     def test_get_networks_shared_false(self):
@@ -128,14 +129,16 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin):
                    status="ACTIVE")
         with self._stubs(nets=[net]) as net_find:
             self.plugin.get_networks(self.context, {"shared": [False]})
-            net_find.assert_called_with(self.context, None, defaults=[])
+            net_find.assert_called_with(self.context, None, join_subnets=True,
+                                        defaults=[])
 
     def test_get_networks_no_shared(self):
         net = dict(id=1, tenant_id=self.context.tenant_id, name="mynet",
                    status="ACTIVE")
         with self._stubs(nets=[net]) as net_find:
             self.plugin.get_networks(self.context, {})
-            net_find.assert_called_with(self.context, None, defaults=[])
+            net_find.assert_called_with(self.context, None, join_subnets=True,
+                                        defaults=[])
 
 
 class TestQuarkGetNetworkCount(test_quark_plugin.TestQuarkPlugin):

quark/tests/plugin_modules/test_subnets.py

@@ -117,6 +117,37 @@ class TestQuarkGetSubnets(test_quark_plugin.TestQuarkPlugin):
                 self.assertEqual(routes[0][key], expected_route[key])
 
 
+class TestQuarkGetSubnetsHideAllocPools(test_quark_plugin.TestQuarkPlugin):
+    @contextlib.contextmanager
+    def _stubs(self, subnets=None):
+        if isinstance(subnets, list):
+            subnet_models = []
+            for subnet in subnets:
+                s_dict = subnet.copy()
+                s = models.Subnet(network=models.Network())
+                s.update(s_dict)
+                subnet_models.append(s)
+
+        cfg.CONF.set_override('show_allocation_pools', False, "QUARK")
+        with mock.patch("quark.db.api.subnet_find") as subnet_find:
+            subnet_find.return_value = subnet_models
+            yield
+        cfg.CONF.set_override('show_allocation_pools', True, "QUARK")
+
+    def test_subnets_list(self):
+        subnet_id = str(uuid.uuid4())
+
+        subnet = dict(id=subnet_id, network_id=1, name=subnet_id,
+                      tenant_id=self.context.tenant_id, ip_version=4,
+                      cidr="192.168.0.0/24", gateway_ip="192.168.0.1",
+                      dns_nameservers=[],
+                      enable_dhcp=None)
+
+        with self._stubs(subnets=[subnet]):
+            res = self.plugin.get_subnets(self.context, {}, {})
+            self.assertEqual(res[0]["allocation_pools"], [])
+
+
 class TestQuarkCreateSubnetOverlapping(test_quark_plugin.TestQuarkPlugin):
     @contextlib.contextmanager
     def _stubs(self, subnets=None):

quark/utils.py

@@ -13,7 +13,16 @@
 # License for the specific language governing permissions and limitations
 #  under the License.
 
+
+import cProfile as profiler
+import gc
+try:
+    import pstats
+except Exception:
+    # Don't want to force pstats into the venv if it's not always used
+    pass
 import sys
+import time
 
 import contextlib
 
@@ -27,6 +36,59 @@ def attr_specified(param):
     return param is not attributes.ATTR_NOT_SPECIFIED
 
 
+def timed(fn):
+    def _wrapped(*args, **kwargs):
+        began = time.time()
+        res = fn(*args, **kwargs)
+        elapsed = time.time() - began
+        LOG.info("Time for %s = %s" % (fn, elapsed))
+        return res
+    return _wrapped
+
+
+def profile(output):
+    def _inner(fn):
+        def _wrapped(*args, **kw):
+            result = _profile(output, fn, *args, **kw)
+            # uncomment this to see who's calling what
+            # stats.print_callers()
+            return result
+        return _wrapped
+    return _inner
+
+
+def live_profile(fn):
+    def _wrapped(*args, **kw):
+        elapsed, stat_loader, result = _live_profile(fn, *args, **kw)
+        stats = stat_loader()
+        stats.sort_stats('cumulative')
+        stats.print_stats()
+        # uncomment this to see who's calling what
+        # stats.print_callers()
+        return result
+    return _wrapped
+
+
+def _profile(filename, fn, *args, **kw):
+    gc.collect()
+
+    profiler.runctx('result = fn(*args, **kw)', globals(), locals(),
+                    filename=filename)
+
+    return locals()['result']
+
+
+def _live_profile(fn, *args, **kw):
+    load_stats = lambda: pstats.Stats()
+    gc.collect()
+
+    began = time.time()
+    profiler.runctx('result = fn(*args, **kw)', globals(), locals())
+    ended = time.time()
+
+    return ended - began, load_stats, locals()['result']
+
+
 def pop_param(attrs, param, default=None):
     val = attrs.pop(param, default)
     if attr_specified(val):