x/quark
Code Issues Proposed changes

More optimizations for #75

Browse Source 
Condenses many of the SQL queries down to a single query per call,
dramatically speeding up quark in several instances. Additionally, adds
the profiling decorators I've been using in testing to util.py for
future usage and optimization.
This commit is contained in:
Matt Dietz
2014-03-03 18:15:43 +00:00
parent 983c095b55
commit 1be2787a63
8 changed files with 140 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
quark/db/api.py
View File

@@ -173,6 +173,9 @@ def port_find(context, **filters):
if filters.get("device_id"):
model_filters.append(models.Port.device_id.in_(filters["device_id"]))
if "join_security_groups" in filters:
query = query.options(orm.joinedload(models.Port.security_groups))
return query.filter(*model_filters).order_by(asc(models.Port.created_at))
@@ -235,7 +238,7 @@ def ip_address_find(context, lock_mode=False, **filters):
if lock_mode:
stmt = stmt.with_lockmode("update")
stmt = stmt.outerjoin(models.port_ip_association_table)
stmt = stmt.group_by(models.IPAddress).subquery()
stmt = stmt.group_by(models.IPAddress.id).subquery()
query = query.outerjoin(stmt, stmt.c.id == models.IPAddress.id)
@@ -345,6 +348,9 @@ def _network_find(context, fields, defaults=None, **filters):
else:
query = query.filter(*model_filters)
if "join_subnets" in filters:
query = query.options(orm.joinedload(models.Network.subnets))
return query
@@ -381,7 +387,7 @@ def subnet_find_allocation_counts(context, net_id, **filters):
label("count")).with_lockmode('update')
query = query.filter_by(do_not_use=False)
query = query.outerjoin(models.Subnet.generated_ips)
query = query.group_by(models.Subnet)
query = query.group_by(models.Subnet.id)
query = query.order_by("count DESC")
query = query.filter(models.Subnet.network_id == net_id)
@@ -399,9 +405,15 @@ def subnet_find_allocation_counts(context, net_id, **filters):
def subnet_find(context, **filters):
if "shared" in filters and True in filters["shared"]:
return []
query = context.session.query(models.Subnet).\
options(orm.joinedload(models.Subnet.routes))
query = context.session.query(models.Subnet)
model_filters = _model_query(context, models.Subnet, filters)
if "join_dns" in filters:
query = query.options(orm.joinedload(models.Subnet.dns_nameservers))
if "join_routes" in filters:
query = query.options(orm.joinedload(models.Subnet.routes))
return query.filter(*model_filters)

5
quark/plugin_modules/networks.py
View File

@@ -155,7 +155,8 @@ def get_network(context, id, fields=None):
LOG.info("get_network %s for tenant %s fields %s" %
(id, context.tenant_id, fields))
network = db_api.network_find(context, id=id, scope=db_api.ONE)
network = db_api.network_find(context, id=id, join_subnets=True,
scope=db_api.ONE)
if not network:
raise exceptions.NetworkNotFound(net_id=id)
return v._make_network_dict(network)
@@ -182,7 +183,7 @@ def get_networks(context, filters=None, fields=None):
"""
LOG.info("get_networks for tenant %s with filters %s, fields %s" %
(context.tenant_id, filters, fields))
nets = db_api.network_find(context, **filters) or []
nets = db_api.network_find(context, join_subnets=True, **filters) or []
nets = [v._make_network_dict(net) for net in nets]
return nets

5
quark/plugin_modules/ports.py
View File

@@ -383,7 +383,8 @@ def get_ports(context, filters=None, fields=None):
(context.tenant_id, filters, fields))
if filters is None:
filters = {}
query = db_api.port_find(context, fields=fields, **filters)
query = db_api.port_find(context, fields=fields, join_security_groups=True,
**filters)
return v._make_ports_list(query, fields)
@@ -406,7 +407,7 @@ def get_ports_count(context, filters=None):
"""
LOG.info("get_ports_count for tenant %s filters %s" %
(context.tenant_id, filters))
return db_api.port_count_all(context, **filters)
return db_api.port_count_all(context, join_security_groups=True, **filters)
def delete_port(context, id):

6
quark/plugin_modules/subnets.py
View File

@@ -231,7 +231,8 @@ def get_subnet(context, id, fields=None):
"""
LOG.info("get_subnet %s for tenant %s with fields %s" %
(id, context.tenant_id, fields))
subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE)
subnet = db_api.subnet_find(context, id=id, join_dns=True,
join_routes=True, scope=db_api.ONE)
if not subnet:
raise exceptions.SubnetNotFound(subnet_id=id)
@@ -264,7 +265,8 @@ def get_subnets(context, filters=None, fields=None):
"""
LOG.info("get_subnets for tenant %s with filters %s fields %s" %
(context.tenant_id, filters, fields))
subnets = db_api.subnet_find(context, **filters)
subnets = db_api.subnet_find(context, join_dns=True, join_routes=True,
**filters)
return v._make_subnets_list(subnets, fields=fields,
default_route=routes.DEFAULT_ROUTE)

17
quark/plugin_views.py
View File

@@ -21,15 +21,26 @@ import netaddr
from neutron.extensions import securitygroup as sg_ext
from neutron.openstack.common import log as logging
from oslo.config import cfg
from quark.db import api as db_api
from quark.db import models
from quark import network_strategy
from quark import utils
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
STRATEGY = network_strategy.STRATEGY
quark_view_opts = [
cfg.BoolOpt('show_allocation_pools',
default=True,
help=_('Controls whether or not to calculate and display'
'allocation pools or not'))
]
CONF.register_opts(quark_view_opts, "QUARK")
def _make_network_dict(network, fields=None):
shared_net = STRATEGY.is_parent_network(network["id"])
@@ -85,12 +96,16 @@ def _make_subnet_dict(subnet, default_route=None, fields=None):
"tenant_id": subnet.get("tenant_id"),
"network_id": net_id,
"ip_version": subnet.get("ip_version"),
"allocation_pools": _allocation_pools(subnet),
"dns_nameservers": dns_nameservers or [],
"cidr": subnet.get("cidr"),
"shared": STRATEGY.is_parent_network(net_id),
"enable_dhcp": None}
if CONF.QUARK.show_allocation_pools:
res["allocation_pools"] = _allocation_pools(subnet)
else:
res["allocation_pools"] = []
def _host_route(route):
return {"destination": route["cidr"],
"nexthop": route["gateway"]}

7
quark/tests/plugin_modules/test_networks.py
View File

@@ -121,6 +121,7 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin):
with self._stubs(nets=[net]) as net_find:
self.plugin.get_networks(self.context, {"shared": [True]})
net_find.assert_called_with(self.context, None,
join_subnets=True,
defaults=["public_network"])
def test_get_networks_shared_false(self):
@@ -128,14 +129,16 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin):
status="ACTIVE")
with self._stubs(nets=[net]) as net_find:
self.plugin.get_networks(self.context, {"shared": [False]})
net_find.assert_called_with(self.context, None, defaults=[])
net_find.assert_called_with(self.context, None, join_subnets=True,
defaults=[])
def test_get_networks_no_shared(self):
net = dict(id=1, tenant_id=self.context.tenant_id, name="mynet",
status="ACTIVE")
with self._stubs(nets=[net]) as net_find:
self.plugin.get_networks(self.context, {})
net_find.assert_called_with(self.context, None, defaults=[])
net_find.assert_called_with(self.context, None, join_subnets=True,
defaults=[])
class TestQuarkGetNetworkCount(test_quark_plugin.TestQuarkPlugin):

31
quark/tests/plugin_modules/test_subnets.py
View File

@@ -117,6 +117,37 @@ class TestQuarkGetSubnets(test_quark_plugin.TestQuarkPlugin):
self.assertEqual(routes[0][key], expected_route[key])
class TestQuarkGetSubnetsHideAllocPools(test_quark_plugin.TestQuarkPlugin):
@contextlib.contextmanager
def _stubs(self, subnets=None):
if isinstance(subnets, list):
subnet_models = []
for subnet in subnets:
s_dict = subnet.copy()
s = models.Subnet(network=models.Network())
s.update(s_dict)
subnet_models.append(s)
cfg.CONF.set_override('show_allocation_pools', False, "QUARK")
with mock.patch("quark.db.api.subnet_find") as subnet_find:
subnet_find.return_value = subnet_models
yield
cfg.CONF.set_override('show_allocation_pools', True, "QUARK")
def test_subnets_list(self):
subnet_id = str(uuid.uuid4())
subnet = dict(id=subnet_id, network_id=1, name=subnet_id,
tenant_id=self.context.tenant_id, ip_version=4,
cidr="192.168.0.0/24", gateway_ip="192.168.0.1",
dns_nameservers=[],
enable_dhcp=None)
with self._stubs(subnets=[subnet]):
res = self.plugin.get_subnets(self.context, {}, {})
self.assertEqual(res[0]["allocation_pools"], [])
class TestQuarkCreateSubnetOverlapping(test_quark_plugin.TestQuarkPlugin):
@contextlib.contextmanager
def _stubs(self, subnets=None):

62
quark/utils.py
View File

@@ -13,7 +13,16 @@
# License for the specific language governing permissions and limitations
# under the License.
import cProfile as profiler
import gc
try:
import pstats
except Exception:
# Don't want to force pstats into the venv if it's not always used
pass
import sys
import time
import contextlib
@@ -27,6 +36,59 @@ def attr_specified(param):
return param is not attributes.ATTR_NOT_SPECIFIED
def timed(fn):
def _wrapped(*args, **kwargs):
began = time.time()
res = fn(*args, **kwargs)
elapsed = time.time() - began
LOG.info("Time for %s = %s" % (fn, elapsed))
return res
return _wrapped
def profile(output):
def _inner(fn):
def _wrapped(*args, **kw):
result = _profile(output, fn, *args, **kw)
# uncomment this to see who's calling what
# stats.print_callers()
return result
return _wrapped
return _inner
def live_profile(fn):
def _wrapped(*args, **kw):
elapsed, stat_loader, result = _live_profile(fn, *args, **kw)
stats = stat_loader()
stats.sort_stats('cumulative')
stats.print_stats()
# uncomment this to see who's calling what
# stats.print_callers()
return result
return _wrapped
def _profile(filename, fn, *args, **kw):
gc.collect()
profiler.runctx('result = fn(*args, **kw)', globals(), locals(),
filename=filename)
return locals()['result']
def _live_profile(fn, *args, **kw):
load_stats = lambda: pstats.Stats()
gc.collect()
began = time.time()
profiler.runctx('result = fn(*args, **kw)', globals(), locals())
ended = time.time()
return ended - began, load_stats, locals()['result']
def pop_param(attrs, param, default=None):
val = attrs.pop(param, default)
if attr_specified(val):