diff --git a/quark/db/api.py b/quark/db/api.py index 399a194..bb93c27 100644 --- a/quark/db/api.py +++ b/quark/db/api.py @@ -173,6 +173,9 @@ def port_find(context, **filters): if filters.get("device_id"): model_filters.append(models.Port.device_id.in_(filters["device_id"])) + if "join_security_groups" in filters: + query = query.options(orm.joinedload(models.Port.security_groups)) + return query.filter(*model_filters).order_by(asc(models.Port.created_at)) @@ -235,7 +238,7 @@ def ip_address_find(context, lock_mode=False, **filters): if lock_mode: stmt = stmt.with_lockmode("update") stmt = stmt.outerjoin(models.port_ip_association_table) - stmt = stmt.group_by(models.IPAddress).subquery() + stmt = stmt.group_by(models.IPAddress.id).subquery() query = query.outerjoin(stmt, stmt.c.id == models.IPAddress.id) @@ -345,6 +348,9 @@ def _network_find(context, fields, defaults=None, **filters): else: query = query.filter(*model_filters) + if "join_subnets" in filters: + query = query.options(orm.joinedload(models.Network.subnets)) + return query @@ -381,7 +387,7 @@ def subnet_find_allocation_counts(context, net_id, **filters): label("count")).with_lockmode('update') query = query.filter_by(do_not_use=False) query = query.outerjoin(models.Subnet.generated_ips) - query = query.group_by(models.Subnet) + query = query.group_by(models.Subnet.id) query = query.order_by("count DESC") query = query.filter(models.Subnet.network_id == net_id) @@ -399,9 +405,15 @@ def subnet_find_allocation_counts(context, net_id, **filters): def subnet_find(context, **filters): if "shared" in filters and True in filters["shared"]: return [] - query = context.session.query(models.Subnet).\ - options(orm.joinedload(models.Subnet.routes)) + query = context.session.query(models.Subnet) model_filters = _model_query(context, models.Subnet, filters) + + if "join_dns" in filters: + query = query.options(orm.joinedload(models.Subnet.dns_nameservers)) + + if "join_routes" in filters: + query = query.options(orm.joinedload(models.Subnet.routes)) + return query.filter(*model_filters) diff --git a/quark/plugin_modules/networks.py b/quark/plugin_modules/networks.py index 26ff8d1..839bd9c 100644 --- a/quark/plugin_modules/networks.py +++ b/quark/plugin_modules/networks.py @@ -155,7 +155,8 @@ def get_network(context, id, fields=None): LOG.info("get_network %s for tenant %s fields %s" % (id, context.tenant_id, fields)) - network = db_api.network_find(context, id=id, scope=db_api.ONE) + network = db_api.network_find(context, id=id, join_subnets=True, + scope=db_api.ONE) if not network: raise exceptions.NetworkNotFound(net_id=id) return v._make_network_dict(network) @@ -182,7 +183,7 @@ def get_networks(context, filters=None, fields=None): """ LOG.info("get_networks for tenant %s with filters %s, fields %s" % (context.tenant_id, filters, fields)) - nets = db_api.network_find(context, **filters) or [] + nets = db_api.network_find(context, join_subnets=True, **filters) or [] nets = [v._make_network_dict(net) for net in nets] return nets diff --git a/quark/plugin_modules/ports.py b/quark/plugin_modules/ports.py index 775ff2c..cb62e9b 100644 --- a/quark/plugin_modules/ports.py +++ b/quark/plugin_modules/ports.py @@ -383,7 +383,8 @@ def get_ports(context, filters=None, fields=None): (context.tenant_id, filters, fields)) if filters is None: filters = {} - query = db_api.port_find(context, fields=fields, **filters) + query = db_api.port_find(context, fields=fields, join_security_groups=True, + **filters) return v._make_ports_list(query, fields) @@ -406,7 +407,7 @@ def get_ports_count(context, filters=None): """ LOG.info("get_ports_count for tenant %s filters %s" % (context.tenant_id, filters)) - return db_api.port_count_all(context, **filters) + return db_api.port_count_all(context, join_security_groups=True, **filters) def delete_port(context, id): diff --git a/quark/plugin_modules/subnets.py b/quark/plugin_modules/subnets.py index da27dfb..36b56e9 100644 --- a/quark/plugin_modules/subnets.py +++ b/quark/plugin_modules/subnets.py @@ -231,7 +231,8 @@ def get_subnet(context, id, fields=None): """ LOG.info("get_subnet %s for tenant %s with fields %s" % (id, context.tenant_id, fields)) - subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE) + subnet = db_api.subnet_find(context, id=id, join_dns=True, + join_routes=True, scope=db_api.ONE) if not subnet: raise exceptions.SubnetNotFound(subnet_id=id) @@ -264,7 +265,8 @@ def get_subnets(context, filters=None, fields=None): """ LOG.info("get_subnets for tenant %s with filters %s fields %s" % (context.tenant_id, filters, fields)) - subnets = db_api.subnet_find(context, **filters) + subnets = db_api.subnet_find(context, join_dns=True, join_routes=True, + **filters) return v._make_subnets_list(subnets, fields=fields, default_route=routes.DEFAULT_ROUTE) diff --git a/quark/plugin_views.py b/quark/plugin_views.py index f9f6d3f..d374dba 100644 --- a/quark/plugin_views.py +++ b/quark/plugin_views.py @@ -21,15 +21,26 @@ import netaddr from neutron.extensions import securitygroup as sg_ext from neutron.openstack.common import log as logging +from oslo.config import cfg from quark.db import api as db_api from quark.db import models from quark import network_strategy from quark import utils +CONF = cfg.CONF LOG = logging.getLogger(__name__) STRATEGY = network_strategy.STRATEGY +quark_view_opts = [ + cfg.BoolOpt('show_allocation_pools', + default=True, + help=_('Controls whether or not to calculate and display' + 'allocation pools or not')) +] + +CONF.register_opts(quark_view_opts, "QUARK") + def _make_network_dict(network, fields=None): shared_net = STRATEGY.is_parent_network(network["id"]) @@ -85,12 +96,16 @@ def _make_subnet_dict(subnet, default_route=None, fields=None): "tenant_id": subnet.get("tenant_id"), "network_id": net_id, "ip_version": subnet.get("ip_version"), - "allocation_pools": _allocation_pools(subnet), "dns_nameservers": dns_nameservers or [], "cidr": subnet.get("cidr"), "shared": STRATEGY.is_parent_network(net_id), "enable_dhcp": None} + if CONF.QUARK.show_allocation_pools: + res["allocation_pools"] = _allocation_pools(subnet) + else: + res["allocation_pools"] = [] + def _host_route(route): return {"destination": route["cidr"], "nexthop": route["gateway"]} diff --git a/quark/tests/plugin_modules/test_networks.py b/quark/tests/plugin_modules/test_networks.py index 7f38a47..bfd9909 100644 --- a/quark/tests/plugin_modules/test_networks.py +++ b/quark/tests/plugin_modules/test_networks.py @@ -121,6 +121,7 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin): with self._stubs(nets=[net]) as net_find: self.plugin.get_networks(self.context, {"shared": [True]}) net_find.assert_called_with(self.context, None, + join_subnets=True, defaults=["public_network"]) def test_get_networks_shared_false(self): @@ -128,14 +129,16 @@ class TestQuarkGetNetworksShared(test_quark_plugin.TestQuarkPlugin): status="ACTIVE") with self._stubs(nets=[net]) as net_find: self.plugin.get_networks(self.context, {"shared": [False]}) - net_find.assert_called_with(self.context, None, defaults=[]) + net_find.assert_called_with(self.context, None, join_subnets=True, + defaults=[]) def test_get_networks_no_shared(self): net = dict(id=1, tenant_id=self.context.tenant_id, name="mynet", status="ACTIVE") with self._stubs(nets=[net]) as net_find: self.plugin.get_networks(self.context, {}) - net_find.assert_called_with(self.context, None, defaults=[]) + net_find.assert_called_with(self.context, None, join_subnets=True, + defaults=[]) class TestQuarkGetNetworkCount(test_quark_plugin.TestQuarkPlugin): diff --git a/quark/tests/plugin_modules/test_subnets.py b/quark/tests/plugin_modules/test_subnets.py index d883e54..6e35bc9 100644 --- a/quark/tests/plugin_modules/test_subnets.py +++ b/quark/tests/plugin_modules/test_subnets.py @@ -117,6 +117,37 @@ class TestQuarkGetSubnets(test_quark_plugin.TestQuarkPlugin): self.assertEqual(routes[0][key], expected_route[key]) +class TestQuarkGetSubnetsHideAllocPools(test_quark_plugin.TestQuarkPlugin): + @contextlib.contextmanager + def _stubs(self, subnets=None): + if isinstance(subnets, list): + subnet_models = [] + for subnet in subnets: + s_dict = subnet.copy() + s = models.Subnet(network=models.Network()) + s.update(s_dict) + subnet_models.append(s) + + cfg.CONF.set_override('show_allocation_pools', False, "QUARK") + with mock.patch("quark.db.api.subnet_find") as subnet_find: + subnet_find.return_value = subnet_models + yield + cfg.CONF.set_override('show_allocation_pools', True, "QUARK") + + def test_subnets_list(self): + subnet_id = str(uuid.uuid4()) + + subnet = dict(id=subnet_id, network_id=1, name=subnet_id, + tenant_id=self.context.tenant_id, ip_version=4, + cidr="192.168.0.0/24", gateway_ip="192.168.0.1", + dns_nameservers=[], + enable_dhcp=None) + + with self._stubs(subnets=[subnet]): + res = self.plugin.get_subnets(self.context, {}, {}) + self.assertEqual(res[0]["allocation_pools"], []) + + class TestQuarkCreateSubnetOverlapping(test_quark_plugin.TestQuarkPlugin): @contextlib.contextmanager def _stubs(self, subnets=None): diff --git a/quark/utils.py b/quark/utils.py index aaa3fd6..28131d3 100644 --- a/quark/utils.py +++ b/quark/utils.py @@ -13,7 +13,16 @@ # License for the specific language governing permissions and limitations # under the License. + +import cProfile as profiler +import gc +try: + import pstats +except Exception: + # Don't want to force pstats into the venv if it's not always used + pass import sys +import time import contextlib @@ -27,6 +36,59 @@ def attr_specified(param): return param is not attributes.ATTR_NOT_SPECIFIED +def timed(fn): + def _wrapped(*args, **kwargs): + began = time.time() + res = fn(*args, **kwargs) + elapsed = time.time() - began + LOG.info("Time for %s = %s" % (fn, elapsed)) + return res + return _wrapped + + +def profile(output): + def _inner(fn): + def _wrapped(*args, **kw): + result = _profile(output, fn, *args, **kw) + # uncomment this to see who's calling what + # stats.print_callers() + return result + return _wrapped + return _inner + + +def live_profile(fn): + def _wrapped(*args, **kw): + elapsed, stat_loader, result = _live_profile(fn, *args, **kw) + stats = stat_loader() + stats.sort_stats('cumulative') + stats.print_stats() + # uncomment this to see who's calling what + # stats.print_callers() + return result + return _wrapped + + +def _profile(filename, fn, *args, **kw): + gc.collect() + + profiler.runctx('result = fn(*args, **kw)', globals(), locals(), + filename=filename) + + return locals()['result'] + + +def _live_profile(fn, *args, **kw): + load_stats = lambda: pstats.Stats() + gc.collect() + + began = time.time() + profiler.runctx('result = fn(*args, **kw)', globals(), locals()) + ended = time.time() + + return ended - began, load_stats, locals()['result'] + + def pop_param(attrs, param, default=None): val = attrs.pop(param, default) if attr_specified(val):