From 25732260b54e5b4f14115abc475bfd1e4e0b1eb2 Mon Sep 17 00:00:00 2001 From: Tetsuro Nakamura Date: Mon, 19 Aug 2019 06:31:46 +0000 Subject: [PATCH] Get usages in _build_provider_summaries() For performance optimization purpose, we explored which provider is already processed both in extend_usages_by_provider_tree() and in _build_provider_summaries(), and we iterated all usage information in _build_provider_summaries(), which could cost even when only one request group has been given. This patch refactors it and move get_usages_by_provider_trees() into _build_provider_summaries() to combine the exploration parts into one. Change-Id: Ic36d453adf92b1f475960eb1b2796919d635da10 --- placement/objects/allocation_candidate.py | 66 +++++++++++------------ placement/objects/research_context.py | 31 +---------- 2 files changed, 32 insertions(+), 65 deletions(-) diff --git a/placement/objects/allocation_candidate.py b/placement/objects/allocation_candidate.py index cd145b94f..3b69ed978 100644 --- a/placement/objects/allocation_candidate.py +++ b/placement/objects/allocation_candidate.py @@ -308,9 +308,6 @@ def _alloc_candidates_multiple_providers(rg_ctx, rw_ctx, rp_candidates): # they have their "anchor" providers for the second value. root_ids = rp_candidates.all_rps - # Update rw_ctx.usages summaries for each provider in the trees - rw_ctx.extend_usages_by_provider_tree(root_ids) - # Get a dict, keyed by resource provider internal ID, of trait string names # that provider has associated with it prov_traits = trait_obj.get_traits_by_provider_tree( @@ -318,7 +315,7 @@ def _alloc_candidates_multiple_providers(rg_ctx, rw_ctx, rp_candidates): # Extend rw_ctx.summaries_by_id dict, keyed by resource provider internal # ID, of ProviderSummary objects for all providers - _build_provider_summaries(rg_ctx.context, rw_ctx, prov_traits) + _build_provider_summaries(rg_ctx.context, rw_ctx, root_ids, prov_traits) # Get a dict, keyed by root provider internal ID, of a dict, keyed by # resource class internal ID, of lists of AllocationRequestResource objects @@ -408,9 +405,6 @@ def _alloc_candidates_single_provider(rg_ctx, rw_ctx, rp_tuples): # Get all root resource provider IDs. root_ids = set(p[1] for p in rp_tuples) - # Update rw_ctx.usages summaries for each provider - rw_ctx.extend_usages_by_provider_tree(root_ids) - # Get a dict, keyed by resource provider internal ID, of trait string names # that provider has associated with it prov_traits = trait_obj.get_traits_by_provider_tree( @@ -418,7 +412,7 @@ def _alloc_candidates_single_provider(rg_ctx, rw_ctx, rp_tuples): # Extend rw_ctx.summaries_by_id dict, keyed by resource provider internal # ID, of ProviderSummary objects for all providers - _build_provider_summaries(rg_ctx.context, rw_ctx, prov_traits) + _build_provider_summaries(rg_ctx.context, rw_ctx, root_ids, prov_traits) # Next, build up a list of allocation requests. These allocation requests # are AllocationRequest objects, containing resource provider UUIDs, @@ -484,7 +478,7 @@ def _allocation_request_for_provider(context, requested_resources, provider, mappings=mappings) -def _build_provider_summaries(context, rw_ctx, prov_traits): +def _build_provider_summaries(context, rw_ctx, root_ids, prov_traits): """Given a list of dicts of usage information and a map of providers to their associated string traits, returns a dict, keyed by resource provider ID, of ProviderSummary objects. @@ -494,36 +488,37 @@ def _build_provider_summaries(context, rw_ctx, prov_traits): :param context: placement.context.RequestContext object :param rw_ctx: placement.research_context.RequestWideSearchContext + :param root_ids: A set of root resource provider ids :param prov_traits: A dict, keyed by internal resource provider ID, of string trait names associated with that provider """ - # Create a pared-down list of usages we care about -- only those for which - # we haven't already created summaries - pared_usages = [] - rp_ids = set() - for usage in rw_ctx.usages: - rp_id = usage['resource_provider_id'] - if rp_id not in rw_ctx.summaries_by_id: - pared_usages.append(usage) - rp_ids.add(rp_id) - - if not rp_ids: + # Filter resource providers by those we haven't seen yet. + new_roots = root_ids - set(rw_ctx.summaries_by_id) + if not new_roots: return + # Get a dict-like usage information of resource providers in a tree where + # at least one member of the tree is contributing resources or traits to + # an allocation candidate, which has the following structure: + # { + # 'resource_provider_id': , + # 'resource_provider_uuid': , + # 'resource_class_id': , + # 'total': integer, + # 'reserved': integer, + # 'allocation_ratio': float, + # } + usages = res_ctx.get_usages_by_provider_trees(context, new_roots) + # Before we go creating provider summary objects, first grab all the # provider information (including root, parent and UUID information) for - # all the providers that we haven't yet looked at. Above, `usages` includes - # information for every provider in a tree of providers where at least one - # member of the tree is contributing resources or traits to an allocation - # candidate. At this stage, any tree which we have previously touched has - # been fully summarized already and any trees left are fully present in - # rp_ids. See _get_usages_by_provider_trees for additional detail. - provider_ids = _provider_ids_from_rp_ids(context, rp_ids) + # the providers. + provider_ids = _provider_ids_from_root_ids(context, new_roots) # Build up a dict, keyed by internal resource provider ID, of # ProviderSummary objects containing one or more ProviderSummaryResource # objects representing the resources the provider has inventory for. - for usage in pared_usages: + for usage in usages: rp_id = usage['resource_provider_id'] summary = rw_ctx.summaries_by_id.get(rp_id) if not summary: @@ -896,11 +891,12 @@ def _get_ancestors_by_one_uuid( parent_uuid, parent_uuid_by_rp_uuid, ancestors=ancestors) -def _provider_ids_from_rp_ids(context, rp_ids): - """Given an iterable of internal resource provider IDs, returns a dict, - keyed by internal provider Id, of sqla objects describing those providers. +def _provider_ids_from_root_ids(context, root_ids): + """Given an iterable of internal root resource provider IDs, returns a + dict, keyed by internal provider Id, of sqla objects describing those + providers under the given root providers. - :param rp_ids: iterable of internal provider IDs to look up + :param root_ids: iterable of root provider IDs for trees to look up :returns: dict, keyed by internal provider Id, of sqla objects with the following attributes: @@ -913,7 +909,7 @@ def _provider_ids_from_rp_ids(context, rp_ids): # SELECT # rp.id, rp.uuid, rp.parent_provider_id, rp.root_provider.id # FROM resource_providers AS rp - # WHERE rp.id IN ($rp_ids) + # WHERE rp.root_provider_id IN ($root_ids) me = sa.alias(_RP_TBL, name="me") cols = [ me.c.id, @@ -922,9 +918,9 @@ def _provider_ids_from_rp_ids(context, rp_ids): me.c.root_provider_id.label('root_id'), ] sel = sa.select(cols).where( - me.c.id.in_(sa.bindparam('rps', expanding=True))) + me.c.root_provider_id.in_(sa.bindparam('root_ids', expanding=True))) ret = {} - for r in context.session.execute(sel, {'rps': list(rp_ids)}): + for r in context.session.execute(sel, {'root_ids': list(root_ids)}): ret[r['id']] = r return ret diff --git a/placement/objects/research_context.py b/placement/objects/research_context.py index 52089ee2e..ee40b0ccc 100644 --- a/placement/objects/research_context.py +++ b/placement/objects/research_context.py @@ -190,23 +190,6 @@ class RequestWideSearchContext(object): # Used as a cache of ProviderSummaries created in this request to # avoid duplication. self.summaries_by_id = {} - # A list of usage information for each resource provider considered - # in this request. Items in the list are dicts with the following - # structure: - # { - # 'resource_provider_id': , - # 'resource_provider_uuid': , - # 'resource_class_id': , - # 'total': integer, - # 'reserved': integer, - # 'allocation_ratio': float, - # 'max_unit': integer, - # 'used': integer, - # } - self.usages = [] - # A set of root provider ids for which usage summaries have already - # been calculated during this request. - self.usage_roots = set() # A set of resource classes that were requested in more than one group self.multi_group_rcs = set() # A mapping of resource provider uuid to parent provider uuid, used @@ -330,18 +313,6 @@ class RequestWideSearchContext(object): return alloc_request_objs, summary_objs - def extend_usages_by_provider_tree(self, root_ids): - """Extend our record of usages by the provided root_ids.""" - # filter root_ids by those we haven't seen yet - root_ids = set(root_ids) - self.usage_roots - - # Do nothing if we have nothing to look at - if not root_ids: - return - - self.usages.extend(_get_usages_by_provider_trees(self._ctx, root_ids)) - self.usage_roots.update(root_ids) - def copy_arr_if_needed(self, arr): """Copy or return arr, depending on the search context. @@ -1257,7 +1228,7 @@ def _has_provider_trees(ctx): return len(res) > 0 -def _get_usages_by_provider_trees(ctx, root_ids): +def get_usages_by_provider_trees(ctx, root_ids): """Returns a row iterator of usage records grouped by provider ID for all resource providers in all trees indicated in the ``root_ids``. """