From d4d9457aa10c3309cae00b40598b3cec8b1c793a Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Thu, 1 Aug 2019 15:22:53 +0100 Subject: [PATCH] Use expanding bindparam in provider_ids_from_rp_ids in_ This was explored as part of discussion with zzzeek on various ways to make a statement with a large in_ be more performant. Batching the queries by chunking rp_ids did not help, but an expanding bindparam did: it makes the surrounding method about 50% faster [1]. bindparam expanding was added in sqlalchemy 1.2.0 so we bump the requirements here to the latest 1.2.x bugfix release, 1.2.19. [1] With the caveat that profiling and benchmarking are very different things, and in this case it is profiling results which are being analysed. Change-Id: Ic4e0cdd87f8f2d76b921059ac4bf16a838913abf --- lower-constraints.txt | 2 +- placement/objects/research_context.py | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 7e46d4cf1..dc433179e 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -76,7 +76,7 @@ rfc3986==0.3.1 Routes==2.3.1 six==1.10.0 smmap2==2.0.3 -SQLAlchemy==1.0.10 +SQLAlchemy==1.2.19 sqlparse==0.2.4 statsd==3.2.2 stestr==1.0.0 diff --git a/placement/objects/research_context.py b/placement/objects/research_context.py index 16d81f7b2..b272dbe8f 100644 --- a/placement/objects/research_context.py +++ b/placement/objects/research_context.py @@ -331,10 +331,10 @@ def provider_ids_from_rp_ids(context, rp_ids): me_to_root, parent, me.c.parent_provider_id == parent.c.id) sel = sa.select(cols).select_from(me_to_parent) - sel = sel.where(me.c.id.in_(rp_ids)) + sel = sel.where(me.c.id.in_(sa.bindparam('rps', expanding=True))) ret = {} - for r in context.session.execute(sel): + for r in context.session.execute(sel, {'rps': list(rp_ids)}): ret[r['id']] = r return ret diff --git a/requirements.txt b/requirements.txt index b339a0221..3c30aba39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. pbr!=2.1.0,>=2.0.0 # Apache-2.0 -SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT +SQLAlchemy>=1.2.19 # MIT keystonemiddleware>=4.18.0 # Apache-2.0 Routes>=2.3.1 # MIT WebOb>=1.8.2 # MIT