db: Replace implicit conversion of SELECT into FROM

Resolve the following warnings issued in recent versions of SQLAlchemy:

  SADeprecationWarning: Implicit coercion of SELECT and textual SELECT
  constructs into FROM clauses is deprecated; please call .subquery() on
  any Core select or ORM Query object in order to produce a subquery
  object.

The resolution is simple: instead of using 'FromClause.alias', use
'Query.subquery'. A warning filter is included to prevent us
reintroducing this issue in the future. It can be dropped once we start
using SQLAlchemy 2.0 where this behavior will be removed.

This change requires bumping SQLAlchemy to 1.4.0, which is the first
version to introduce the selectable '.subquery()' method. We must also
bump oslo.db to 8.6.0, which is the first version to support SQLAlchemy
1.4.x. The alternative would be to introduce a switch based on
SQLAlchemy versions, but that's tech debt that'll have to be cleaned up.

Change-Id: I6300be962dc3ce391f70d7c7e8af7cb4d6ce9baf
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

lower-constraints.txt

@@ -40,7 +40,7 @@ os-traits==2.7.0
 oslo.concurrency==3.26.0
 oslo.config==6.7.0
 oslo.context==2.22.0
-oslo.db==4.40.0
+oslo.db==8.6.0
 oslo.i18n==3.20.0
 oslo.log==4.3.0
 oslo.middleware==3.31.0
@@ -74,7 +74,7 @@ requestsexceptions==1.4.0
 rfc3986==0.3.1
 Routes==2.3.1
 smmap2==2.0.3
-SQLAlchemy==1.2.19
+SQLAlchemy==1.4.0
 setuptools==21.0.0
 sqlparse==0.2.4
 statsd==3.2.2

placement/objects/allocation.py

@@ -127,7 +127,7 @@ def _check_capacity_exceeded(ctx, allocs):
                 _ALLOC_TBL.c.resource_provider_id.in_(provider_ids)))
     usage = usage.group_by(_ALLOC_TBL.c.resource_provider_id,
                            _ALLOC_TBL.c.resource_class_id)
-    usage = sa.alias(usage, name='usage')
+    usage = usage.subquery(name='usage')
 
     inv_join = sql.join(
         _RP_TBL, _INV_TBL,

placement/objects/research_context.py

@@ -422,7 +422,7 @@ def _usage_select(rc_ids):
     usage = usage.where(_ALLOC_TBL.c.resource_class_id.in_(rc_ids))
     usage = usage.group_by(_ALLOC_TBL.c.resource_provider_id,
                            _ALLOC_TBL.c.resource_class_id)
-    return sa.alias(usage, name='usage')
+    return usage.subquery(name='usage')
 
 
 def _capacity_check_clause(amount, usage, inv_tbl=_INV_TBL):
@@ -1350,16 +1350,14 @@ def get_usages_by_provider_trees(ctx, root_ids):
                 _RP_TBL.c.root_provider_id.in_(sa.bindparam(
                     'root_ids', expanding=True)))
     )
-    usage = sa.alias(
+    usage = sa.select([
-        sa.select([
+        _ALLOC_TBL.c.resource_provider_id,
-            _ALLOC_TBL.c.resource_provider_id,
+        _ALLOC_TBL.c.resource_class_id,
-            _ALLOC_TBL.c.resource_class_id,
+        sql.func.sum(_ALLOC_TBL.c.used).label('used'),
-            sql.func.sum(_ALLOC_TBL.c.used).label('used'),
+    ]).select_from(derived_alloc_to_rp).group_by(
-        ]).select_from(derived_alloc_to_rp).group_by(
+        _ALLOC_TBL.c.resource_provider_id,
-            _ALLOC_TBL.c.resource_provider_id,
+        _ALLOC_TBL.c.resource_class_id
-            _ALLOC_TBL.c.resource_class_id
+    ).subquery(name='usage')
-        ),
-        name='usage')
     # Build a join between the resource providers and inventories table
     rpt_inv_join = sa.outerjoin(rpt, inv,
                                 rpt.c.id == inv.c.resource_provider_id)

placement/tests/functional/fixtures/capture.py

@@ -17,6 +17,7 @@ import warnings
 
 import fixtures
 from oslotest import log
+from sqlalchemy import exc as sqla_exc
 
 
 class NullHandler(logging.Handler):
@@ -69,13 +70,30 @@ class WarningsFixture(fixtures.Fixture):
     def setUp(self):
         super(WarningsFixture, self).setUp()
 
+        warnings.simplefilter("once", DeprecationWarning)
+
         # Ignore policy scope warnings.
-        warnings.filterwarnings('ignore',
+        warnings.filterwarnings(
-                                message="Policy .* failed scope check",
+            'ignore',
-                                category=UserWarning)
+            message="Policy .* failed scope check",
+            category=UserWarning)
+
         # The UUIDFields emits a warning if the value is not a  valid UUID.
         # Let's escalate that to an exception in the test to prevent adding
         # violations.
         warnings.filterwarnings('error', message=".*invalid UUID.*")
 
+        # Prevent us introducing unmapped columns
+        warnings.filterwarnings(
+            'error',
+            message='Evaluating non-mapped column expression',
+            category=sqla_exc.SAWarning)
+
+        # TODO(stephenfin): Remove once we're using sqlalchemy 2.0 which should
+        # remove this functionality entirely
+        warnings.filterwarnings(
+            'error',
+            message='Implicit coercion of SELECT and textual SELECT .*',
+            category=sqla_exc.SADeprecationWarning)
+
         self.addCleanup(warnings.resetwarnings)

requirements.txt

@@ -3,7 +3,7 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=3.1.1 # Apache-2.0
-SQLAlchemy>=1.2.19 # MIT
+SQLAlchemy>=1.4.0 # MIT
 keystonemiddleware>=4.18.0 # Apache-2.0
 Routes>=2.3.1 # MIT
 WebOb>=1.8.2 # MIT
@@ -16,7 +16,7 @@ oslo.context>=2.22.0 # Apache-2.0
 oslo.log>=4.3.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.utils>=4.5.0 # Apache-2.0
-oslo.db>=4.40.0 # Apache-2.0
+oslo.db>=8.6.0 # Apache-2.0
 oslo.policy>=3.7.0 # Apache-2.0
 oslo.middleware>=3.31.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0 # Apache-2.0