Fix migration for SQLAlchemy 1.4

This patch fixes a database migration for SQLAlchemy 1.4.

In tacker repository following issues are encounter:

1. Replace deprecated SQLAlchemy "with_lockmode" method.
   The method was deprecated in SQLAlchemy 0.9 [1][2]
   and finally removed in version 1.4.
   This patch replaces "with_lockmode" with new method
   "with_for_update", which has no end-user impact.

2. The clause column IN, in the new version it requires parameters
   passed to operator either be a list of literal value
   or a tuple, or an empty list [3].

3. Observe an argument error in 'update' query.
   Error: sqlalchemy.exc.ArgumentError: subject table for an INSERT,
   UPDATE or DELETE expected, got Column('id', Uuid(length=36),
   table=<vnf_software_images>, primary_key=True, nullable=False,
   default=ColumnDefault(<function generate_uuid at 0x7fb5be371d30>))

4. The INSERT statement requires values to add in table as a list,
   tuple or dictionary [4].
   Error: sqlalchemy.exc.ArgumentError: mapping or sequence expected
          for parameters
   This patch creates a dictionary of fields to be insert in
   VnfLcmOpOccs table.

5. Query fails on applying filters.
   When applying query to list vnf package or vnf instance using
   apply_filters() method exported from sqlalchemy_filters,
   it failed.
   AttributeError: 'Query' object has no attribute '_join_entities'

   In SQLAlchemy 1.4 unification of "query" and "select" construct
   took place [5][6].
   The issue observe in tacker is an open bug of SQLAlchemy [7].
   This patch drops "sqlalchemy_filters" and add customize method
   to apply filters in query.

Note: This patch address sqlalchemy errors, SAWarnings will be resolve
in future patches.

[1] https://docs.sqlalchemy.org/en/13/changelog/migration_09.html?highlight=with_lockmode#new-for-update-support-on-select-query
[2] https://docs.sqlalchemy.org/en/13/orm/query.html#sqlalchemy.orm.query.Query.with_lockmode
[3] https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.ColumnElement.in_
[4] https://docs.sqlalchemy.org/en/14/core/dml.html#dml-class-documentation-constructors
[5] https://github.com/sqlalchemy/sqlalchemy/issues/5159
[6] https://gerrit.sqlalchemy.org/c/sqlalchemy/sqlalchemy/+/1599/
[7] https://github.com/juliotrigo/sqlalchemy-filters/issues/61

Co-Authored-By: Ayumu Ueha <ueha.ayumu@fujitsu.com>

Change-Id: Ifd1ae2753c639f22fc1afa020222416fe79469ef
This commit is contained in:
Manpreet Kaur 2021-06-13 08:50:05 +05:30
parent 8e79746b28
commit e0c09499a6
12 changed files with 111 additions and 23 deletions

View File

@ -129,7 +129,6 @@ rsa==3.4.2
setuptools==21.0.0 setuptools==21.0.0
simplejson==3.13.2 simplejson==3.13.2
snowballstemmer==1.2.1 snowballstemmer==1.2.1
sqlalchemy-filters==0.10.0
sqlalchemy-migrate==0.11.0 sqlalchemy-migrate==0.11.0
SQLAlchemy==1.3.11 SQLAlchemy==1.3.11
sqlparse==0.2.4 sqlparse==0.2.4

View File

@ -15,7 +15,6 @@ keystonemiddleware>=4.17.0 # Apache-2.0
kombu>=4.3.0 # BSD kombu>=4.3.0 # BSD
netaddr>=0.7.18 # BSD netaddr>=0.7.18 # BSD
SQLAlchemy>=1.3.11 # MIT SQLAlchemy>=1.3.11 # MIT
sqlalchemy-filters>=0.10.0
WebOb>=1.7.1 # MIT WebOb>=1.7.1 # MIT
python-heatclient>=1.10.0 # Apache-2.0 python-heatclient>=1.10.0 # Apache-2.0
python-keystoneclient>=3.8.0 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0

View File

@ -141,7 +141,7 @@ class CommonDbMixin(object):
for key, value in filters.items(): for key, value in filters.items():
column = getattr(model, key, None) column = getattr(model, key, None)
if column: if column:
query = query.filter(column.in_(value)) query = query.filter(column.in_([value]))
model_hooks = self._model_query_hooks.get(model, {}) model_hooks = self._model_query_hooks.get(model, {})
for _name, hooks in model_hooks.items(): for _name, hooks in model_hooks.items():
result_filter = hooks.get('result_filters', None) result_filter = hooks.get('result_filters', None)

View File

@ -160,8 +160,7 @@ class NfvoPluginDb(nfvo.NFVOPluginBase, db_base.CommonDbMixin):
{'placement_attr': vim.get('placement_attr')}) {'placement_attr': vim.get('placement_attr')})
vim_auth_db = (self._model_query( vim_auth_db = (self._model_query(
context, nfvo_db.VimAuth).filter( context, nfvo_db.VimAuth).filter(
nfvo_db.VimAuth.vim_id == vim_id).with_lockmode( nfvo_db.VimAuth.vim_id == vim_id).with_for_update().one())
'update').one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.VimNotFoundException(vim_id=vim_id) raise nfvo.VimNotFoundException(vim_id=vim_id)
vim_auth_db.update({'auth_cred': vim_cred, 'password': vim_auth_db.update({'auth_cred': vim_cred, 'password':
@ -181,7 +180,7 @@ class NfvoPluginDb(nfvo.NFVOPluginBase, db_base.CommonDbMixin):
with context.session.begin(subtransactions=True): with context.session.begin(subtransactions=True):
try: try:
vim_db = (self._model_query(context, nfvo_db.Vim).filter( vim_db = (self._model_query(context, nfvo_db.Vim).filter(
nfvo_db.Vim.id == vim_id).with_lockmode('update').one()) nfvo_db.Vim.id == vim_id).with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.VimNotFoundException(vim_id=vim_id) raise nfvo.VimNotFoundException(vim_id=vim_id)
vim_db.update({'status': status, vim_db.update({'status': status,

View File

@ -138,7 +138,7 @@ class NSPluginDb(network_service.NSPluginBase, db_base.CommonDbMixin):
self._model_query(context, NS). self._model_query(context, NS).
filter(NS.id == ns_id). filter(NS.id == ns_id).
filter(NS.status.in_(current_statuses)). filter(NS.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise network_service.NSNotFound(ns_id=ns_id) raise network_service.NSNotFound(ns_id=ns_id)
return ns_db return ns_db

View File

@ -1207,7 +1207,7 @@ class VnffgPluginDbMixin(vnffg.VNFFGPluginBase, db_base.CommonDbMixin):
self._model_query(context, Vnffg). self._model_query(context, Vnffg).
filter(Vnffg.id == vnffg_id). filter(Vnffg.id == vnffg_id).
filter(Vnffg.status.in_(current_statuses)). filter(Vnffg.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.VnffgNotFoundException(vnffg_id=vnffg_id) raise nfvo.VnffgNotFoundException(vnffg_id=vnffg_id)
if vnffg_db.status == constants.PENDING_UPDATE: if vnffg_db.status == constants.PENDING_UPDATE:
@ -1221,7 +1221,7 @@ class VnffgPluginDbMixin(vnffg.VNFFGPluginBase, db_base.CommonDbMixin):
self._model_query(context, VnffgNfp). self._model_query(context, VnffgNfp).
filter(VnffgNfp.id == nfp_id). filter(VnffgNfp.id == nfp_id).
filter(VnffgNfp.status.in_(current_statuses)). filter(VnffgNfp.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.NfpNotFoundException(nfp_id=nfp_id) raise nfvo.NfpNotFoundException(nfp_id=nfp_id)
if nfp_db.status == constants.PENDING_UPDATE: if nfp_db.status == constants.PENDING_UPDATE:
@ -1235,7 +1235,7 @@ class VnffgPluginDbMixin(vnffg.VNFFGPluginBase, db_base.CommonDbMixin):
self._model_query(context, VnffgChain). self._model_query(context, VnffgChain).
filter(VnffgChain.id == sfc_id). filter(VnffgChain.id == sfc_id).
filter(VnffgChain.status.in_(current_statuses)). filter(VnffgChain.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.SfcNotFoundException(sfc_id=sfc_id) raise nfvo.SfcNotFoundException(sfc_id=sfc_id)
if sfc_db.status == constants.PENDING_UPDATE: if sfc_db.status == constants.PENDING_UPDATE:
@ -1249,7 +1249,7 @@ class VnffgPluginDbMixin(vnffg.VNFFGPluginBase, db_base.CommonDbMixin):
self._model_query(context, VnffgClassifier). self._model_query(context, VnffgClassifier).
filter(VnffgClassifier.id == fc_id). filter(VnffgClassifier.id == fc_id).
filter(VnffgClassifier.status.in_(current_statuses)). filter(VnffgClassifier.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise nfvo.ClassifierNotFoundException(fc_id=fc_id) raise nfvo.ClassifierNotFoundException(fc_id=fc_id)
if fc_db.status == constants.PENDING_UPDATE: if fc_db.status == constants.PENDING_UPDATE:

View File

@ -491,7 +491,7 @@ class VNFMPluginDb(vnfm.VNFMPluginBase, db_base.CommonDbMixin):
self._model_query(context, VNF). self._model_query(context, VNF).
filter(VNF.id == vnf_id). filter(VNF.id == vnf_id).
filter(VNF.status.in_(current_statuses)). filter(VNF.status.in_(current_statuses)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
raise vnfm.VNFNotFound(vnf_id=vnf_id) raise vnfm.VNFNotFound(vnf_id=vnf_id)
return vnf_db return vnf_db
@ -799,7 +799,7 @@ class VNFMPluginDb(vnfm.VNFMPluginBase, db_base.CommonDbMixin):
self._model_query(context, VNF). self._model_query(context, VNF).
filter(VNF.id == vnf_id). filter(VNF.id == vnf_id).
filter(~VNF.status.in_(exclude_status)). filter(~VNF.status.in_(exclude_status)).
with_lockmode('update').one()) with_for_update().one())
except orm_exc.NoResultFound: except orm_exc.NoResultFound:
LOG.warning('no vnf found %s', vnf_id) LOG.warning('no vnf found %s', vnf_id)
return False return False

65
tacker/objects/common.py Normal file
View File

@ -0,0 +1,65 @@
# Copyright (C) 2021 NEC Corp
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tacker.db.db_sqlalchemy import models
def apply_filters(query, filters):
"""Apply filters to a SQLAlchemy query.
:param query: The query object to which we apply filters.
:param filters: A dict or an iterable of dicts, where each one includes
the necesary information to create a filter to be applied
to the query. There are single query filters, such as
filters = {'model': 'Foo', 'field': 'name', 'op': '==',
'value': 'foo'}. And multiple query filters, such as
filters = {'and': [
{'field': 'name', 'model': 'Foo', 'value': 'foo',
'op': '=='},
{'field': 'id', 'model': 'Bar', 'value': 'bar',
'op': '=='}
]}
"""
def apply_filter(query, filter):
value = filter.get('value')
op = filter.get('op')
model = getattr(models, filter.get('model'))
column_attr = getattr(model, filter.get('field'))
if 'in' == op:
query = query.filter(column_attr.in_(value))
elif 'not_in' == op:
query = query.filter(~column_attr.in_(value))
elif '!=' == op:
query = query.filter(column_attr != value)
elif '>' == op:
query = query.filter(column_attr > value)
elif '>=' == op:
query = query.filter(column_attr >= value)
elif '<' == op:
query = query.filter(column_attr < value)
elif '<=' == op:
query = query.filter(column_attr <= value)
elif '==' == op:
query = query.filter(column_attr == value)
return query
if 'and' in filters:
for filter in filters.get('and'):
query = apply_filter(query, filter)
else:
query = apply_filter(query, filters)
return query

View File

@ -20,7 +20,6 @@ from oslo_utils import uuidutils
from oslo_versionedobjects import base as ovoo_base from oslo_versionedobjects import base as ovoo_base
from sqlalchemy import exc from sqlalchemy import exc
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
from sqlalchemy_filters import apply_filters
from tacker._i18n import _ from tacker._i18n import _
from tacker.common import exceptions from tacker.common import exceptions
@ -31,6 +30,7 @@ from tacker.db.db_sqlalchemy import models
from tacker.db.vnfm import vnfm_db from tacker.db.vnfm import vnfm_db
from tacker import objects from tacker import objects
from tacker.objects import base from tacker.objects import base
from tacker.objects import common
from tacker.objects import fields from tacker.objects import fields
from tacker.objects import vnf_instantiated_info from tacker.objects import vnf_instantiated_info
from tacker.objects import vnf_package as vnf_package_obj from tacker.objects import vnf_package as vnf_package_obj
@ -120,7 +120,7 @@ def _vnf_instance_list_by_filter(context, columns_to_join=None,
query = query.options(joinedload(column)) query = query.options(joinedload(column))
if filters: if filters:
query = apply_filters(query, filters) query = common.apply_filters(query, filters)
return query.all() return query.all()

View File

@ -18,7 +18,6 @@ from oslo_utils import timeutils
from oslo_versionedobjects import base as ovoo_base from oslo_versionedobjects import base as ovoo_base
from sqlalchemy import exc from sqlalchemy import exc
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
from sqlalchemy_filters import apply_filters
from tacker.common import exceptions from tacker.common import exceptions
from tacker.common import utils from tacker.common import utils
@ -27,6 +26,7 @@ from tacker.db.db_sqlalchemy import api
from tacker.db.db_sqlalchemy import models from tacker.db.db_sqlalchemy import models
from tacker import objects from tacker import objects
from tacker.objects import base from tacker.objects import base
from tacker.objects import common
from tacker.objects import fields from tacker.objects import fields
@ -35,9 +35,29 @@ LOG = logging.getLogger(__name__)
@db_api.context_manager.writer @db_api.context_manager.writer
def _vnf_lcm_op_occ_create(context, values): def _vnf_lcm_op_occ_create(context, values):
fields = {
'id': values.id,
'operation_state': values.operation_state,
'state_entered_time': values.state_entered_time,
'start_time': values.start_time,
'vnf_instance_id': values.vnf_instance_id,
'operation': values.operation,
'is_automatic_invocation': values.is_automatic_invocation,
'is_cancel_pending': values.is_cancel_pending,
'error': values.error,
'resource_changes': values.resource_changes,
'changed_info': values.changed_info,
'changed_ext_connectivity': values.changed_ext_connectivity,
'error_point': values.error_point,
}
if 'grant_id' in values:
fields['grant_id'] = values.grant_id
if 'operation_params' in values:
fields['operation_params'] = values.operation_params
context.session.execute( context.session.execute(
models.VnfLcmOpOccs.__table__.insert(None), models.VnfLcmOpOccs.__table__.insert(None),
values) fields)
@db_api.context_manager.writer @db_api.context_manager.writer
@ -125,7 +145,7 @@ def _vnf_lcm_op_occs_get_by_filters(context, read_deleted=None,
read_deleted=read_deleted, project_only=True) read_deleted=read_deleted, project_only=True)
if filters: if filters:
query = apply_filters(query, filters) query = common.apply_filters(query, filters)
return query.all() return query.all()

View File

@ -16,7 +16,6 @@ from oslo_utils import timeutils
from oslo_versionedobjects import base as ovoo_base from oslo_versionedobjects import base as ovoo_base
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
from sqlalchemy.sql import text from sqlalchemy.sql import text
from sqlalchemy_filters import apply_filters
from tacker.common import exceptions from tacker.common import exceptions
from tacker.common import utils from tacker.common import utils
@ -27,6 +26,7 @@ from tacker.db.db_sqlalchemy import api
from tacker.db.db_sqlalchemy import models from tacker.db.db_sqlalchemy import models
from tacker import objects from tacker import objects
from tacker.objects import base from tacker.objects import base
from tacker.objects import common
from tacker.objects import fields from tacker.objects import fields
_NO_DATA_SENTINEL = object() _NO_DATA_SENTINEL = object()
@ -341,7 +341,7 @@ def _vnf_lcm_subscription_list_by_filters(context,
converted_value = utils.str_to_bytes(filters['value']) converted_value = utils.str_to_bytes(filters['value'])
filters.update({'value': converted_value}) filters.update({'value': converted_value})
query = apply_filters(query, filters) query = common.apply_filters(query, filters)
if nextpage_opaque_marker: if nextpage_opaque_marker:
start_offset = CONF.vnf_lcm.subscription_num * nextpage_opaque_marker start_offset = CONF.vnf_lcm.subscription_num * nextpage_opaque_marker

View File

@ -22,7 +22,6 @@ from oslo_utils import versionutils
from oslo_versionedobjects import base as ovoo_base from oslo_versionedobjects import base as ovoo_base
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
from sqlalchemy.sql import func from sqlalchemy.sql import func
from sqlalchemy_filters import apply_filters
from tacker._i18n import _ from tacker._i18n import _
from tacker.common import exceptions from tacker.common import exceptions
@ -32,6 +31,7 @@ from tacker.db.db_sqlalchemy import api
from tacker.db.db_sqlalchemy import models from tacker.db.db_sqlalchemy import models
from tacker import objects from tacker import objects
from tacker.objects import base from tacker.objects import base
from tacker.objects import common
from tacker.objects import fields from tacker.objects import fields
from tacker.objects import vnf_artifact from tacker.objects import vnf_artifact
from tacker.objects import vnf_software_image from tacker.objects import vnf_software_image
@ -185,7 +185,10 @@ def _vnf_package_list_by_filters(context, read_deleted=None, filters=None):
if 'VnfPackageArtifactInfo' in filter_data: if 'VnfPackageArtifactInfo' in filter_data:
query = query.join(models.VnfPackageArtifactInfo) query = query.join(models.VnfPackageArtifactInfo)
query = apply_filters(query, filters) if 'VnfPackageVnfd' in filter_data:
query = query.join(models.VnfPackageVnfd)
query = common.apply_filters(query, filters)
return query.all() return query.all()
@ -232,7 +235,10 @@ def _destroy_vnf_package(context, package_uuid):
software_images_query.subquery())).update( software_images_query.subquery())).update(
updated_values, synchronize_session=False) updated_values, synchronize_session=False)
software_images_query.update(updated_values, synchronize_session=False) api.model_query(context, models.VnfSoftwareImage). \
filter(models.VnfSoftwareImage.id.in_(
flavour_query.subquery())).update(
updated_values, synchronize_session=False)
api.model_query(context, models.VnfPackageUserData). \ api.model_query(context, models.VnfPackageUserData). \
filter_by(package_uuid=package_uuid). \ filter_by(package_uuid=package_uuid). \