Merge "db: Remove final users of 'get_session'"

This commit is contained in:
Zuul 2022-06-09 17:04:18 +00:00 committed by Gerrit Code Review
commit 872b7fa3d9
5 changed files with 153 additions and 131 deletions

View File

@ -372,54 +372,63 @@ class QuotaCommands(object):
""" """
self._check_sync(project_id, do_fix=True) self._check_sync(project_id, do_fix=True)
def _get_quota_projects(self, ctxt, project_id): @db_api.main_context_manager.reader
def _get_quota_projects(self, context, project_id):
"""Get project ids that have quota_usage entries.""" """Get project ids that have quota_usage entries."""
if project_id: if project_id:
model = models.QuotaUsage model = models.QuotaUsage
session = db_api.get_session()
# If the project does not exist # If the project does not exist
if not session.query(db_api.sql.exists().where( if not context.session.query(
db_api.and_(model.project_id == project_id, db_api.sql.exists()
~model.deleted))).scalar(): .where(
print('Project id %s has no quota usage. Nothing to do.' % db_api.and_(
project_id) model.project_id == project_id,
~model.deleted,
),
)
).scalar():
print(
'Project id %s has no quota usage. Nothing to do.' %
project_id,
)
return [] return []
return [project_id] return [project_id]
projects = db_api.model_query(context, projects = db_api.model_query(
models.QuotaUsage, context,
read_deleted="no").\ models.QuotaUsage,
with_entities('project_id').\ read_deleted="no"
distinct().\ ).with_entities('project_id').distinct().all()
all()
project_ids = [row.project_id for row in projects] project_ids = [row.project_id for row in projects]
return project_ids return project_ids
def _get_usages(self, ctxt, session, resources, project_id): def _get_usages(self, context, resources, project_id):
"""Get data necessary to check out of sync quota usage. """Get data necessary to check out of sync quota usage.
Returns a list QuotaUsage instances for the specific project Returns a list QuotaUsage instances for the specific project
""" """
usages = db_api.model_query(ctxt, usages = db_api.model_query(
db_api.models.QuotaUsage, context,
read_deleted="no", db_api.models.QuotaUsage,
session=session).\ read_deleted="no",
filter_by(project_id=project_id).\ ).filter_by(project_id=project_id).with_for_update().all()
with_for_update().\
all()
return usages return usages
def _get_reservations(self, ctxt, session, project_id, usage_id): def _get_reservations(self, context, project_id, usage_id):
"""Get reservations for a given project and usage id.""" """Get reservations for a given project and usage id."""
reservations = db_api.model_query(ctxt, models.Reservation, reservations = (
read_deleted="no", db_api.model_query(
session=session).\ context,
filter_by(project_id=project_id, usage_id=usage_id).\ models.Reservation,
with_for_update().\ read_deleted="no",
all() )
.filter_by(project_id=project_id, usage_id=usage_id)
.with_for_update()
.all()
)
return reservations return reservations
def _check_duplicates(self, ctxt, session, usages, do_fix): def _check_duplicates(self, context, usages, do_fix):
"""Look for duplicated quota used entries (bug#1484343) """Look for duplicated quota used entries (bug#1484343)
If we have duplicates and we are fixing them, then we reassign the If we have duplicates and we are fixing them, then we reassign the
@ -443,15 +452,17 @@ class QuotaCommands(object):
# Each of the duplicates can have reservations # Each of the duplicates can have reservations
reassigned = 0 reassigned = 0
for usage in resource_usages[1:]: for usage in resource_usages[1:]:
reservations = self._get_reservations(ctxt, session, reservations = self._get_reservations(
usage.project_id, context,
usage.id) usage.project_id,
usage.id,
)
reassigned += len(reservations) reassigned += len(reservations)
for reservation in reservations: for reservation in reservations:
reservation.usage_id = keep_usage.id reservation.usage_id = keep_usage.id
keep_usage.in_use += usage.in_use keep_usage.in_use += usage.in_use
keep_usage.reserved += usage.reserved keep_usage.reserved += usage.reserved
usage.delete(session=session) usage.delete(context.session)
print('duplicates removed & %s reservations reassigned' % print('duplicates removed & %s reservations reassigned' %
reassigned) reassigned)
else: else:
@ -471,59 +482,83 @@ class QuotaCommands(object):
# projects removed will just turn nothing on the quota usage. # projects removed will just turn nothing on the quota usage.
projects = self._get_quota_projects(ctxt, project_id) projects = self._get_quota_projects(ctxt, project_id)
session = db_api.get_session()
action_msg = ' - fixed' if do_fix else ''
discrepancy = False discrepancy = False
# NOTE: It's important to always get the quota first and then the # NOTE: It's important to always get the quota first and then the
# reservations to prevent deadlocks with quota commit and rollback from # reservations to prevent deadlocks with quota commit and rollback from
# running Cinder services. # running Cinder services.
for project in projects: for project in projects:
with session.begin(): discrepancy &= self._check_project_sync(
print('Processing quota usage for project %s' % project) ctxt,
# We only want to sync existing quota usage rows project,
usages = self._get_usages(ctxt, session, resources, project) do_fix,
resources,
)
# Check for duplicated entries (bug#1484343)
usages, duplicates_found = self._check_duplicates(ctxt,
session,
usages,
do_fix)
if duplicates_found:
discrepancy = True
# Check quota and reservations
for usage in usages:
resource_name = usage.resource
# Get the correct value for this quota usage resource
updates = db_api._get_sync_updates(ctxt, project, session,
resources,
resource_name)
in_use = updates[resource_name]
if in_use != usage.in_use:
print('\t%s: invalid usage saved=%s actual=%s%s' %
(resource_name, usage.in_use, in_use,
action_msg))
discrepancy = True
if do_fix:
usage.in_use = in_use
reservations = self._get_reservations(ctxt, session,
project, usage.id)
num_reservations = sum(r.delta for r in reservations
if r.delta > 0)
if num_reservations != usage.reserved:
print('\t%s: invalid reserved saved=%s actual=%s%s' %
(resource_name, usage.reserved,
num_reservations, action_msg))
discrepancy = True
if do_fix:
usage.reserved = num_reservations
print('Action successfully completed') print('Action successfully completed')
return discrepancy return discrepancy
@db_api.main_context_manager.reader
def _check_project_sync(self, context, project, do_fix, resources):
print('Processing quota usage for project %s' % project)
discrepancy = False
action_msg = ' - fixed' if do_fix else ''
# We only want to sync existing quota usage rows
usages = self._get_usages(context, resources, project)
# Check for duplicated entries (bug#1484343)
usages, duplicates_found = self._check_duplicates(
context, usages, do_fix,
)
if duplicates_found:
discrepancy = True
# Check quota and reservations
for usage in usages:
resource_name = usage.resource
# Get the correct value for this quota usage resource
updates = db_api._get_sync_updates(
context,
project,
resources,
resource_name,
)
in_use = updates[resource_name]
if in_use != usage.in_use:
print(
'\t%s: invalid usage saved=%s actual=%s%s' %
(resource_name, usage.in_use, in_use, action_msg)
)
discrepancy = True
if do_fix:
usage.in_use = in_use
reservations = self._get_reservations(
context,
project,
usage.id,
)
num_reservations = sum(
r.delta for r in reservations if r.delta > 0
)
if num_reservations != usage.reserved:
print(
'\t%s: invalid reserved saved=%s actual=%s%s' %
(
resource_name,
usage.reserved,
num_reservations,
action_msg,
)
)
discrepancy = True
if do_fix:
usage.reserved = num_reservations
return discrepancy
class VersionCommands(object): class VersionCommands(object):
"""Class for exposing the codebase version.""" """Class for exposing the codebase version."""

View File

@ -86,13 +86,7 @@ def configure(conf):
def get_engine(): def get_engine():
return main_context_manager._factory.get_legacy_facade().get_engine() return main_context_manager.writer.get_engine()
def get_session(**kwargs):
return main_context_manager._factory.get_legacy_facade().get_session(
**kwargs
)
def dispose_engine(): def dispose_engine():

View File

@ -100,21 +100,30 @@ class DBCommonFilterTestCase(BaseTest):
@mock.patch('sqlalchemy.orm.query.Query.filter') @mock.patch('sqlalchemy.orm.query.Query.filter')
def test__process_model_like_filter(self, mock_filter): def test__process_model_like_filter(self, mock_filter):
filters = {'display_name': 'fake_name', filters = {
'display_description': 'fake_description', 'display_name': 'fake_name',
'host': 123, 'display_description': 'fake_description',
'status': []} 'host': 123,
session = sqlalchemy_api.get_session() 'status': [],
query = session.query(models.Volume) }
with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
query = self.ctxt.session.query(models.Volume)
mock_filter.return_value = query mock_filter.return_value = query
with mock.patch.object(operators.Operators, 'op') as mock_op: with mock.patch.object(operators.Operators, 'op') as mock_op:
def fake_operator(value): def fake_operator(value):
return value return value
mock_op.return_value = fake_operator mock_op.return_value = fake_operator
sqlalchemy_api._process_model_like_filter(models.Volume, sqlalchemy_api._process_model_like_filter(
query, filters) models.Volume, query, filters,
calls = [call('%fake_description%'), )
call('%fake_name%'), call('%123%')] calls = [
call('%fake_description%'),
call('%fake_name%'),
call('%123%'),
]
mock_filter.assert_has_calls(calls, any_order=True) mock_filter.assert_has_calls(calls, any_order=True)
@ddt.data({'handler': [db.volume_create, db.volume_get_all], @ddt.data({'handler': [db.volume_create, db.volume_get_all],
@ -2395,11 +2404,9 @@ class DBAPIVolumeTypeTestCase(BaseTest):
# NOTE(dulek): Bug 1496747 uncovered problems when deleting accesses # NOTE(dulek): Bug 1496747 uncovered problems when deleting accesses
# with id column higher than 128. This is regression test for that # with id column higher than 128. This is regression test for that
# case. # case.
with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
session = sqlalchemy_api.get_session() vta.id = 150
vta.id = 150 vta.save(self.ctxt.session)
vta.save(session=session)
session.close()
db.volume_type_access_remove(self.ctxt, vt['id'], 'fake_project') db.volume_type_access_remove(self.ctxt, vt['id'], 'fake_project')
vtas = db.volume_type_access_get_all(self.ctxt, vt['id']) vtas = db.volume_type_access_get_all(self.ctxt, vt['id'])
@ -2602,11 +2609,10 @@ class DBAPIReservationTestCase(BaseTest):
reservations = _quota_reserve(self.ctxt, project, volumes=2) reservations = _quota_reserve(self.ctxt, project, volumes=2)
# Force a smaller reserved value in quota_usages table # Force a smaller reserved value in quota_usages table
session = sqlalchemy_api.get_session() vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
with session.begin(): with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
vol_usage.reserved -= 1 vol_usage.reserved -= 1
vol_usage.save(session=session) vol_usage.save(self.ctxt.session)
# When committing 2 volumes from reserved to used reserved should not # When committing 2 volumes from reserved to used reserved should not
# go from 1 to -1 but from 1 to 0, but in-use should still increase by # go from 1 to -1 but from 1 to 0, but in-use should still increase by
@ -2623,11 +2629,10 @@ class DBAPIReservationTestCase(BaseTest):
reservations = _quota_reserve(self.ctxt, project, volumes=-2) reservations = _quota_reserve(self.ctxt, project, volumes=-2)
# Force a smaller in_use than the one the reservation will decrease # Force a smaller in_use than the one the reservation will decrease
session = sqlalchemy_api.get_session() vol_usage = db.quota_usage_get(self.ctxt, 'project1', 'volumes')
with session.begin(): with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
vol_usage = db.quota_usage_get(self.ctxt, 'project1', 'volumes')
vol_usage.in_use = 1 vol_usage.in_use = 1
vol_usage.save(session=session) vol_usage.save(self.ctxt.session)
# When committing -2 volumes from reserved to in-use they should not # When committing -2 volumes from reserved to in-use they should not
# make in-use go from 1 to -1, but from 1 to 0 # make in-use go from 1 to -1, but from 1 to 0
@ -2663,11 +2668,10 @@ class DBAPIReservationTestCase(BaseTest):
reservations = _quota_reserve(self.ctxt, project, volumes=2) reservations = _quota_reserve(self.ctxt, project, volumes=2)
# Force a smaller reserved value in quota_usages table # Force a smaller reserved value in quota_usages table
session = sqlalchemy_api.get_session() vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
with session.begin(): with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
vol_usage.reserved -= 1 vol_usage.reserved -= 1
vol_usage.save(session=session) vol_usage.save(self.ctxt.session)
# When rolling back 2 volumes from reserved when there's only 1 in the # When rolling back 2 volumes from reserved when there's only 1 in the
# quota usage's reserved field, reserved should not go from 1 to -1 # quota usage's reserved field, reserved should not go from 1 to -1
@ -2698,11 +2702,10 @@ class DBAPIReservationTestCase(BaseTest):
_quota_reserve(self.ctxt, project, volumes=2) _quota_reserve(self.ctxt, project, volumes=2)
# Force a smaller reserved value in quota_usages table # Force a smaller reserved value in quota_usages table
session = sqlalchemy_api.get_session() vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
with session.begin(): with sqlalchemy_api.main_context_manager.writer.using(self.ctxt):
vol_usage = db.quota_usage_get(self.ctxt, project, 'volumes')
vol_usage.reserved -= 1 vol_usage.reserved -= 1
vol_usage.save(session=session) vol_usage.save(self.ctxt.session)
# When expiring 2 volumes from reserved when there's only 1 in the # When expiring 2 volumes from reserved when there's only 1 in the
# quota usage's reserved field, reserved should not go from 1 to -1 # quota usage's reserved field, reserved should not go from 1 to -1
@ -3537,12 +3540,7 @@ class DBAPIGenericTestCase(BaseTest):
# one is not being used) to confirm that the DB exists subquery is # one is not being used) to confirm that the DB exists subquery is
# properly formulated and doesn't result in multiple rows, as such # properly formulated and doesn't result in multiple rows, as such
# case would raise an exception when converting the result to an # case would raise an exception when converting the result to an
# scalar. This would happen if for example the query wasn't generated # scalar.
# directly using get_session but using model_query like this:
# query = model_query(context, model,
# sql.exists().where(and_(*conditions)))
# Instead of what we do:
# query = get_session().query(sql.exists().where(and_(*conditions)))
db.volume_create(self.ctxt, {'id': fake.VOLUME_ID, db.volume_create(self.ctxt, {'id': fake.VOLUME_ID,
'volume_type_id': fake.VOLUME_TYPE_ID}) 'volume_type_id': fake.VOLUME_TYPE_ID})
db.volume_create(self.ctxt, {'id': fake.VOLUME2_ID, db.volume_create(self.ctxt, {'id': fake.VOLUME2_ID,
@ -3647,8 +3645,7 @@ class EngineFacadeTestCase(BaseTest):
self.project_id = fake.PROJECT_ID self.project_id = fake.PROJECT_ID
self.context = context.RequestContext(self.user_id, self.project_id) self.context = context.RequestContext(self.user_id, self.project_id)
@mock.patch.object(sqlalchemy_api, 'get_session') def test_use_single_context_session_writer(self):
def test_use_single_context_session_writer(self, mock_get_session):
# Checks that session in context would not be overwritten by # Checks that session in context would not be overwritten by
# annotation @sqlalchemy_api.main_context_manager.writer if annotation # annotation @sqlalchemy_api.main_context_manager.writer if annotation
# is used twice. # is used twice.
@ -3667,8 +3664,7 @@ class EngineFacadeTestCase(BaseTest):
parent_session, child_session = fake_parent_method(self.context) parent_session, child_session = fake_parent_method(self.context)
self.assertEqual(parent_session, child_session) self.assertEqual(parent_session, child_session)
@mock.patch.object(sqlalchemy_api, 'get_session') def test_use_single_context_session_reader(self):
def test_use_single_context_session_reader(self, mock_get_session):
# Checks that session in context would not be overwritten by # Checks that session in context would not be overwritten by
# annotation @sqlalchemy_api.main_context_manager.reader if annotation # annotation @sqlalchemy_api.main_context_manager.reader if annotation
# is used twice. # is used twice.

View File

@ -1392,9 +1392,6 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase):
self.usages_created = {} self.usages_created = {}
self.reservations_created = {} self.reservations_created = {}
def fake_get_session():
return FakeSession()
def fake_get_quota_usages(context, project_id, resources=None): def fake_get_quota_usages(context, project_id, resources=None):
return self.usages.copy() return self.usages.copy()
@ -1418,8 +1415,6 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase):
return reservation_ref return reservation_ref
self.mock_object(sqa_api, 'get_session',
fake_get_session)
self.mock_object(sqa_api, '_get_quota_usages', self.mock_object(sqa_api, '_get_quota_usages',
fake_get_quota_usages) fake_get_quota_usages)
self.mock_object(sqa_api, '_quota_usage_create', self.mock_object(sqa_api, '_quota_usage_create',

View File

@ -217,8 +217,10 @@ class VolumeTypeTestCase(test.TestCase):
def test_get_all_volume_types(self): def test_get_all_volume_types(self):
"""Ensures that all volume types can be retrieved.""" """Ensures that all volume types can be retrieved."""
session = db_api.get_session() with db_api.main_context_manager.writer.using(self.ctxt):
total_volume_types = session.query(models.VolumeType).count() total_volume_types = self.ctxt.session.query(
models.VolumeType,
).count()
vol_types = volume_types.get_all_types(self.ctxt) vol_types = volume_types.get_all_types(self.ctxt)
self.assertEqual(total_volume_types, len(vol_types)) self.assertEqual(total_volume_types, len(vol_types))