Add online migration to move quotas to API database
This moves quota limits and classes from the main database to the API database in an online migration. Part of blueprint cells-quota-api-db Change-Id: I64b600b30f6e54db0ec9083c6c176e895c6d0cc2
This commit is contained in:
melanie witt
Matt Riedemann
parent
b31062803e
commit
b291061cb1
@ -90,6 +90,7 @@ from nova.objects import host_mapping as host_mapping_obj
|
||||
from nova.objects import instance as instance_obj
|
||||
from nova.objects import instance_group as instance_group_obj
|
||||
from nova.objects import keypair as keypair_obj
|
||||
from nova.objects import quotas as quotas_obj
|
||||
from nova.objects import request_spec
|
||||
from nova import quota
|
||||
from nova import rpc
|
||||
@ -646,6 +647,10 @@ class DbCommands(object):
|
||||
build_request_obj.delete_build_requests_with_no_instance_uuid,
|
||||
# Added in Pike
|
||||
db.service_uuids_online_data_migration,
|
||||
# Added in Pike
|
||||
quotas_obj.migrate_quota_limits_to_api_db,
|
||||
# Added in Pike
|
||||
quotas_obj.migrate_quota_classes_to_api_db,
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
|
||||
@ -19,6 +19,7 @@ from oslo_db import exception as db_exc
|
||||
from nova import db
|
||||
from nova.db.sqlalchemy import api as db_api
|
||||
from nova.db.sqlalchemy import api_models
|
||||
from nova.db.sqlalchemy import models as main_models
|
||||
from nova import exception
|
||||
from nova.objects import base
|
||||
from nova.objects import fields
|
||||
@ -515,3 +516,131 @@ class QuotasNoOp(Quotas):
|
||||
|
||||
def check_deltas(cls, context, deltas, *count_args, **count_kwargs):
|
||||
pass
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_per_project_limits(context, limit):
|
||||
return context.session.query(main_models.Quota).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_per_user_limits(context, limit):
|
||||
return context.session.query(main_models.ProjectUserQuota).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_per_project_limits(context, project_id, resource):
|
||||
context.session.query(main_models.Quota).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(project_id=project_id).\
|
||||
filter_by(resource=resource).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_per_user_limits(context, project_id, resource, user_id):
|
||||
context.session.query(main_models.ProjectUserQuota).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(project_id=project_id).\
|
||||
filter_by(user_id=user_id).\
|
||||
filter_by(resource=resource).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.api_context_manager.writer
|
||||
def _create_limits_in_api_db(context, db_limits, per_user=False):
|
||||
for db_limit in db_limits:
|
||||
user_id = db_limit.user_id if per_user else None
|
||||
Quotas._create_limit_in_db(context, db_limit.project_id,
|
||||
db_limit.resource, db_limit.hard_limit,
|
||||
user_id=user_id)
|
||||
|
||||
|
||||
def migrate_quota_limits_to_api_db(context, count):
|
||||
# Migrate per project limits
|
||||
main_per_project_limits = _get_main_per_project_limits(context, count)
|
||||
done = 0
|
||||
try:
|
||||
# Create all the limits in a single transaction.
|
||||
_create_limits_in_api_db(context, main_per_project_limits)
|
||||
except exception.QuotaExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# limits were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the limits separately.
|
||||
for db_limit in main_per_project_limits:
|
||||
_destroy_main_per_project_limits(context, db_limit.project_id,
|
||||
db_limit.resource)
|
||||
done += 1
|
||||
if done == count:
|
||||
return len(main_per_project_limits), done
|
||||
# Migrate per user limits
|
||||
count -= done
|
||||
main_per_user_limits = _get_main_per_user_limits(context, count)
|
||||
try:
|
||||
# Create all the limits in a single transaction.
|
||||
_create_limits_in_api_db(context, main_per_user_limits, per_user=True)
|
||||
except exception.QuotaExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# limits were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the limits separately.
|
||||
for db_limit in main_per_user_limits:
|
||||
_destroy_main_per_user_limits(context, db_limit.project_id,
|
||||
db_limit.resource, db_limit.user_id)
|
||||
done += 1
|
||||
return len(main_per_project_limits) + len(main_per_user_limits), done
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_quota_classes(context, limit):
|
||||
return context.session.query(main_models.QuotaClass).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_quota_classes(context, db_classes):
|
||||
for db_class in db_classes:
|
||||
context.session.query(main_models.QuotaClass).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(id=db_class.id).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.api_context_manager.writer
|
||||
def _create_classes_in_api_db(context, db_classes):
|
||||
for db_class in db_classes:
|
||||
Quotas._create_class_in_db(context, db_class.class_name,
|
||||
db_class.resource, db_class.hard_limit)
|
||||
|
||||
|
||||
def migrate_quota_classes_to_api_db(context, count):
|
||||
main_quota_classes = _get_main_quota_classes(context, count)
|
||||
done = 0
|
||||
try:
|
||||
# Create all the classes in a single transaction.
|
||||
_create_classes_in_api_db(context, main_quota_classes)
|
||||
except exception.QuotaClassExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# classes were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the classes in a single transaction.
|
||||
_destroy_main_quota_classes(context, main_quota_classes)
|
||||
found = done = len(main_quota_classes)
|
||||
return found, done
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
# under the License.
|
||||
|
||||
from nova import context
|
||||
from nova.db.sqlalchemy import api as db_api
|
||||
from nova import exception
|
||||
from nova.objects import quotas
|
||||
from nova import test
|
||||
@ -206,3 +207,113 @@ class QuotasObjectTestCase(test.TestCase,
|
||||
self.assertEqual('foo', limits_dict['class_name'])
|
||||
self.assertEqual(5, limits_dict['instances'])
|
||||
self.assertEqual(10, limits_dict['cores'])
|
||||
|
||||
def test_migrate_quota_limits(self):
|
||||
# Create a limit in api db
|
||||
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
|
||||
'instances', 5, user_id='fake-user')
|
||||
# Create 4 limits in main db
|
||||
db_api.quota_create(self.context, 'fake-project', 'cores', 10,
|
||||
user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'ram', 8192,
|
||||
user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'fixed_ips', 10)
|
||||
db_api.quota_create(self.context, 'fake-project', 'floating_ips', 10)
|
||||
|
||||
# Migrate with a count/limit of 3
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(self.context, 3)
|
||||
self.assertEqual(3, total)
|
||||
self.assertEqual(3, done)
|
||||
|
||||
# This only fetches from the api db. There should now be 4 limits.
|
||||
api_user_limits = quotas.Quotas._get_all_from_db(self.context,
|
||||
'fake-project')
|
||||
api_proj_limits_dict = quotas.Quotas._get_all_from_db_by_project(
|
||||
self.context, 'fake-project')
|
||||
api_proj_limits_dict.pop('project_id', None)
|
||||
self.assertEqual(4,
|
||||
len(api_user_limits) + len(api_proj_limits_dict))
|
||||
|
||||
# This only fetches from the main db. There should be one left.
|
||||
main_user_limits = db_api.quota_get_all(self.context, 'fake-project')
|
||||
main_proj_limits_dict = db_api.quota_get_all_by_project(self.context,
|
||||
'fake-project')
|
||||
main_proj_limits_dict.pop('project_id', None)
|
||||
self.assertEqual(1, len(main_user_limits) + len(main_proj_limits_dict))
|
||||
|
||||
self.assertEqual((1, 1),
|
||||
quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100))
|
||||
self.assertEqual((0, 0),
|
||||
quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100))
|
||||
|
||||
def test_migrate_quota_limits_skips_existing(self):
|
||||
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
|
||||
'instances', 5, user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'instances', 5,
|
||||
user_id='fake-user')
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(1, total)
|
||||
self.assertEqual(1, done)
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(0, total)
|
||||
self.assertEqual(0, done)
|
||||
self.assertEqual(1, len(quotas.Quotas._get_all_from_db(
|
||||
self.context, 'fake-project')))
|
||||
|
||||
def test_migrate_quota_classes(self):
|
||||
# Create a class in api db
|
||||
quotas.Quotas._create_class_in_db(self.context, 'foo', 'instances', 5)
|
||||
# Create 3 classes in main db
|
||||
db_api.quota_class_create(self.context, 'foo', 'cores', 10)
|
||||
db_api.quota_class_create(self.context, db_api._DEFAULT_QUOTA_NAME,
|
||||
'instances', 10)
|
||||
db_api.quota_class_create(self.context, 'foo', 'ram', 8192)
|
||||
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(self.context, 2)
|
||||
self.assertEqual(2, total)
|
||||
self.assertEqual(2, done)
|
||||
|
||||
# This only fetches from the api db
|
||||
api_foo_dict = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, 'foo')
|
||||
api_foo_dict.pop('class_name', None)
|
||||
api_default_dict = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, db_api._DEFAULT_QUOTA_NAME)
|
||||
api_default_dict.pop('class_name', None)
|
||||
self.assertEqual(3,
|
||||
len(api_foo_dict) + len(api_default_dict))
|
||||
|
||||
# This only fetches from the main db
|
||||
main_foo_dict = db_api.quota_class_get_all_by_name(self.context, 'foo')
|
||||
main_foo_dict.pop('class_name', None)
|
||||
main_default_dict = db_api.quota_class_get_default(self.context)
|
||||
main_default_dict.pop('class_name', None)
|
||||
self.assertEqual(1, len(main_foo_dict) + len(main_default_dict))
|
||||
|
||||
self.assertEqual((1, 1),
|
||||
quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100))
|
||||
self.assertEqual((0, 0),
|
||||
quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100))
|
||||
|
||||
def test_migrate_quota_classes_skips_existing(self):
|
||||
quotas.Quotas._create_class_in_db(self.context, 'foo-class',
|
||||
'instances', 5)
|
||||
db_api.quota_class_create(self.context, 'foo-class', 'instances', 7)
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(1, total)
|
||||
self.assertEqual(1, done)
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(0, total)
|
||||
self.assertEqual(0, done)
|
||||
# Existing class should not be overwritten in the result
|
||||
db_class = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, 'foo-class')
|
||||
self.assertEqual(5, db_class['instances'])
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
---
|
||||
upgrade:
|
||||
- |
|
||||
Quota limits and classes are being moved to the API database for Cells v2.
|
||||
In this release, the online data migrations will move any quota limits and
|
||||
classes you have in your main database to the API database, retaining all
|
||||
attributes.
|
||||
|
||||
.. note:: Quota limits and classes can no longer be soft-deleted as the API
|
||||
database does not replicate the legacy soft-delete functionality from the
|
||||
main database. As such, deleted quota limits and classes are not migrated
|
||||
and the behavior users will experience will be the same as if a purge of
|
||||
deleted records was performed.
|
||||
Loading…
Reference in New Issue
Block a user