openstack/nova
Code Issues Proposed changes

Move resource usage sync functions to db backend

Browse Source 
Resource usage sync functions was declared in nova/quota.py, and
using db.api public methods. This functions was moved to database
backend implementation, so now sync functions can use private
methods of database backend, and session attribute can be removed
from this public methods, and even some public methods can be removed.

Blueprint: db-session-cleanup

Change-Id: I63eeb18b8ede4b4263ee553d4e70633de463e0c4
This commit is contained in:
Sergey Skripnick
2013-06-07 16:09:45 +03:00
committed by Boris Pavlovic
parent 536f37906d
commit 3bda4c0585
5 changed files with 135 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
nova/db/api.py
View File

@@ -272,12 +272,6 @@ def floating_ip_create(context, values):
return IMPL.floating_ip_create(context, values)
def floating_ip_count_by_project(context, project_id, session=None):
"""Count floating ips used by project."""
return IMPL.floating_ip_count_by_project(context, project_id,
session=session)
def floating_ip_deallocate(context, address):
"""Deallocate a floating ip by address."""
return IMPL.floating_ip_deallocate(context, address)
@@ -521,11 +515,6 @@ def fixed_ip_update(context, address, values):
return IMPL.fixed_ip_update(context, address, values)
def fixed_ip_count_by_project(context, project_id, session=None):
"""Count fixed ips used by project."""
return IMPL.fixed_ip_count_by_project(context, project_id,
session=session)
####################
@@ -580,12 +569,6 @@ def instance_create(context, values):
return IMPL.instance_create(context, values)
def instance_data_get_for_project(context, project_id, session=None):
"""Get (instance_count, total_cores, total_ram) for project."""
return IMPL.instance_data_get_for_project(context, project_id,
session=session)
def instance_destroy(context, instance_uuid, constraint=None,
update_cells=True):
"""Destroy the instance or raise if it does not exist."""
@@ -1208,12 +1191,6 @@ def security_group_destroy(context, security_group_id):
return IMPL.security_group_destroy(context, security_group_id)
def security_group_count_by_project(context, project_id, session=None):
"""Count number of security groups in a project."""
return IMPL.security_group_count_by_project(context, project_id,
session=session)
####################

43
nova/db/sqlalchemy/api.py
View File

@@ -279,6 +279,34 @@ def convert_datetimes(values, *datetime_keys):
values[key] = timeutils.parse_strtime(values[key])
return values
def _sync_instances(context, project_id, session):
return dict(zip(('instances', 'cores', 'ram'),
_instance_data_get_for_project(
context, project_id, session)))
def _sync_floating_ips(context, project_id, session):
return dict(floating_ips=_floating_ip_count_by_project(
context, project_id, session))
def _sync_fixed_ips(context, project_id, session):
return dict(fixed_ips=_fixed_ip_count_by_project(
context, project_id, session))
def _sync_security_groups(context, project_id, session):
return dict(security_groups=_security_group_count_by_project(
context, project_id, session))
QUOTA_SYNC_FUNCTIONS = {
'_sync_instances': _sync_instances,
'_sync_floating_ips': _sync_floating_ips,
'_sync_fixed_ips': _sync_fixed_ips,
'_sync_security_groups': _sync_security_groups,
}
###################
@@ -743,8 +771,7 @@ def floating_ip_create(context, values):
return floating_ip_ref
@require_context
def floating_ip_count_by_project(context, project_id, session=None):
def _floating_ip_count_by_project(context, project_id, session=None):
nova.context.authorize_project_context(context, project_id)
# TODO(tr3buchet): why leave auto_assigned floating IPs out?
return model_query(context, models.FloatingIp, read_deleted="no",
@@ -1286,8 +1313,7 @@ def fixed_ip_update(context, address, values):
update(values)
@require_context
def fixed_ip_count_by_project(context, project_id, session=None):
def _fixed_ip_count_by_project(context, project_id, session=None):
nova.context.authorize_project_context(context, project_id)
return model_query(context, models.FixedIp.id,
base_model=models.FixedIp, read_deleted="no",
@@ -1499,8 +1525,7 @@ def instance_create(context, values):
return instance_ref
@require_admin_context
def instance_data_get_for_project(context, project_id, session=None):
def _instance_data_get_for_project(context, project_id, session=None):
result = model_query(context,
func.count(models.Instance.id),
func.sum(models.Instance.vcpus),
@@ -2870,7 +2895,7 @@ def quota_reserve(context, resources, quotas, deltas, expire,
# OK, refresh the usage
if refresh:
# Grab the sync routine
sync = resources[resource].sync
sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync]
updates = sync(elevated, project_id, session)
for res, in_use in updates.items():
@@ -3459,14 +3484,14 @@ def security_group_destroy(context, security_group_id):
soft_delete()
@require_context
def security_group_count_by_project(context, project_id, session=None):
def _security_group_count_by_project(context, project_id, session=None):
nova.context.authorize_project_context(context, project_id)
return model_query(context, models.SecurityGroup, read_deleted="no",
session=session).\
filter_by(project_id=project_id).\
count()
###################

60
nova/quota.py
View File

@@ -218,9 +218,9 @@ class DbQuotaDriver(object):
:param resources: A dictionary of the registered resources.
:param keys: A list of the desired quotas to retrieve.
:param has_sync: If True, indicates that the resource must
have a sync attribute; if False, indicates
have a sync function; if False, indicates
that the resource must NOT have a sync
attribute.
function.
:param project_id: Specify the project_id if current context
is admin and admin wants to impact on
common user's tenant.
@@ -716,14 +716,16 @@ class ReservableResource(BaseResource):
"""Describe a reservable resource."""
def __init__(self, name, sync, flag=None):
"""
Initializes a ReservableResource.
"""Initializes a ReservableResource.
Reservable resources are those resources which directly
correspond to objects in the database, i.e., instances, cores,
etc. A ReservableResource must be constructed with a usage
synchronization function, which will be called to determine the
current counts of one or more resources.
correspond to objects in the database, i.e., instances,
cores, etc.
Usage synchronization function must be associated with each
object. This function will be called to determine the current
counts of one or more resources. This association is done in
database backend.
The usage synchronization function will be passed three
arguments: an admin context, the project ID, and an opaque
@@ -735,15 +737,14 @@ class ReservableResource(BaseResource):
synchronization functions may be associated with more than one
ReservableResource.
:param name: The name of the resource, i.e., "instances".
:param sync: A callable which returns a dictionary to
resynchronize the in_use count for one or more
:param name: The name of the resource, i.e., "volumes".
:param sync: A dbapi methods name which returns a dictionary
to resynchronize the in_use count for one or more
resources, as described above.
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(ReservableResource, self).__init__(name, flag=flag)
self.sync = sync
@@ -1063,45 +1064,24 @@ class QuotaEngine(object):
return sorted(self._resources.keys())
def _sync_instances(context, project_id, session):
return dict(zip(('instances', 'cores', 'ram'),
db.instance_data_get_for_project(
context, project_id, session=session)))
def _sync_floating_ips(context, project_id, session):
return dict(floating_ips=db.floating_ip_count_by_project(
context, project_id, session=session))
def _sync_fixed_ips(context, project_id, session):
return dict(fixed_ips=db.fixed_ip_count_by_project(
context, project_id, session=session))
def _sync_security_groups(context, project_id, session):
return dict(security_groups=db.security_group_count_by_project(
context, project_id, session=session))
QUOTAS = QuotaEngine()
resources = [
ReservableResource('instances', _sync_instances, 'quota_instances'),
ReservableResource('cores', _sync_instances, 'quota_cores'),
ReservableResource('ram', _sync_instances, 'quota_ram'),
ReservableResource('floating_ips', _sync_floating_ips,
ReservableResource('instances', '_sync_instances', 'quota_instances'),
ReservableResource('cores', '_sync_instances', 'quota_cores'),
ReservableResource('ram', '_sync_instances', 'quota_ram'),
ReservableResource('security_groups', '_sync_security_groups',
'quota_security_groups'),
ReservableResource('floating_ips', '_sync_floating_ips',
'quota_floating_ips'),
ReservableResource('fixed_ips', _sync_fixed_ips, 'quota_fixed_ips'),
ReservableResource('fixed_ips', '_sync_fixed_ips', 'quota_fixed_ips'),
AbsoluteResource('metadata_items', 'quota_metadata_items'),
AbsoluteResource('injected_files', 'quota_injected_files'),
AbsoluteResource('injected_file_content_bytes',
'quota_injected_file_content_bytes'),
AbsoluteResource('injected_file_path_bytes',
'quota_injected_file_path_bytes'),
ReservableResource('security_groups', _sync_security_groups,
'quota_security_groups'),
CountableResource('security_group_rules',
db.security_group_rule_count_by_group,
'quota_security_group_rules'),

188
nova/tests/db/test_db_api.py
View File

@@ -47,6 +47,7 @@ from nova.openstack.common.db.sqlalchemy import session as db_session
from nova.openstack.common import timeutils
from nova.openstack.common import uuidutils
from nova.quota import ReservableResource
from nova.quota import resources
from nova import test
from nova.tests import matchers
from nova import utils
@@ -77,11 +78,15 @@ def _quota_reserve(context, project_id):
resources = {}
deltas = {}
for i in range(3):
resource = 'res%d' % i
resource = 'resource%d' % i
sync_name = '_sync_%s' % resource
quotas[resource] = db.quota_create(context, project_id, resource, i)
resources[resource] = ReservableResource(resource,
get_sync(resource, i), 'quota_res_%d' % i)
resources[resource] = ReservableResource(
resource, sync_name, 'quota_res_%d' % i)
deltas[resource] = i
setattr(sqlalchemy_api, sync_name, get_sync(resource, i))
sqlalchemy_api.QUOTA_SYNC_FUNCTIONS[sync_name] = getattr(
sqlalchemy_api, sync_name)
return db.quota_reserve(context, resources, quotas, deltas,
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
datetime.timedelta(days=1), project_id)
@@ -806,33 +811,6 @@ class ReservationTestCase(test.TestCase, ModelsObjectComparatorMixin):
datetime.timedelta(days=1),
'usage': {'id': 1}}
def _quota_reserve(self):
"""Create sample Quota, QuotaUsage and Reservation objects.
There is no method db.quota_usage_create(), so we have to use
db.quota_reserve() for creating QuotaUsage objects.
Returns reservations uuids.
"""
def get_sync(resource, usage):
def sync(elevated, project_id, session):
return {resource: usage}
return sync
quotas = {}
resources = {}
deltas = {}
for i in range(3):
resource = 'resource%d' % i
quotas[resource] = db.quota_create(self.ctxt, 'project1',
resource, i)
resources[resource] = ReservableResource(resource,
get_sync(resource, i), 'quota_res_%d' % i)
deltas[resource] = i
return db.quota_reserve(self.ctxt, resources, quotas, deltas,
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
datetime.timedelta(days=1), self.values['project_id'])
def test_reservation_create(self):
reservation = db.reservation_create(self.ctxt, **self.values)
self._assertEqualObjects(self.values, reservation, ignored_keys=(
@@ -852,7 +830,7 @@ class ReservationTestCase(test.TestCase, ModelsObjectComparatorMixin):
self.ctxt, 'non-exitent-resevation-uuid')
def test_reservation_commit(self):
reservations = self._quota_reserve()
reservations = _quota_reserve(self.ctxt, 'project1')
expected = {'project_id': 'project1',
'resource0': {'reserved': 0, 'in_use': 0},
'resource1': {'reserved': 1, 'in_use': 1},
@@ -871,7 +849,7 @@ class ReservationTestCase(test.TestCase, ModelsObjectComparatorMixin):
self.ctxt, 'project1'))
def test_reservation_rollback(self):
reservations = self._quota_reserve()
reservations = _quota_reserve(self.ctxt, 'project1')
expected = {'project_id': 'project1',
'resource0': {'reserved': 0, 'in_use': 0},
'resource1': {'reserved': 1, 'in_use': 1},
@@ -892,7 +870,7 @@ class ReservationTestCase(test.TestCase, ModelsObjectComparatorMixin):
def test_reservation_expire(self):
self.values['expire'] = datetime.datetime.utcnow() + datetime.\
timedelta(days=1)
self._quota_reserve()
_quota_reserve(self.ctxt, 'project1')
db.reservation_expire(self.ctxt)
expected = {'project_id': 'project1',
@@ -1150,22 +1128,6 @@ class SecurityGroupTestCase(test.TestCase, ModelsObjectComparatorMixin):
self._assertEqualListsOfObjects(security_groups, real,
ignored_keys=['instances'])
def test_security_group_count_by_project(self):
values = [
{'name': 'fake1', 'project_id': 'fake_proj1'},
{'name': 'fake2', 'project_id': 'fake_proj1'},
{'name': 'fake3', 'project_id': 'fake_proj2'},
]
for vals in values:
self._create_security_group(vals)
real = []
for project in ('fake_proj1', 'fake_proj2'):
real.append(db.security_group_count_by_project(self.ctxt, project))
expected = [2, 1]
self.assertEquals(expected, real)
def test_security_group_in_use(self):
instance = db.instance_create(self.ctxt, dict(host='foo'))
values = [
@@ -1628,6 +1590,13 @@ class InstanceTestCase(test.TestCase, ModelsObjectComparatorMixin):
del meta['gigawatts']
set_and_check(meta)
def test_security_group_in_use(self):
instance = db.instance_create(self.ctxt, dict(host='foo'))
values = [
{'instances': [instance]},
{'instances': []},
]
def test_instance_update_updates_system_metadata(self):
# Ensure that system_metadata is updated during instance_update
self._test_instance_update_updates_metadata('system_metadata')
@@ -2877,32 +2846,6 @@ class FixedIPTestCase(BaseInstanceTypeTestCase):
ips_list = db.fixed_ips_by_virtual_interface(self.ctxt, vif.id)
self.assertEquals(0, len(ips_list))
def test_fixed_ip_count_by_project_one_ip(self):
PROJECT_ID = "project_id"
instance_uuid = self._create_instance(project_id=PROJECT_ID)
db.fixed_ip_create(self.ctxt, dict(
instance_uuid=instance_uuid, address='address'))
ips_count = db.fixed_ip_count_by_project(self.ctxt, PROJECT_ID)
self.assertEquals(1, ips_count)
def test_fixed_ip_count_by_project_two_ips_for_different_instances(self):
PROJECT_ID = "project_id"
instance_uuid = self._create_instance(project_id=PROJECT_ID)
db.fixed_ip_create(self.ctxt, dict(
instance_uuid=instance_uuid, address='address_1'))
another_instance_for_this_project =\
db.instance_create(self.ctxt, dict(project_id=PROJECT_ID))
db.fixed_ip_create(self.ctxt, dict(
instance_uuid=another_instance_for_this_project['uuid'],
address='address_2'))
ips_count = db.fixed_ip_count_by_project(self.ctxt, PROJECT_ID)
self.assertEquals(2, ips_count)
def create_fixed_ip(self, **params):
default_params = {'address': '192.168.0.1'}
default_params.update(params)
@@ -3343,26 +3286,6 @@ class FloatingIpTestCase(test.TestCase, ModelsObjectComparatorMixin):
self.assertRaises(exception.FloatingIpExists,
self._create_floating_ip, {})
def test_floating_ip_count_by_project(self):
projects = {
'project1': ['1.1.1.1', '2.2.2.2', '3.3.3.3'],
'project2': ['4.4.4.4', '5.5.5.5'],
'project3': ['6.6.6.6']
}
for project_id, addresses in projects.iteritems():
for address in addresses:
self._create_floating_ip({'project_id': project_id,
'address': address})
for project_id, addresses in projects.iteritems():
real_count = db.floating_ip_count_by_project(self.ctxt, project_id)
self.assertEqual(len(addresses), real_count)
def test_floating_ip_count_by_project_not_authorized(self):
ctxt = context.RequestContext(user_id='a', project_id='abc',
is_admin=False)
self.assertRaises(exception.NotAuthorized,
db.floating_ip_count_by_project, ctxt, 'def')
def _create_fixed_ip(self, params):
default_params = {'address': '192.168.0.1'}
default_params.update(params)
@@ -4708,14 +4631,56 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin):
self.assertRaises(exception.ProjectQuotaNotFound,
db.quota_get, self.ctxt, 'project1', 'resource1')
def test_quota_reserve(self):
reservations = _quota_reserve(self.ctxt, 'project1')
self.assertEqual(len(reservations), 3)
res_names = ['res0', 'res1', 'res2']
for uuid in reservations:
reservation = db.reservation_get(self.ctxt, uuid)
self.assertTrue(reservation.resource in res_names)
res_names.remove(reservation.resource)
def test_quota_reserve_all_resources(self):
quotas = {}
deltas = {}
reservable_resources = {}
for i, resource in enumerate(resources):
if isinstance(resource, ReservableResource):
quotas[resource.name] = db.quota_create(self.ctxt, 'project1',
resource.name, 100)
deltas[resource.name] = i
reservable_resources[resource.name] = resource
usages = {'instances': 3, 'cores': 6, 'ram': 9}
instances = []
for i in range(3):
instances.append(db.instance_create(self.ctxt,
{'vcpus': 2, 'memory_mb': 3,
'project_id': 'project1'}))
usages['fixed_ips'] = 2
network = db.network_create_safe(self.ctxt, {})
for i in range(2):
address = '192.168.0.%d' % i
ip = db.fixed_ip_create(self.ctxt, {'project_id': 'project1',
'address': address,
'network_id': network['id']})
db.fixed_ip_associate(self.ctxt, address,
instances[0].uuid, network['id'])
usages['floating_ips'] = 5
for i in range(5):
db.floating_ip_create(self.ctxt, {'project_id': 'project1'})
usages['security_groups'] = 3
for i in range(3):
db.security_group_create(self.ctxt, {'project_id': 'project1'})
reservations_uuids = db.quota_reserve(self.ctxt, reservable_resources,
quotas, deltas, None,
None, None, 'project1')
resources_names = reservable_resources.keys()
for reservation_uuid in reservations_uuids:
reservation = db.reservation_get(self.ctxt, reservation_uuid)
usage = db.quota_usage_get(self.ctxt, 'project1',
reservation.resource)
self.assertEqual(usage.in_use, usages[reservation.resource],
'Resource: %s' % reservation.resource)
self.assertEqual(usage.reserved, deltas[reservation.resource])
self.assertIn(reservation.resource, resources_names)
resources_names.remove(reservation.resource)
self.assertEqual(len(resources_names), 0)
def test_quota_destroy_all_by_project(self):
reservations = _quota_reserve(self.ctxt, 'project1')
@@ -4735,8 +4700,8 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin):
def test_quota_usage_get(self):
_quota_reserve(self.ctxt, 'p1')
quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'res0')
expected = {'resource': 'res0', 'project_id': 'p1',
quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'resource0')
expected = {'resource': 'resource0', 'project_id': 'p1',
'in_use': 0, 'reserved': 0, 'total': 0}
for key, value in expected.iteritems():
self.assertEqual(value, quota_usage[key])
@@ -4744,9 +4709,9 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin):
def test_quota_usage_get_all_by_project(self):
_quota_reserve(self.ctxt, 'p1')
expected = {'project_id': 'p1',
'res0': {'in_use': 0, 'reserved': 0},
'res1': {'in_use': 1, 'reserved': 1},
'res2': {'in_use': 2, 'reserved': 2}}
'resource0': {'in_use': 0, 'reserved': 0},
'resource1': {'in_use': 1, 'reserved': 1},
'resource2': {'in_use': 2, 'reserved': 2}}
self.assertEqual(expected, db.quota_usage_get_all_by_project(
self.ctxt, 'p1'))
@@ -4756,9 +4721,10 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin):
def test_quota_usage_update(self):
_quota_reserve(self.ctxt, 'p1')
db.quota_usage_update(self.ctxt, 'p1', 'res0', in_use=42, reserved=43)
quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'res0')
expected = {'resource': 'res0', 'project_id': 'p1',
db.quota_usage_update(self.ctxt, 'p1', 'resource0', in_use=42,
reserved=43)
quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'resource0')
expected = {'resource': 'resource0', 'project_id': 'p1',
'in_use': 42, 'reserved': 43, 'total': 85}
for key, value in expected.iteritems():
self.assertEqual(value, quota_usage[key])

36
nova/tests/test_quota.py
View File

@@ -447,36 +447,6 @@ class QuotaEngineTestCase(test.TestCase):
test_resource3=resources[2],
))
def test_sync_predeclared(self):
quota_obj = quota.QuotaEngine()
def spam(*args, **kwargs):
pass
resource = quota.ReservableResource('test_resource', spam)
quota_obj.register_resource(resource)
self.assertEqual(resource.sync, spam)
def test_sync_multi(self):
quota_obj = quota.QuotaEngine()
def spam(*args, **kwargs):
pass
resources = [
quota.ReservableResource('test_resource1', spam),
quota.ReservableResource('test_resource2', spam),
quota.ReservableResource('test_resource3', spam),
quota.ReservableResource('test_resource4', spam),
]
quota_obj.register_resources(resources[:2])
self.assertEqual(resources[0].sync, spam)
self.assertEqual(resources[1].sync, spam)
self.assertEqual(resources[2].sync, spam)
self.assertEqual(resources[3].sync, spam)
def test_get_by_project(self):
context = FakeContext('test_project', 'test_class')
driver = FakeDriver(by_project=dict(
@@ -1433,10 +1403,12 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase):
return {res_name: self.usages[res_name].in_use - 1}
return {res_name: 0}
return sync
self.resources = {}
for res_name in ('instances', 'cores', 'ram'):
res = quota.ReservableResource(res_name, make_sync(res_name))
method_name = '_sync_%s' % res_name
sqa_api.QUOTA_SYNC_FUNCTIONS[method_name] = make_sync(res_name)
res = quota.ReservableResource(res_name, '_sync_%s' % res_name)
self.resources[res_name] = res
self.expire = timeutils.utcnow() + datetime.timedelta(seconds=3600)