x
/
mogan
Code Issues Proposed changes

Support quota in mogan(part two) 

This patch introduces the quota operation in
creating and deleting instances process.

Change-Id: If6573fafc5acb805cf727acdc8f0f9872fc1a717
Implements: blueprint quota-support
This commit is contained in:
wanghao 2017-03-09 15:53:38 +08:00
parent 3736848733
commit efa63af620
9 changed files with 167 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mogan/conf/quota.py
View File

@ -20,6 +20,7 @@ from mogan.common.i18n import _
quota_opts = [
cfg.StrOpt('quota_driver',
default="database",
help=_("Specify the quota driver which is used in Mogan "
"service.")),
cfg.IntOpt('reservation_expire',

6
mogan/db/sqlalchemy/alembic/versions/91941bf1ebc9_initial_migration.py
View File

@ -183,7 +183,7 @@ def upgrade():
sa.Column('until_refresh', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('resource_name', 'project_id',
name='uniq_quotas0resource_name'),
name='uniq_quotas_usages0resource_name'),
mysql_ENGINE='InnoDB',
mysql_DEFAULT_CHARSET='UTF8'
)
@ -193,8 +193,8 @@ def upgrade():
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('usage_id', sa.Integer(), nullable=False),
sa.Column('allocated_id', sa.Integer(), nullable=False),
sa.Column('usage_id', sa.Integer(), nullable=True),
sa.Column('allocated_id', sa.Integer(), nullable=True),
sa.Column('project_id', sa.String(length=36), nullable=True),
sa.Column('resource_name', sa.String(length=255), nullable=True),
sa.Column('delta', sa.Integer(), nullable=True),

2
mogan/db/sqlalchemy/models.py
View File

@ -239,7 +239,7 @@ class Quota(Base):
id = Column(Integer, primary_key=True)
resource_name = Column(String(255), nullable=False)
project_id = Column(String(36), nullable=False)
hard_limit = Column(Integer, nullable=False)
hard_limit = Column(Integer, default=0)
allocated = Column(Integer, default=0)

39
mogan/engine/api.py
View File

@ -28,6 +28,7 @@ from mogan.engine import rpcapi
from mogan import image
from mogan import network
from mogan import objects
from mogan.objects import quota
LOG = log.getLogger(__name__)
@ -58,6 +59,8 @@ class API(object):
self.image_api = image_api or image.API()
self.engine_rpcapi = rpcapi.EngineAPI()
self.network_api = network.API()
self.quota = quota.Quota()
self.quota.register_resource(objects.quota.InstanceResource())
def _get_image(self, context, image_uuid):
return self.image_api.get(context, image_uuid)
@ -133,15 +136,32 @@ class API(object):
return instance
def _check_num_instances_quota(self, context, min_count, max_count):
# TODO(little): check quotas and return reserved quotas
return max_count, None
ins_resource = self.quota.resources['instances']
quotas = self.quota.get_quota_limit_and_usage(context,
{'instances':
ins_resource},
context.tenant)
limit = quotas['instances']['limit']
in_use = quotas['instances']['in_use']
reserved = quotas['instances']['reserved']
available_quota = limit - in_use - reserved
if max_count <= available_quota:
return max_count
elif min_count <= available_quota and max_count > available_quota:
return available_quota
else:
raise exception.OverQuota(overs='instances')
def _provision_instances(self, context, base_options,
min_count, max_count):
# TODO(little): finish to return num_instances according quota
num_instances, quotas = self._check_num_instances_quota(
# Return num_instances according quota
num_instances = self._check_num_instances_quota(
context, min_count, max_count)
# Create the instances reservations
reserve_opts = {'instances': num_instances}
reservations = self.quota.reserve(context, **reserve_opts)
LOG.debug("Going to run %s instances...", num_instances)
instances = []
@ -164,8 +184,11 @@ class API(object):
except exception.ObjectActionError:
pass
finally:
# TODO(little): quota release
pass
self.quota.rollback(context, reservations)
# Commit instances reservations
if reservations:
self.quota.commit(context, reservations)
return instances
@ -261,6 +284,10 @@ class API(object):
LOG.debug("Instance is not found while deleting",
instance=instance)
return
reserve_opts = {'instances': -1}
reservations = self.quota.reserve(context, **reserve_opts)
if reservations:
self.quota.commit(context, reservations)
self.engine_rpcapi.delete_instance(context, instance)
@check_instance_lock

15
mogan/engine/manager.py
View File

@ -30,6 +30,7 @@ from mogan.engine.flows import create_instance
from mogan.notifications import base as notifications
from mogan import objects
from mogan.objects import fields
from mogan.objects import quota
LOG = log.getLogger(__name__)
@ -41,6 +42,11 @@ class EngineManager(base_manager.BaseEngineManager):
target = messaging.Target(version=RPC_API_VERSION)
def __init__(self, *args, **kwargs):
super(EngineManager, self).__init__(*args, **kwargs)
self.quota = quota.Quota()
self.quota.register_resource(objects.quota.InstanceResource())
def _get_compute_port(self, context, port_uuid):
"""Gets compute port by the uuid."""
try:
@ -298,6 +304,12 @@ class EngineManager(base_manager.BaseEngineManager):
for port in ports:
self.network_api.delete_port(context, port, instance.uuid)
def _rollback_instances_quota(self, context, number):
reserve_opts = {'instances': number}
reservations = self.quota.reserve(context, **reserve_opts)
if reservations:
self.quota.commit(context, reservations)
def create_instance(self, context, instance, requested_networks,
request_spec=None, filter_properties=None):
"""Perform a deployment."""
@ -353,6 +365,7 @@ class EngineManager(base_manager.BaseEngineManager):
)
except Exception:
utils.process_event(fsm, instance, event='error')
self._rollback_instances_quota(context, -1)
msg = _("Create manager instance flow failed.")
LOG.exception(msg)
raise exception.MoganException(msg)
@ -370,6 +383,7 @@ class EngineManager(base_manager.BaseEngineManager):
except Exception as e:
instance.power_state = states.NOSTATE
utils.process_event(fsm, instance, event='error')
self._rollback_instances_quota(context, -1)
LOG.error("Created instance %(uuid)s failed."
"Exception: %(exception)s",
{"uuid": instance.uuid,
@ -426,6 +440,7 @@ class EngineManager(base_manager.BaseEngineManager):
instance=instance)
instance.power_state = states.NOSTATE
utils.process_event(fsm, instance, event='error')
self._rollback_instances_quota(context, 1)
# Issue delete request to driver only if instance is associated with
# a underlying node.

71
mogan/objects/quota.py
View File

@ -19,10 +19,10 @@
import datetime
from oslo_config import cfg
from oslo_utils import importutils
from oslo_utils import timeutils
from oslo_versionedobjects import base as object_base
import six
from stevedore import driver
from mogan.common import exception
from mogan.db import api as dbapi
@ -50,7 +50,9 @@ class Quota(base.MoganObject, object_base.VersionedObjectDictCompat):
def __init__(self, *args, **kwargs):
super(Quota, self).__init__(*args, **kwargs)
self.quota_driver = importutils.import_object(CONF.quota.quota_driver)
self.quota_driver = driver.DriverManager(
'mogan.quota.backend_driver', CONF.quota.quota_driver,
invoke_on_load=True).driver
self._resources = {}
@property
@ -106,9 +108,8 @@ class Quota(base.MoganObject, object_base.VersionedObjectDictCompat):
def reserve(self, context, expire=None, project_id=None, **deltas):
"""reserve the Quota."""
return self.quota_driver.reserver(context, self.resources, deltas,
expire=expire,
project_id=project_id)
return self.quota_driver.reserve(context, self.resources, deltas,
expire=expire, project_id=project_id)
def commit(self, context, reservations, project_id=None):
self.quota_driver.commit(context, reservations, project_id=project_id)
@ -150,6 +151,10 @@ class Quota(base.MoganObject, object_base.VersionedObjectDictCompat):
for resource in resources:
self.register_resource(resource)
def get_quota_limit_and_usage(self, context, resources, project_id):
return self.quota_driver.get_project_quotas(context, resources,
project_id, usages=True)
class DbQuotaDriver(object):
@ -159,9 +164,9 @@ class DbQuotaDriver(object):
The default driver utilizes the local database.
"""
def get_project_quotas(self, context, resources, project_id,
quota_class=None, defaults=True,
usages=True):
dbapi = dbapi.get_instance()
def get_project_quotas(self, context, resources, project_id, usages=True):
"""Retrieve quotas for a project.
Given a list of resources, retrieve the quotas for the given
@ -170,32 +175,31 @@ class DbQuotaDriver(object):
:param context: The request context, for access checks.
:param resources: A dictionary of the registered resources.
:param project_id: The ID of the project to return quotas for.
:param quota_class: If project_id != context.tenant, the
quota class cannot be determined. This
parameter allows it to be specified. It
will be ignored if project_id ==
context.tenant.
:param defaults: If True, the quota class value (or the
default value, if there is no value from the
quota class) will be reported if there is no
specific value for the resource.
:param usages: If True, the current in_use, reserved and allocated
counts will also be returned.
"""
quotas = {}
project_quotas = dbapi.quota_get_all_by_project(context, project_id)
project_quotas = {}
res = self.dbapi.quota_get_all_by_project(context, project_id)
for p_quota in res:
project_quotas[p_quota.resource_name] = p_quota.hard_limit
if project_quotas == {}:
self.dbapi.quota_create(context, {'resource_name': 'instances',
'project_id': project_id,
'hard_limit': 10,
'allocated': 0})
project_quotas['instances'] = 10
allocated_quotas = None
if usages:
project_usages = dbapi.quota_usage_get_all_by_project(context,
project_id)
allocated_quotas = dbapi.quota_allocated_get_all_by_project(
project_usages = self.dbapi.quota_usage_get_all_by_project(
context, project_id)
allocated_quotas = self.dbapi.quota_allocated_get_all_by_project(
context, project_id)
allocated_quotas.pop('project_id')
for resource in resources.values():
# Omit default/quota class values
if not defaults and resource.name not in project_quotas:
if resource.name not in project_quotas:
continue
quota_val = project_quotas.get(resource.name)
@ -250,8 +254,7 @@ class DbQuotaDriver(object):
# Grab and return the quotas (without usages)
quotas = self.get_project_quotas(context, sub_resources,
project_id,
context.quota_class, usages=False)
project_id, usages=False)
return {k: v['limit'] for k, v in quotas.items()}
@ -313,10 +316,11 @@ class DbQuotaDriver(object):
project_id)
def _reserve(self, context, resources, quotas, deltas, expire, project_id):
return dbapi.quota_reserve(context, resources, quotas, deltas, expire,
CONF.quota.until_refresh,
CONF.quota.max_age,
project_id=project_id)
return self.dbapi.quota_reserve(context, resources, quotas, deltas,
expire,
CONF.quota.until_refresh,
CONF.quota.max_age,
project_id)
def commit(self, context, reservations, project_id=None):
"""Commit reservations.
@ -332,7 +336,8 @@ class DbQuotaDriver(object):
if project_id is None:
project_id = context.tenant
dbapi.reservation_commit(context, reservations, project_id=project_id)
self.dbapi.reservation_commit(context, reservations,
project_id=project_id)
def rollback(self, context, reservations, project_id=None):
"""Roll back reservations.
@ -348,8 +353,8 @@ class DbQuotaDriver(object):
if project_id is None:
project_id = context.tenant
dbapi.reservation_rollback(context, reservations,
project_id=project_id)
self.dbapi.reservation_rollback(context, reservations,
project_id=project_id)
def expire(self, context):
"""Expire reservations.
@ -360,7 +365,7 @@ class DbQuotaDriver(object):
:param context: The request context, for access checks.
"""
dbapi.reservation_expire(context)
self.dbapi.reservation_expire(context)
class BaseResource(object):

37
mogan/tests/functional/api/v1/test_instances.py
View File

@ -71,7 +71,11 @@ class TestInstances(v1_test.APITestV1):
INSTANCE_UUIDS = ['59f1b681-6ca4-4a17-b784-297a7285004e',
'2b32fc87-576c-481b-880e-bef8c7351746',
'482decff-7561-41ad-9bfb-447265b26972',
'427693e1-a820-4d7d-8a92-9f5fe2849399']
'427693e1-a820-4d7d-8a92-9f5fe2849399',
'253b2878-ec60-4793-ad19-e65496ec7aab',
'f26f181d-7891-4720-b022-b074ec1733ef',
'02f53bd8-3514-485b-ba60-2722ef09c016',
'8f7495fe-5e44-4f33-81af-4b28e9b2952f']
def setUp(self):
self.rpc_api = mock.Mock()
@ -113,8 +117,12 @@ class TestInstances(v1_test.APITestV1):
@mock.patch('oslo_utils.uuidutils.generate_uuid')
def _prepare_instance(self, amount, mocked):
mocked.side_effect = self.INSTANCE_UUIDS[:amount]
# NOTE(wanghao): Since we added quota reserve in creation option,
# there is one more generate_uuid out of provision_instances, so
# amount should *2 here.
mocked.side_effect = self.INSTANCE_UUIDS[:(amount * 2)]
responses = []
headers = self.gen_headers(self.context)
for i in six.moves.xrange(amount):
test_body = {
"name": "test_instance_" + str(i),
@ -127,14 +135,15 @@ class TestInstances(v1_test.APITestV1):
'extra': {'fake_key': 'fake_value'}
}
responses.append(
self.post_json('/instances', test_body, status=201))
self.post_json('/instances', test_body, headers=headers,
status=201))
return responses
def test_instance_post(self):
resp = self._prepare_instance(1)[0].json
self.assertEqual('test_instance_0', resp['name'])
self.assertEqual('building', resp['status'])
self.assertEqual(self.INSTANCE_UUIDS[0], resp['uuid'])
self.assertEqual(self.INSTANCE_UUIDS[1], resp['uuid'])
self.assertEqual('just test instance 0', resp['description'])
self.assertEqual(self.INSTANCE_TYPE_UUID, resp['instance_type_uuid'])
self.assertEqual('b8f82429-3a13-4ffe-9398-4d1abdc256a8',
@ -151,10 +160,12 @@ class TestInstances(v1_test.APITestV1):
def test_instance_show(self):
self._prepare_instance(1)
resp = self.get_json('/instances/%s' % self.INSTANCE_UUIDS[0])
headers = self.gen_headers(self.context)
resp = self.get_json('/instances/%s' % self.INSTANCE_UUIDS[1],
headers=headers)
self.assertEqual('test_instance_0', resp['name'])
self.assertEqual('building', resp['status'])
self.assertEqual(self.INSTANCE_UUIDS[0], resp['uuid'])
self.assertEqual(self.INSTANCE_UUIDS[1], resp['uuid'])
self.assertEqual('just test instance 0', resp['description'])
self.assertEqual(self.INSTANCE_TYPE_UUID, resp['instance_type_uuid'])
self.assertEqual('b8f82429-3a13-4ffe-9398-4d1abdc256a8',
@ -171,7 +182,8 @@ class TestInstances(v1_test.APITestV1):
def test_instance_list(self):
self._prepare_instance(4)
resps = self.get_json('/instances')['instances']
headers = self.gen_headers(self.context)
resps = self.get_json('/instances', headers=headers)['instances']
self.assertEqual(4, len(resps))
self.assertEqual('test_instance_0', resps[0]['name'])
self.assertEqual('just test instance 0', resps[0]['description'])
@ -179,7 +191,9 @@ class TestInstances(v1_test.APITestV1):
def test_instance_list_with_details(self):
self._prepare_instance(4)
resps = self.get_json('/instances/detail')['instances']
headers = self.gen_headers(self.context)
resps = self.get_json('/instances/detail',
headers=headers)['instances']
self.assertEqual(4, len(resps))
self.assertEqual(16, len(resps[0].keys()))
self.assertEqual('test_instance_0', resps[0]['name'])
@ -192,6 +206,9 @@ class TestInstances(v1_test.APITestV1):
def test_instance_delete(self):
self._prepare_instance(4)
self.delete('/instances/' + self.INSTANCE_UUIDS[0], status=204)
resp = self.get_json('/instances/%s' % self.INSTANCE_UUIDS[0])
headers = self.gen_headers(self.context)
self.delete('/instances/' + self.INSTANCE_UUIDS[1], headers=headers,
status=204)
resp = self.get_json('/instances/%s' % self.INSTANCE_UUIDS[1],
headers=headers)
self.assertEqual('deleting', resp['status'])

52
mogan/tests/unit/engine/test_engine_api.py
View File

@ -88,12 +88,11 @@ class ComputeAPIUnitTest(base.DbTestCase):
mock_inst_create.assert_has_calls(calls)
@mock.patch.object(engine_rpcapi.EngineAPI, 'create_instance')
@mock.patch('mogan.engine.api.API._provision_instances')
@mock.patch('mogan.engine.api.API._get_image')
@mock.patch('mogan.engine.api.API._validate_and_build_base_options')
@mock.patch.object(engine_rpcapi.EngineAPI, 'list_availability_zones')
def test_create(self, mock_list_az, mock_validate, mock_get_image,
mock_provision, mock_create):
mock_create):
instance_type = self._create_instance_type()
base_options = {'image_uuid': 'fake-uuid',
@ -109,12 +108,15 @@ class ComputeAPIUnitTest(base.DbTestCase):
max_count = 2
mock_validate.return_value = (base_options, max_count)
mock_get_image.side_effect = None
mock_provision.return_value = [mock.MagicMock()
for i in range(max_count)]
mock_create.return_value = mock.MagicMock()
mock_list_az.return_value = {'availability_zones': ['test_az']}
requested_networks = [{'uuid': 'fake'}]
res = self.dbapi._get_quota_usages(self.context, self.project_id)
before_in_use = 0
if res.get('instances') is not None:
before_in_use = res.get('instances').in_use
self.engine_api.create(
self.context,
instance_type=instance_type,
@ -132,10 +134,11 @@ class ComputeAPIUnitTest(base.DbTestCase):
self.context, instance_type, 'fake-uuid', 'fake-name',
'fake-descritpion', 'test_az', {'k1', 'v1'}, requested_networks,
max_count)
mock_provision.assert_called_once_with(self.context, base_options,
min_count, max_count)
self.assertTrue(mock_create.called)
self.assertTrue(mock_get_image.called)
res = self.dbapi._get_quota_usages(self.context, self.project_id)
after_in_use = res.get('instances').in_use
self.assertEqual(before_in_use + 2, after_in_use)
@mock.patch.object(engine_rpcapi.EngineAPI, 'list_availability_zones')
def test_create_with_invalid_az(self, mock_list_az):
@ -156,6 +159,43 @@ class ComputeAPIUnitTest(base.DbTestCase):
mock_list_az.assert_called_once_with(self.context)
@mock.patch('mogan.engine.api.API._get_image')
@mock.patch('mogan.engine.api.API._validate_and_build_base_options')
@mock.patch.object(engine_rpcapi.EngineAPI, 'list_availability_zones')
def test_create_over_quota_limit(self, mock_list_az, mock_validate,
mock_get_image):
instance_type = self._create_instance_type()
base_options = {'image_uuid': 'fake-uuid',
'status': states.BUILDING,
'user_id': 'fake-user',
'project_id': 'fake-project',
'instance_type_uuid': 'fake-type-uuid',
'name': 'fake-name',
'description': 'fake-description',
'extra': {'k1', 'v1'},
'availability_zone': 'test_az'}
min_count = 11
max_count = 20
mock_validate.return_value = (base_options, max_count)
mock_get_image.side_effect = None
mock_list_az.return_value = {'availability_zones': ['test_az']}
requested_networks = [{'uuid': 'fake'}]
self.assertRaises(
exception.OverQuota,
self.engine_api.create,
self.context,
instance_type,
'fake-uuid',
'fake-name',
'fake-descritpion',
'test_az',
{'k1', 'v1'},
requested_networks,
min_count,
max_count)
def _create_fake_instance_obj(self, fake_instance):
fake_instance_obj = objects.Instance(self.context, **fake_instance)
fake_instance_obj.create(self.context)

3
setup.cfg
View File

@ -51,6 +51,9 @@ mogan.database.migration_backend =
tempest.test_plugins =
mogan_tests = mogan.tests.tempest.plugin:MoganTempestPlugin
mogan.quota.backend_driver =
database = mogan.objects.quota:DbQuotaDriver
[build_sphinx]
source-dir = doc/source
build-dir = doc/build