From e0e8f3d13d2e1230cd6bbe26ffb3e96846b831df Mon Sep 17 00:00:00 2001 From: Jiangyuan Date: Tue, 24 Apr 2018 15:48:28 +0800 Subject: [PATCH] Add cpu and memory_size options support for function Allow users to specify cpu/memory_size when creating function, and allow to update their values saved in function database. This patch is api part of customized cpu/mem in qinling server and is based on patch [0]. [0]: https://review.openstack.org/#/c/562507/ Story: 2001586 Task: 14366 Change-Id: I7a245f93a445a00c2722238d3f94d3a960f16af4 --- qinling/api/controllers/v1/function.py | 34 +++++++++- qinling/api/controllers/v1/resources.py | 2 + qinling/config.py | 35 +++++++++++ .../unit/api/controllers/v1/test_function.py | 62 +++++++++++++++++++ qinling/tests/unit/base.py | 3 + qinling/utils/common.py | 17 +++++ 6 files changed, 152 insertions(+), 1 deletion(-) diff --git a/qinling/api/controllers/v1/function.py b/qinling/api/controllers/v1/function.py index cb9c3859..723925ec 100644 --- a/qinling/api/controllers/v1/function.py +++ b/qinling/api/controllers/v1/function.py @@ -34,6 +34,7 @@ from qinling.db import api as db_api from qinling import exceptions as exc from qinling import rpc from qinling.storage import base as storage_base +from qinling.utils import common from qinling.utils import constants from qinling.utils import etcd_util from qinling.utils.openstack import keystone as keystone_util @@ -45,7 +46,8 @@ CONF = cfg.CONF POST_REQUIRED = set(['code']) CODE_SOURCE = set(['package', 'swift', 'image']) -UPDATE_ALLOWED = set(['name', 'description', 'code', 'package', 'entry']) +UPDATE_ALLOWED = set(['name', 'description', 'code', 'package', 'entry', + 'cpu', 'memory_size']) class FunctionWorkerController(rest.RestController): @@ -147,8 +149,22 @@ class FunctionsController(rest.RestController): 'runtime_id': kwargs.get('runtime_id'), 'code': json.loads(kwargs['code']), 'entry': kwargs.get('entry', 'main.main'), + 'cpu': kwargs.get('cpu', CONF.resource_limits.default_cpu), + 'memory_size': kwargs.get( + 'memory_size', CONF.resource_limits.default_memory + ), } + # Check cpu and memory_size values. + common.validate_int_in_range( + 'cpu', values['cpu'], CONF.resource_limits.min_cpu, + CONF.resource_limits.max_cpu + ) + common.validate_int_in_range( + 'memory', values['memory_size'], CONF.resource_limits.min_memory, + CONF.resource_limits.max_memory + ) + source = values['code'].get('source') if not source or source not in CODE_SOURCE: raise exc.InputException( @@ -302,6 +318,22 @@ class FunctionsController(rest.RestController): else: source = values.get('code', {}).get('source') md5sum = values.get('code', {}).get('md5sum') + cpu = values.get('cpu') + memory_size = values.get('memory_size') + + # Check cpu and memory_size values when updating. + if cpu is not None: + common.validate_int_in_range( + 'cpu', values['cpu'], CONF.resource_limits.min_cpu, + CONF.resource_limits.max_cpu + ) + if memory_size is not None: + common.validate_int_in_range( + 'memory', values['memory_size'], + CONF.resource_limits.min_memory, + CONF.resource_limits.max_memory + ) + with db_api.transaction(): pre_func = db_api.get_function(id) diff --git a/qinling/api/controllers/v1/resources.py b/qinling/api/controllers/v1/resources.py index d4fda7b7..717720cb 100644 --- a/qinling/api/controllers/v1/resources.py +++ b/qinling/api/controllers/v1/resources.py @@ -164,6 +164,7 @@ class Function(Resource): id = wtypes.text name = wtypes.text description = wtypes.text + cpu = int memory_size = int timeout = int runtime_id = wsme.wsattr(types.uuid, readonly=True) @@ -181,6 +182,7 @@ class Function(Resource): id='123e4567-e89b-12d3-a456-426655440000', name='hello_world', description='this is the first function.', + cpu=1, memory_size=1, timeout=1, runtime_id='123e4567-e89b-12d3-a456-426655440001', diff --git a/qinling/config.py b/qinling/config.py index 78d22e29..88ab60af 100644 --- a/qinling/config.py +++ b/qinling/config.py @@ -184,6 +184,40 @@ etcd_opts = [ cfg.PortOpt('port', default=2379, help='Etcd service port.'), ] +RLIMITS_GROUP = 'resource_limits' +rlimits_opts = [ + cfg.IntOpt( + 'default_cpu', + default=100, + help='Default cpu resource(unit: millicpu).' + ), + cfg.IntOpt( + 'min_cpu', + default=100, + help='Minimum cpu resource(unit: millicpu).' + ), + cfg.IntOpt( + 'max_cpu', + default=300, + help='Maximum cpu resource(unit: millicpu).' + ), + cfg.IntOpt( + 'default_memory', + default=33554432, + help='Default memory resource(unit: bytes).' + ), + cfg.IntOpt( + 'min_memory', + default=33554432, + help='Minimum memory resource(unit: bytes).' + ), + cfg.IntOpt( + 'max_memory', + default=134217728, + help='Maximum memory resource(unit: bytes).' + ), +] + def list_opts(): keystone_middleware_opts = auth_token.list_opts() @@ -198,6 +232,7 @@ def list_opts(): (STORAGE_GROUP, storage_opts), (KUBERNETES_GROUP, kubernetes_opts), (ETCD_GROUP, etcd_opts), + (RLIMITS_GROUP, rlimits_opts), (None, [launch_opt]), (None, default_opts), ] diff --git a/qinling/tests/unit/api/controllers/v1/test_function.py b/qinling/tests/unit/api/controllers/v1/test_function.py index a78f3a65..11bf3c18 100644 --- a/qinling/tests/unit/api/controllers/v1/test_function.py +++ b/qinling/tests/unit/api/controllers/v1/test_function.py @@ -17,6 +17,7 @@ import json import tempfile import mock +from oslo_config import cfg from qinling import status from qinling.tests.unit.api import base @@ -97,6 +98,8 @@ class TestFunctionController(base.APITest): "name": db_func.name, 'entry': 'main.main', "project_id": unit_base.DEFAULT_PROJECT_ID, + "cpu": cfg.CONF.resource_limits.default_cpu, + "memory_size": cfg.CONF.resource_limits.default_memory, } resp = self.app.get('/v1/functions/%s' % db_func.id) @@ -111,6 +114,8 @@ class TestFunctionController(base.APITest): "name": db_func.name, 'entry': 'main.main', "project_id": unit_base.DEFAULT_PROJECT_ID, + "cpu": cfg.CONF.resource_limits.default_cpu, + "memory_size": cfg.CONF.resource_limits.default_memory, } resp = self.app.get('/v1/functions') @@ -173,6 +178,63 @@ class TestFunctionController(base.APITest): self.assertEqual(400, resp.status_int) + def test_put_cpu_with_type_error(self): + db_func = self.create_function(runtime_id=self.runtime_id) + + # Check for type of cpu values. + resp = self.app.put_json( + '/v1/functions/%s' % db_func.id, {'cpu': 'non-int'}, + expect_errors=True + ) + + self.assertEqual(400, resp.status_int) + self.assertIn( + 'Invalid cpu resource specified. An integer is required.', + resp.json['faultstring'] + ) + + def test_put_cpu_with_overrun_error(self): + db_func = self.create_function(runtime_id=self.runtime_id) + + # Check for cpu error with input out of range. + resp = self.app.put_json( + '/v1/functions/%s' % db_func.id, {'cpu': 0}, + expect_errors=True + ) + + self.assertEqual(400, resp.status_int) + self.assertIn( + 'cpu resource limitation not within the allowable range', + resp.json['faultstring'] + ) + + @mock.patch('qinling.utils.etcd_util.delete_function') + @mock.patch('qinling.rpc.EngineClient.delete_function') + def test_put_cpu_and_memorysize( + self, mock_delete_func, mock_etcd_del + ): + # Test for updating cpu/mem with good input values. + db_func = self.create_function(runtime_id=self.runtime_id) + + req_body = { + 'cpu': str(cfg.CONF.resource_limits.default_cpu + 1), + 'memory_size': str(cfg.CONF.resource_limits.default_memory + 1) + } + + resp = self.app.put_json('/v1/functions/%s' % db_func.id, req_body) + + self.assertEqual(200, resp.status_int) + self.assertEqual( + cfg.CONF.resource_limits.default_cpu + 1, + resp.json['cpu'] + ) + self.assertEqual( + cfg.CONF.resource_limits.default_memory + 1, + resp.json['memory_size'] + ) + mock_delete_func.assert_called_once_with(db_func.id) + mock_etcd_del.assert_called_once_with(db_func.id) + @mock.patch('qinling.utils.etcd_util.delete_function') @mock.patch('qinling.rpc.EngineClient.delete_function') @mock.patch('qinling.storage.file_system.FileSystemStorage.delete') diff --git a/qinling/tests/unit/base.py b/qinling/tests/unit/base.py index 2747fdd9..bc4d704f 100644 --- a/qinling/tests/unit/base.py +++ b/qinling/tests/unit/base.py @@ -139,6 +139,7 @@ class DbTestCase(BaseTest): (config.STORAGE_GROUP, config.storage_opts), (config.KUBERNETES_GROUP, config.kubernetes_opts), (config.ETCD_GROUP, config.etcd_opts), + (config.RLIMITS_GROUP, config.rlimits_opts), (None, [config.launch_opt]), (None, config.default_opts) ] @@ -193,6 +194,8 @@ class DbTestCase(BaseTest): # 'auth_enable' is disabled by default, we create runtime for # default tenant. 'project_id': DEFAULT_PROJECT_ID, + 'cpu': cfg.CONF.resource_limits.default_cpu, + 'memory_size': cfg.CONF.resource_limits.default_memory, } ) diff --git a/qinling/utils/common.py b/qinling/utils/common.py index 2bc237a4..3fa9f8b5 100644 --- a/qinling/utils/common.py +++ b/qinling/utils/common.py @@ -20,6 +20,7 @@ import warnings from oslo_utils import uuidutils import six +from qinling import exceptions as exc from qinling import version @@ -91,6 +92,22 @@ def generate_unicode_uuid(dashed=True): return uuidutils.generate_uuid(dashed=dashed) +def validate_int_in_range(name, value, min_allowed, max_allowed): + try: + value_int = int(value) + except ValueError: + raise exc.InputException( + 'Invalid %s resource specified. An integer is required.' % name + ) + + if (value_int < min_allowed or value_int > max_allowed): + raise exc.InputException( + '%s resource limitation not within the allowable range: ' + '%s ~ %s(%s).' % (name, min_allowed, max_allowed, + 'millicpu' if name == 'cpu' else 'bytes') + ) + + def disable_ssl_warnings(func): @functools.wraps(func) def wrapper(*args, **kwargs):