diff --git a/lower-constraints.txt b/lower-constraints.txt index 41afef13c798..93e757994e25 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -73,6 +73,7 @@ oslo.context==3.4.0 oslo.db==10.0.0 oslo.i18n==5.1.0 oslo.log==4.6.1 +oslo.limit==1.5.0 oslo.messaging==10.3.0 oslo.middleware==3.31.0 oslo.policy==3.7.0 diff --git a/mypy-files.txt b/mypy-files.txt index 898eee25c7cd..5dcd3044207a 100644 --- a/mypy-files.txt +++ b/mypy-files.txt @@ -1,5 +1,6 @@ nova/compute/manager.py nova/crypto.py +nova/limit/local.py nova/network/neutron.py nova/pci nova/privsep/path.py diff --git a/nova/exception.py b/nova/exception.py index eeba922ced77..4588898aae9d 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -135,6 +135,10 @@ class GlanceConnectionFailed(NovaException): "%(reason)s") +class KeystoneConnectionFailed(NovaException): + msg_fmt = _("Connection to keystone host failed: %(reason)s") + + class CinderConnectionFailed(NovaException): msg_fmt = _("Connection to cinder host failed: %(reason)s") @@ -1281,6 +1285,14 @@ class PortLimitExceeded(OverQuota): msg_fmt = _("Maximum number of ports exceeded") +class ServerGroupLimitExceeded(OverQuota): + msg_fmt = _("Quota exceeded, too many server groups.") + + +class GroupMemberLimitExceeded(OverQuota): + msg_fmt = _("Quota exceeded, too many servers in group") + + class AggregateNotFound(NotFound): msg_fmt = _("Aggregate %(aggregate_id)s could not be found.") diff --git a/nova/limit/__init__.py b/nova/limit/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/limit/local.py b/nova/limit/local.py new file mode 100644 index 000000000000..5a02a2d5ed43 --- /dev/null +++ b/nova/limit/local.py @@ -0,0 +1,174 @@ +# Copyright 2022 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import typing as ty + +from oslo_limit import exception as limit_exceptions +from oslo_limit import limit +from oslo_log import log as logging + +import nova.conf +from nova import context as nova_context +from nova import exception +from nova import objects + +LOG = logging.getLogger(__name__) +CONF = nova.conf.CONF + +# Entity types for API Limits, same as names of config options prefixed with +# "server_" to disambiguate them in keystone +SERVER_METADATA_ITEMS = "server_metadata_items" +INJECTED_FILES = "server_injected_files" +INJECTED_FILES_CONTENT = "server_injected_file_content_bytes" +INJECTED_FILES_PATH = "server_injected_file_path_bytes" +API_LIMITS = set([ + SERVER_METADATA_ITEMS, + INJECTED_FILES, + INJECTED_FILES_CONTENT, + INJECTED_FILES_PATH, +]) + +# Entity types for all DB limits, same as names of config options prefixed with +# "server_" to disambiguate them in keystone +KEY_PAIRS = "server_key_pairs" +SERVER_GROUPS = "server_groups" +SERVER_GROUP_MEMBERS = "server_group_members" + +# Checks only happen when we are using the unified limits driver +UNIFIED_LIMITS_DRIVER = "nova.quota.UnifiedLimitsDriver" + +# Map entity types to the exception we raise in the case that the resource is +# over the allowed limit. Each of these should be a subclass of +# exception.OverQuota. +EXCEPTIONS = { + KEY_PAIRS: exception.KeypairLimitExceeded, + INJECTED_FILES_CONTENT: exception.OnsetFileContentLimitExceeded, + INJECTED_FILES_PATH: exception.OnsetFilePathLimitExceeded, + INJECTED_FILES: exception.OnsetFileLimitExceeded, + SERVER_METADATA_ITEMS: exception.MetadataLimitExceeded, + SERVER_GROUPS: exception.ServerGroupLimitExceeded, + SERVER_GROUP_MEMBERS: exception.GroupMemberLimitExceeded, +} + + +def always_zero_usage( + project_id: str, resource_names: ty.List[str] +) -> ty.Dict[str, int]: + """Called by oslo_limit's enforcer""" + # Return usage of 0 for API limits. Values in API requests will be used as + # the deltas. + return {resource_name: 0 for resource_name in resource_names} + + +def enforce_api_limit(entity_type: str, count: int) -> None: + """Check if the values given are over the limit for that key. + + This is generally used for limiting the size of certain API requests + that eventually get stored in the database. + """ + if CONF.quota.driver != UNIFIED_LIMITS_DRIVER: + return + + if entity_type not in API_LIMITS: + fmt = "%s is not a valid API limit: %s" + raise ValueError(fmt % (entity_type, API_LIMITS)) + + try: + enforcer = limit.Enforcer(always_zero_usage) + except limit_exceptions.SessionInitError as e: + msg = ("Failed to connect to keystone while enforcing %s quota limit." + % entity_type) + LOG.error(msg + " Error: " + str(e)) + raise exception.KeystoneConnectionFailed(msg) + + try: + enforcer.enforce(None, {entity_type: count}) + except limit_exceptions.ProjectOverLimit as e: + # Copy the exception message to a OverQuota to propagate to the + # API layer. + raise EXCEPTIONS.get(entity_type, exception.OverQuota)(str(e)) + + +def enforce_db_limit( + context: nova_context.RequestContext, + entity_type: str, + entity_scope: ty.Any, + delta: int +) -> None: + """Check provided delta does not put resource over limit. + + Firstly we count the current usage given the specified scope. + We then add that count to the specified delta to see if we + are over the limit for that kind of entity. + + Note previously we used to recheck these limits. + However these are really soft DDoS protections, + not hard resource limits, so we don't do the recheck for these. + + The scope is specific to the limit type: + * key_pairs scope is context.user_id + * server_groups scope is context.project_id + * server_group_members scope is server_group_uuid + """ + if CONF.quota.driver != UNIFIED_LIMITS_DRIVER: + return + + if entity_type not in DB_COUNT_FUNCTION.keys(): + fmt = "%s does not have a DB count function defined: %s" + raise ValueError(fmt % (entity_type, DB_COUNT_FUNCTION.keys())) + if delta < 0: + raise ValueError("delta must be a positive integer") + + count_function = DB_COUNT_FUNCTION[entity_type] + + try: + enforcer = limit.Enforcer( + functools.partial(count_function, context, entity_scope)) + except limit_exceptions.SessionInitError as e: + msg = ("Failed to connect to keystone while enforcing %s quota limit." + % entity_type) + LOG.error(msg + " Error: " + str(e)) + raise exception.KeystoneConnectionFailed(msg) + + try: + enforcer.enforce(None, {entity_type: delta}) + except limit_exceptions.ProjectOverLimit as e: + # Copy the exception message to a OverQuota to propagate to the + # API layer. + raise EXCEPTIONS.get(entity_type, exception.OverQuota)(str(e)) + + +def _keypair_count(context, user_id, *args): + count = objects.KeyPairList.get_count_by_user(context, user_id) + return {'server_key_pairs': count} + + +def _server_group_count(context, project_id, *args): + raw_counts = objects.InstanceGroupList.get_counts(context, project_id) + return {'server_groups': raw_counts['project']['server_groups']} + + +def _server_group_members_count(context, server_group_uuid, *args): + # NOTE(johngarbutt) we used to count members added per user + server_group = objects.InstanceGroup.get_by_uuid(context, + server_group_uuid) + return {'server_group_members': len(server_group.members)} + + +DB_COUNT_FUNCTION = { + KEY_PAIRS: _keypair_count, + SERVER_GROUPS: _server_group_count, + SERVER_GROUP_MEMBERS: _server_group_members_count +} diff --git a/nova/tests/unit/limit/__init__.py b/nova/tests/unit/limit/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/tests/unit/limit/test_local.py b/nova/tests/unit/limit/test_local.py new file mode 100644 index 000000000000..bd0fc0c776b6 --- /dev/null +++ b/nova/tests/unit/limit/test_local.py @@ -0,0 +1,204 @@ +# Copyright 2022 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslo_config import cfg +from oslo_limit import exception as limit_exceptions +from oslo_limit import fixture as limit_fixture +from oslo_utils.fixture import uuidsentinel as uuids + +from nova import context +from nova import exception +from nova.limit import local as local_limit +from nova import objects +from nova import test + +CONF = cfg.CONF + + +class TestLocalLimits(test.NoDBTestCase): + def setUp(self): + super(TestLocalLimits, self).setUp() + self.flags(driver=local_limit.UNIFIED_LIMITS_DRIVER, group="quota") + self.context = context.RequestContext() + + def test_enforce_api_limit_metadata(self): + # default max is 128 + self.useFixture(limit_fixture.LimitFixture( + {local_limit.SERVER_METADATA_ITEMS: 128}, {})) + local_limit.enforce_api_limit(local_limit.SERVER_METADATA_ITEMS, 128) + + e = self.assertRaises(exception.MetadataLimitExceeded, + local_limit.enforce_api_limit, + local_limit.SERVER_METADATA_ITEMS, 129) + msg = ("Resource %s is over limit" % local_limit.SERVER_METADATA_ITEMS) + self.assertIn(msg, str(e)) + + def test_enforce_api_limit_skip(self): + self.flags(driver="nova.quota.NoopQuotaDriver", group="quota") + local_limit.enforce_api_limit(local_limit.SERVER_METADATA_ITEMS, 200) + + @mock.patch('oslo_limit.limit.Enforcer') + def test_enforce_api_limit_session_init_error(self, mock_util): + mock_util.side_effect = limit_exceptions.SessionInitError('error') + + e = self.assertRaises(exception.KeystoneConnectionFailed, + local_limit.enforce_api_limit, + local_limit.SERVER_METADATA_ITEMS, 42) + expected = ('Failed to connect to keystone while enforcing ' + 'server_metadata_items quota limit.') + self.assertIn(expected, str(e)) + + def test_enforce_api_limit_raises_for_invalid_entity(self): + e = self.assertRaises(ValueError, + local_limit.enforce_api_limit, + local_limit.KEY_PAIRS, 42) + expected = '%s is not a valid API limit: %s' % ( + local_limit.KEY_PAIRS, local_limit.API_LIMITS) + self.assertEqual(expected, str(e)) + + def test_enforce_api_limit_no_registered_limit_found(self): + self.useFixture(limit_fixture.LimitFixture({}, {})) + e = self.assertRaises(exception.MetadataLimitExceeded, + local_limit.enforce_api_limit, + local_limit.SERVER_METADATA_ITEMS, 42) + msg = ("Resource %s is over limit" % local_limit.SERVER_METADATA_ITEMS) + self.assertIn(msg, str(e)) + + def test_enforce_injected_files(self): + reglimits = {local_limit.INJECTED_FILES: 5, + local_limit.INJECTED_FILES_CONTENT: 10 * 1024, + local_limit.INJECTED_FILES_PATH: 255} + self.useFixture(limit_fixture.LimitFixture(reglimits, {})) + + local_limit.enforce_api_limit(local_limit.INJECTED_FILES, 5) + local_limit.enforce_api_limit(local_limit.INJECTED_FILES_CONTENT, + 10 * 1024) + local_limit.enforce_api_limit(local_limit.INJECTED_FILES_PATH, 255) + + e = self.assertRaises(exception.OnsetFileLimitExceeded, + local_limit.enforce_api_limit, + local_limit.INJECTED_FILES, 6) + msg = ("Resource %s is over limit" % local_limit.INJECTED_FILES) + self.assertIn(msg, str(e)) + e = self.assertRaises(exception.OnsetFileContentLimitExceeded, + local_limit.enforce_api_limit, + local_limit.INJECTED_FILES_CONTENT, + 10 * 1024 + 1) + msg = ( + "Resource %s is over limit" % local_limit.INJECTED_FILES_CONTENT) + self.assertIn(msg, str(e)) + e = self.assertRaises(exception.OnsetFilePathLimitExceeded, + local_limit.enforce_api_limit, + local_limit.INJECTED_FILES_PATH, 256) + msg = ("Resource %s is over limit" % local_limit.INJECTED_FILES_PATH) + self.assertIn(msg, str(e)) + + @mock.patch.object(objects.KeyPairList, "get_count_by_user") + def test_enforce_db_limit_keypairs(self, mock_count): + self.useFixture(limit_fixture.LimitFixture( + {local_limit.KEY_PAIRS: 100}, {})) + + mock_count.return_value = 99 + local_limit.enforce_db_limit(self.context, local_limit.KEY_PAIRS, + uuids.user_id, 1) + mock_count.assert_called_once_with(self.context, uuids.user_id) + + self.assertRaises(exception.KeypairLimitExceeded, + local_limit.enforce_db_limit, + self.context, local_limit.KEY_PAIRS, + uuids.user_id, 2) + + mock_count.return_value = 100 + local_limit.enforce_db_limit(self.context, local_limit.KEY_PAIRS, + uuids.user_id, 0) + mock_count.return_value = 101 + self.assertRaises(exception.KeypairLimitExceeded, + local_limit.enforce_db_limit, + self.context, local_limit.KEY_PAIRS, + uuids.user_id, 0) + + def test_enforce_db_limit_skip(self): + self.flags(driver="nova.quota.NoopQuotaDriver", group="quota") + local_limit.enforce_db_limit(self.context, local_limit.KEY_PAIRS, + uuids.user_id, 1) + + @mock.patch('oslo_limit.limit.Enforcer') + def test_enforce_db_limit_session_init_error(self, mock_util): + mock_util.side_effect = limit_exceptions.SessionInitError( + test.TestingException()) + + e = self.assertRaises(exception.KeystoneConnectionFailed, + local_limit.enforce_db_limit, self.context, + local_limit.KEY_PAIRS, uuids.user_id, 42) + expected = ('Failed to connect to keystone while enforcing ' + 'server_key_pairs quota limit.') + self.assertEqual(expected, str(e)) + + def test_enforce_db_limit_raise_on_invalid(self): + e = self.assertRaises(ValueError, local_limit.enforce_db_limit, + self.context, local_limit.INJECTED_FILES, + uuids.user_id, 1) + fmt = '%s does not have a DB count function defined: %s' + expected = fmt % ( + local_limit.INJECTED_FILES, local_limit.DB_COUNT_FUNCTION.keys()) + self.assertEqual(expected, str(e)) + + @mock.patch.object(objects.KeyPairList, "get_count_by_user") + def test_enforce_db_limit_no_registered_limit_found(self, mock_count): + self.useFixture(limit_fixture.LimitFixture({}, {})) + mock_count.return_value = 5 + e = self.assertRaises(exception.KeypairLimitExceeded, + local_limit.enforce_db_limit, self.context, + local_limit.KEY_PAIRS, uuids.user_id, 42) + msg = ("Resource %s is over limit" % local_limit.KEY_PAIRS) + self.assertIn(msg, str(e)) + + def test_enforce_db_limit_raise_bad_delta(self): + e = self.assertRaises(ValueError, local_limit.enforce_db_limit, + self.context, local_limit.KEY_PAIRS, + uuids.user_id, -1) + self.assertEqual("delta must be a positive integer", str(e)) + + @mock.patch.object(objects.InstanceGroupList, "get_counts") + def test_enforce_db_limit_server_groups(self, mock_count): + self.useFixture(limit_fixture.LimitFixture( + {local_limit.SERVER_GROUPS: 10}, {})) + + mock_count.return_value = {'project': {'server_groups': 9}} + local_limit.enforce_db_limit(self.context, local_limit.SERVER_GROUPS, + uuids.project_id, 1) + mock_count.assert_called_once_with(self.context, uuids.project_id) + + self.assertRaises(exception.ServerGroupLimitExceeded, + local_limit.enforce_db_limit, + self.context, local_limit.SERVER_GROUPS, + uuids.project_id, 2) + + @mock.patch.object(objects.InstanceGroup, "get_by_uuid") + def test_enforce_db_limit_server_group_members(self, mock_get): + self.useFixture(limit_fixture.LimitFixture( + {local_limit.SERVER_GROUP_MEMBERS: 10}, {})) + + mock_get.return_value = objects.InstanceGroup(members=[]) + local_limit.enforce_db_limit(self.context, + local_limit.SERVER_GROUP_MEMBERS, + uuids.server_group, 10) + mock_get.assert_called_once_with(self.context, uuids.server_group) + + self.assertRaises(exception.GroupMemberLimitExceeded, + local_limit.enforce_db_limit, + self.context, local_limit.SERVER_GROUP_MEMBERS, + uuids.server_group, 11) diff --git a/requirements.txt b/requirements.txt index dc5860d6e630..1a83a0813a4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ oslo.concurrency>=4.5.0 # Apache-2.0 oslo.config>=8.6.0 # Apache-2.0 oslo.context>=3.4.0 # Apache-2.0 oslo.log>=4.6.1 # Apache-2.0 +oslo.limit>=1.5.0 # Apache-2.0 oslo.reports>=1.18.0 # Apache-2.0 oslo.serialization>=4.2.0 # Apache-2.0 oslo.upgradecheck>=1.3.0