diff --git a/lower-constraints.txt b/lower-constraints.txt index 52b41781f25c..8da94c941756 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -15,7 +15,7 @@ coverage==4.0 cryptography==2.7 cursive==0.2.1 dataclasses==0.7 -ddt==1.0.1 +ddt==1.2.1 debtcollector==1.19.0 decorator==3.4.0 deprecation==2.0 diff --git a/nova/compute/provider_config.py b/nova/compute/provider_config.py new file mode 100644 index 000000000000..e6fd7fbe5c32 --- /dev/null +++ b/nova/compute/provider_config.py @@ -0,0 +1,278 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jsonschema +import logging +import microversion_parse +import yaml + +from nova import exception as nova_exc +from nova.i18n import _ + +LOG = logging.getLogger(__name__) + +# A dictionary with keys for all supported major versions with lists of +# corresponding minor versions as values. +SUPPORTED_SCHEMA_VERSIONS = { + 1: {0} +} + +# Supported provider config file schema +SCHEMA_V1 = { + # This defintion uses JSON Schema Draft 7. + # https://json-schema.org/draft-07/json-schema-release-notes.html + 'type': 'object', + 'properties': { + # This property is used to track where the provider.yaml file + # originated. It is reserved for internal use and should never be + # set in a provider.yaml file supplied by an end user. + '__source_file': {'not': {}}, + 'meta': { + 'type': 'object', + 'properties': { + # Version ($Major, $minor) of the schema must successfully + # parse documents conforming to ($Major, 0..N). + # Any breaking schema change (e.g. removing fields, adding + # new required fields, imposing a stricter pattern on a value, + # etc.) must bump $Major. + 'schema_version': { + 'type': 'string', + 'pattern': '^1.([0-9]|[1-9][0-9]+)$' + } + }, + 'required': ['schema_version'], + 'additionalProperties': True + }, + 'providers': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'identification': { + '$ref': '#/$defs/providerIdentification' + }, + 'inventories': { + '$ref': '#/$defs/providerInventories' + }, + 'traits': { + '$ref': '#/$defs/providerTraits' + } + }, + 'required': ['identification'], + 'additionalProperties': True + } + } + }, + 'required': ['meta'], + 'additionalProperties': True, + '$defs': { + 'providerIdentification': { + # Identify a single provider to configure. + # Exactly one identification method should be used. Currently + # `uuid` or `name` are supported, but future versions may + # support others. The uuid can be set to the sentinel value + # `$COMPUTE_NODE` which will cause the consuming compute service to + # apply the configuration to all compute node root providers + # it manages that are not otherwise specified using a uuid or name. + 'type': 'object', + 'properties': { + 'uuid': { + 'oneOf': [ + { + # TODO(sean-k-mooney): replace this with type uuid + # when we can depend on a version of the jsonschema + # lib that implements draft 8 or later of the + # jsonschema spec. + 'type': 'string', + 'pattern': + '^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-' + '[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-' + '[0-9A-Fa-f]{12}$' + }, + { + 'type': 'string', + 'const': '$COMPUTE_NODE' + } + ] + }, + 'name': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 200 + } + }, + # This introduces the possibility of an unsupported key name being + # used to get by schema validation, but is necessary to support + # forward compatibility with new identification methods. + # This should be checked after schema validation. + 'minProperties': 1, + 'maxProperties': 1, + 'additionalProperties': False + }, + 'providerInventories': { + # Allows the admin to specify various adjectives to create and + # manage providers' inventories. This list of adjectives can be + # extended in the future as the schema evolves to meet new use + # cases. As of v1.0, only one adjective, `additional`, is + # supported. + 'type': 'object', + 'properties': { + 'additional': { + 'type': 'array', + 'items': { + 'patternProperties': { + # Allows any key name matching the resource class + # pattern, check to prevent conflicts with virt + # driver owned resouces classes will be done after + # schema validation. + '^[A-Z0-9_]{1,255}$': { + 'type': 'object', + 'properties': { + # Any optional properties not populated + # will be given a default value by + # placement. If overriding a pre-existing + # provider values will not be preserved + # from the existing inventory. + 'total': { + 'type': 'integer' + }, + 'reserved': { + 'type': 'integer' + }, + 'min_unit': { + 'type': 'integer' + }, + 'max_unit': { + 'type': 'integer' + }, + 'step_size': { + 'type': 'integer' + }, + 'allocation_ratio': { + 'type': 'number' + } + }, + 'required': ['total'], + # The defined properties reflect the current + # placement data model. While defining those + # in the schema and not allowing additional + # properties means we will need to bump the + # schema version if they change, that is likely + # to be part of a large change that may have + # other impacts anyway. The benefit of stricter + # validation of property names outweighs the + # (small) chance of having to bump the schema + # version as described above. + 'additionalProperties': False + } + }, + # This ensures only keys matching the pattern + # above are allowed. + 'additionalProperties': False + } + } + }, + 'additionalProperties': True + }, + 'providerTraits': { + # Allows the admin to specify various adjectives to create and + # manage providers' traits. This list of adjectives can be extended + # in the future as the schema evolves to meet new use cases. + # As of v1.0, only one adjective, `additional`, is supported. + 'type': 'object', + 'properties': { + 'additional': { + 'type': 'array', + 'items': { + # Allows any value matching the trait pattern here, + # additional validation will be done after schema + # validation. + 'type': 'string', + 'pattern': '^[A-Z0-9_]{1,255}$' + } + } + }, + 'additionalProperties': True + } + } +} + + +def _load_yaml_file(path): + """Loads and parses a provider.yaml config file into a dict. + + :param path: Path to the yaml file to load. + :return: Dict representing the yaml file requested. + :raise: ProviderConfigException if the path provided cannot be read + or the file is not valid yaml. + """ + try: + with open(path) as open_file: + try: + return yaml.safe_load(open_file) + except yaml.YAMLError as ex: + message = _("Unable to load yaml file: %s ") % ex + if hasattr(ex, 'problem_mark'): + pos = ex.problem_mark + message += _("File: %s ") % open_file.name + message += _("Error position: (%s:%s)") % ( + pos.line + 1, pos.column + 1) + raise nova_exc.ProviderConfigException(error=message) + except OSError: + message = _("Unable to read yaml config file: %s") % path + raise nova_exc.ProviderConfigException(error=message) + + +def _parse_provider_yaml(path): + """Loads schema, parses a provider.yaml file and validates the content. + + :param path: File system path to the file to parse. + :return: dict representing the contents of the file. + :raise ProviderConfigException: If the specified file does + not validate against the schema, the schema version is not supported, + or if unable to read configuration or schema files. + """ + yaml_file = _load_yaml_file(path) + + try: + schema_version = microversion_parse.parse_version_string( + yaml_file['meta']['schema_version']) + except (KeyError, TypeError): + message = _("Unable to detect schema version: %s") % yaml_file + raise nova_exc.ProviderConfigException(error=message) + + if schema_version.major not in SUPPORTED_SCHEMA_VERSIONS: + message = _( + "Unsupported schema major version: %d") % schema_version.major + raise nova_exc.ProviderConfigException(error=message) + + if schema_version.minor not in \ + SUPPORTED_SCHEMA_VERSIONS[schema_version.major]: + # TODO(sean-k-mooney): We should try to provide a better + # message that identifies which fields may be ignored + # and the max minor version supported by this version of nova. + message = ( + "Provider config file [%(path)s] is at schema version " + "%(schema_version)s. Nova supports the major version, " + "but not the minor. Some fields may be ignored." + % {"path": path, "schema_version": schema_version}) + LOG.warning(message) + + try: + jsonschema.validate(yaml_file, SCHEMA_V1) + except jsonschema.exceptions.ValidationError as e: + message = _( + "The provider config file %(path)s did not pass validation " + "for schema version %(schema_version)s: %(reason)s") % { + "path": path, "schema_version": schema_version, "reason": e} + raise nova_exc.ProviderConfigException(error=message) + return yaml_file diff --git a/nova/exception.py b/nova/exception.py index a5dff45bacb9..d60b15ec83f7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -2358,3 +2358,13 @@ class MixedInstanceNotSupportByComputeService(NovaException): class InvalidMixedInstanceDedicatedMask(Invalid): msg_fmt = _("Mixed instance must have at least 1 pinned vCPU and 1 " "unpinned vCPU. See 'hw:cpu_dedicated_mask'.") + + +class ProviderConfigException(NovaException): + """Exception indicating an error occurred processing provider config files. + + This class is used to avoid a raised exception inadvertently being caught + and mishandled by the resource tracker. + """ + msg_fmt = _("An error occurred while processing " + "a provider config file: %(error)s") diff --git a/nova/tests/unit/compute/provider_config_data/v1/validation_error_test_data.yaml b/nova/tests/unit/compute/provider_config_data/v1/validation_error_test_data.yaml new file mode 100644 index 000000000000..278b77cae684 --- /dev/null +++ b/nova/tests/unit/compute/provider_config_data/v1/validation_error_test_data.yaml @@ -0,0 +1,204 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# expected_messages is a list of matches. If the test matches _all_ of the +# values in the list, it will pass. + +no_metadata: + config: {} + expected_messages: ['Unable to detect schema version:'] +no_schema_version: + config: + meta: {} + expected_messages: ['Unable to detect schema version:'] +invalid_schema_version: + config: + meta: + schema_version: '99.99' + expected_messages: ['Unsupported schema major version: 99'] +property__source_file_present_value: + config: + meta: + schema_version: '1.0' + __source_file: "present" + expected_messages: + - "{} is not allowed for" + - "validating 'not' in schema['properties']['__source_file']" +property__source_file_present_null: + config: + meta: + schema_version: '1.0' + __source_file: null + expected_messages: + - "{} is not allowed for" + - "validating 'not' in schema['properties']['__source_file']" +provider_invalid_uuid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: not quite a uuid + expected_messages: + - "'not quite a uuid'" + - "Failed validating" + - "'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'" +provider_null_uuid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: null + expected_messages: + - "The provider config file test_path did not pass validation for schema version 1.0" + - "None is not" + - "'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'" + - "'type': 'string'" +provider_empty_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + name: '' + expected_messages: ["'' is too short"] +provider_null_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + name: null + expected_messages: ["None is not of type 'string'"] +provider_no_name_or_uuid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + expected_messages: ["Failed validating 'type' in schema['properties']['providers']['items']['properties']['identification']"] +provider_uuid_and_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + name: custom_provider + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + expected_messages: + - "'name': 'custom_provider'" + - "'uuid': 'aa884151-b4e2-4e82-9fd4-81cfcd01abb9'" + - "has too many properties" +provider_no_identification: + config: + meta: + schema_version: '1.0' + providers: + - {} + expected_messages: ["'identification' is a required property"] +inventories_additional_resource_class_no_total: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - RESOURCE1: {} + expected_messages: ["'total' is a required property"] +inventories_additional_resource_class_invalid_total: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - RESOURCE1: + total: invalid_total + expected_messages: ["'invalid_total' is not of type 'integer'"] +inventories_additional_resource_class_additional_property: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - RESOURCE1: + total: 1 + additional_property: 2 + expected_messages: ["Additional properties are not allowed ('additional_property' was unexpected)"] +inventories_one_invalid_additional_resource_class: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - RESOURCE1: + total: 1 + - RESOURCE2: {} + expected_messages: ["'total' is a required property"] +inventories_invalid_additional_resource_class_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - INVALID_RESOURCE_CLASS_NAME_!@#$%^&*()_+: + total: 1 + expected_messages: ["'INVALID_RESOURCE_CLASS_NAME_!@#$%^&*()_+' does not match any of the regexes"] +traits_one_additional_trait_invalid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + additional: + - TRAIT1: invalid_trait + expected_messages: ["{'TRAIT1': 'invalid_trait'} is not of type 'string'"] +traits_multiple_additional_traits_two_invalid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + additional: + - TRAIT1: invalid + - TRAIT2 + - TRAIT3: invalid + expected_messages: ["{'TRAIT1': 'invalid'} is not of type 'string'"] +traits_invalid_trait_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + additional: + - INVALID_TRAIT_NAME_!@#$%^&*()_+ + expected_messages: ["'INVALID_TRAIT_NAME_!@#$%^&*()_+' does not match '^[A-Z0-9_]{1,255}$'"] diff --git a/nova/tests/unit/compute/provider_config_data/v1/validation_success_test_data.yaml b/nova/tests/unit/compute/provider_config_data/v1/validation_success_test_data.yaml new file mode 100644 index 000000000000..5453f2bc3765 --- /dev/null +++ b/nova/tests/unit/compute/provider_config_data/v1/validation_success_test_data.yaml @@ -0,0 +1,113 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +provider_by_uuid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 +provider_magic_uuid: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: "$COMPUTE_NODE" +provider_by_name: + config: + meta: + schema_version: '1.0' + providers: + - identification: + name: custom_provider +inventories_additional_resource_class: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - CUSTOM_RESOURCE1: + total: 1 +inventories_unknown_adjective: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + invalid_adjective: + - CUSTOM_RESOURCE1: + total: 1 +inventories_multiple_additional_resource_classes: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - CUSTOM_RESOURCE1: + total: 1 + - CUSTOM_RESOURCE2: + total: 1 +traits_one_additional_trait: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + additional: + - CUSTOM_TRAIT1 +traits_multiple_additional_traits: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + additional: + - CUSTOM_TRAIT1 + - CUSTOM_TRAIT2 +traits_unknown_adjective: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + traits: + invalid: + - CUSTOM_TRAIT1 +inventories_and_traits_additional_resource_class_and_trait: + config: + meta: + schema_version: '1.0' + providers: + - identification: + uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9 + inventories: + additional: + - CUSTOM_RESOURCE1: + total: 1 + traits: + additional: + - CUSTOM_TRAIT1 diff --git a/nova/tests/unit/compute/test_provider_config.py b/nova/tests/unit/compute/test_provider_config.py new file mode 100644 index 000000000000..46372d2764de --- /dev/null +++ b/nova/tests/unit/compute/test_provider_config.py @@ -0,0 +1,124 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import fixtures +import microversion_parse + +from oslotest import base + +from nova.compute import provider_config +from nova import exception as nova_exc + + +class SchemaValidationMixin(base.BaseTestCase): + """This class provides the basic methods for running schema validation test + cases. It can be used along with ddt.file_data to test a specific schema + version using tests defined in yaml files. See SchemaValidationTestCasesV1 + for an example of how this was done for schema version 1. + + Because decorators can only access class properties of the class they are + defined in (even when overriding values in the subclass), the decorators + need to be placed in the subclass. This is why there are test_ functions in + the subclass that call the run_test_ methods in this class. This should + keep things simple as more schema versions are added. + """ + def setUp(self): + super(SchemaValidationMixin, self).setUp() + self.mock_load_yaml = self.useFixture( + fixtures.MockPatchObject( + provider_config, '_load_yaml_file')).mock + self.mock_LOG = self.useFixture( + fixtures.MockPatchObject( + provider_config, 'LOG')).mock + + def set_config(self, config=None): + data = config or {} + self.mock_load_yaml.return_value = data + return data + + def run_test_validation_errors(self, config, expected_messages): + self.set_config(config=config) + + actual_msg = self.assertRaises( + nova_exc.ProviderConfigException, + provider_config._parse_provider_yaml, 'test_path').message + + for msg in expected_messages: + self.assertIn(msg, actual_msg) + + def run_test_validation_success(self, config): + reference = self.set_config(config=config) + + actual = provider_config._parse_provider_yaml('test_path') + + self.assertEqual(reference, actual) + + def run_schema_version_matching( + self, min_schema_version, max_schema_version): + # note _load_yaml_file is mocked so the value is not important + # however it may appear in logs messages so changing it could + # result in tests failing unless the expected_messages field + # is updated in the test data. + path = 'test_path' + + # test exactly min and max versions are supported + self.set_config(config={ + 'meta': {'schema_version': str(min_schema_version)}}) + provider_config._parse_provider_yaml(path) + self.set_config(config={ + 'meta': {'schema_version': str(max_schema_version)}}) + provider_config._parse_provider_yaml(path) + + self.mock_LOG.warning.assert_not_called() + + # test max major+1 raises + higher_major = microversion_parse.Version( + major=max_schema_version.major + 1, minor=max_schema_version.minor) + self.set_config(config={'meta': {'schema_version': str(higher_major)}}) + + self.assertRaises(nova_exc.ProviderConfigException, + provider_config._parse_provider_yaml, path) + + # test max major with max minor+1 is logged + higher_minor = microversion_parse.Version( + major=max_schema_version.major, minor=max_schema_version.minor + 1) + expected_log_call = ( + "Provider config file [%(path)s] is at schema version " + "%(schema_version)s. Nova supports the major version, but " + "not the minor. Some fields may be ignored." % { + "path": path, "schema_version": higher_minor}) + self.set_config(config={'meta': {'schema_version': str(higher_minor)}}) + + provider_config._parse_provider_yaml(path) + + self.mock_LOG.warning.assert_called_once_with(expected_log_call) + + +@ddt.ddt +class SchemaValidationTestCasesV1(SchemaValidationMixin): + MIN_SCHEMA_VERSION = microversion_parse.Version(1, 0) + MAX_SCHEMA_VERSION = microversion_parse.Version(1, 0) + + @ddt.unpack + @ddt.file_data('provider_config_data/v1/validation_error_test_data.yaml') + def test_validation_errors(self, config, expected_messages): + self.run_test_validation_errors(config, expected_messages) + + @ddt.unpack + @ddt.file_data('provider_config_data/v1/validation_success_test_data.yaml') + def test_validation_success(self, config): + self.run_test_validation_success(config) + + def test_schema_version_matching(self): + self.run_schema_version_matching(self.MIN_SCHEMA_VERSION, + self.MAX_SCHEMA_VERSION) diff --git a/requirements.txt b/requirements.txt index 94d25467b7e9..674381864e01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,3 +71,4 @@ zVMCloudConnector>=1.3.0;sys_platform!='win32' # Apache 2.0 License futurist>=1.8.0 # Apache-2.0 openstacksdk>=0.35.0 # Apache-2.0 dataclasses>=0.7;python_version=='3.6' # Apache 2.0 License +PyYAML>=3.12 # MIT diff --git a/test-requirements.txt b/test-requirements.txt index 5ec94aec9cc2..2e6a61ad5fde 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=3.1.0,<3.2.0 # Apache-2.0 mypy>=0.761 # MIT coverage!=4.4,>=4.0 # Apache-2.0 -ddt>=1.0.1 # MIT +ddt>=1.2.1 # MIT fixtures>=3.0.0 # Apache-2.0/BSD mock>=3.0.0 # BSD psycopg2>=2.7 # LGPL/ZPL