diff --git a/.zuul.yaml b/.zuul.yaml index 2681790bf..09a704b74 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -198,6 +198,10 @@ period: 120 watcher_cluster_data_model_collectors.storage: period: 120 + compute_model: + enable_extended_attributes: true + nova_client: + api_version: "2.96" test-config: $TEMPEST_CONFIG: compute: @@ -217,12 +221,14 @@ ceilometer_polling_interval: 15 optimize: datasource: prometheus + extended_attributes_nova_microversion: "2.96" + data_model_collectors_period: 120 tempest_plugins: - watcher-tempest-plugin # All tests inside watcher_tempest_plugin.tests.scenario with tag "strategy" - # or test_execute_strategies file + # and test_execute_strategies, test_data_model files # excluding tests with tag "real_load" - tempest_test_regex: (watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(watcher_tempest_plugin.tests.scenario.test_execute_strategies) + tempest_test_regex: (watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(watcher_tempest_plugin.tests.scenario.(test_execute_strategies|test_data_model)) tempest_exclude_regex: .*\[.*\breal_load\b.*\].* tempest_concurrency: 1 tox_envlist: all diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index c8cd2a3b7..eff542ccd 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -552,6 +552,13 @@ server_disk: in: body required: true type: integer +server_flavor_extra_specs: + description: | + The flavor extra specs of the server. + in: body + required: true + type: JSON + min_version: 1.6 server_locked: description: | Whether the server is locked. @@ -576,6 +583,13 @@ server_name: in: body required: true type: string +server_pinned_az: + description: | + The pinned availability zone of the server. + in: body + required: true + type: string + min_version: 1.6 server_project_id: description: | The project ID of the server. diff --git a/api-ref/source/samples/datamodel-list-response.json b/api-ref/source/samples/datamodel-list-response.json index 919569c7b..9e412dffc 100644 --- a/api-ref/source/samples/datamodel-list-response.json +++ b/api-ref/source/samples/datamodel-list-response.json @@ -11,6 +11,10 @@ "server_project_id": "baea342fc74b4a1785b4a40c69a8d958", "server_locked":false, "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb", + "server_pinned_az": "nova", + "server_flavor_extra_specs": { + "hw_rng:allowed": true + }, "node_hostname": "localhost.localdomain", "node_status": "enabled", "node_disabled_reason": null, @@ -37,6 +41,8 @@ "server_project_id": "baea342fc74b4a1785b4a40c69a8d958", "server_locked": false, "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4", + "server_pinned_az": "nova", + "server_flavor_extra_specs": {}, "node_hostname": "localhost.localdomain", "node_status": "enabled", "node_disabled_reason": null, diff --git a/api-ref/source/watcher-api-v1-datamodel.inc b/api-ref/source/watcher-api-v1-datamodel.inc index 57c5093b5..be944fb34 100644 --- a/api-ref/source/watcher-api-v1-datamodel.inc +++ b/api-ref/source/watcher-api-v1-datamodel.inc @@ -45,6 +45,8 @@ Response - server_project_id: server_project_id - server_locked: server_locked - server_uuid: server_uuid + - server_pinned_az: server_pinned_az + - server_flavor_extra_specs: server_flavor_extra_specs - node_hostname: node_hostname - node_status: node_status - node_disabled_reason: node_disabled_reason diff --git a/devstack/lib/watcher b/devstack/lib/watcher index fe04d4c11..962070999 100644 --- a/devstack/lib/watcher +++ b/devstack/lib/watcher @@ -268,7 +268,7 @@ function configure_tempest_for_watcher { # Please make sure to update this when the microversion is updated, otherwise # new tests may be skipped. TEMPEST_WATCHER_MIN_MICROVERSION=${TEMPEST_WATCHER_MIN_MICROVERSION:-"1.0"} - TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.5"} + TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.6"} # Set microversion options in tempest.conf iniset $TEMPEST_CONFIG optimize min_microversion $TEMPEST_WATCHER_MIN_MICROVERSION diff --git a/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml b/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml new file mode 100644 index 000000000..952fce49b --- /dev/null +++ b/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + The compute model was extended with additional server attributes to provide + more detailed information about compute instances. These additions will + enable strategies to make more precise decisions by considering more server + placement constraints. The new attributes are ``flavor extra specs`` and + ``pinned availability zone``. Each new attribute depends on a minimal + microversion to be supported in nova and configured in the watcher + configuration, at ``nova_client`` section. Please refer to the nova api-ref + documentation for more details on which microversion is required: + https://docs.openstack.org/api-ref/compute/ + A new configuration option was added to allow the user to enable or disable + the extended attributes collection, which is disabled by default. diff --git a/test-requirements.txt b/test-requirements.txt index 005f0704e..f668bcc01 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ coverage>=4.5.1 # Apache-2.0 +ddt>=1.2.1 # MIT freezegun>=0.3.10 # Apache-2.0 oslotest>=3.3.0 # Apache-2.0 testscenarios>=0.5.0 # Apache-2.0/BSD diff --git a/watcher/api/controllers/v1/data_model.py b/watcher/api/controllers/v1/data_model.py index a3c794ade..9e6d164f5 100644 --- a/watcher/api/controllers/v1/data_model.py +++ b/watcher/api/controllers/v1/data_model.py @@ -30,6 +30,22 @@ from watcher.common import policy from watcher.decision_engine import rpcapi +def hide_fields_in_newer_versions(obj): + """This method hides fields that were added in newer API versions. + + Certain node fields were introduced at certain API versions. + These fields are only made available when the request's API version + matches or exceeds the versions when these fields were introduced. + """ + if not utils.allow_list_extend_compute_model(): + # NOTE(dviroel): the content returned by the rpc is + # a list of dicts, so we need to remove the elements based + # on the api version. + for elem in obj.get('context', []): + elem.pop('server_pinned_az', None) + elem.pop('server_flavor_extra_specs', None) + + class DataModelController(rest.RestController): """REST controller for data model""" @@ -63,4 +79,5 @@ class DataModelController(rest.RestController): context, data_model_type, audit_uuid) + hide_fields_in_newer_versions(rpc_all_data_model) return rpc_all_data_model diff --git a/watcher/api/controllers/v1/utils.py b/watcher/api/controllers/v1/utils.py index 23cce2920..790160f74 100644 --- a/watcher/api/controllers/v1/utils.py +++ b/watcher/api/controllers/v1/utils.py @@ -203,3 +203,13 @@ def allow_skipped_action(): """ return pecan.request.version.minor >= ( versions.VERSIONS.MINOR_5_SKIPPED_ACTION.value) + + +def allow_list_extend_compute_model(): + """Check if it should list extended compute model attributes. + + Version 1.6 of the API added support to additional attributes + to the compute data model. + """ + return pecan.request.version.minor >= ( + versions.VERSIONS.MINOR_6_EXT_COMPUTE_MODEL.value) diff --git a/watcher/api/controllers/v1/versions.py b/watcher/api/controllers/v1/versions.py index f7dfbae77..ddb526959 100644 --- a/watcher/api/controllers/v1/versions.py +++ b/watcher/api/controllers/v1/versions.py @@ -24,7 +24,8 @@ class VERSIONS(enum.Enum): MINOR_3_DATAMODEL = 3 # v1.3: Add list datamodel API MINOR_4_WEBHOOK_API = 4 # v1.4: Add webhook trigger API MINOR_5_SKIPPED_ACTION = 5 # v1.5: Add skipped action support - MINOR_MAX_VERSION = 5 + MINOR_6_EXT_COMPUTE_MODEL = 6 # v1.6: Extend compute data model API + MINOR_MAX_VERSION = 6 # This is the version 1 API diff --git a/watcher/common/nova_helper.py b/watcher/common/nova_helper.py index 997414daa..e8719d8ca 100644 --- a/watcher/common/nova_helper.py +++ b/watcher/common/nova_helper.py @@ -43,6 +43,19 @@ class NovaHelper(object): self.cinder = self.osc.cinder() self.nova = self.osc.nova() self.glance = self.osc.glance() + self._is_pinned_az_available = None + + def is_pinned_az_available(self): + """Check if pinned AZ is available in GET /servers/detail response. + + :returns: True if is available, False otherwise. + """ + if self._is_pinned_az_available is None: + self._is_pinned_az_available = ( + api_versions.APIVersion( + version_str=CONF.nova_client.api_version) >= + api_versions.APIVersion(version_str='2.96')) + return self._is_pinned_az_available def get_compute_node_list(self): hypervisors = self.nova.hypervisors.list() diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index 12dcb35ae..337339e6d 100644 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -36,6 +36,7 @@ from watcher.conf import grafana_translators from watcher.conf import ironic_client from watcher.conf import keystone_client from watcher.conf import maas_client +from watcher.conf import models from watcher.conf import monasca_client from watcher.conf import neutron_client from watcher.conf import nova_client @@ -59,6 +60,7 @@ applier.register_opts(CONF) decision_engine.register_opts(CONF) maas_client.register_opts(CONF) monasca_client.register_opts(CONF) +models.register_opts(CONF) nova_client.register_opts(CONF) glance_client.register_opts(CONF) gnocchi_client.register_opts(CONF) diff --git a/watcher/conf/models.py b/watcher/conf/models.py new file mode 100644 index 000000000..03671cea3 --- /dev/null +++ b/watcher/conf/models.py @@ -0,0 +1,40 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from oslo_config import cfg + +compute_model = cfg.OptGroup(name='compute_model', + title='Configuration Options for Compute Model', + help="Additional configuration options for the " + "compute model.") + +COMPUTE_MODEL_OPTS = [ + cfg.BoolOpt( + 'enable_extended_attributes', + default=False, + help="Enable the collection of compute model extended attributes. " + "Note that some attributes require a more recent api " + "microversion to be configured in nova_client section." + ), +] + + +def register_opts(conf): + conf.register_group(compute_model) + conf.register_opts(COMPUTE_MODEL_OPTS, group=compute_model) + + +def list_opts(): + return [(compute_model, COMPUTE_MODEL_OPTS)] diff --git a/watcher/decision_engine/model/collector/nova.py b/watcher/decision_engine/model/collector/nova.py index 154b8b94d..b65631031 100644 --- a/watcher/decision_engine/model/collector/nova.py +++ b/watcher/decision_engine/model/collector/nova.py @@ -471,13 +471,22 @@ class NovaModelBuilder(base.BaseModelBuilder): "state": getattr(instance, "OS-EXT-STS:vm_state"), "metadata": instance.metadata, "project_id": instance.tenant_id, - "locked": instance.locked} + "locked": instance.locked, + # NOTE(dviroel): new attributes are updated if + # extended feature is enabled + "flavor_extra_specs": {}, + "pinned_az": "", + } + + if self.model.extended_attributes_enabled: + instance_attributes.update({ + "flavor_extra_specs": flavor["extra_specs"], + }) + if self.nova_helper.is_pinned_az_available(): + instance_attributes["pinned_az"] = ( + instance.pinned_availability_zone + ) - # node_attributes = dict() - # node_attributes["layer"] = "virtual" - # node_attributes["category"] = "compute" - # node_attributes["type"] = "compute" - # node_attributes["attributes"] = instance_attributes return element.Instance(**instance_attributes) def _merge_compute_scope(self, compute_scope): diff --git a/watcher/decision_engine/model/element/instance.py b/watcher/decision_engine/model/element/instance.py index f2e9aeadf..bb892e874 100644 --- a/watcher/decision_engine/model/element/instance.py +++ b/watcher/decision_engine/model/element/instance.py @@ -54,6 +54,9 @@ class Instance(compute_resource.ComputeResource): "metadata": wfields.JsonField(), "project_id": wfields.UUIDField(), "locked": wfields.BooleanField(default=False), + # New fields for extended compute model + "pinned_az": wfields.StringField(default=""), + "flavor_extra_specs": wfields.JsonField(default={}), } def accept(self, visitor): diff --git a/watcher/decision_engine/model/model_root.py b/watcher/decision_engine/model/model_root.py index c05101ec8..4a7b740e5 100644 --- a/watcher/decision_engine/model/model_root.py +++ b/watcher/decision_engine/model/model_root.py @@ -17,9 +17,11 @@ Openstack implementation of the cluster graph. """ import ast + from lxml import etree # nosec: B410 import networkx as nx from oslo_concurrency import lockutils +from oslo_config import cfg from oslo_log import log from watcher._i18n import _ @@ -28,6 +30,7 @@ from watcher.decision_engine.model import base from watcher.decision_engine.model import element LOG = log.getLogger(__name__) +CONF = cfg.CONF class ModelRoot(nx.DiGraph, base.Model): @@ -36,12 +39,24 @@ class ModelRoot(nx.DiGraph, base.Model): def __init__(self, stale=False): super(ModelRoot, self).__init__() self.stale = stale + self._extended_attributes_enabled = None def __nonzero__(self): return not self.stale __bool__ = __nonzero__ + @property + def extended_attributes_enabled(self): + if self._extended_attributes_enabled is None: + self._extended_attributes_enabled = ( + CONF.compute_model.enable_extended_attributes) + return self._extended_attributes_enabled + + @extended_attributes_enabled.setter + def extended_attributes_enabled(self, value): + self._extended_attributes_enabled = value + @staticmethod def assert_node(obj): if not isinstance(obj, element.ComputeNode): diff --git a/watcher/decision_engine/model/notification/nova.py b/watcher/decision_engine/model/notification/nova.py index eb776f2f1..7a11645bd 100644 --- a/watcher/decision_engine/model/notification/nova.py +++ b/watcher/decision_engine/model/notification/nova.py @@ -16,8 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from novaclient import api_versions import os_resource_classes as orc from oslo_log import log + from watcher.common import exception from watcher.common import nova_helper from watcher.common import placement_helper @@ -72,7 +74,7 @@ class NovaNotification(base.NotificationEndpoint): return instance def update_instance(self, instance, data): - n_version = float(data['nova_object.version']) + n_version = api_versions.APIVersion(data['nova_object.version']) instance_data = data['nova_object.data'] instance_flavor_data = instance_data['flavor']['nova_object.data'] @@ -91,12 +93,20 @@ class NovaNotification(base.NotificationEndpoint): 'vcpus': num_cores, 'disk': disk_gb, 'metadata': instance_metadata, - 'project_id': instance_data['tenant_id'] + 'project_id': instance_data['tenant_id'], }) + # locked was added in nova notification payload version 1.1 - if n_version > 1.0: + if n_version > api_versions.APIVersion(version_str='1.0'): instance.update({'locked': instance_data['locked']}) + # NOTE(dviroel): extra_specs can change due to a resize operation. + # 'extra_specs' was added in nova notification payload version 1.2 + if (n_version > api_versions.APIVersion(version_str='1.1') and + self.cluster_data_model.extended_attributes_enabled): + extra_specs = instance_flavor_data.get("extra_specs", {}) + instance.update({'flavor_extra_specs': extra_specs}) + try: node = self.get_or_create_node(instance_data['host']) except exception.ComputeNodeNotFound as exc: diff --git a/watcher/tests/api/v1/test_data_model.py b/watcher/tests/api/v1/test_data_model.py index d6e59e490..d2f4e9bee 100644 --- a/watcher/tests/api/v1/test_data_model.py +++ b/watcher/tests/api/v1/test_data_model.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt from unittest import mock from http import HTTPStatus @@ -30,15 +31,16 @@ class TestListDataModel(api_base.FunctionalTest): super(TestListDataModel, self).setUp() p_dcapi = mock.patch.object(deapi, 'DecisionEngineAPI') self.mock_dcapi = p_dcapi.start() - self.mock_dcapi().get_data_model_info.return_value = \ - 'fake_response_value' + self.fake_response = {'context': [{'server_uuid': 'fake_uuid'}]} + self.mock_dcapi().get_data_model_info.return_value = ( + self.fake_response) self.addCleanup(p_dcapi.stop) def test_get_all(self): response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': 'infra-optim 1.3'}) - self.assertEqual('fake_response_value', response) + self.assertEqual(self.fake_response, response) def test_get_all_not_acceptable(self): response = self.get_json( @@ -48,6 +50,7 @@ class TestListDataModel(api_base.FunctionalTest): self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int) +@ddt.ddt class TestListDataModelResponse(api_base.FunctionalTest): NODE_FIELDS_1_3 = [ @@ -67,6 +70,13 @@ class TestListDataModelResponse(api_base.FunctionalTest): 'node_uuid' ] + # Map of API version to expected node fields + NODE_FIELDS_MAP = { + '1.3': NODE_FIELDS_1_3, + '1.6': NODE_FIELDS_1_3, + 'latest': NODE_FIELDS_1_3, + } + SERVER_FIELDS_1_3 = [ 'server_watcher_exclude', 'server_name', @@ -80,8 +90,17 @@ class TestListDataModelResponse(api_base.FunctionalTest): 'server_uuid' ] - NODE_FIELDS_LATEST = NODE_FIELDS_1_3 - SERVER_FIELDS_LATEST = SERVER_FIELDS_1_3 + SERVER_FIELDS_1_6 = [ + 'server_pinned_az', + 'server_flavor_extra_specs' + ] + SERVER_FIELDS_1_3 + + # Map of API version to expected server fields + SERVER_FIELDS_MAP = { + '1.3': SERVER_FIELDS_1_3, + '1.6': SERVER_FIELDS_1_6, + 'latest': SERVER_FIELDS_1_6, + } def setUp(self): super(TestListDataModelResponse, self).setUp() @@ -89,36 +108,39 @@ class TestListDataModelResponse(api_base.FunctionalTest): self.mock_dcapi = p_dcapi.start() self.addCleanup(p_dcapi.stop) - def test_model_list_compute_no_instance(self): + @ddt.data("1.3", "1.6", versions.max_version_string()) + def test_model_list_compute_no_instance(self, version): fake_cluster = faker_cluster_state.FakerModelCollector() model = fake_cluster.generate_scenario_11_with_1_node_no_instance() get_model_resp = {'context': model.to_list()} self.mock_dcapi().get_data_model_info.return_value = get_model_resp - infra_max_version = 'infra-optim ' + versions.max_version_string() + infra_max_version = 'infra-optim ' + version response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': infra_max_version}) server_info = response.get("context")[0] - expected_keys = self.NODE_FIELDS_LATEST + expected_keys = self.NODE_FIELDS_MAP[version] self.assertEqual(len(response.get("context")), 1) self.assertEqual(set(expected_keys), set(server_info.keys())) - def test_model_list_compute_with_instances(self): + @ddt.data("1.3", "1.6", versions.max_version_string()) + def test_model_list_compute_with_instances(self, version): fake_cluster = faker_cluster_state.FakerModelCollector() model = fake_cluster.generate_scenario_11_with_2_nodes_2_instances() get_model_resp = {'context': model.to_list()} self.mock_dcapi().get_data_model_info.return_value = get_model_resp - infra_max_version = 'infra-optim ' + versions.max_version_string() + infra_max_version = 'infra-optim ' + version response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': infra_max_version}) server_info = response.get("context")[0] - expected_keys = self.NODE_FIELDS_LATEST + self.SERVER_FIELDS_LATEST + expected_keys = (self.NODE_FIELDS_MAP[version] + + self.SERVER_FIELDS_MAP[version]) self.assertEqual(len(response.get("context")), 2) self.assertEqual(set(expected_keys), set(server_info.keys())) diff --git a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py index 749e1c068..cb76d0197 100644 --- a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py +++ b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py @@ -16,16 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt import os_resource_classes as orc from unittest import mock from watcher.common import nova_helper from watcher.common import placement_helper from watcher.decision_engine.model.collector import nova +from watcher.decision_engine.model import model_root from watcher.tests import base from watcher.tests import conf_fixture +@ddt.ddt class TestNovaClusterDataModelCollector(base.TestCase): def setUp(self): @@ -35,8 +38,17 @@ class TestNovaClusterDataModelCollector(base.TestCase): @mock.patch('keystoneclient.v3.client.Client', mock.Mock()) @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, 'NovaHelper') - def test_nova_cdmc_execute(self, m_nova_helper_cls, + @mock.patch.object(model_root.ModelRoot, 'extended_attributes_enabled', + new_callable=mock.PropertyMock) + @ddt.data(False, True) + def test_nova_cdmc_execute(self, + extended_attr_enabled, + m_extended_attr_enabled, + m_nova_helper_cls, m_placement_helper_cls): + # Set the extended_attributes_enabled configuration value + m_extended_attr_enabled.return_value = extended_attr_enabled + m_placement_helper = mock.Mock(name="placement_helper") m_placement_helper.get_inventories.return_value = { orc.VCPU: { @@ -117,12 +129,15 @@ class TestNovaClusterDataModelCollector(base.TestCase): fake_instance = mock.Mock( id='ef500f7e-dac8-470f-960c-169486fce71b', name='fake_instance', - flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1}, + flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1, + 'extra_specs': {'hw_rng:allowed': 'True'}}, metadata={'hi': 'hello'}, tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b', + pinned_availability_zone='nova', ) setattr(fake_instance, 'OS-EXT-STS:vm_state', 'VM_STATE') setattr(fake_instance, 'name', 'fake_instance') + # Returns the hypervisors with details (service) but no servers. m_nova_helper.get_compute_node_list.return_value = [fake_compute_node] # Returns the hypervisor with servers and details (service). @@ -161,12 +176,21 @@ class TestNovaClusterDataModelCollector(base.TestCase): vcpus_total = (node.vcpus-node.vcpu_reserved)*node.vcpu_ratio self.assertEqual(node.vcpu_capacity, vcpus_total) + if extended_attr_enabled: + self.assertEqual('nova', instance.pinned_az) + self.assertEqual({'hw_rng:allowed': 'True'}, + instance.flavor_extra_specs) + else: + self.assertEqual('', instance.pinned_az) + self.assertEqual({}, instance.flavor_extra_specs) + m_nova_helper.get_compute_node_by_name.assert_called_once_with( minimal_node['hypervisor_hostname'], servers=True, detailed=True) m_nova_helper.get_instance_list.assert_called_once_with( filters={'host': fake_compute_node.service['host']}, limit=1) +@ddt.ddt class TestNovaModelBuilder(base.TestCase): @mock.patch.object(nova_helper, 'NovaHelper', mock.MagicMock()) @@ -180,11 +204,15 @@ class TestNovaModelBuilder(base.TestCase): tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b') setattr(inst1, 'OS-EXT-STS:vm_state', 'deleted') setattr(inst1, 'name', 'instance1') + setattr(inst1, 'pinned_availability_zone', 'nova') + inst2 = mock.MagicMock( id='ef500f7e-dac8-470f-960c-169486fce722', tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b') setattr(inst2, 'OS-EXT-STS:vm_state', 'active') setattr(inst2, 'name', 'instance2') + setattr(inst2, 'pinned_availability_zone', 'nova') + mock_instances = [inst1, inst2] model_builder.nova_helper.get_instance_list.return_value = ( mock_instances) @@ -203,6 +231,49 @@ class TestNovaModelBuilder(base.TestCase): model_builder.nova_helper.get_instance_list.assert_called_with( filters={'host': mock_host}, limit=-1) + @ddt.unpack + # unpacks (extended_attr_enabled, pinned_az_available) + @ddt.data((True, True), (True, False), (False, True)) + @mock.patch.object(nova_helper.NovaHelper, 'is_pinned_az_available') + def test__build_instance_node_extended_fields(self, + extended_attr_enabled, + pinned_az_available, + m_is_pinned_az_available): + model_builder = nova.NovaModelBuilder(osc=mock.MagicMock()) + model_builder.model = mock.MagicMock() + model_builder.model.extended_attributes_enabled = extended_attr_enabled + m_is_pinned_az_available.return_value = pinned_az_available + + inst1 = mock.MagicMock( + id='ef500f7e-dac8-470f-960c-169486fce711', + tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b', + flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1, + 'extra_specs': {'hw_rng:allowed': 'True'}},) + setattr(inst1, 'OS-EXT-STS:vm_state', 'active') + setattr(inst1, 'name', 'instance1') + setattr(inst1, 'pinned_availability_zone', 'nova') + + fake_instance = model_builder._build_instance_node(inst1) + + self.assertEqual(fake_instance.uuid, + 'ef500f7e-dac8-470f-960c-169486fce711') + self.assertEqual(fake_instance.name, 'instance1') + self.assertEqual(fake_instance.state, 'active') + self.assertEqual(fake_instance.memory, 333) + self.assertEqual(fake_instance.disk, 222) + self.assertEqual(fake_instance.vcpus, 4) + self.assertEqual(fake_instance.project_id, + 'ff560f7e-dbc8-771f-960c-164482fce21b') + + if extended_attr_enabled: + expected_pinned_az = 'nova' if pinned_az_available else '' + self.assertEqual(expected_pinned_az, fake_instance.pinned_az) + self.assertEqual({'hw_rng:allowed': 'True'}, + fake_instance.flavor_extra_specs) + else: + self.assertEqual('', fake_instance.pinned_az) + self.assertEqual({}, fake_instance.flavor_extra_specs) + def test_check_model(self): """Initialize collector ModelBuilder and test check model""" diff --git a/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json b/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json new file mode 100644 index 000000000..3b5f92b0b --- /dev/null +++ b/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json @@ -0,0 +1,90 @@ +{ + "event_type": "instance.update", + "payload": { + "nova_object.data": { + "action_initiator_project": "6f70656e737461636b20342065766572", + "action_initiator_user": "fake", + "architecture": "x86_64", + "audit_period": { + "nova_object.data": { + "audit_period_beginning": "2012-10-01T00:00:00Z", + "audit_period_ending": "2012-10-29T13:42:11Z" + }, + "nova_object.name": "AuditPeriodPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + }, + "auto_disk_config": "MANUAL", + "availability_zone": "nova", + "block_devices": [], + "created_at": "2012-10-29T13:42:11Z", + "deleted_at": null, + "display_description": "some-server", + "display_name": "some-server", + "host": "compute", + "host_name": "some-server", + "image_uuid": "155d900f-4e14-4e4c-a73d-069cbf4541e6", + "ip_addresses": [], + "kernel_id": "", + "key_name": "my-key", + "launched_at": null, + "locked": false, + "locked_reason": null, + "metadata": {}, + "node": "fake-mini", + "old_display_name": null, + "os_type": null, + "power_state": "pending", + "progress": 0, + "ramdisk_id": "", + "request_id": "req-5b6c791d-5709-4f36-8fbe-c3e02869e35d", + "reservation_id": "r-npxv0e40", + "shares": [], + "state": "paused", + "state_update": { + "nova_object.data": { + "new_task_state": null, + "old_state": null, + "old_task_state": null, + "state": "active"}, + "nova_object.name": "InstanceStateUpdatePayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0"}, + "tags": [], + "task_state": "scheduling", + "tenant_id": "6f70656e737461636b20342065766572", + "flavor": { + "nova_object.data": { + "description": null, + "disabled": false, + "ephemeral_gb": 0, + "extra_specs": { + "hw:watchdog_action": "disabled" + }, + "flavorid": "a22d5517-147c-4147-a0d1-e698df5cd4e3", + "is_public": true, + "memory_mb": 512, + "name": "test_flavor", + "projects": null, + "root_gb": 1, + "rxtx_factor": 1.0, + "swap": 0, + "vcpu_weight": 0, + "vcpus": 1 + }, + "nova_object.name": "FlavorPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.4" + }, + "terminated_at": null, + "updated_at": null, + "user_id": "fake", + "uuid": "73b09e16-35b7-4922-804e-e8f5d9b740fc" + }, + "nova_object.name": "InstanceUpdatePayload", + "nova_object.namespace": "nova", + "nova_object.version": "2.1" + }, + "priority": "INFO", + "publisher_id": "nova-compute:compute" +} diff --git a/watcher/tests/decision_engine/model/notification/test_nova_notifications.py b/watcher/tests/decision_engine/model/notification/test_nova_notifications.py index d5b9f6e38..1d94f8fb2 100644 --- a/watcher/tests/decision_engine/model/notification/test_nova_notifications.py +++ b/watcher/tests/decision_engine/model/notification/test_nova_notifications.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt import os import os_resource_classes as orc from unittest import mock @@ -110,6 +111,7 @@ class TestReceiveNovaNotifications(NotificationTestCase): expected_message, self.FAKE_METADATA) +@ddt.ddt class TestNovaNotifications(NotificationTestCase): FAKE_METADATA = {'message_id': None, 'timestamp': None} @@ -285,8 +287,10 @@ class TestNovaNotifications(NotificationTestCase): @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, "NovaHelper") + @ddt.data(False, True) def test_nova_instance_update_notfound_still_creates( - self, m_nova_helper_cls, m_placement_helper): + self, extended_attr_enabled, m_nova_helper_cls, + m_placement_helper): mock_placement = mock.Mock(name="placement_helper") mock_placement.get_inventories.return_value = dict() mock_placement.get_usages_for_resource_provider.return_value = { @@ -313,6 +317,10 @@ class TestNovaNotifications(NotificationTestCase): name='m_nova_helper') compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + # Set the extended_attributes_enabled configuration value + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = '9966d6bd-a45c-4e1c-9d57-3054899a3ec7' @@ -333,6 +341,10 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(1, instance0.vcpus) self.assertEqual(1, instance0.disk) self.assertEqual(512, instance0.memory) + # NOTE(dviroel): pinned_az is not yet available in nova notifications + # and flavor_extra_specs is not available in nova notifications 1.0 + self.assertEqual({}, instance0.flavor_extra_specs) + self.assertEqual('', instance0.pinned_az) m_get_compute_node_by_hostname.assert_called_once_with('Node_2') node_2 = compute_model.get_node_by_name('Node_2') @@ -380,7 +392,10 @@ class TestNovaNotifications(NotificationTestCase): @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, 'NovaHelper') - def test_nova_instance_create(self, m_nova_helper_cls, + @ddt.data(False, True) + def test_nova_instance_create(self, + extended_attr_enabled, + m_nova_helper_cls, m_placement_helper): mock_placement = mock.Mock(name="placement_helper") mock_placement.get_inventories.return_value = dict() @@ -410,6 +425,10 @@ class TestNovaNotifications(NotificationTestCase): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + # Set the extended_attributes_enabled configuration value + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = 'c03c0bf9-f46e-4e4f-93f1-817568567ee2' @@ -440,6 +459,14 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(1, instance0.vcpus) self.assertEqual(1, instance0.disk) self.assertEqual(512, instance0.memory) + if extended_attr_enabled: + self.assertEqual({'hw:watchdog_action': 'disabled'}, + instance0.flavor_extra_specs) + # NOTE(dviroel): pinned_az is not available in nova notifications + self.assertEqual('', instance0.pinned_az) + else: + self.assertEqual({}, instance0.flavor_extra_specs) + self.assertEqual('', instance0.pinned_az) def test_nova_instance_delete_end(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() @@ -564,6 +591,34 @@ class TestNovaNotifications(NotificationTestCase): self.assertFalse(instance0.locked) + @ddt.data(False, True) + def test_nova_instance_update_extra_specs(self, extended_attr_enabled): + compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() + self.fake_cdmc.cluster_data_model = compute_model + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) + + instance0_uuid = '73b09e16-35b7-4922-804e-e8f5d9b740fc' + instance0 = compute_model.get_instance_by_uuid(instance0_uuid) + + message = self.load_message('instance-update-2-1.json') + + self.assertEqual({}, instance0.flavor_extra_specs) + + handler.info( + ctxt=self.context, + publisher_id=message['publisher_id'], + event_type=message['event_type'], + payload=message['payload'], + metadata=self.FAKE_METADATA, + ) + if extended_attr_enabled: + self.assertEqual({'hw:watchdog_action': 'disabled'}, + instance0.flavor_extra_specs) + else: + self.assertEqual({}, instance0.flavor_extra_specs) + def test_nova_instance_pause(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model @@ -685,14 +740,22 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(element.InstanceState.ACTIVE.value, instance0.state) - def test_instance_resize_confirm_end(self): + @ddt.data(False, True) + def test_instance_resize_confirm_end(self, extended_attr_enabled): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = '73b09e16-35b7-4922-804e-e8f5d9b740fc' instance0 = compute_model.get_instance_by_uuid(instance0_uuid) node = compute_model.get_node_by_instance_uuid(instance0_uuid) + self.assertEqual('fa69c544-906b-4a6a-a9c6-c1f7a8078c73', node.uuid) + # NOTE(dviroel): extra_specs are empty generated scenario + self.assertEqual({}, instance0.flavor_extra_specs) + message = self.load_message( 'instance-resize_confirm-end.json') handler.info( @@ -706,6 +769,11 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual('fa69c544-906b-4a6a-a9c6-c1f7a8078c73', node.uuid) self.assertEqual(element.InstanceState.ACTIVE.value, instance0.state) + expected_extra_specs = { + 'hw:watchdog_action': 'disabled' + } if extended_attr_enabled else {} + self.assertEqual(expected_extra_specs, instance0.flavor_extra_specs) + def test_nova_instance_restore_end(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model @@ -846,6 +914,7 @@ class TestNovaNotifications(NotificationTestCase): def test_fake_instance_create(self): self.fake_cdmc.cluster_data_model = mock.Mock() + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = False handler = novanotification.VersionedNotification(self.fake_cdmc) message = self.load_message('instance-create-end.json') diff --git a/watcher/tests/decision_engine/model/test_element.py b/watcher/tests/decision_engine/model/test_element.py index 91dbb737f..d7a060902 100644 --- a/watcher/tests/decision_engine/model/test_element.py +++ b/watcher/tests/decision_engine/model/test_element.py @@ -60,6 +60,16 @@ class TestElement(base.TestCase): 'vcpus': 222, 'disk': 333, })), + ("Instance_with_extended_fields", dict( + cls=element.Instance, + data={ + 'uuid': 'FAKE_UUID', + 'state': 'state', + 'vcpus': 222, + 'disk': 333, + 'pinned_az': 'nova', + 'flavor_extra_specs': {"spec1": "value1", "spec2": "value2"}, + })), ] def test_as_xml_element(self):