diff --git a/ceilometer/compute/discovery.py b/ceilometer/compute/discovery.py index 76f926dfae..dd18ba96b1 100644 --- a/ceilometer/compute/discovery.py +++ b/ceilometer/compute/discovery.py @@ -62,7 +62,18 @@ OPTS = [ "The minimum should be the value of the config option " "of resource_update_interval. This option is only used " "for agent polling to Nova API, so it will work only " - "when 'instance_discovery_method' is set to 'naive'.") + "when 'instance_discovery_method' is set to 'naive'."), + cfg.BoolOpt('fetch_extra_metadata', + default=True, + help="Whether or not additional instance attributes that " + "require Nova API queries should be fetched. Currently " + "the only value that requires fetching from Nova API is " + "'metadata', the attribute storing user-configured " + "server metadata, which is used to fill out some " + "optional fields such as the server group of an " + "instance. fetch_extra_metadata is currently set to " + "True by default, but to reduce the load on Nova API " + "this will be changed to False in a future release."), ] LOG = log.getLogger(__name__) @@ -93,7 +104,8 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): self.expiration_time = conf.compute.resource_update_interval self.cache_expiry = conf.compute.resource_cache_expiry if self.method == "libvirt_metadata": - # 4096 instances on a compute should be enough :) + # 4096 resources on a compute should be enough :) + self._flavor_id_cache = cachetools.LRUCache(4096) self._server_cache = cachetools.LRUCache(4096) else: self.lock = threading.Lock() @@ -119,8 +131,49 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): return int(elem.text) return 0 + def _get_flavor_id(self, flavor_xml, instance_id): + flavor_name = flavor_xml.attrib["name"] + # Flavor ID is available in libvirt metadata from 2025.2 onwards. + flavor_id = flavor_xml.attrib.get("id") + if flavor_id: + return flavor_id + # If not found in libvirt metadata, fallback to API queries. + # If we already have the server metadata get the flavor ID from there. + if self.conf.compute.fetch_extra_metadata: + server = self.get_server(instance_id) + if server: + return server.flavor["id"] + # If server metadata is not otherwise fetched, or the query failed, + # query just the flavor for better cache hit rates. + return (self.get_flavor_id(flavor_name) or flavor_name) + + def _get_flavor_extra_specs(self, flavor_xml): + # Extra specs are available in libvirt metadata from 2025.2 onwards. + # Note that this checks for existence of the element, not whether + # or not it is empty, as it *can* exist but have nothing set in it. + extra_specs = flavor_xml.find("./extraSpecs") + if extra_specs is not None: + return { + extra_spec.attrib["name"]: extra_spec.text + for extra_spec in extra_specs.findall("./extraSpec")} + # If not found in libvirt metadata, return None to signify + # "not fetched", as we don't support performing additional + # API queries just for the extra specs. + return None + + @cachetools.cachedmethod(operator.attrgetter('_flavor_id_cache')) + def get_flavor_id(self, name): + LOG.debug("Querying metadata for flavor %s from Nova API", name) + try: + return self.nova_cli.nova_client.flavors.find( + name=name, + is_public=None).id + except exceptions.NotFound: + return None + @cachetools.cachedmethod(operator.attrgetter('_server_cache')) def get_server(self, uuid): + LOG.debug("Querying metadata for instance %s from Nova API", uuid) try: return self.nova_cli.nova_client.servers.get(uuid) except exceptions.NotFound: @@ -130,6 +183,7 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): def discover_libvirt_polling(self, manager, param=None): instances = [] for domain in self.connection.listAllDomains(): + instance_id = domain.UUIDString() xml_string = libvirt_utils.instance_metadata(domain) if xml_string is None: continue @@ -138,15 +192,6 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): os_type_xml = full_xml.find("./os/type") metadata_xml = etree.fromstring(xml_string) - # TODO(sileht, jwysogla): We don't have the flavor ID - # and server metadata here. We currently poll nova to get - # the flavor ID, but storing the - # flavor_id doesn't have any sense because the flavor description - # can change over the time, we should store the detail of the - # flavor. this is why nova doesn't put the id in the libvirt - # metadata. I think matadata field could be eventually added to - # the libvirt matadata created by nova. - try: flavor_xml = metadata_xml.find( "./flavor") @@ -158,11 +203,10 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): "./name").text instance_arch = os_type_xml.attrib["arch"] - server = self.get_server(domain.UUIDString()) - flavor_id = (server.flavor["id"] if server is not None - else flavor_xml.attrib["name"]) + extra_specs = self._get_flavor_extra_specs(flavor_xml) + flavor = { - "id": flavor_id, + "id": self._get_flavor_id(flavor_xml, instance_id), "name": flavor_xml.attrib["name"], "vcpus": self._safe_find_int(flavor_xml, "vcpus"), "ram": self._safe_find_int(flavor_xml, "memory"), @@ -170,18 +214,28 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): "ephemeral": self._safe_find_int(flavor_xml, "ephemeral"), "swap": self._safe_find_int(flavor_xml, "swap"), } + if extra_specs is not None: + flavor["extra_specs"] = extra_specs # The image description is partial, but Gnocchi only care about # the id, so we are fine image_xml = metadata_xml.find("./root[@type='image']") image = ({'id': image_xml.attrib['uuid']} if image_xml is not None else None) - metadata = server.metadata if server is not None else {} + + # Getting the server metadata requires expensive Nova API + # queries, and may potentially contain sensitive user info, + # so it is only fetched when configured to do so. + if self.conf.compute.fetch_extra_metadata: + server = self.get_server(instance_id) + metadata = server.metadata if server is not None else {} + else: + metadata = {} except AttributeError: LOG.error( "Fail to get domain uuid %s metadata: " "metadata was missing expected attributes", - domain.UUIDString()) + instance_id) continue dom_state = domain.state()[0] @@ -194,7 +248,7 @@ class InstanceDiscovery(plugin_base.DiscoveryBase): (project_id + self.conf.host).encode('utf-8')).hexdigest() instance_data = { - "id": domain.UUIDString(), + "id": instance_id, "name": instance_name, "flavor": flavor, "image": image, diff --git a/ceilometer/tests/unit/compute/pollsters/base.py b/ceilometer/tests/unit/compute/pollsters/base.py index c49f2f2e9f..90e88a2f57 100644 --- a/ceilometer/tests/unit/compute/pollsters/base.py +++ b/ceilometer/tests/unit/compute/pollsters/base.py @@ -37,8 +37,13 @@ class TestPollsterBase(base.BaseTestCase): 'active') setattr(self.instance, 'OS-EXT-STS:task_state', None) self.instance.id = 1 - self.instance.flavor = {'name': 'm1.small', 'id': 2, 'vcpus': 1, - 'ram': 512, 'disk': 20, 'ephemeral': 0} + self.instance.flavor = {'name': 'm1.small', + 'id': 'eba4213d-3c6c-4b5f-8158-dd0022d71d62', + 'vcpus': 1, + 'ram': 512, + 'disk': 20, + 'ephemeral': 0, + 'extra_specs': {'hw_rng:allowed': 'true'}} self.instance.status = 'active' self.instance.metadata = { 'fqdn': 'vm_fqdn', diff --git a/ceilometer/tests/unit/compute/pollsters/test_cpu.py b/ceilometer/tests/unit/compute/pollsters/test_cpu.py index 83046b9754..14a4d9e91b 100644 --- a/ceilometer/tests/unit/compute/pollsters/test_cpu.py +++ b/ceilometer/tests/unit/compute/pollsters/test_cpu.py @@ -65,6 +65,8 @@ class TestCPUPollster(base.TestPollsterBase): self.assertEqual('active', samples[0].resource_metadata['status']) self.assertEqual('active', samples[0].resource_metadata['state']) self.assertIsNone(samples[0].resource_metadata['task_state']) + self.assertEqual(self.instance.flavor, + samples[0].resource_metadata['flavor']) def test_get_reserved_metadata_with_keys(self): self.CONF.set_override('reserved_metadata_keys', ['fqdn']) diff --git a/ceilometer/tests/unit/compute/pollsters/test_location_metadata.py b/ceilometer/tests/unit/compute/pollsters/test_location_metadata.py index 2b35de219b..6f195ab34a 100644 --- a/ceilometer/tests/unit/compute/pollsters/test_location_metadata.py +++ b/ceilometer/tests/unit/compute/pollsters/test_location_metadata.py @@ -69,11 +69,15 @@ class TestLocationMetadata(base.BaseTestCase): 'hostId': '1234-5678', 'OS-EXT-SRV-ATTR:host': 'host-test', 'flavor': {'name': 'm1.tiny', - 'id': 1, + 'id': ('eba4213d-3c6c-' + '4b5f-8158-' + 'dd0022d71d62'), 'disk': 20, 'ram': 512, 'vcpus': 2, - 'ephemeral': 0}, + 'ephemeral': 0, + 'extra_specs': { + 'hw_rng:allowed': 'true'}}, 'metadata': {'metering.autoscale.group': 'X' * 512, 'metering.ephemeral_gb': 42}} diff --git a/ceilometer/tests/unit/compute/test_discovery.py b/ceilometer/tests/unit/compute/test_discovery.py index 9b3708d509..fe831cc3d4 100644 --- a/ceilometer/tests/unit/compute/test_discovery.py +++ b/ceilometer/tests/unit/compute/test_discovery.py @@ -10,6 +10,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import argparse import datetime from unittest import mock @@ -24,6 +26,29 @@ from ceilometer.tests import base LIBVIRT_METADATA_XML = """ + + + test.dom.com + 2016-11-16 07:35:06 + + 512 + 1 + 0 + 0 + 1 + + true + + + + admin + admin + + + +""" + +LIBVIRT_METADATA_XML_OLD = """ test.dom.com @@ -43,6 +68,93 @@ LIBVIRT_METADATA_XML = """ """ +LIBVIRT_METADATA_XML_EMPTY_FLAVOR_ID = """ + + + test.dom.com + 2016-11-16 07:35:06 + + 512 + 1 + 0 + 0 + 1 + + true + + + + admin + admin + + + +""" + +LIBVIRT_METADATA_XML_NO_FLAVOR_ID = """ + + + test.dom.com + 2016-11-16 07:35:06 + + 512 + 1 + 0 + 0 + 1 + + true + + + + admin + admin + + + +""" + +LIBVIRT_METADATA_XML_EMPTY_FLAVOR_EXTRA_SPECS = """ + + + test.dom.com + 2016-11-16 07:35:06 + + 512 + 1 + 0 + 0 + 1 + + + + admin + admin + + + +""" + +LIBVIRT_METADATA_XML_NO_FLAVOR_EXTRA_SPECS = """ + + + test.dom.com + 2016-11-16 07:35:06 + + 512 + 1 + 0 + 0 + 1 + + + admin + admin + + + +""" + LIBVIRT_DESC_XML = """ instance-00000001 @@ -75,6 +187,10 @@ LIBVIRT_MANUAL_INSTANCE_DESC_XML = """ class FakeDomain: + def __init__(self, desc=None, metadata=None): + self._desc = desc or LIBVIRT_DESC_XML + self._metadata = metadata or LIBVIRT_METADATA_XML + def state(self): return [1, 2] @@ -85,15 +201,18 @@ class FakeDomain: return "a75c2fa5-6c03-45a8-bbf7-b993cfcdec27" def XMLDesc(self): - return LIBVIRT_DESC_XML + return self._desc def metadata(self, flags, url): - return LIBVIRT_METADATA_XML + return self._metadata class FakeConn: + def __init__(self, domains=None): + self._domains = domains or [FakeDomain()] + def listAllDomains(self): - return [FakeDomain()] + return list(self._domains) def isAlive(self): return True @@ -149,8 +268,13 @@ class TestDiscovery(base.BaseTestCase): # FIXME(sileht): This is wrong, this should be a uuid # The internal id of nova can't be retrieved via API or notification self.instance.id = 1 - self.instance.flavor = {'name': 'm1.small', 'id': 2, 'vcpus': 1, - 'ram': 512, 'disk': 20, 'ephemeral': 0} + self.instance.flavor = {'name': 'm1.small', + 'id': 'eba4213d-3c6c-4b5f-8158-dd0022d71d62', + 'vcpus': 1, + 'ram': 512, + 'disk': 20, + 'ephemeral': 0, + 'extra_specs': {'hw_rng:allowed': 'true'}} self.instance.status = 'active' self.instance.metadata = { 'fqdn': 'vm_fqdn', @@ -220,16 +344,26 @@ class TestDiscovery(base.BaseTestCase): self.client.instance_get_all_by_host.assert_called_once_with( self.CONF.host, "2016-01-01T00:00:00+00:00") + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") @mock.patch("ceilometer.compute.virt.libvirt.utils." "refresh_libvirt_connection") - def test_discovery_with_libvirt(self, mock_libvirt_conn): + def test_discovery_with_libvirt( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): self.CONF.set_override("instance_discovery_method", "libvirt_metadata", group="compute") mock_libvirt_conn.return_value = FakeConn() + mock_get_server.return_value = argparse.Namespace( + metadata={"metering.server_group": "group1"}) dsc = discovery.InstanceDiscovery(self.CONF) resources = dsc.discover(mock.MagicMock()) + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_called_with( + "a75c2fa5-6c03-45a8-bbf7-b993cfcdec27") + self.assertEqual(1, len(resources)) r = list(resources)[0] s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", @@ -254,6 +388,15 @@ class TestDiscovery(base.BaseTestCase): self.assertEqual("a75c2fa5-6c03-45a8-bbf7-b993cfcdec27", metadata["instance_id"]) self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {"hw_rng:allowed": "true"}}, + metadata["flavor"]) self.assertEqual( "4d0bc931ea7f0513da2efd9acb4cf3a273c64b7bcc544e15c070e662", metadata["host"]) @@ -262,6 +405,330 @@ class TestDiscovery(base.BaseTestCase): self.assertEqual("running", metadata["state"]) self.assertEqual("hvm", metadata["os_type"]) self.assertEqual("x86_64", metadata["architecture"]) + self.assertEqual({"server_group": "group1"}, + metadata["user_metadata"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_old( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain(metadata=LIBVIRT_METADATA_XML_OLD)]) + mock_get_server.return_value = argparse.Namespace( + flavor={"id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62"}, + metadata={"metering.server_group": "group1"}) + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_called_with( + "a75c2fa5-6c03-45a8-bbf7-b993cfcdec27") + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + self.assertEqual("a75c2fa5-6c03-45a8-bbf7-b993cfcdec27", + s.resource_id) + self.assertEqual("d99c829753f64057bc0f2030da309943", + s.project_id) + self.assertEqual("a1f4684e58bd4c88aefd2ecb0783b497", + s.user_id) + + metadata = s.resource_metadata + self.assertEqual(1, metadata["vcpus"]) + self.assertEqual(512, metadata["memory_mb"]) + self.assertEqual(1, metadata["disk_gb"]) + self.assertEqual(0, metadata["ephemeral_gb"]) + self.assertEqual(1, metadata["root_gb"]) + self.assertEqual("bdaf114a-35e9-4163-accd-226d5944bf11", + metadata["image_ref"]) + self.assertEqual("test.dom.com", metadata["display_name"]) + self.assertEqual("instance-00000001", metadata["name"]) + self.assertEqual("a75c2fa5-6c03-45a8-bbf7-b993cfcdec27", + metadata["instance_id"]) + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1}, + metadata["flavor"]) + self.assertEqual( + "4d0bc931ea7f0513da2efd9acb4cf3a273c64b7bcc544e15c070e662", + metadata["host"]) + self.assertEqual(self.CONF.host, metadata["instance_host"]) + self.assertEqual("active", metadata["status"]) + self.assertEqual("running", metadata["state"]) + self.assertEqual("hvm", metadata["os_type"]) + self.assertEqual("x86_64", metadata["architecture"]) + self.assertEqual({"server_group": "group1"}, + metadata["user_metadata"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_no_extra_metadata( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", False, group="compute") + mock_libvirt_conn.return_value = FakeConn() + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_not_called() + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertNotIn("user_metadata", metadata) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_empty_flavor_id_get_by_flavor( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", False, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain( + metadata=LIBVIRT_METADATA_XML_EMPTY_FLAVOR_ID)]) + mock_get_flavor_id.return_value = ( + "eba4213d-3c6c-4b5f-8158-dd0022d71d62") + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_called_with("m1.tiny") + mock_get_server.assert_not_called() + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {"hw_rng:allowed": "true"}}, + metadata["flavor"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_empty_flavor_id_get_by_server( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", True, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain( + metadata=LIBVIRT_METADATA_XML_EMPTY_FLAVOR_ID)]) + mock_get_server.return_value = argparse.Namespace( + flavor={"id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62"}, + metadata={}) + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_called_with( + "a75c2fa5-6c03-45a8-bbf7-b993cfcdec27") + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {"hw_rng:allowed": "true"}}, + metadata["flavor"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_no_flavor_id_get_by_flavor( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", False, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain(metadata=LIBVIRT_METADATA_XML_NO_FLAVOR_ID)]) + mock_get_flavor_id.return_value = ( + "eba4213d-3c6c-4b5f-8158-dd0022d71d62") + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_called_with("m1.tiny") + mock_get_server.assert_not_called() + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {"hw_rng:allowed": "true"}}, + metadata["flavor"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_no_flavor_id_get_by_server( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", True, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain(metadata=LIBVIRT_METADATA_XML_NO_FLAVOR_ID)]) + mock_get_server.return_value = argparse.Namespace( + flavor={"id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62"}, + metadata={}) + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_called_with( + "a75c2fa5-6c03-45a8-bbf7-b993cfcdec27") + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {"hw_rng:allowed": "true"}}, + metadata["flavor"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_empty_flavor_extra_specs( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", False, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain( + metadata=LIBVIRT_METADATA_XML_EMPTY_FLAVOR_EXTRA_SPECS)]) + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_not_called() + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1, + "extra_specs": {}}, + metadata["flavor"]) + + @mock.patch.object(discovery.InstanceDiscovery, "get_server") + @mock.patch.object(discovery.InstanceDiscovery, "get_flavor_id") + @mock.patch("ceilometer.compute.virt.libvirt.utils." + "refresh_libvirt_connection") + def test_discovery_with_libvirt_no_flavor_extra_specs( + self, mock_libvirt_conn, + mock_get_flavor_id, mock_get_server): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.CONF.set_override("fetch_extra_metadata", False, group="compute") + mock_libvirt_conn.return_value = FakeConn( + domains=[FakeDomain( + metadata=LIBVIRT_METADATA_XML_NO_FLAVOR_EXTRA_SPECS)]) + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + + mock_get_flavor_id.assert_not_called() + mock_get_server.assert_not_called() + + self.assertEqual(1, len(resources)) + r = list(resources)[0] + s = util.make_sample_from_instance(self.CONF, r, "metric", "delta", + "carrot", 1) + + metadata = s.resource_metadata + self.assertEqual("m1.tiny", metadata["instance_type"]) + self.assertEqual({"name": "m1.tiny", + "id": "eba4213d-3c6c-4b5f-8158-dd0022d71d62", + "ram": 512, + "disk": 1, + "swap": 0, + "ephemeral": 0, + "vcpus": 1}, + metadata["flavor"]) def test_discovery_with_legacy_resource_cache_cleanup(self): self.CONF.set_override("instance_discovery_method", "naive", @@ -301,6 +768,25 @@ class TestDiscovery(base.BaseTestCase): resources = dsc.discover(mock.MagicMock()) self.assertEqual(0, len(resources)) + def test_get_flavor_id(self): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + fake_flavor = argparse.Namespace( + id="eba4213d-3c6c-4b5f-8158-dd0022d71d62") + self.client.nova_client.flavors.find.return_value = fake_flavor + dsc = discovery.InstanceDiscovery(self.CONF) + self.assertEqual(fake_flavor.id, dsc.get_flavor_id("m1.tiny")) + + def test_get_flavor_id_notfound(self): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + self.client.nova_client.flavors.find.side_effect = ( + exceptions.NotFound(404)) + dsc = discovery.InstanceDiscovery(self.CONF) + self.assertIsNone(dsc.get_flavor_id("m1.tiny")) + def test_get_server(self): self.client.nova_client = mock.MagicMock() self.client.nova_client.servers = mock.MagicMock() diff --git a/releasenotes/notes/get-more-flavor-info-from-libvirt-c8db26fe410abe6e.yaml b/releasenotes/notes/get-more-flavor-info-from-libvirt-c8db26fe410abe6e.yaml new file mode 100644 index 0000000000..a0276b00df --- /dev/null +++ b/releasenotes/notes/get-more-flavor-info-from-libvirt-c8db26fe410abe6e.yaml @@ -0,0 +1,35 @@ +--- +features: + - | + When using Nova 2025.2 and later, the flavor ID for an instance is now + available from the libvirt domain metadata. Ceilometer now takes advantage + of this and populates the flavor ID from metadata instead of querying + Nova, when the value is available. If not available from metadata, + Ceilometer will fallback to querying Nova API for the flavor ID. + - | + When using Nova 2025.2 and later, the extra specs for the flavor and + instance is running is now available from the libvirt domain metadata. + Ceilometer now adds the flavor's extra specs to compute sample metadata + when found. + - | + Added the ``[compute]/fetch_extra_metadata`` configuration option, which + allows configuration of whether or not Ceilometer fetches additional + compute instance metadata attributes that require Nova API queries. + This mainly affects the ``user_metadata`` attributes populated with + metering-related values such as the server group an instance is part of. + When ``fetch_extra_metadata`` is set to ``False``, Ceilometer Compute + Agent will not query Nova API for anything unless absolutely necessary. +upgrade: + - | + After Nova has been upgraded to 2025.2 or later, new instances will + start providing additional flavor metadata for Ceilometer to use. + Instances already running at the time of the upgrade are not updated + as part of the process; to update those instances they will need to + be cold restarted, cold migrated or shelved-and-unshelved (until this + happens, Nova API queries will continue to be performed for those + instances). +deprecations: + - | + The newly added ``[compute]/fetch_extra_metadata`` option is set to + ``True`` by default, but to reduce the amount of load Ceilometer places + on Nova this will be changed to ``False`` by default in a future release.