From 56ce75d897e5e8261eec79d3552b95a60ed8663c Mon Sep 17 00:00:00 2001 From: Callum Dickinson Date: Fri, 31 Jan 2025 11:06:58 +1300 Subject: [PATCH] Add storage_policy attribute for Swift containers This exposes a Swift container's storage policy as a metadata attribute in Ceilometer samples (via the storage.containers.objects and storage.containers.objects.size meters) and the swift_account Gnocchi resources, named storage_policy. The storage policy determines the location and replication parameters for a Swift container, making it a useful metric for billing alongside the size of the container. The storage policy for a container cannot be changed unless the container is recreated, meaning that it does not change often (making it suitable to be stored in Gnocchi as a metadata attribute). The reason why this was not possible before was that Swift did not return the storage policy for each container in the GET account request, but a new proposed change to Swift [1] adds it to the response as the storage_policy attribute. This change for Ceilometer takes advantage of this and adds it to Ceilometer samples and the Gnocchi resource type. For this to work, the Swift change needs to be accepted and merged. [1]: https://review.opendev.org/c/openstack/swift/+/940601 Depends-On: I52b37cfa49cac8675f5087bcbcfe18db0b46d887 Change-Id: I25f74df61ef0a6a4b50f1357ddf86d5acc6d71e7 --- ceilometer/gnocchi_client.py | 9 ++++ ceilometer/objectstore/swift.py | 16 +++++-- .../publisher/data/gnocchi_resources.yaml | 2 + .../tests/unit/objectstore/test_swift.py | 42 +++++++++++++++---- ...age_policy-attribute-322fbb5716c5bb10.yaml | 22 ++++++++++ 5 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/add-swift-storage_policy-attribute-322fbb5716c5bb10.yaml diff --git a/ceilometer/gnocchi_client.py b/ceilometer/gnocchi_client.py index ea3dcbe5f2..24d985be81 100644 --- a/ceilometer/gnocchi_client.py +++ b/ceilometer/gnocchi_client.py @@ -209,6 +209,15 @@ resources_update_operations = [ "value": {"type": "string", "min_length": 0, "max_length": 255, "required": False} }]}, + {"desc": "add storage_policy to swift_account", + "type": "update_attribute_type", + "resource_type": "swift_account", + "data": [{ + "op": "add", + "path": "/attributes/storage_policy", + "value": {"type": "string", "min_length": 0, "max_length": 255, + "required": False} # Only containers have a storage policy + }]}, ] diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index 62acb37580..1a7c5f8039 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -111,6 +111,14 @@ class _Base(plugin_base.PollsterBase): 'v1/' + reseller_prefix + tenant_id) +class _ContainersBase(_Base): + FIELDS = ("storage_policy",) + + def _get_resource_metadata(self, container): + # NOTE(callumdickinson): Sets value to None if a field is not found. + return {f: container.get(f) for f in self.FIELDS} + + class ObjectsPollster(_Base): """Collect the total objects count for each project""" def get_samples(self, manager, cache, resources): @@ -165,7 +173,7 @@ class ObjectsContainersPollster(_Base): ) -class ContainersObjectsPollster(_Base): +class ContainersObjectsPollster(_ContainersBase): """Collect the objects count per container for each project""" METHOD = 'get' @@ -184,11 +192,11 @@ class ContainersObjectsPollster(_Base): user_id=None, project_id=tenant, resource_id=tenant + '/' + container['name'], - resource_metadata=None, + resource_metadata=self._get_resource_metadata(container), ) -class ContainersSizePollster(_Base): +class ContainersSizePollster(_ContainersBase): """Collect the total objects size per container for each project""" METHOD = 'get' @@ -207,5 +215,5 @@ class ContainersSizePollster(_Base): user_id=None, project_id=tenant, resource_id=tenant + '/' + container['name'], - resource_metadata=None, + resource_metadata=self._get_resource_metadata(container), ) diff --git a/ceilometer/publisher/data/gnocchi_resources.yaml b/ceilometer/publisher/data/gnocchi_resources.yaml index afbedc6a23..2c7eb3d2c4 100644 --- a/ceilometer/publisher/data/gnocchi_resources.yaml +++ b/ceilometer/publisher/data/gnocchi_resources.yaml @@ -226,6 +226,8 @@ resources: storage.objects.containers: storage.containers.objects: storage.containers.objects.size: + attributes: + storage_policy: resource_metadata.storage_policy - resource_type: volume metrics: diff --git a/ceilometer/tests/unit/objectstore/test_swift.py b/ceilometer/tests/unit/objectstore/test_swift.py index 312830875c..8091180433 100644 --- a/ceilometer/tests/unit/objectstore/test_swift.py +++ b/ceilometer/tests/unit/objectstore/test_swift.py @@ -13,6 +13,7 @@ # under the License. import collections +import itertools from unittest import mock import fixtures @@ -44,10 +45,15 @@ GET_ACCOUNTS = [('tenant-000', ({'x-account-object-count': 10, }, [{'count': 10, 'bytes': 123123, - 'name': 'my_container'}, + 'name': 'my_container', + 'storage_policy': 'Policy-0', + }, {'count': 0, 'bytes': 0, - 'name': 'new_container' + 'name': 'new_container', + # NOTE(callumdickinson): No storage policy, + # to test backwards compatibility with older + # versions of Swift. }])), ('tenant-001', ({'x-account-object-count': 0, 'x-account-bytes-used': 0, @@ -81,15 +87,27 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios, # pollsters. scenarios = [ ('storage.objects', - {'factory': swift.ObjectsPollster}), + {'factory': swift.ObjectsPollster, 'resources': {}}), ('storage.objects.size', - {'factory': swift.ObjectsSizePollster}), + {'factory': swift.ObjectsSizePollster, 'resources': {}}), ('storage.objects.containers', - {'factory': swift.ObjectsContainersPollster}), + {'factory': swift.ObjectsContainersPollster, 'resources': {}}), ('storage.containers.objects', - {'factory': swift.ContainersObjectsPollster}), + {'factory': swift.ContainersObjectsPollster, + 'resources': { + f"{project_id}/{container['name']}": container + for project_id, container in itertools.chain.from_iterable( + itertools.product([acc[0]], acc[1][1]) + for acc in GET_ACCOUNTS) + }}), ('storage.containers.objects.size', - {'factory': swift.ContainersSizePollster}), + {'factory': swift.ContainersSizePollster, + 'resources': { + f"{project_id}/{container['name']}": container + for project_id, container in itertools.chain.from_iterable( + itertools.product([acc[0]], acc[1][1]) + for acc in GET_ACCOUNTS) + }}), ] @staticmethod @@ -174,6 +192,16 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios, ASSIGNED_TENANTS)) self.assertEqual(2, len(samples), self.pollster.__class__) + for resource_id, resource in self.resources.items(): + for field in getattr(self.pollster, 'FIELDS', []): + with self.subTest(f'{resource_id}-{field}'): + sample = next(s for s in samples + if s.resource_id == resource_id) + if field in resource: + self.assertEqual(resource[field], + sample.resource_metadata[field]) + else: + self.assertIsNone(sample.resource_metadata[field]) def test_get_meter_names(self): with fixtures.MockPatchObject(self.factory, '_iter_accounts', diff --git a/releasenotes/notes/add-swift-storage_policy-attribute-322fbb5716c5bb10.yaml b/releasenotes/notes/add-swift-storage_policy-attribute-322fbb5716c5bb10.yaml new file mode 100644 index 0000000000..34c54b0411 --- /dev/null +++ b/releasenotes/notes/add-swift-storage_policy-attribute-322fbb5716c5bb10.yaml @@ -0,0 +1,22 @@ +--- +features: + - | + The ``storage_policy`` resource metadata attribute has been added to the + ``swift.containers.objects`` and ``swift.containers.objects.size`` meters, + populated from already performed Swift account ``GET`` requests. + This functionality requires using a new version of Swift that adds the + ``storage_policy`` attribute when listing containers in an account. + Ceilometer is backwards compatible with Swift versions that do not + provide this functionality, but ``storage_policy`` will be set to + ``None`` in samples and Gnocchi resources. + - | + An optional ``storage_policy`` attribute has been added to the + ``swift_account`` Gnocchi resource type, to store the storage policy for + Swift containers in Gnocchi. For Swift accounts, ``storage_policy`` will + be set to ``None``. +upgrade: + - | + To publish the ``storage_policy`` attribute for Swift containers, + ``gnocchi_resources.yaml`` will need to be updated to the latest version. + Swift in the target OpenStack cloud will also need upgrading to add + support for providing the storage policy when listing containers.