From e3d77b77be8a3a544ce353acf6354a42808e74ea Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Fri, 25 Aug 2023 16:52:16 +0530 Subject: [PATCH] Fix configuration issues * Add pipeline.yaml * Add event_pipeline.yaml * Add service_credentials to ceilometer.conf * Add action ceilometer-upgrade * Run tox -e fmt for formatting code Change-Id: I8c1edc0bdaa71ee48a272f8c4d4aa6fae6642f4e --- charms/ceilometer-k8s/actions.yaml | 9 +- charms/ceilometer-k8s/pyproject.toml | 20 ++- charms/ceilometer-k8s/src/charm.py | 120 ++++++++++++++++-- .../src/templates/ceilometer.conf | 10 +- .../src/templates/event_pipeline.yaml | 12 ++ .../parts/section-service-credentials | 14 ++ .../src/templates/pipeline.yaml | 89 +++++++++++++ charms/ceilometer-k8s/tests/actions.yaml | 1 + .../tests/integration/test_charm.py | 41 +++++- 9 files changed, 282 insertions(+), 34 deletions(-) create mode 100644 charms/ceilometer-k8s/src/templates/event_pipeline.yaml create mode 100644 charms/ceilometer-k8s/src/templates/parts/section-service-credentials create mode 100644 charms/ceilometer-k8s/src/templates/pipeline.yaml create mode 120000 charms/ceilometer-k8s/tests/actions.yaml diff --git a/charms/ceilometer-k8s/actions.yaml b/charms/ceilometer-k8s/actions.yaml index 88e6195d..a19df938 100644 --- a/charms/ceilometer-k8s/actions.yaml +++ b/charms/ceilometer-k8s/actions.yaml @@ -1,2 +1,7 @@ -# NOTE: no actions yet! -{ } +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +ceilometer-upgrade: + description: | + Perform ceilometer-upgrade. This action will update Ceilometer data store + configuration. diff --git a/charms/ceilometer-k8s/pyproject.toml b/charms/ceilometer-k8s/pyproject.toml index 2edc519a..30821404 100644 --- a/charms/ceilometer-k8s/pyproject.toml +++ b/charms/ceilometer-k8s/pyproject.toml @@ -1,3 +1,6 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + # Testing tools configuration [tool.coverage.run] branch = true @@ -11,23 +14,26 @@ log_cli_level = "INFO" # Formatting tools configuration [tool.black] -line-length = 99 -target-version = ["py38"] +line-length = 79 [tool.isort] -line_length = 99 profile = "black" +multi_line_output = 3 +force_grid_wrap = true # Linting tools configuration [tool.flake8] -max-line-length = 99 +max-line-length = 79 max-doc-length = 99 max-complexity = 10 exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ -ignore = ["W503", "E501", "D107"] -# D100, D101, D102, D103: Ignore missing docstrings in tests -per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] +ignore = ["W503", "E501", "D107", "E402"] +per-file-ignores = [] docstring-convention = "google" +# Check for properly formatted copyright header in each file +copyright-check = "True" +copyright-author = "Canonical Ltd." +copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" diff --git a/charms/ceilometer-k8s/src/charm.py b/charms/ceilometer-k8s/src/charm.py index 83416218..8f1c4ab5 100755 --- a/charms/ceilometer-k8s/src/charm.py +++ b/charms/ceilometer-k8s/src/charm.py @@ -1,4 +1,18 @@ #!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Ceilometer Operator Charm. This charm provide Ceilometer services as part of an OpenStack deployment @@ -6,13 +20,20 @@ This charm provide Ceilometer services as part of an OpenStack deployment import logging import uuid -from typing import List +from typing import ( + List, +) import ops.framework import ops_sunbeam.charm as sunbeam_charm import ops_sunbeam.container_handlers as container_handlers -import ops_sunbeam.core as core -from ops.main import main +import ops_sunbeam.core as sunbeam_core +from ops.charm import ( + ActionEvent, +) +from ops.main import ( + main, +) logger = logging.getLogger(__name__) @@ -44,8 +65,20 @@ class CeilometerCentralPebbleHandler(container_handlers.ServicePebbleHandler): }, } + def default_container_configs( + self, + ) -> List[sunbeam_core.ContainerConfigFile]: + """Container configurations for handler. -class CeilometerNotificationPebbleHandler(container_handlers.ServicePebbleHandler): + :returns: Container configuration files + :rtype: List[ContainerConfigFile] + """ + return self.charm.container_configs + + +class CeilometerNotificationPebbleHandler( + container_handlers.ServicePebbleHandler +): """Pebble handler for ceilometer-notification service.""" def get_layer(self) -> dict: @@ -69,6 +102,33 @@ class CeilometerNotificationPebbleHandler(container_handlers.ServicePebbleHandle }, } + def default_container_configs( + self, + ) -> List[sunbeam_core.ContainerConfigFile]: + """Container configurations for handler. + + :returns: Container configuration files + :rtype: List[ContainerConfigFile] + """ + _cconfigs = self.charm.container_configs + _cconfigs.extend( + [ + sunbeam_core.ContainerConfigFile( + "/etc/ceilometer/pipeline.yaml", + self.charm.service_user, + self.charm.service_group, + 0o640, + ), + sunbeam_core.ContainerConfigFile( + "/etc/ceilometer/event_pipeline.yaml", + self.charm.service_user, + self.charm.service_group, + 0o640, + ), + ] + ) + return _cconfigs + class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): """Charm the service.""" @@ -78,6 +138,12 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): mandatory_relations = {"amqp", "identity-credentials"} + def __init__(self, framework: ops.framework): + super().__init__(framework) + self.framework.observe( + self.on.ceilometer_upgrade_action, self._ceilometer_upgrade_action + ) + def get_shared_meteringsecret(self): """Return the shared metering secret.""" return self.leader_get(self.shared_metering_secret_key) @@ -103,13 +169,23 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): super().configure_charm(event) @property - def container_configs(self) -> List[core.ContainerConfigFile]: + def service_user(self) -> str: + """Service user file and directory ownership.""" + return "ceilometer" + + @property + def service_group(self) -> str: + """Service group file and directory ownership.""" + return "ceilometer" + + @property + def container_configs(self) -> List[sunbeam_core.ContainerConfigFile]: """Container configurations for the operator.""" _cconfigs = [ - core.ContainerConfigFile( + sunbeam_core.ContainerConfigFile( "/etc/ceilometer/ceilometer.conf", - "root", - "ceilometer", + self.service_user, + self.service_group, 0o640, ), ] @@ -122,7 +198,7 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): self, CEILOMETER_CENTRAL_CONTAINER, "ceilometer-central", - self.container_configs, + [], self.template_dir, self.configure_charm, ), @@ -130,12 +206,36 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): self, CEILOMETER_NOTIFICATION_CONTAINER, "ceilometer-notification", - self.container_configs, + [], self.template_dir, self.configure_charm, ), ] + def _ceilometer_upgrade_action(self, event: ActionEvent) -> None: + """Run ceilometer-upgrade. + + This action will upgrade the data store configuration in gnocchi. + """ + try: + logger.info("Syncing database...") + cmd = ["ceilometer-upgrade"] + container = self.unit.get_container( + CEILOMETER_NOTIFICATION_CONTAINER + ) + process = container.exec(cmd, timeout=5 * 60) + out, warnings = process.wait_output() + logging.debug("Output from database sync: \n%s", out) + if warnings: + for line in warnings.splitlines(): + logger.warning("DB Sync Out: %s", line.strip()) + event.fail(f"Error in running ceilometer-upgrade: {warnings}") + else: + event.set_results({"message": "ceilometer-upgrade successful"}) + except Exception as e: + logger.exception(e) + event.fail(f"Error in running ceilometer-updgrade: {e}") + if __name__ == "__main__": main(CeilometerOperatorCharm) diff --git a/charms/ceilometer-k8s/src/templates/ceilometer.conf b/charms/ceilometer-k8s/src/templates/ceilometer.conf index f406d734..93e2255d 100644 --- a/charms/ceilometer-k8s/src/templates/ceilometer.conf +++ b/charms/ceilometer-k8s/src/templates/ceilometer.conf @@ -2,18 +2,10 @@ debug = {{ options.debug }} # event_pipeline_cfg_file = /etc/ceilometer/event_pipeline.yaml -meter_dispatchers = gnocchi -event_dispatchers = gnocchi - {% if amqp.transport_url -%} transport_url = {{ amqp.transport_url }} {%- endif %} -[notification] -{% if amqp.transport_url -%} -messaging_urls = {{ amqp.transport_url }} -{% endif %} - [polling] batch_size = 50 @@ -29,3 +21,5 @@ archive_policy = low {% include "parts/identity-data-id-creds" %} {% include "parts/section-service-user-id-creds" %} + +{% include "parts/section-service-credentials" %} diff --git a/charms/ceilometer-k8s/src/templates/event_pipeline.yaml b/charms/ceilometer-k8s/src/templates/event_pipeline.yaml new file mode 100644 index 00000000..dd5f14cd --- /dev/null +++ b/charms/ceilometer-k8s/src/templates/event_pipeline.yaml @@ -0,0 +1,12 @@ +--- +sources: + - name: event_source + events: + - "*" + sinks: + - event_sink +sinks: + - name: event_sink + transformers: + publishers: + - notifier://?topic=alarm.all diff --git a/charms/ceilometer-k8s/src/templates/parts/section-service-credentials b/charms/ceilometer-k8s/src/templates/parts/section-service-credentials new file mode 100644 index 00000000..f2e6871c --- /dev/null +++ b/charms/ceilometer-k8s/src/templates/parts/section-service-credentials @@ -0,0 +1,14 @@ +{% if identity_credentials.project_domain_id -%} +[service_credentials] +{% if identity_credentials.internal_auth_url -%} +auth_url = {{ identity_credentials.internal_auth_url }} +{% elif identity_credentials.internal_host -%} +auth_url = {{ identity_credentials.internal_protocol }}://{{ identity_credentials.internal_host }}:{{ identity_credentials.internal_port }} +{% endif -%} +auth_type = password +project_domain_id = {{ identity_credentials.project_domain_id }} +user_domain_id = {{ identity_credentials.user_domain_id }} +project_name = {{ identity_credentials.project_name }} +username = {{ identity_credentials.username }} +password = {{ identity_credentials.password }} +{% endif -%} diff --git a/charms/ceilometer-k8s/src/templates/pipeline.yaml b/charms/ceilometer-k8s/src/templates/pipeline.yaml new file mode 100644 index 00000000..6e361df0 --- /dev/null +++ b/charms/ceilometer-k8s/src/templates/pipeline.yaml @@ -0,0 +1,89 @@ +--- +sources: + - name: meter_source + meters: + - "*" + sinks: + - meter_sink + - name: cpu_source + meters: + - "cpu" + sinks: + - cpu_sink + - cpu_delta_sink + - name: disk_source + meters: + - "disk.read.bytes" + - "disk.read.requests" + - "disk.write.bytes" + - "disk.write.requests" + - "disk.device.read.bytes" + - "disk.device.read.requests" + - "disk.device.write.bytes" + - "disk.device.write.requests" + sinks: + - disk_sink + - name: network_source + meters: + - "network.incoming.bytes" + - "network.incoming.packets" + - "network.outgoing.bytes" + - "network.outgoing.packets" + sinks: + - network_sink +sinks: + - name: meter_sink + transformers: + publishers: + - gnocchi:// + - name: cpu_sink + transformers: + - name: "rate_of_change" + parameters: + target: + name: "cpu_util" + unit: "%" + type: "gauge" + max: 100 + scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))" + publishers: + - gnocchi:// + - name: cpu_delta_sink + transformers: + - name: "delta" + parameters: + target: + name: "cpu.delta" + growth_only: True + publishers: + - gnocchi:// + - name: disk_sink + transformers: + - name: "rate_of_change" + parameters: + source: + map_from: + name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)" + unit: "(B|request)" + target: + map_to: + name: "\\1.\\2.\\3.rate" + unit: "\\1/s" + type: "gauge" + publishers: + - gnocchi:// + - name: network_sink + transformers: + - name: "rate_of_change" + parameters: + source: + map_from: + name: "network\\.(incoming|outgoing)\\.(bytes|packets)" + unit: "(B|packet)" + target: + map_to: + name: "network.\\1.\\2.rate" + unit: "\\1/s" + type: "gauge" + publishers: + - gnocchi:// diff --git a/charms/ceilometer-k8s/tests/actions.yaml b/charms/ceilometer-k8s/tests/actions.yaml new file mode 120000 index 00000000..9adaf92e --- /dev/null +++ b/charms/ceilometer-k8s/tests/actions.yaml @@ -0,0 +1 @@ +../actions.yaml \ No newline at end of file diff --git a/charms/ceilometer-k8s/tests/integration/test_charm.py b/charms/ceilometer-k8s/tests/integration/test_charm.py index 18f24d39..b3466b4c 100644 --- a/charms/ceilometer-k8s/tests/integration/test_charm.py +++ b/charms/ceilometer-k8s/tests/integration/test_charm.py @@ -1,14 +1,32 @@ #!/usr/bin/env python3 -# Copyright 2023 liam -# See LICENSE file for licensing details. + +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for ceilometer charm.""" import asyncio import logging -from pathlib import Path +from pathlib import ( + Path, +) import pytest import yaml -from pytest_operator.plugin import OpsTest +from pytest_operator.plugin import ( + OpsTest, +) logger = logging.getLogger(__name__) @@ -24,12 +42,21 @@ async def test_build_and_deploy(ops_test: OpsTest): """ # Build and deploy charm from local source folder charm = await ops_test.build_charm(".") - resources = {"httpbin-image": METADATA["resources"]["httpbin-image"]["upstream-source"]} + resources = { + "httpbin-image": METADATA["resources"]["httpbin-image"][ + "upstream-source" + ] + } # Deploy the charm and wait for active/idle status await asyncio.gather( - ops_test.model.deploy(await charm, resources=resources, application_name=APP_NAME), + ops_test.model.deploy( + await charm, resources=resources, application_name=APP_NAME + ), ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 + apps=[APP_NAME], + status="active", + raise_on_blocked=True, + timeout=1000, ), )