Fix configuration issues

* Add pipeline.yaml
* Add event_pipeline.yaml
* Add service_credentials to ceilometer.conf
* Add action ceilometer-upgrade
* Run tox -e fmt for formatting code

Change-Id: I8c1edc0bdaa71ee48a272f8c4d4aa6fae6642f4e

charms/ceilometer-k8s/actions.yaml

@@ -1,2 +1,7 @@
-# NOTE: no actions yet!
+# Copyright 2023 Canonical Ltd.
-{ }
+# See LICENSE file for licensing details.
+
+ceilometer-upgrade:
+  description: |
+    Perform ceilometer-upgrade. This action will update Ceilometer data store
+    configuration.

charms/ceilometer-k8s/pyproject.toml

@@ -1,3 +1,6 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 # Testing tools configuration
 [tool.coverage.run]
 branch = true
@@ -11,23 +14,26 @@ log_cli_level = "INFO"
 
 # Formatting tools configuration
 [tool.black]
-line-length = 99
+line-length = 79
-target-version = ["py38"]
 
 [tool.isort]
-line_length = 99
 profile = "black"
+multi_line_output = 3
+force_grid_wrap = true
 
 # Linting tools configuration
 [tool.flake8]
-max-line-length = 99
+max-line-length = 79
 max-doc-length = 99
 max-complexity = 10
 exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"]
 select = ["E", "W", "F", "C", "N", "R", "D", "H"]
 # Ignore W503, E501 because using black creates errors with this
 # Ignore D107 Missing docstring in __init__
-ignore = ["W503", "E501", "D107"]
+ignore = ["W503", "E501", "D107", "E402"]
-# D100, D101, D102, D103: Ignore missing docstrings in tests
+per-file-ignores = []
-per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"]
 docstring-convention = "google"
+# Check for properly formatted copyright header in each file
+copyright-check = "True"
+copyright-author = "Canonical Ltd."
+copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s"

charms/ceilometer-k8s/src/charm.py

@@ -1,4 +1,18 @@
 #!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Ceilometer Operator Charm.
 
 This charm provide Ceilometer services as part of an OpenStack deployment
@@ -6,13 +20,20 @@ This charm provide Ceilometer services as part of an OpenStack deployment
 
 import logging
 import uuid
-from typing import List
+from typing import (
+    List,
+)
 
 import ops.framework
 import ops_sunbeam.charm as sunbeam_charm
 import ops_sunbeam.container_handlers as container_handlers
-import ops_sunbeam.core as core
+import ops_sunbeam.core as sunbeam_core
-from ops.main import main
+from ops.charm import (
+    ActionEvent,
+)
+from ops.main import (
+    main,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +65,20 @@ class CeilometerCentralPebbleHandler(container_handlers.ServicePebbleHandler):
             },
         }
 
+    def default_container_configs(
+        self,
+    ) -> List[sunbeam_core.ContainerConfigFile]:
+        """Container configurations for handler.
 
-class CeilometerNotificationPebbleHandler(container_handlers.ServicePebbleHandler):
+        :returns: Container configuration files
+        :rtype: List[ContainerConfigFile]
+        """
+        return self.charm.container_configs
+
+
+class CeilometerNotificationPebbleHandler(
+    container_handlers.ServicePebbleHandler
+):
     """Pebble handler for ceilometer-notification service."""
 
     def get_layer(self) -> dict:
@@ -69,6 +102,33 @@ class CeilometerNotificationPebbleHandler(container_handlers.ServicePebbleHandle
             },
         }
 
+    def default_container_configs(
+        self,
+    ) -> List[sunbeam_core.ContainerConfigFile]:
+        """Container configurations for handler.
+
+        :returns: Container configuration files
+        :rtype: List[ContainerConfigFile]
+        """
+        _cconfigs = self.charm.container_configs
+        _cconfigs.extend(
+            [
+                sunbeam_core.ContainerConfigFile(
+                    "/etc/ceilometer/pipeline.yaml",
+                    self.charm.service_user,
+                    self.charm.service_group,
+                    0o640,
+                ),
+                sunbeam_core.ContainerConfigFile(
+                    "/etc/ceilometer/event_pipeline.yaml",
+                    self.charm.service_user,
+                    self.charm.service_group,
+                    0o640,
+                ),
+            ]
+        )
+        return _cconfigs
+
 
 class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
     """Charm the service."""
@@ -78,6 +138,12 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
 
     mandatory_relations = {"amqp", "identity-credentials"}
 
+    def __init__(self, framework: ops.framework):
+        super().__init__(framework)
+        self.framework.observe(
+            self.on.ceilometer_upgrade_action, self._ceilometer_upgrade_action
+        )
+
     def get_shared_meteringsecret(self):
         """Return the shared metering secret."""
         return self.leader_get(self.shared_metering_secret_key)
@@ -103,13 +169,23 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
         super().configure_charm(event)
 
     @property
-    def container_configs(self) -> List[core.ContainerConfigFile]:
+    def service_user(self) -> str:
+        """Service user file and directory ownership."""
+        return "ceilometer"
+
+    @property
+    def service_group(self) -> str:
+        """Service group file and directory ownership."""
+        return "ceilometer"
+
+    @property
+    def container_configs(self) -> List[sunbeam_core.ContainerConfigFile]:
         """Container configurations for the operator."""
         _cconfigs = [
-            core.ContainerConfigFile(
+            sunbeam_core.ContainerConfigFile(
                 "/etc/ceilometer/ceilometer.conf",
-                "root",
+                self.service_user,
-                "ceilometer",
+                self.service_group,
                 0o640,
             ),
         ]
@@ -122,7 +198,7 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
                 self,
                 CEILOMETER_CENTRAL_CONTAINER,
                 "ceilometer-central",
-                self.container_configs,
+                [],
                 self.template_dir,
                 self.configure_charm,
             ),
@@ -130,12 +206,36 @@ class CeilometerOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
                 self,
                 CEILOMETER_NOTIFICATION_CONTAINER,
                 "ceilometer-notification",
-                self.container_configs,
+                [],
                 self.template_dir,
                 self.configure_charm,
             ),
         ]
 
+    def _ceilometer_upgrade_action(self, event: ActionEvent) -> None:
+        """Run ceilometer-upgrade.
+
+        This action will upgrade the data store configuration in gnocchi.
+        """
+        try:
+            logger.info("Syncing database...")
+            cmd = ["ceilometer-upgrade"]
+            container = self.unit.get_container(
+                CEILOMETER_NOTIFICATION_CONTAINER
+            )
+            process = container.exec(cmd, timeout=5 * 60)
+            out, warnings = process.wait_output()
+            logging.debug("Output from database sync: \n%s", out)
+            if warnings:
+                for line in warnings.splitlines():
+                    logger.warning("DB Sync Out: %s", line.strip())
+                event.fail(f"Error in running ceilometer-upgrade: {warnings}")
+            else:
+                event.set_results({"message": "ceilometer-upgrade successful"})
+        except Exception as e:
+            logger.exception(e)
+            event.fail(f"Error in running ceilometer-updgrade: {e}")
+
 
 if __name__ == "__main__":
     main(CeilometerOperatorCharm)

charms/ceilometer-k8s/src/templates/ceilometer.conf

@@ -2,18 +2,10 @@
 debug = {{ options.debug }}
 # event_pipeline_cfg_file = /etc/ceilometer/event_pipeline.yaml
 
-meter_dispatchers = gnocchi
-event_dispatchers = gnocchi
-
 {% if amqp.transport_url -%}
 transport_url = {{ amqp.transport_url }}
 {%- endif %}
 
-[notification]
-{% if amqp.transport_url -%}
-messaging_urls = {{ amqp.transport_url }}
-{% endif %}
-
 [polling]
 batch_size = 50
 
@@ -29,3 +21,5 @@ archive_policy = low
 {% include "parts/identity-data-id-creds" %}
 
 {% include "parts/section-service-user-id-creds" %}
+
+{% include "parts/section-service-credentials" %}

charms/ceilometer-k8s/src/templates/event_pipeline.yaml

@@ -0,0 +1,12 @@
+---
+sources:
+  - name: event_source
+    events:
+     - "*"
+    sinks:
+     - event_sink
+sinks:
+  - name: event_sink
+    transformers:
+    publishers:
+     - notifier://?topic=alarm.all

charms/ceilometer-k8s/src/templates/parts/section-service-credentials

@@ -0,0 +1,14 @@
+{% if identity_credentials.project_domain_id -%}
+[service_credentials]
+{% if identity_credentials.internal_auth_url -%}
+auth_url = {{ identity_credentials.internal_auth_url }}
+{% elif identity_credentials.internal_host -%}
+auth_url = {{ identity_credentials.internal_protocol }}://{{ identity_credentials.internal_host }}:{{ identity_credentials.internal_port }}
+{% endif -%}
+auth_type = password
+project_domain_id = {{ identity_credentials.project_domain_id }}
+user_domain_id = {{ identity_credentials.user_domain_id }}
+project_name = {{ identity_credentials.project_name }}
+username = {{ identity_credentials.username }}
+password = {{ identity_credentials.password }}
+{% endif -%}

charms/ceilometer-k8s/src/templates/pipeline.yaml

@@ -0,0 +1,89 @@
+---
+sources:
+    - name: meter_source
+      meters:
+          - "*"
+      sinks:
+          - meter_sink
+    - name: cpu_source
+      meters:
+          - "cpu"
+      sinks:
+          - cpu_sink
+          - cpu_delta_sink
+    - name: disk_source
+      meters:
+          - "disk.read.bytes"
+          - "disk.read.requests"
+          - "disk.write.bytes"
+          - "disk.write.requests"
+          - "disk.device.read.bytes"
+          - "disk.device.read.requests"
+          - "disk.device.write.bytes"
+          - "disk.device.write.requests"
+      sinks:
+          - disk_sink
+    - name: network_source
+      meters:
+          - "network.incoming.bytes"
+          - "network.incoming.packets"
+          - "network.outgoing.bytes"
+          - "network.outgoing.packets"
+      sinks:
+          - network_sink
+sinks:
+    - name: meter_sink
+      transformers:
+      publishers:
+          - gnocchi://
+    - name: cpu_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                target:
+                    name: "cpu_util"
+                    unit: "%"
+                    type: "gauge"
+                    max: 100
+                    scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))"
+      publishers:
+          - gnocchi://
+    - name: cpu_delta_sink
+      transformers:
+          - name: "delta"
+            parameters:
+                target:
+                    name: "cpu.delta"
+                growth_only: True
+      publishers:
+          - gnocchi://
+    - name: disk_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                    map_from:
+                        name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)"
+                        unit: "(B|request)"
+                target:
+                    map_to:
+                        name: "\\1.\\2.\\3.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - gnocchi://
+    - name: network_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                   map_from:
+                       name: "network\\.(incoming|outgoing)\\.(bytes|packets)"
+                       unit: "(B|packet)"
+                target:
+                    map_to:
+                        name: "network.\\1.\\2.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - gnocchi://

charms/ceilometer-k8s/tests/actions.yaml

@@ -0,0 +1 @@
+../actions.yaml

charms/ceilometer-k8s/tests/integration/test_charm.py

@@ -1,14 +1,32 @@
 #!/usr/bin/env python3
-# Copyright 2023 liam
+
-# See LICENSE file for licensing details.
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for ceilometer charm."""
 
 import asyncio
 import logging
-from pathlib import Path
+from pathlib import (
+    Path,
+)
 
 import pytest
 import yaml
-from pytest_operator.plugin import OpsTest
+from pytest_operator.plugin import (
+    OpsTest,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -24,12 +42,21 @@ async def test_build_and_deploy(ops_test: OpsTest):
     """
     # Build and deploy charm from local source folder
     charm = await ops_test.build_charm(".")
-    resources = {"httpbin-image": METADATA["resources"]["httpbin-image"]["upstream-source"]}
+    resources = {
+        "httpbin-image": METADATA["resources"]["httpbin-image"][
+            "upstream-source"
+        ]
+    }
 
     # Deploy the charm and wait for active/idle status
     await asyncio.gather(
-        ops_test.model.deploy(await charm, resources=resources, application_name=APP_NAME),
+        ops_test.model.deploy(
+            await charm, resources=resources, application_name=APP_NAME
+        ),
         ops_test.model.wait_for_idle(
-            apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000
+            apps=[APP_NAME],
+            status="active",
+            raise_on_blocked=True,
+            timeout=1000,
         ),
     )