[masakari-k8s] Add masakari hostmonitor container

Add container masakari-hostmonitor
Add relations consul-management, consul-tenant,
consul-stoage to integrate with consul-k8s charm.
Update masakarimonitor configuraiton and consul
matrix yaml.

Change-Id: I238f67942b1a831143d3bff0f095340dd3155386
This commit is contained in:
Hemanth Nakkina 2024-10-08 16:28:08 +05:30
parent 26c7ef9193
commit bd137e4c39
No known key found for this signature in database
GPG Key ID: 2E4970F7B143168E
9 changed files with 655 additions and 16 deletions

View File

@ -1,4 +1,5 @@
external-libraries:
- charms.consul_k8s.v0.consul_cluster
- charms.data_platform_libs.v0.data_interfaces
- charms.rabbitmq_k8s.v0.rabbitmq
- charms.traefik_k8s.v2.ingress
@ -14,4 +15,4 @@ templates:
- parts/section-identity
- parts/section-database
- parts/identity-data
- ca-bundle.pem.j2
- ca-bundle.pem.j2

View File

@ -47,6 +47,8 @@ containers:
resource: masakari-image
masakari-engine:
resource: masakari-image
masakari-hostmonitor:
resource: masakari-image
resources:
masakari-image:
@ -83,18 +85,18 @@ requires:
interface: tracing
limit: 1
optional: true
# Note(mylesjp): consul disabled until charm is published
# consul-management:
# interface: consul-client
# limit: 1
# consul-tenant: # Name TBD
# interface: consul-client
# limit: 1
# optional: true
# consul-storage:
# interface: consul-client
# limit: 1
# optional: true
consul-management:
interface: consul-cluster
limit: 1
optional: true
consul-tenant:
interface: consul-cluster
limit: 1
optional: true
consul-storage:
interface: consul-cluster
limit: 1
optional: true
peers:
peers:

View File

@ -18,22 +18,38 @@ This charm provide Masakari services as part of an OpenStack deployment
"""
import logging
from collections import (
OrderedDict,
)
from typing import (
Callable,
)
import ops.framework
import ops.model
import ops.pebble
import ops_sunbeam.charm as sunbeam_charm
import ops_sunbeam.config_contexts as config_contexts
import ops_sunbeam.container_handlers as sunbeam_chandlers
import ops_sunbeam.core as sunbeam_core
import ops_sunbeam.relation_handlers as sunbeam_rhandlers
import ops_sunbeam.tracing as sunbeam_tracing
from ops.main import (
import yaml
from charms.consul_k8s.v0.consul_cluster import (
ConsulEndpointsRequirer,
)
from ops import (
main,
)
from ops.model import (
BlockedStatus,
)
logger = logging.getLogger(__name__)
MASAKARI_API_CONTAINER = "masakari-api"
MASAKARI_ENGINE_CONTAINER = "masakari-engine"
MASAKARI_HOSTMONITOR_CONTAINER = "masakari-hostmonitor"
def exec(container: ops.model.Container, cmd: str):
@ -50,6 +66,143 @@ def exec(container: ops.model.Container, cmd: str):
logger.exception(f"Command {cmd!r} failed")
@sunbeam_tracing.trace_type
class ConsulEndpointsRequirerHandler(sunbeam_rhandlers.RelationHandler):
"""Handle consul cluster relation on the requires side."""
interface: "ConsulEndpointsRequirer"
def __init__(
self,
charm: "sunbeam_charm.OSBaseOperatorCharm",
relation_name: str,
callback_f: Callable,
mandatory: bool = False,
):
"""Create a new consul-cluster handler.
Create a new ConsulEndpointsRequirerHandler that handles initial
events from the relation and invokes the provided callbacks based on
the event raised.
:param charm: the Charm class the handler is for
:type charm: ops.charm.CharmBase
:param relation_name: the relation the handler is bound to
:type relation_name: str
:param callback_f: the function to call when the nodes are connected
:type callback_f: Callable
:param mandatory: If the relation is mandatory to proceed with
configuring charm
:type mandatory: bool
"""
super().__init__(charm, relation_name, callback_f, mandatory)
def setup_event_handler(self) -> ops.framework.Object:
"""Configure event handlers for consul-cluster relation."""
logger.debug(f"Setting up {self.relation_name} event handler")
svc = sunbeam_tracing.trace_type(ConsulEndpointsRequirer)(
self.charm,
self.relation_name,
)
self.framework.observe(
svc.on.endpoints_changed,
self._on_consul_cluster_endpoints_changed,
)
self.framework.observe(
svc.on.goneaway,
self._on_consul_cluster_goneaway,
)
return svc
def _on_consul_cluster_endpoints_changed(
self, event: ops.framework.EventBase
) -> None:
"""Handle endpoints_changed event."""
logger.debug(
f"Consul cluster endpoints changed event received for relation {self.relation_name}"
)
self.callback_f(event)
def _on_consul_cluster_goneaway(
self, event: ops.framework.EventBase
) -> None:
"""Handle gone_away event."""
logger.debug(
f"Consul cluster gone away event received for relation {self.relation_name}"
)
self.callback_f(event)
if self.mandatory:
self.status.set(BlockedStatus("integration missing"))
@property
def ready(self) -> bool:
"""Whether handler is ready for use."""
return bool(self.interface.internal_http_endpoint)
@sunbeam_tracing.trace_type
class MasakariConfigurationContext(config_contexts.ConfigContext):
"""Configuration context for Masakar."""
def construct_consul_matrix(self) -> str | None:
"""Construct Consul matrix yaml."""
agent_handlers_map = OrderedDict(
{
"manage": self.charm.consul_management.ready,
"tenant": self.charm.consul_tenant.ready,
"storage": self.charm.consul_storage.ready,
}
)
sequence = [k for k, v in agent_handlers_map.items() if v]
active_agents_count = len(sequence)
# Do not set any matrix if no agents are active
# Leave it to DEFAULTS from masakarimonitors if all the consul agents are enabled
# and so do not set any matrix
# https://opendev.org/openstack/masakari-monitors/src/commit/21a78c65d3e0536500ab55a7868c1edb99131b67/masakarimonitors/hostmonitor/consul_check/matrix_helper.py#L26 # noqa
if active_agents_count in {0, 3}:
return None
matrix = []
if active_agents_count == 1:
up = {"health": "up", "action": []}
down = {"health": "down", "action": ["recovery"]}
matrix.extend([up, down])
elif active_agents_count == 2:
# Defaults for 2*2 matrix with no actions
up_up = {"health": ["up", "up"], "action": []}
up_down = {"health": ["up", "down"], "action": []}
down_up = {"health": ["down", "up"], "action": []}
down_down = {"health": ["down", "down"], "action": []}
# Actions should be recovery if storage is down
# If storage not present, consider management handles storage as well
if sequence == ["manage", "tenant"]:
down_up["action"] = ["recovery"]
down_down["action"] = ["recovery"]
elif sequence == ["manage", "storage"]:
up_down["action"] = ["recovery"]
down_down["action"] = ["recovery"]
elif sequence == ["tenant", "storage"]:
up_down["action"] = ["recovery"]
down_down["action"] = ["recovery"]
matrix.extend([up_up, up_down, down_up, down_down])
matrix_yaml = yaml.safe_dump({"sequence": sequence, "matrix": matrix})
return matrix_yaml
def context(self) -> dict:
"""Generate context information for masakari config."""
ctx = {}
matrix = self.construct_consul_matrix()
if matrix:
ctx["consul_matrix"] = matrix
logger.debug(f"Context from masakari {ctx}")
return ctx
@sunbeam_tracing.trace_type
class MasakariWSGIPebbleHandler(sunbeam_chandlers.WSGIPebbleHandler):
"""Pebble handler for Masakari API container."""
@ -88,6 +241,48 @@ class MasakariEnginePebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
}
@sunbeam_tracing.trace_type
class MasakariHostMonitorPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
"""Pebble handler for Masakari Host monitor container."""
def get_layer(self):
"""Pebble layer for Masakari Host monitor service.
:returns: pebble service layer config for masakari host monitor service
:rtype: dict
"""
return {
"summary": "masakari host monitor layer",
"description": "pebble configuration for masakari host monitor service",
"services": {
"masakari-hostmonitor": {
"override": "replace",
"summary": "masakari host monitor",
"command": "masakari-hostmonitor --config-file /etc/masakari/masakarimonitors.conf",
"user": self.charm.service_user,
"group": self.charm.service_group,
}
},
}
def default_container_configs(
self,
) -> list[sunbeam_core.ContainerConfigFile]:
"""Container configurations for handler."""
return [
sunbeam_core.ContainerConfigFile(
"/etc/masakari/masakarimonitors.conf",
self.charm.service_user,
self.charm.service_group,
),
sunbeam_core.ContainerConfigFile(
"/etc/masakari/matrix.yaml",
self.charm.service_user,
self.charm.service_group,
),
]
@sunbeam_tracing.trace_sunbeam_charm
class MasakariOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
"""Charm the service."""
@ -110,6 +305,18 @@ class MasakariOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
]
]
# Initialise custom event handlers
consul_management: ConsulEndpointsRequirerHandler | None = None
consul_tenant: ConsulEndpointsRequirerHandler | None = None
consul_storage: ConsulEndpointsRequirerHandler | None = None
@property
def config_contexts(self) -> list[config_contexts.ConfigContext]:
"""Configuration contexts for the operator."""
contexts = super().config_contexts
contexts.append(MasakariConfigurationContext(self, "masakari_config"))
return contexts
@property
def container_configs(self) -> list[sunbeam_core.ContainerConfigFile]:
"""Container configurations for handler."""
@ -126,6 +333,38 @@ class MasakariOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
)
return _cconfigs
def get_relation_handlers(
self, handlers: list[sunbeam_rhandlers.RelationHandler] | None = None
) -> list[sunbeam_rhandlers.RelationHandler]:
"""Relation handlers for the service."""
handlers = handlers or []
self.consul_management = ConsulEndpointsRequirerHandler(
self,
"consul-management",
self.configure_charm,
"consul-management" in self.mandatory_relations,
)
handlers.append(self.consul_management)
self.consul_tenant = ConsulEndpointsRequirerHandler(
self,
"consul-tenant",
self.configure_charm,
"consul-tenant" in self.mandatory_relations,
)
handlers.append(self.consul_tenant)
self.consul_storage = ConsulEndpointsRequirerHandler(
self,
"consul-storage",
self.configure_charm,
"consul-storage" in self.mandatory_relations,
)
handlers.append(self.consul_storage)
handlers = super().get_relation_handlers(handlers)
return handlers
def get_pebble_handlers(self):
"""Pebble handlers for operator."""
pebble_handlers = []
@ -148,6 +387,14 @@ class MasakariOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
self.template_dir,
self.configure_charm,
),
MasakariHostMonitorPebbleHandler(
self,
MASAKARI_HOSTMONITOR_CONTAINER,
"masakari-hostmonitor",
[],
self.template_dir,
self.configure_charm,
),
]
)
return pebble_handlers

View File

@ -0,0 +1,23 @@
[DEFAULT]
debug = {{ options.debug }}
{% if consul_management or consul_tenant or consul_storage -%}
[host]
monitoring_driver = consul
{% endif -%}
[consul]
{% if consul_management and consul_management.internal_http_endpoint -%}
agent_manage = {{ consul_management.internal_http_endpoint }}
{% endif -%}
{% if consul_tenant and consul_tenant.internal_http_endpoint -%}
agent_tenant = {{ consul_tenant.internal_http_endpoint }}
{% endif -%}
{% if consul_storage and consul_storage.internal_http_endpoint -%}
agent_storage = {{ consul_storage.internal_http_endpoint }}
{% endif -%}
matrix_config_file = /etc/masakari/matrix.yaml

View File

@ -0,0 +1,3 @@
{% if masakari_config and masakari_config.consul_matrix -%}
{{ masakari_config.consul_matrix }}
{% endif %}

View File

@ -18,6 +18,9 @@
import charm
import ops_sunbeam.test_utils as test_utils
from ops.testing import (
Harness,
)
class _MasakariOperatorCharm(charm.MasakariOperatorCharm):
@ -66,6 +69,29 @@ class TestMasakariOperatorCharm(test_utils.CharmTestCase):
self.addCleanup(self.harness.cleanup)
def _add_all_consul_cluster_relations(self, harness: Harness):
harness.add_relation(
"consul-management",
"consul-management",
app_data={
"internal_http_address": "consul-management.consul.svc:8500"
},
)
harness.add_relation(
"consul-tenant",
"consul-tenant",
app_data={
"internal_http_address": "consul-tenant.consul.svc:8500"
},
)
harness.add_relation(
"consul-storage",
"consul-storage",
app_data={
"internal_http_address": "consul-storage.consul.svc:8500"
},
)
def test_pebble_ready_handler(self):
"""Test Pebble ready event is captured."""
self.harness.begin()
@ -73,7 +99,7 @@ class TestMasakariOperatorCharm(test_utils.CharmTestCase):
test_utils.set_all_pebbles_ready(self.harness)
self.assertEqual(
self.harness.charm.seen_events,
["PebbleReadyEvent", "PebbleReadyEvent"],
["PebbleReadyEvent", "PebbleReadyEvent", "PebbleReadyEvent"],
)
def test_all_relations(self):
@ -83,6 +109,7 @@ class TestMasakariOperatorCharm(test_utils.CharmTestCase):
test_utils.set_all_pebbles_ready(self.harness)
test_utils.add_api_relations(self.harness)
test_utils.add_complete_ingress_relation(self.harness)
self._add_all_consul_cluster_relations(self.harness)
self.harness.charm.configure_charm(event=None)
@ -106,3 +133,9 @@ class TestMasakariOperatorCharm(test_utils.CharmTestCase):
# Check for rendering of single configuration file in masakari-engine
self.check_file("masakari-engine", "/etc/masakari/masakari.conf")
# Check for rendering masakari-hostmonitor config files
self.check_file(
"masakari-hostmonitor", "/etc/masakari/masakarimonitors.conf"
)
self.check_file("masakari-hostmonitor", "/etc/masakari/matrix.yaml")

View File

@ -0,0 +1,316 @@
"""ConsulCluster Provides and Requires module.
This library contains Provider and Requirer classes for
consul-cluster interface.
The provider side updates relation data with the endpoints
information required by consul agents running in client mode
or consul users/clients.
The requirer side receives the endpoints via relation data.
Example on how to use Requirer side using this library.
Import `ConsulEndpointsRequirer` in your charm, with the charm object and the
relation name:
- self
- "consul-cluster"
Two events are also available to respond to:
- endpoints_changed
- goneaway
A basic example showing the usage of this relation follows:
```
from charms.consul_k8s.v0.consul_cluster import (
ConsulEndpointsRequirer
)
class ConsulClientCharm(CharmBase):
def __init__(self, *args):
super().__init__(*args)
# ConsulCluster Requires
self.consul = ConsulEdnpointsRequirer(
self, "consul-cluster",
)
self.framework.observe(
self.consul.on.endpoints_changed,
self._on_consul_service_endpoints_changed
)
self.framework.observe(
self.consul.on.goneaway,
self._on_consul_service_goneaway
)
def _on_consul_service_endpoints_changed(self, event):
'''React to the Consul service endpoints changed event.
This event happens when consul-cluster relation is added to the
model and relation data is changed.
'''
# Do something with the endpoints provided by relation.
pass
def _on_consul_service_goneaway(self, event):
'''React to the ConsulService goneaway event.
This event happens when consul-cluster relation is removed.
'''
# ConsulService Relation has goneaway.
pass
```
"""
import json
import logging
from ops.charm import CharmBase, RelationBrokenEvent, RelationChangedEvent, RelationEvent
from ops.framework import EventSource, Object, ObjectEvents
from ops.model import Relation
from pydantic import BaseModel, Field, ValidationError, field_validator
# The unique Charmhub library identifier, never change it
LIBID = "f10432d106524b82ba68aa6eddbc3308"
# Increment this major API version when introducing breaking changes
LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 1
DEFAULT_RELATION_NAME = "consul-cluster"
logger = logging.getLogger(__name__)
class ConsulServiceProviderAppData(BaseModel):
"""Cluster endpoints from Consul server."""
datacenter: str = Field("Datacenter cluster name")
# All endpoints are json serialized
internal_gossip_endpoints: list[str] | None = Field(
"Consul server join addresses for internal consul agents"
)
external_gossip_endpoints: list[str] | None = Field(
"Consul server join addresses for external consul agents"
)
internal_http_endpoint: str | None = Field(
"Consul server http address for consul users running in same k8s cluster as consul-server"
)
# This field will be the ingress endpoint. Ingress is not supported yet.
external_http_endpoint: str | None = Field("Consul server http address for external users")
@field_validator("internal_gossip_endpoints", "external_gossip_endpoints", mode="before")
@classmethod
def convert_str_to_list_of_str(cls, v: str) -> list[str]:
"""Convert string field to list of str."""
if not isinstance(v, str):
return v
try:
return json.loads(v)
except json.decoder.JSONDecodeError:
raise ValueError("Field not in json format")
@field_validator("internal_http_endpoint", "external_http_endpoint", mode="before")
@classmethod
def convert_str_null_to_none(cls, v: str) -> str | None:
"""Convert null string to None."""
if v == "null":
return None
return v
class ClusterEndpointsChangedEvent(RelationEvent):
"""Consul cluster endpoints changed event."""
pass
class ClusterServerGoneAwayEvent(RelationEvent):
"""Cluster server relation gone away event."""
pass
class ConsulEndpointsRequirerEvents(ObjectEvents):
"""Consul Cluster requirer events."""
endpoints_changed = EventSource(ClusterEndpointsChangedEvent)
goneaway = EventSource(ClusterServerGoneAwayEvent)
class ConsulEndpointsRequirer(Object):
"""Class to be instantiated on the requirer side of the relation."""
on = ConsulEndpointsRequirerEvents() # pyright: ignore
def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME):
super().__init__(charm, relation_name)
self.charm = charm
self.relation_name = relation_name
events = self.charm.on[relation_name]
self.framework.observe(events.relation_changed, self._on_relation_changed)
self.framework.observe(events.relation_broken, self._on_relation_changed)
def _on_relation_changed(self, event: RelationChangedEvent):
if self._validate_databag_from_relation():
self.on.endpoints_changed.emit(event.relation)
def _on_relation_broken(self, event: RelationBrokenEvent):
"""Handle relation broken event."""
self.on.goneaway.emit()
def _validate_databag_from_relation(self) -> bool:
try:
if self._consul_cluster_rel:
databag = self._consul_cluster_rel.data[self._consul_cluster_rel.app]
ConsulServiceProviderAppData(**databag) # type: ignore
except ValidationError as e:
logger.info(f"Incorrect app databag: {str(e)}")
return False
return True
def _get_app_databag_from_relation(self) -> dict:
try:
if self._consul_cluster_rel:
databag = self._consul_cluster_rel.data[self._consul_cluster_rel.app]
data = ConsulServiceProviderAppData(**databag) # type: ignore
return data.model_dump()
except ValidationError as e:
logger.info(f"Incorrect app databag: {str(e)}")
return {}
@property
def _consul_cluster_rel(self) -> Relation | None:
"""The Consul cluster relation."""
return self.framework.model.get_relation(self.relation_name)
@property
def datacenter(self) -> str | None:
"""Return datacenter name from provider app data."""
data = self._get_app_databag_from_relation()
return data.get("datacenter")
@property
def internal_gossip_endpoints(self) -> list[str] | None:
"""Return internal gossip endpoints from provider app data."""
data = self._get_app_databag_from_relation()
return data.get("internal_gossip_endpoints")
@property
def external_gossip_endpoints(self) -> list[str] | None:
"""Return external gossip endpoints from provider app data."""
data = self._get_app_databag_from_relation()
return data.get("external_gossip_endpoints")
@property
def internal_http_endpoint(self) -> str | None:
"""Return internal http endpoint from provider app data."""
data = self._get_app_databag_from_relation()
return data.get("internal_http_endpoint")
@property
def external_http_endpoint(self) -> str | None:
"""Return external http endpoint from provider app data."""
data = self._get_app_databag_from_relation()
return data.get("external_http_endpoint")
class ClusterEndpointsRequestEvent(RelationEvent):
"""Consul cluster endpoints request event."""
pass
class ConsulServiceProviderEvents(ObjectEvents):
"""Events class for `on`."""
endpoints_request = EventSource(ClusterEndpointsRequestEvent)
class ConsulServiceProvider(Object):
"""Class to be instantiated on the provider side of the relation."""
on = ConsulServiceProviderEvents() # pyright: ignore
def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME):
super().__init__(charm, relation_name)
self.charm = charm
self.relation_name = relation_name
events = self.charm.on[relation_name]
self.framework.observe(events.relation_changed, self._on_relation_changed)
def _on_relation_changed(self, event: RelationChangedEvent):
"""Handle new cluster client connect."""
self.on.endpoints_request.emit(event.relation)
def set_cluster_endpoints(
self,
relation: Relation | None,
datacenter: str,
internal_gossip_endpoints: list[str] | None,
external_gossip_endpoints: list[str] | None,
internal_http_endpoint: str | None,
external_http_endpoint: str | None,
) -> None:
"""Set consul cluster endpoints on the relation.
If relation is None, send cluster endpoints on all related units.
"""
if not self.charm.unit.is_leader():
logging.debug("Not a leader unit, skipping set endpoints")
return
try:
databag = ConsulServiceProviderAppData(
datacenter=datacenter,
internal_gossip_endpoints=internal_gossip_endpoints,
external_gossip_endpoints=external_gossip_endpoints,
internal_http_endpoint=internal_http_endpoint,
external_http_endpoint=external_http_endpoint,
)
except ValidationError as e:
logger.info(f"Provider trying to set incorrect app data {str(e)}")
return
# If relation is not provided send endpoints to all the related
# applications. This happens usually when endpoints data is
# updated by provider and wants to send the data to all
# related applications
_datacenter: str = databag.datacenter
_internal_gossip_endpoints: str = json.dumps(databag.internal_gossip_endpoints)
_external_gossip_endpoints: str = json.dumps(databag.external_gossip_endpoints)
_internal_http_endpoint: str = json.dumps(databag.internal_http_endpoint)
_external_http_endpoint: str = json.dumps(external_http_endpoint)
if relation is None:
logging.debug(
"Sending endpoints to all related applications of relation" f"{self.relation_name}"
)
relations_to_send_endpoints = self.framework.model.relations[self.relation_name]
else:
logging.debug(
f"Sending endpoints on relation {relation.app.name} "
f"{relation.name}/{relation.id}"
)
relations_to_send_endpoints = [relation]
for relation in relations_to_send_endpoints:
if relation:
relation.data[self.charm.app]["datacenter"] = _datacenter
relation.data[self.charm.app]["internal_gossip_endpoints"] = (
_internal_gossip_endpoints
)
relation.data[self.charm.app]["external_gossip_endpoints"] = (
_external_gossip_endpoints
)
relation.data[self.charm.app]["internal_http_endpoint"] = _internal_http_endpoint
relation.data[self.charm.app]["external_http_endpoint"] = _external_http_endpoint

View File

@ -114,6 +114,14 @@ applications:
trust: true
resources:
masakari-image: ghcr.io/canonical/masakari-consolidated:2024.1
consul:
charm: consul-k8s
channel: 1.19/edge
base: ubuntu@24.04
scale: 1
trust: true
resources:
consul-image: ghcr.io/canonical/consul:1.19.2
relations:
- - mysql:database
@ -156,4 +164,7 @@ relations:
- - keystone:identity-service
- masakari:identity-service
- - traefik:ingress
- masakari:ingress-public
- masakari:ingress-public
- - masakari:consul-management
- consul:consul-cluster

View File

@ -56,3 +56,6 @@ target_deploy_status:
masakari:
workload-status: active
workload-status-message-regex: '^$'
consul:
workload-status: active
workload-status-message-regex: '^$'