Add support for healthchecks
Add healthchecks for WSGI, OVN services Check healthcheck in update-status Change-Id: I3df7026f18cdf5ad897f4dcf1d06a3c3f6fd80c4
This commit is contained in:
parent
59967b2a51
commit
87e5ee8a32
|
@ -66,6 +66,8 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
|
||||||
self.relation_handlers = self.get_relation_handlers()
|
self.relation_handlers = self.get_relation_handlers()
|
||||||
self.pebble_handlers = self.get_pebble_handlers()
|
self.pebble_handlers = self.get_pebble_handlers()
|
||||||
self.framework.observe(self.on.config_changed, self._on_config_changed)
|
self.framework.observe(self.on.config_changed, self._on_config_changed)
|
||||||
|
# TODO: change update_status based on compound_status feature
|
||||||
|
self.framework.observe(self.on.update_status, self._on_update_status)
|
||||||
|
|
||||||
def can_add_handler(
|
def can_add_handler(
|
||||||
self,
|
self,
|
||||||
|
@ -235,6 +237,10 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
|
||||||
if self.unit.is_leader() and self.supports_peer_relation:
|
if self.unit.is_leader() and self.supports_peer_relation:
|
||||||
self.set_leader_ready()
|
self.set_leader_ready()
|
||||||
|
|
||||||
|
# Add healthchecks to the plan
|
||||||
|
for ph in self.pebble_handlers:
|
||||||
|
ph.add_healthchecks()
|
||||||
|
|
||||||
self.unit.status = ops.model.ActiveStatus()
|
self.unit.status = ops.model.ActiveStatus()
|
||||||
self._state.bootstrapped = True
|
self._state.bootstrapped = True
|
||||||
|
|
||||||
|
@ -395,6 +401,22 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
|
||||||
self._state.bootstrapped = False
|
self._state.bootstrapped = False
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def _on_update_status(self, event: ops.framework.EventBase) -> None:
|
||||||
|
"""Update status event handler."""
|
||||||
|
status = []
|
||||||
|
for ph in self.pebble_handlers:
|
||||||
|
ph.assess_status()
|
||||||
|
# Below lines are not required with compound status feature
|
||||||
|
if ph.status:
|
||||||
|
status.append(ph.status)
|
||||||
|
|
||||||
|
# Need to be changed once compound status in place
|
||||||
|
if len(status) == 0:
|
||||||
|
self.unit.status = ops.model.ActiveStatus()
|
||||||
|
else:
|
||||||
|
status_msg = ','.join(status)
|
||||||
|
self.unit.status = ops.model.BlockedStatus(status_msg)
|
||||||
|
|
||||||
|
|
||||||
class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
|
class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
|
||||||
"""Base class for OpenStack API operators."""
|
"""Base class for OpenStack API operators."""
|
||||||
|
@ -623,3 +645,8 @@ class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
|
||||||
def db_sync_container_name(self) -> str:
|
def db_sync_container_name(self) -> str:
|
||||||
"""Name of Containerto run db sync from."""
|
"""Name of Containerto run db sync from."""
|
||||||
return self.wsgi_container_name
|
return self.wsgi_container_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def healthcheck_http_url(self) -> str:
|
||||||
|
"""Healthcheck HTTP URL for the service."""
|
||||||
|
return f'http://localhost:{self.default_public_ingress_port}/'
|
||||||
|
|
|
@ -66,6 +66,9 @@ class PebbleHandler(ops.charm.Object):
|
||||||
self.openstack_release = openstack_release
|
self.openstack_release = openstack_release
|
||||||
self.callback_f = callback_f
|
self.callback_f = callback_f
|
||||||
self.setup_pebble_handler()
|
self.setup_pebble_handler()
|
||||||
|
# The structure of status variable and corresponding logic
|
||||||
|
# will change with compund status feature
|
||||||
|
self.status = ""
|
||||||
|
|
||||||
def setup_pebble_handler(self) -> None:
|
def setup_pebble_handler(self) -> None:
|
||||||
"""Configure handler for pebble ready event."""
|
"""Configure handler for pebble ready event."""
|
||||||
|
@ -111,6 +114,10 @@ class PebbleHandler(ops.charm.Object):
|
||||||
"""Pebble configuration layer for the container."""
|
"""Pebble configuration layer for the container."""
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
def get_healthcheck_layer(self) -> dict:
|
||||||
|
"""Pebble configuration for health check layer for the container."""
|
||||||
|
return {}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def directories(self) -> List[ContainerDir]:
|
def directories(self) -> List[ContainerDir]:
|
||||||
"""List of directories to create in container."""
|
"""List of directories to create in container."""
|
||||||
|
@ -194,6 +201,61 @@ class PebbleHandler(ops.charm.Object):
|
||||||
if exception_on_error:
|
if exception_on_error:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def add_healthchecks(self) -> None:
|
||||||
|
"""Add healthcheck layer to the plan."""
|
||||||
|
healthcheck_layer = self.get_healthcheck_layer()
|
||||||
|
if not healthcheck_layer:
|
||||||
|
logger.debug("Healthcheck layer not defined in pebble handler")
|
||||||
|
return
|
||||||
|
|
||||||
|
container = self.charm.unit.get_container(self.container_name)
|
||||||
|
try:
|
||||||
|
plan = container.get_plan()
|
||||||
|
if not plan.checks:
|
||||||
|
logger.debug("Adding healthcheck layer to the plan")
|
||||||
|
container.add_layer(
|
||||||
|
"healthchecks", healthcheck_layer, combine=True)
|
||||||
|
except ops.pebble.ConnectionError as connect_error:
|
||||||
|
logger.error("Not able to add Healthcheck layer")
|
||||||
|
logger.exception(connect_error)
|
||||||
|
|
||||||
|
def assess_status(self) -> str:
|
||||||
|
"""Assess Healthcheck status.
|
||||||
|
|
||||||
|
:return: status message based on healthchecks
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
failed_checks = []
|
||||||
|
container = self.charm.unit.get_container(self.container_name)
|
||||||
|
try:
|
||||||
|
checks = container.get_checks(level=ops.pebble.CheckLevel.READY)
|
||||||
|
for name, check in checks.items():
|
||||||
|
if check.status != ops.pebble.CheckStatus.UP:
|
||||||
|
failed_checks.append(name)
|
||||||
|
|
||||||
|
# Verify alive checks if ready checks are missing
|
||||||
|
if not checks:
|
||||||
|
checks = container.get_checks(
|
||||||
|
level=ops.pebble.CheckLevel.ALIVE)
|
||||||
|
for name, check in checks.items():
|
||||||
|
if check.status != ops.pebble.CheckStatus.UP:
|
||||||
|
failed_checks.append(name)
|
||||||
|
|
||||||
|
except ops.model.ModelError:
|
||||||
|
logger.warning(
|
||||||
|
f'Health check online for {self.container_name} not defined')
|
||||||
|
except ops.pebble.ConnectionError as connect_error:
|
||||||
|
logger.exception(connect_error)
|
||||||
|
failed_checks.append("Pebble Connection Error")
|
||||||
|
|
||||||
|
if failed_checks:
|
||||||
|
self.status = (
|
||||||
|
f'Health check failed for {self.container_name}: '
|
||||||
|
f'{failed_checks}'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.status = ''
|
||||||
|
|
||||||
|
|
||||||
class ServicePebbleHandler(PebbleHandler):
|
class ServicePebbleHandler(PebbleHandler):
|
||||||
"""Container handler for containers which manage a service."""
|
"""Container handler for containers which manage a service."""
|
||||||
|
@ -295,6 +357,33 @@ class WSGIPebbleHandler(PebbleHandler):
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def get_healthcheck_layer(self) -> dict:
|
||||||
|
"""Apache WSGI health check pebble layer.
|
||||||
|
|
||||||
|
:returns: pebble health check layer configuration for wsgi service
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"checks": {
|
||||||
|
"up": {
|
||||||
|
"override": "replace",
|
||||||
|
"level": "alive",
|
||||||
|
"period": "10s",
|
||||||
|
"timeout": "3s",
|
||||||
|
"threshold": 3,
|
||||||
|
"exec": {
|
||||||
|
"command": "service apache2 status"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"online": {
|
||||||
|
"override": "replace",
|
||||||
|
"level": "ready",
|
||||||
|
"http": {
|
||||||
|
"url": self.charm.healthcheck_http_url
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None:
|
def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None:
|
||||||
"""Enable and start WSGI service."""
|
"""Enable and start WSGI service."""
|
||||||
container = self.charm.unit.get_container(self.container_name)
|
container = self.charm.unit.get_container(self.container_name)
|
||||||
|
|
|
@ -28,6 +28,11 @@ class OVNPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
|
||||||
"""Path to OVN service wrapper."""
|
"""Path to OVN service wrapper."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@property
|
||||||
|
def status_command(self) -> str:
|
||||||
|
"""Command to check status of service."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None:
|
def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None:
|
||||||
"""Initialise service ready for use.
|
"""Initialise service ready for use.
|
||||||
|
|
||||||
|
@ -46,7 +51,11 @@ class OVNPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def get_layer(self) -> dict:
|
def get_layer(self) -> dict:
|
||||||
"""Pebble configuration layer for OVN service."""
|
"""Pebble configuration layer for OVN service.
|
||||||
|
|
||||||
|
:returns: pebble layer configuration for service
|
||||||
|
:rtype: dict
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"summary": f"{self.service_description} service",
|
"summary": f"{self.service_description} service",
|
||||||
"description": ("Pebble config layer for "
|
"description": ("Pebble config layer for "
|
||||||
|
@ -61,6 +70,24 @@ class OVNPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def get_healthcheck_layer(self) -> dict:
|
||||||
|
"""Health check pebble layer.
|
||||||
|
|
||||||
|
:returns: pebble health check layer configuration for OVN service
|
||||||
|
:rtype: dict
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"checks": {
|
||||||
|
"online": {
|
||||||
|
"override": "replace",
|
||||||
|
"level": "ready",
|
||||||
|
"exec": {
|
||||||
|
"command": f"{self.status_command}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def directories(self) -> List[sunbeam_chandlers.ContainerDir]:
|
def directories(self) -> List[sunbeam_chandlers.ContainerDir]:
|
||||||
"""Directories to creete in container."""
|
"""Directories to creete in container."""
|
||||||
|
|
|
@ -250,3 +250,8 @@ class MyAPICharm(sunbeam_charm.OSBaseOperatorAPICharm):
|
||||||
def template_dir(self) -> str:
|
def template_dir(self) -> str:
|
||||||
"""Templates dir."""
|
"""Templates dir."""
|
||||||
return self._template_dir
|
return self._template_dir
|
||||||
|
|
||||||
|
@property
|
||||||
|
def healthcheck_http_url(self) -> str:
|
||||||
|
"""Healthcheck HTTP URL for the service."""
|
||||||
|
return f'http://localhost:{self.default_public_ingress_port}/v3'
|
||||||
|
|
Loading…
Reference in New Issue