sunbeam-charms/charms/sunbeam-clusterd/src/charm.py

#!/usr/bin/env python3

# Copyright 2024 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""Sunbeam Clusterd Charm.

This charm manages a clusterd deployment. Clusterd is a service storing
every metadata about a sunbeam deployment.
"""

import hashlib
import logging
import socket
from pathlib import (
    Path,
)

import clusterd
import ops
import ops.framework
import ops_sunbeam.charm as sunbeam_charm
import ops_sunbeam.guard as sunbeam_guard
import ops_sunbeam.relation_handlers as sunbeam_rhandlers
import ops_sunbeam.tracing as sunbeam_tracing
import requests
import tenacity
from charms.operator_libs_linux.v2 import (
    snap,
)
from charms.tls_certificates_interface.v3.tls_certificates import (
    generate_csr,
)
from cryptography import (
    x509,
)
from ops_sunbeam.relation_handlers import (
    RelationHandler,
)
from relation_handlers import (
    ClusterdNewNodeEvent,
    ClusterdNodeAddedEvent,
    ClusterdPeerHandler,
    ClusterdRemoveNodeEvent,
)

logger = logging.getLogger(__name__)


def _identity(x: bool) -> bool:
    return x


class SnapInstallationError(Exception):
    """Custom exception for snap installation failure errors."""


@sunbeam_tracing.trace_type
class ClusterCertificatesHandler(sunbeam_rhandlers.TlsCertificatesHandler):
    """Handler for certificates interface."""

    def get_entity(self) -> ops.Unit | ops.Application:
        """Return the entity for the key store."""
        return self.charm.model.app

    def key_names(self) -> list[str]:
        """Return the key names managed by this relation.

        First key is considered as default key.
        """
        return ["main", "client"]

    def csrs(self) -> dict[str, bytes]:
        """Return a dict of generated csrs for self.key_names().

        The method calling this method will ensure that all keys have a matching
        csr.
        """
        main_key = self._private_keys.get("main")
        client_key = self._private_keys.get("client")
        if not main_key or not client_key:
            return {}
        return {
            "main": generate_csr(
                private_key=main_key.encode(),
                subject=self.charm.app.name,
                sans_ip=self.sans_ips,
                sans_dns=self.sans_dns,
                additional_critical_extensions=[
                    x509.KeyUsage(
                        digital_signature=True,
                        content_commitment=False,
                        key_encipherment=True,
                        data_encipherment=False,
                        key_agreement=False,
                        key_cert_sign=False,
                        crl_sign=False,
                        encipher_only=False,
                        decipher_only=False,
                    ),
                    x509.ExtendedKeyUsage({x509.OID_SERVER_AUTH}),
                ],
            ),
            "client": generate_csr(
                private_key=client_key.encode(),
                subject=self.charm.app.name + "-client",
                additional_critical_extensions=[
                    x509.KeyUsage(
                        digital_signature=True,
                        content_commitment=False,
                        key_encipherment=True,
                        data_encipherment=False,
                        key_agreement=False,
                        key_cert_sign=False,
                        crl_sign=False,
                        encipher_only=False,
                        decipher_only=False,
                    ),
                    x509.ExtendedKeyUsage({x509.OID_CLIENT_AUTH}),
                ],
            ),
        }

    def get_client_keypair(self) -> dict[str, str]:
        """Return client keypair with the CA."""
        client_key = self.store.get_private_key("client")
        client_csr = self.store.get_csr("client")
        if client_key is None or client_csr is None:
            return {}
        for cert in self.get_certs():
            if cert.csr == client_csr:
                return {
                    "certificate-authority": cert.ca,
                    "certificate": cert.certificate,
                    "private-key-secret": client_key,
                }
        return {}


@sunbeam_tracing.trace_sunbeam_charm
class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
    """Charm the service."""

    _state = ops.StoredState()
    service_name = "sunbeam-clusterd"
    clusterd_port = 7000

    def __init__(self, framework: ops.Framework) -> None:
        """Run constructor."""
        super().__init__(framework)
        self._state.set_default(
            channel="config", departed=False, certs_hash=""
        )
        self.framework.observe(self.on.install, self._on_install)
        self.framework.observe(self.on.stop, self._on_stop)
        self.framework.observe(
            self.on.get_credentials_action, self._on_get_credentials_action
        )
        self._clusterd = clusterd.ClusterdClient(
            Path("/var/snap/openstack/common/state/control.socket")
        )

    def get_relation_handlers(
        self, handlers: list[RelationHandler] | None = None
    ) -> list[RelationHandler]:
        """Setup charm relation handlers."""
        handlers = handlers or []
        if self.can_add_handler("peers", handlers):
            self.peers = ClusterdPeerHandler(
                self,
                "peers",
                self.configure_charm,
                "peers" in self.mandatory_relations,
            )
            handlers.append(self.peers)
        if self.can_add_handler("certificates", handlers):
            self.certs = ClusterCertificatesHandler(
                self,
                "certificates",
                self.configure_charm,
                self.get_domain_name_sans(),
                self.get_sans_ips(),
                mandatory="certificates" in self.mandatory_relations,
            )
            handlers.append(self.certs)
        return super().get_relation_handlers(handlers)

    def get_domain_name_sans(self) -> list[str]:
        """Return domain name sans."""
        return [socket.gethostname()]

    def get_sans_ips(self) -> list[str]:
        """Return Subject Alternate Names to use in cert for service."""
        return ["127.0.0.1"]

    def _on_install(self, event: ops.InstallEvent) -> None:
        """Handle install event."""
        try:
            with sunbeam_guard.guard(self, "Ensure snap installation", False):
                self.ensure_snap_present()
        except TimeoutError:
            logger.debug("Snap installation failed, retrying.")
            event.defer()
            return
        self.clusterd_ready()
        self.status.set(
            ops.WaitingStatus("Waiting for clusterd initialization")
        )

    def _on_stop(self, event: ops.StopEvent) -> None:
        """Handle stop event."""
        try:
            self._clusterd.shutdown()
        except clusterd.ClusterdUnavailableError:
            logger.debug("Clusterd not available, skipping shutdown.")
        snap.SnapCache()["openstack"].stop()

    def _on_get_credentials_action(self, event: ops.ActionEvent) -> None:
        """Handle get-credentials action."""
        if not self.peers.interface.state.joined:
            event.fail("Clusterd not joined yet")
            return

        credentials = {}
        if relation := self.model.get_relation(self.certs.relation_name):
            if relation.active:
                credentials = self.certs.get_client_keypair()
            if not credentials:
                event.fail("No credentials found yet")
                return

        event.set_results(
            {
                "url": "https://"
                + self._binding_address()
                + ":"
                + str(self.clusterd_port),
                **credentials,
            }
        )

    def _binding_address(self) -> str:
        """Return the binding address."""
        relation = self.model.get_relation("peers")

        if relation is None:
            raise ValueError("Missing relation peers")

        binding = self.model.get_binding(relation)

        if binding is None:
            raise ValueError("Missing binding peers")

        if binding.network.bind_address is None:
            raise ValueError("Missing binding address")

        return str(binding.network.bind_address)

    def ensure_snap_present(self):
        """Install/refresh snap if needed."""
        config = self.model.config.get
        snap_channel = config("snap-channel")

        try:
            cache = snap.SnapCache()
            openstack = cache["openstack"]
            if not openstack.present or snap_channel != openstack.channel:
                openstack.ensure(snap.SnapState.Latest, channel=snap_channel)
                self._state.channel = openstack.channel
                self.set_workload_version()
        except (snap.SnapError, snap.SnapNotFoundError) as e:
            logger.error(
                "An exception occurred when installing snap. Reason: %s",
                e.message,
            )
            raise SnapInstallationError("openstack snap installation failed")

    def set_workload_version(self):
        """Set workload version."""
        cache = snap.SnapCache()
        openstack = cache["openstack"]
        if not openstack.present:
            return
        version = openstack.channel + f"(rev {openstack.revision})"
        self.unit.set_workload_version(version)

    def configure_app_leader(self, event: ops.EventBase):
        """Configure leader unit."""
        if not self.clusterd_ready():
            logger.debug("Clusterd not ready yet.")
            event.defer()
            raise sunbeam_guard.WaitingExceptionError("Clusterd not ready yet")
        if not self.is_leader_ready():
            self.bootstrap_cluster()
            self.peers.interface.state.joined = True
        self.configure_certificates()
        super().configure_app_leader(event)
        if isinstance(event, ClusterdNewNodeEvent):
            self.add_node_to_cluster(event)

    def configure_app_non_leader(self, event: ops.EventBase):
        """Configure non-leader unit."""
        super().configure_app_non_leader(event)
        if isinstance(event, ClusterdNodeAddedEvent):
            self.join_node_to_cluster(event)

    def configure_unit(self, event: ops.EventBase):
        """Configure unit."""
        super().configure_unit(event)
        if isinstance(event, ClusterdRemoveNodeEvent):
            self.remove_node_from_cluster(event)
        self.ensure_snap_present()
        config = self.model.config.get
        snap_data = {
            "daemon.debug": config("debug", False),
        }
        self.set_snap_data(snap_data)

    def configure_certificates(self):
        """Configure certificates."""
        if not self.unit.is_leader():
            logger.debug("Not leader, skipping certificate configuration.")
            return
        if not self.certs.ready:
            logger.debug("Certificates not ready yet.")
            return
        certs = self.certs.context()
        certs_hash = hashlib.sha256(bytes(str(certs), "utf-8")).hexdigest()
        if certs_hash == self._state.certs_hash:
            logger.debug("Certificates have not changed.")
            return
        self._clusterd.set_certs(
            ca=certs["ca_cert_main"],
            key=certs["key_main"],
            cert=certs["cert_main"],
        )
        self._state.certs_hash = certs_hash

    def set_snap_data(self, snap_data: dict):
        """Set snap data on local snap."""
        cache = snap.SnapCache()
        openstack = cache["openstack"]
        new_settings = {}
        for k in sorted(snap_data.keys()):
            try:
                if snap_data[k] != openstack.get(k):
                    new_settings[k] = snap_data[k]
            except snap.SnapError:
                # Trying to retrieve an unset parameter results in a snapError
                # so assume the snap.SnapError means there is missing config
                # that needs setting.
                new_settings[k] = snap_data[k]
        if new_settings:
            logger.debug(f"Applying new snap settings {new_settings}")
            openstack.set(new_settings, typed=True)
        else:
            logger.debug("Snap settings do not need updating")

    @tenacity.retry(
        stop=tenacity.stop_after_attempt(10),
        retry=(
            tenacity.retry_if_exception_type(clusterd.ClusterdUnavailableError)
            | tenacity.retry_if_not_result(_identity)
        ),
        after=tenacity.after_log(logger, logging.WARNING),
        wait=tenacity.wait_exponential(multiplier=1, min=1, max=30),
    )
    def clusterd_ready(self) -> bool:
        """Check whether clusterd is ready."""
        if not self._clusterd.ready():
            return False
        return True

    def bootstrap_cluster(self):
        """Bootstrap the cluster."""
        logger.info("Bootstrapping the cluster")
        self._clusterd.bootstrap(
            self.unit.name.replace("/", "-"),
            self._binding_address() + ":" + str(self.clusterd_port),
        )

    def add_node_to_cluster(self, event: ClusterdNewNodeEvent) -> None:
        """Generate token for node joining."""
        if event.unit is None:
            logger.debug("No unit to add")
            return
        unit_key = f"{event.unit.name}.join_token"
        if self.peers.get_app_data(unit_key):
            logger.debug(f"Already generated token for {event.unit.name}")
            return

        try:
            token = self._clusterd.generate_token(
                event.unit.name.replace("/", "-")
            )
        except requests.exceptions.HTTPError as e:
            if e.response is not None and e.response.status_code >= 500:
                logger.error(f"Clusterd error: {str(e)}")
                logger.debug("Failed to generate token, retrying.")
                event.defer()
                return
            raise e
        self.peers.set_app_data({unit_key: token})

    def remove_node_from_cluster(self, event: ClusterdRemoveNodeEvent) -> None:
        """Remove node from cluster."""
        if event.departing_unit is None:
            logger.debug("No unit to remove")
            return

        self_departing = event.departing_unit.name == self.unit.name
        already_left = self._wait_for_roles_to_settle_before_removal(
            event, self_departing
        )
        if already_left:
            return

        logger.debug(f"Departing unit: {event.departing_unit.name}")
        self._remove_member_from_cluster(event.departing_unit.name)
        if self.model.unit.is_leader():
            departing_key = f"{event.departing_unit.name}.join_token"
            self.peers.interface._app_data_bag.pop(
                departing_key,
                None,
            )
        if self_departing:
            self.status.set(ops.WaitingStatus("Waiting for removal"))
            member_left = self._wait_until_local_member_left_cluster()
            if member_left:
                return
            logger.warning(
                "Member %s has not left the cluster yet",
                event.departing_unit.name,
            )
            event.defer()
            raise sunbeam_guard.WaitingExceptionError(
                "Waiting for member to leave cluster"
            )
        self.status.set(ops.WaitingStatus("Waiting for roles to settle"))
        if not self._wait_until_roles_are_settled():
            logger.debug("Roles not settled yet")
            event.defer()
            raise sunbeam_guard.WaitingExceptionError(
                "Waiting for roles to settle"
            )

    def _wait_for_roles_to_settle_before_removal(
        self, event: ops.EventBase, self_departing: bool
    ) -> bool:
        """This method waits for rols to settle before removing a member.

        Returns true if the member has already left the cluster.
        """
        if self_departing:
            # We are the departing unit, and we might be the leader
            message = "Waiting for roles to settle before leaving cluster"
        else:
            # We are the leader, not the departing unit
            message = "Waiting for roles to settle before removing member"
        self.status.set(ops.WaitingStatus(message))
        # Leaving while the roles are not settled can cause the cluster to
        # be in an inconsistent state. So we wait until the roles are
        # settled before leaving.
        try:
            if not self._wait_until_roles_are_settled():
                logger.debug("Roles not settled yet")
                event.defer()
                raise sunbeam_guard.WaitingExceptionError(message)
        except requests.exceptions.HTTPError as e:
            if (
                e.response is not None
                and "Daemon not yet initialized" in e.response.text
            ):
                if self_departing:
                    logger.debug("Member already left cluster")
                    return True
        return False

    def _remove_member_from_cluster(self, departing_unit: str):
        """Helper method to remove a member from the cluster."""
        member_name = departing_unit.replace("/", "-")
        self_departing = departing_unit == self.unit.name
        try:
            logger.debug(f"Removing member {member_name}")
            self._clusterd.remove_node(
                member_name,
                force=True,
                allow_not_found=True,
            )
        except clusterd.ClusterdUnavailableError as e:
            if "Remote end closed connection without response" in str(e):
                logger.debug(
                    "Forwarded request failed, most likely because member was leader"
                    " and this member was removed."
                )
            else:
                raise e
        except requests.exceptions.HTTPError as e:
            if e.response is None:
                raise e
            is_503 = e.response.status_code == 503
            is_500 = e.response.status_code == 500
            if not self_departing or not (is_503 or is_500):
                raise e
            logger.debug(
                "Clusterd is not initialized, most likely because"
                " leader has already removed this unit from clusterd."
                " Error: %s",
                e.response.text,  # type: ignore
            )

    def join_node_to_cluster(self, event: ClusterdNodeAddedEvent) -> None:
        """Join node to cluster."""
        self.status.set(ops.MaintenanceStatus("Joining cluster"))
        token = self.peers.get_app_data(f"{self.unit.name}.join_token")
        if token is None:
            logger.warning("No token found for unit %s", self.unit.name)
            return
        member = self.unit.name.replace("/", "-")
        if not self.peers.interface.state.joined:
            self._clusterd.join(
                member,
                self._binding_address() + ":" + str(self.clusterd_port),
                token,
            )
            self.peers.interface.state.joined = True
            self.peers.set_unit_data({"joined": "true"})

        self.status.set(ops.WaitingStatus("Waiting for clusterd role"))
        is_role_set = self._wait_until_role_set(member)
        if not is_role_set:
            logger.debug("Member %s is still pending", member)
            event.defer()
            return

    def _wait_until_role_set(self, name: str) -> bool:
        @tenacity.retry(
            wait=tenacity.wait_fixed(5),
            stop=tenacity.stop_after_delay(300),
            retry=tenacity.retry_if_not_result(_identity),
        )
        def _wait_until_role_set(name: str) -> bool:
            member = self._clusterd.get_member(name)
            role = member.get("role")
            logger.debug(f"Member {name} role: {role}")
            if role == "PENDING":
                return False
            return True

        try:
            return _wait_until_role_set(name)
        except tenacity.RetryError:
            return False

    def _wait_until_roles_are_settled(self) -> bool:
        """Wait until cluster has odd number of voters."""

        @tenacity.retry(
            wait=tenacity.wait_fixed(5),
            stop=tenacity.stop_after_delay(60),
            retry=tenacity.retry_if_not_result(_identity),
        )
        def _wait_until_roles_are_settled() -> bool:
            members = self._clusterd.get_members()
            voter = 0
            for member in members:
                if member.get("role") == "voter":
                    voter += 1
            if voter % 2 == 0:
                return False
            return True

        try:
            return _wait_until_roles_are_settled()

        except tenacity.RetryError:
            return False

    def _wait_until_local_member_left_cluster(self) -> bool:
        """Wait until local node has left the cluster."""

        @tenacity.retry(
            wait=tenacity.wait_fixed(5),
            stop=tenacity.stop_after_delay(60),
            retry=tenacity.retry_if_not_result(_identity),
        )
        def _wait_until_local_member_left_cluster() -> bool:
            member_name = self.unit.name.replace("/", "-")
            try:
                self._clusterd.get_member(member_name)
                return False
            except requests.exceptions.HTTPError as e:
                if e.response is None:
                    raise e
                db_closed = "database is closed" in e.response.text
                clusterd_not_initialized = (
                    "Daemon not yet initialized" in e.response.text
                )
                if db_closed or clusterd_not_initialized:
                    logger.debug(
                        "Clusterd returned a known error while waiting for removal."
                        ". Skipping."
                        " Error: %s",
                        e.response.text,
                    )
                    return True
                raise e

        try:
            return _wait_until_local_member_left_cluster()
        except tenacity.RetryError:
            return False


if __name__ == "__main__":  # pragma: nocover
    ops.main(SunbeamClusterdCharm)