Remove LoadBalancer service patch

Stop changing the service type to a LoadBalancer type. Prefer using and
ingress controller and letting the loadbalancer sit at the edge. This
also removes the nginx ingress support in favor of using the traefik
ingress support.

Signed-off-by: Billy Olsen <billy.olsen@gmail.com>
This commit is contained in:
Billy Olsen 2022-04-17 13:37:21 -07:00 committed by James Page
parent a213360317
commit 492dd40871
12 changed files with 494 additions and 82 deletions

View File

@ -110,7 +110,7 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
"ingress",
self.service_name,
self.default_public_ingress_port,
self.configure_charm,
self._ingress_changed,
)
handlers.append(self.ingress)
if self.can_add_handler("peers", handlers):
@ -189,6 +189,20 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
else:
return None
def _ingress_changed(self, event: ops.framework.EventBase) -> None:
"""Ingress changed callback.
Invoked when the data on the ingress relation has changed. This will
update the relevant endpoints with the identity service, and then
call the configure_charm.
"""
logger.debug('Received an ingress_changed event')
if hasattr(self, 'id_svc') and hasattr(self, 'service_endpoints'):
logger.debug('Updating service endpoints after ingress relation '
'changed.')
self.id_svc.register_services(self.service_endpoints, self.region)
self.configure_charm(event)
def configure_charm(self, event: ops.framework.EventBase) -> None:
"""Catchall handler to configure charm services."""
if self.supports_peer_relation and not (
@ -337,7 +351,7 @@ class OSBaseOperatorCharm(ops.charm.CharmBase):
logger.warning('DB Sync Out: %s', line.strip())
logging.debug(f'Output from database sync: \n{out}')
else:
logger.warn(
logger.warning(
"Not DB sync ran. Charm does not specify self.db_sync_cmds")
def _do_bootstrap(self) -> None:
@ -362,7 +376,6 @@ class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
self._state.set_default(db_ready=False)
self.service_patcher = kube_svc_patch.KubernetesServicePatch(
self,
service_type="LoadBalancer",
ports=[(f"{self.app.name}", self.default_public_ingress_port)],
)
@ -375,7 +388,10 @@ class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
self, handlers: List[sunbeam_rhandlers.RelationHandler] = None
) -> List[sunbeam_rhandlers.RelationHandler]:
"""Relation handlers for the service."""
handlers = handlers or []
handlers = super().get_relation_handlers(handlers or [])
# Note: intentionally get the parent relations first, as the identity
# requires handler has an implicit dependency on the ingress relation.
# This should be fixed.
if self.can_add_handler("identity-service", handlers):
self.id_svc = sunbeam_rhandlers.IdentityServiceRequiresHandler(
self,
@ -385,7 +401,6 @@ class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
self.model.config["region"],
)
handlers.append(self.id_svc)
handlers = super().get_relation_handlers(handlers)
return handlers
def service_url(self, hostname: str) -> str:
@ -410,14 +425,27 @@ class OSBaseOperatorAPICharm(OSBaseOperatorCharm):
if load_balancer_status:
ingress_addresses = load_balancer_status.ingress
if ingress_addresses:
logger.debug('Found ingress addresses on loadbalancer '
'status')
ingress_address = ingress_addresses[0]
return ingress_address.hostname or ingress_address.ip
addr = ingress_address.hostname or ingress_address.ip
if addr:
logger.debug('Using ingress address from loadbalancer '
f'as {addr}')
return ingress_address.hostname or ingress_address.ip
return None
hostname = self.model.get_binding(
'identity-service'
).network.ingress_address
return hostname
@property
def public_url(self) -> str:
"""Url for accessing the public endpoint for this service."""
if hasattr(self, 'ingress') and self.ingress.url:
logger.debug('Ingress relation found, returning ingress.url of: '
f'{self.ingress.url}')
return self.ingress.url
return self.service_url(self.public_ingress_address)
@property

View File

@ -18,6 +18,7 @@ import json
import logging
import cryptography.hazmat.primitives.serialization as serialization
from typing import Callable, List, Tuple
from urllib.parse import urlparse
import ops.charm
import ops.framework
@ -118,36 +119,37 @@ class IngressHandler(RelationHandler):
logger.debug("Setting up ingress event handler")
# Lazy import to ensure this lib is only required if the charm
# has this relation.
import charms.nginx_ingress_integrator.v0.ingress as ingress
interface = ingress.IngressRequires(self.charm, self.ingress_config)
import charms.traefik_k8s.v0.ingress as ingress
interface = ingress.IngressPerAppRequirer(
self.charm,
self.relation_name,
port=self.default_public_ingress_port,
)
return interface
@property
def ingress_config(self) -> dict:
"""Ingress controller configuration dictionary."""
# Most charms probably won't (or shouldn't) expose service-port
# but use it if its there.
port = self.model.config.get(
"service-port", self.default_public_ingress_port
)
svc_hostname = self.model.config.get(
"os-public-hostname", self.service_name
)
return {
"service-hostname": svc_hostname,
"service-name": self.charm.app.name,
"service-port": port,
}
@property
def ready(self) -> bool:
"""Whether the handler is ready for use."""
# Nothing to wait for
return True
if self.interface.url:
return True
return False
@property
def url(self) -> str:
"""Return the URL used by the remote ingress service."""
if not self.ready:
return None
return self.interface.url
def context(self) -> dict:
"""Context containing ingress data."""
return {}
parse_result = urlparse(self.url)
return {
'ingress_path': parse_result.path,
}
class DBHandler(RelationHandler):

View File

@ -7,5 +7,6 @@ charmcraft fetch-lib charms.sunbeam_keystone_operator.v0.identity_service
charmcraft fetch-lib charms.sunbeam_rabbitmq_operator.v0.amqp
charmcraft fetch-lib charms.sunbeam_ovn_central_operator.v0.ovsdb
charmcraft fetch-lib charms.observability_libs.v0.kubernetes_service_patch
charmcraft fetch-lib charms.traefik_k8s.v0.ingress
echo "Copying libs to to unit_test dir"
rsync --recursive --delete lib/ unit_tests/lib/

View File

@ -6,4 +6,7 @@ python-keystoneclient
git+https://opendev.org/openstack/charm-ops-openstack#egg=ops_openstack
git+https://github.com/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client
lightkube
lightkube-models
lightkube-models
# included for interfacing with traefik
serialized-data-interface>=0.4.0

View File

@ -1,8 +1,11 @@
Listen {{ wsgi_config.public_port }}
<VirtualHost *:{{ wsgi_config.public_port }}>
WSGIDaemonProcess glance processes=3 threads=1 user={{ wsgi_config.user }} group={{ wsgi_config.group }} \
WSGIDaemonProcess {{ wsgi_config.group }} processes=3 threads=1 user={{ wsgi_config.user }} group={{ wsgi_config.group }} \
display-name=%{GROUP}
WSGIProcessGroup glance
WSGIProcessGroup {{ wsgi_config.group }}
{% if ingress.ingress_path -%}
WSGIScriptAlias {{ ingress.ingress_path }} {{ wsgi_config.wsgi_public_script }}
{% endif -%}
WSGIScriptAlias / {{ wsgi_config.wsgi_public_script }}
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On

View File

@ -1,6 +1,5 @@
flake8-annotations
flake8-docstrings
charm-tools>=2.4.4
coverage>=3.6
mock>=1.2
flake8

View File

@ -47,16 +47,21 @@ basepython = python3
deps = -r{toxinidir}/cookie-requirements.txt
commands = /bin/true
[testenv:py3.8]
[testenv:py38]
basepython = python3.8
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py3.9]
[testenv:py39]
basepython = python3.9
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py310]
basepython = python3.10
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py3]
basepython = python3
deps = -r{toxinidir}/requirements.txt

View File

@ -15,6 +15,7 @@ Import `IngressRequires` in your charm, with two required options:
- limit-rps
- limit-whitelist
- max-body-size
- owasp-modsecurity-crs
- path-routes
- retry-errors
- rewrite-enabled
@ -65,7 +66,7 @@ LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 9
LIBPATCH = 10
logger = logging.getLogger(__name__)
@ -80,13 +81,14 @@ OPTIONAL_INGRESS_RELATION_FIELDS = {
"limit-rps",
"limit-whitelist",
"max-body-size",
"owasp-modsecurity-crs",
"path-routes",
"retry-errors",
"rewrite-target",
"rewrite-enabled",
"service-namespace",
"session-cookie-max-age",
"tls-secret-name",
"path-routes",
}
@ -94,10 +96,15 @@ class IngressAvailableEvent(EventBase):
pass
class IngressBrokenEvent(EventBase):
pass
class IngressCharmEvents(CharmEvents):
"""Custom charm events."""
ingress_available = EventSource(IngressAvailableEvent)
ingress_broken = EventSource(IngressBrokenEvent)
class IngressRequires(Object):
@ -134,7 +141,7 @@ class IngressRequires(Object):
if missing:
logger.error(
"Ingress relation error, missing required key(s) in config dictionary: %s",
", ".join(missing),
", ".join(sorted(missing)),
)
self.model.unit.status = BlockedStatus(blocked_message)
return True
@ -173,6 +180,7 @@ class IngressProvides(Object):
# Observe the relation-changed hook event and bind
# self.on_relation_changed() to handle the event.
self.framework.observe(charm.on["ingress"].relation_changed, self._on_relation_changed)
self.framework.observe(charm.on["ingress"].relation_broken, self._on_relation_broken)
self.charm = charm
def _on_relation_changed(self, event):
@ -209,3 +217,11 @@ class IngressProvides(Object):
# Create an event that our charm can use to decide it's okay to
# configure the ingress.
self.charm.on.ingress_available.emit()
def _on_relation_broken(self, _):
"""Handle a relation-broken event in the ingress relation."""
if not self.model.unit.is_leader():
return
# Create an event that our charm can use to remove the ingress resource.
self.charm.on.ingress_broken.emit()

View File

@ -37,7 +37,7 @@ LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 1
LIBPATCH = 2
# TODO: add your code here! Happy coding!
@ -122,8 +122,9 @@ class OVSDBCMSRequires(Object):
"""Retrieve value for key from all related units."""
values = []
relation = self.framework.model.get_relation(self.relation_name)
for unit in relation.units:
values.append(relation.data[unit].get(key))
if relation:
for unit in relation.units:
values.append(relation.data[unit].get(key))
return values
@ -196,7 +197,8 @@ class OVSDBCMSProvides(Object):
def set_unit_data(self, settings: typing.Dict[str, str]) -> None:
"""Publish settings on the peer unit data bag."""
relation = self.framework.model.get_relation(self.relation_name)
for k, v in settings.items():
relation.data[self.model.unit][k] = v
relations = self.framework.model.relations[self.relation_name]
for relation in relations:
for k, v in settings.items():
relation.data[self.model.unit][k] = v

View File

@ -75,10 +75,9 @@ LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 3
LIBPATCH = 4
import logging
import requests
from ops.framework import (
StoredState,
@ -160,7 +159,7 @@ class AMQPRequires(Object):
def _on_amqp_relation_changed(self, event):
"""AMQP relation changed."""
logging.debug("RabbitMQAMQPRequires on_changed")
logging.debug("RabbitMQAMQPRequires on_changed/departed")
if self.password:
self.on.ready.emit()
@ -237,10 +236,11 @@ class AMQPProvides(Object):
on = AMQPClientEvents()
_stored = StoredState()
def __init__(self, charm, relation_name):
def __init__(self, charm, relation_name, callback):
super().__init__(charm, relation_name)
self.charm = charm
self.relation_name = relation_name
self.callback = callback
self.framework.observe(
self.charm.on[relation_name].relation_joined,
self._on_amqp_relation_joined,
@ -256,19 +256,24 @@ class AMQPProvides(Object):
def _on_amqp_relation_joined(self, event):
"""Handle AMQP joined."""
logging.debug("RabbitMQAMQPProvides on_joined")
logging.debug("RabbitMQAMQPProvides on_joined data={}"
.format(event.relation.data))
self.on.has_amqp_clients.emit()
def _on_amqp_relation_changed(self, event):
"""Handle AMQP changed."""
logging.debug("RabbitMQAMQPProvides on_changed")
logging.debug("RabbitMQAMQPProvides on_changed data={}"
.format(event.relation.data))
# Validate data on the relation
if self.username(event) and self.vhost(event):
self.on.ready_amqp_clients.emit()
if self.charm.unit.is_leader():
self.set_amqp_credentials(
event, self.username(event), self.vhost(event)
)
self.callback(event, self.username(event), self.vhost(event))
else:
logging.warning("Received AMQP changed event without the "
"expected keys ('username', 'vhost') in the "
"application data bag. Incompatible charm in "
"other end of relation?")
def _on_amqp_relation_broken(self, event):
"""Handle AMQP broken."""
@ -282,33 +287,3 @@ class AMQPProvides(Object):
def vhost(self, event):
"""Return the AMQP vhost from the client side of the relation."""
return event.relation.data[event.relation.app].get("vhost")
def set_amqp_credentials(self, event, username, vhost):
"""Set AMQP Credentials.
:param event: The current event
:type EventsBase
:param username: The requested username
:type username: str
:param vhost: The requested vhost
:type vhost: str
:returns: None
:rtype: None
"""
# TODO: Can we move this into the charm code?
# TODO TLS Support. Existing interfaces set ssl_port and ssl_ca
logging.debug("Setting amqp connection information.")
try:
if not self.charm.does_vhost_exist(vhost):
self.charm.create_vhost(vhost)
password = self.charm.create_user(username)
self.charm.set_user_permissions(username, vhost)
event.relation.data[self.charm.app]["password"] = password
event.relation.data[self.charm.app][
"hostname"
] = self.charm.hostname
except requests.exceptions.ConnectionError as e:
logging.warning(
"Rabbitmq is not ready. Defering. Errno: {}".format(e.errno)
)
event.defer()

View File

@ -0,0 +1,377 @@
# Copyright 2022 Canonical Ltd.
# See LICENSE file for licensing details.
r"""# Interface Library for ingress.
This library wraps relation endpoints using the `ingress` interface
and provides a Python API for both requesting and providing per-application
ingress, with load-balancing occurring across all units.
## Getting Started
To get started using the library, you just need to fetch the library using `charmcraft`.
**Note that you also need to add the `serialized_data_interface` dependency to your
charm's `requirements.txt`.**
```shell
cd some-charm
charmcraft fetch-lib charms.traefik_k8s.v0.ingress
echo -e "serialized_data_interface\n" >> requirements.txt
```
In the `metadata.yaml` of the charm, add the following:
```yaml
requires:
ingress:
interface: ingress
limit: 1
```
Then, to initialise the library:
```python
# ...
from charms.traefik_k8s.v0.ingress import IngressPerAppRequirer
class SomeCharm(CharmBase):
def __init__(self, *args):
# ...
self.ingress = IngressPerAppRequirer(self, port=80)
# The following event is triggered when the ingress URL to be used
# by this deployment of the `SomeCharm` changes or there is no longer
# an ingress URL available, that is, `self.ingress_per_unit` would
# return `None`.
self.framework.observe(
self.ingress.on.ingress_changed, self._handle_ingress
)
# ...
def _handle_ingress(self, event):
logger.info("This app's ingress URL: %s", self.ingress.url)
```
"""
import logging
from typing import Optional
from ops.charm import CharmBase, RelationBrokenEvent, RelationEvent, RelationRole
from ops.framework import EventSource, StoredState
from ops.model import Relation
try:
from serialized_data_interface import EndpointWrapper
from serialized_data_interface.errors import RelationDataError, UnversionedRelation
from serialized_data_interface.events import EndpointWrapperEvents
except ImportError:
import os
library_name = os.path.basename(__file__)
raise ModuleNotFoundError(
"To use the '{}' library, you must include "
"the '{}' package in your dependencies".format(library_name, "serialized_data_interface")
) from None # Suppress original ImportError
# The unique Charmhub library identifier, never change it
LIBID = "e6de2a5cd5b34422a204668f3b8f90d2"
# Increment this major API version when introducing breaking changes
LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 5
log = logging.getLogger(__name__)
INGRESS_SCHEMA = {
"v1": {
"requires": {
"app": {
"type": "object",
"properties": {
"model": {"type": "string"},
"name": {"type": "string"},
"host": {"type": "string"},
"port": {"type": "integer"},
},
"required": ["model", "name", "host", "port"],
},
},
"provides": {
"app": {
"type": "object",
"properties": {
"ingress": {
"type": "object",
"properties": {
"url": {"type": "string"},
},
}
},
"required": ["ingress"],
},
},
}
}
class IngressPerAppRequestEvent(RelationEvent):
"""Event representing an incoming request.
This is equivalent to the "ready" event, but is more semantically meaningful.
"""
class IngressPerAppProviderEvents(EndpointWrapperEvents):
"""Container for IUP events."""
request = EventSource(IngressPerAppRequestEvent)
class IngressPerAppProvider(EndpointWrapper):
"""Implementation of the provider of ingress."""
ROLE = RelationRole.provides.name
INTERFACE = "ingress"
SCHEMA = INGRESS_SCHEMA
on = IngressPerAppProviderEvents()
def __init__(self, charm: CharmBase, endpoint: str = None):
"""Constructor for IngressPerAppProvider.
Args:
charm: The charm that is instantiating the instance.
endpoint: The name of the relation endpoint to bind to
(defaults to "ingress").
"""
super().__init__(charm, endpoint)
self.framework.observe(self.on.ready, self._emit_request_event)
def _emit_request_event(self, event):
self.on.request.emit(event.relation)
def get_request(self, relation: Relation):
"""Get the IngressPerAppRequest for the given Relation."""
return IngressPerAppRequest(self, relation)
def is_failed(self, relation: Relation = None):
"""Checks whether the given relation, or any relation if not specified, has an error."""
if relation is None:
return any(self.is_failed(relation) for relation in self.relations)
if super().is_failed(relation):
return True
try:
data = self.unwrap(relation)
except UnversionedRelation:
return False
prev_fields = None
other_app = relation.app
new_fields = {
field: data[other_app][field]
for field in ("model", "port")
if field in data[other_app]
}
if prev_fields is None:
prev_fields = new_fields
if new_fields != prev_fields:
raise RelationDataMismatchError(relation, other_app)
return False
@property
def proxied_endpoints(self):
"""Returns the ingress settings provided to applications by this IngressPerAppProvider.
For example, when this IngressPerAppProvider has provided the
`http://foo.bar/my-model.my-app` URL to the my-app application, the returned dictionary
will be:
```
{
"my-app": {
"url": "http://foo.bar/my-model.my-app"
}
}
```
"""
return {
ingress_relation.app.name: self.unwrap(ingress_relation)[self.charm.app].get(
"ingress", {}
)
for ingress_relation in self.charm.model.relations[self.endpoint]
}
class IngressPerAppRequest:
"""A request for per-application ingress."""
def __init__(self, provider: IngressPerAppProvider, relation: Relation):
"""Construct an IngressRequest."""
self._provider = provider
self._relation = relation
self._data = provider.unwrap(relation)
@property
def model(self):
"""The name of the model the request was made from."""
return self._data[self.app].get("model")
@property
def app(self):
"""The remote application."""
return self._relation.app
@property
def app_name(self):
"""The name of the remote app.
Note: This is not the same as `self.app.name` when using CMR relations,
since `self.app.name` is replaced by a `remote-{UUID}` pattern.
"""
return self._relation.app.name
@property
def host(self):
"""The hostname to be used to route to the application."""
return self._data[self.app].get("host")
@property
def port(self):
"""The port to be used to route to the application."""
return self._data[self.app].get("port")
def respond(self, url: str):
"""Send URL back for the application.
Note: only the leader can send URLs.
"""
ingress = self._data[self._provider.charm.app].setdefault("ingress", {})
ingress["url"] = url
self._provider.wrap(self._relation, self._data)
class RelationDataMismatchError(RelationDataError):
"""Data from different units do not match where they should."""
class IngressPerAppConfigurationChangeEvent(RelationEvent):
"""Event representing a change in the data sent by the ingress."""
class IngressPerAppRequirerEvents(EndpointWrapperEvents):
"""Container for IUP events."""
ingress_changed = EventSource(IngressPerAppConfigurationChangeEvent)
class IngressPerAppRequirer(EndpointWrapper):
"""Implementation of the requirer of the ingress relation."""
on = IngressPerAppRequirerEvents()
_stored = StoredState()
ROLE = RelationRole.requires.name
INTERFACE = "ingress"
SCHEMA = INGRESS_SCHEMA
LIMIT = 1
def __init__(
self,
charm: CharmBase,
endpoint: str = None,
*,
host: str = None,
port: int = None,
):
"""Constructor for IngressRequirer.
The request args can be used to specify the ingress properties when the
instance is created. If any are set, at least `port` is required, and
they will be sent to the ingress provider as soon as it is available.
All request args must be given as keyword args.
Args:
charm: the charm that is instantiating the library.
endpoint: the name of the relation endpoint to bind to (defaults to `ingress`);
relation must be of interface type `ingress` and have "limit: 1")
host: Hostname to be used by the ingress provider to address the requiring
application; if unspecified, the default Kubernetes service name will be used.
Request Args:
port: the port of the service
"""
super().__init__(charm, endpoint)
# Workaround for SDI not marking the EndpointWrapper as not
# ready upon a relation broken event
self.is_relation_broken = False
self._stored.set_default(current_url=None)
if port and charm.unit.is_leader():
self.auto_data = self._complete_request(host or "", port)
self.framework.observe(
self.charm.on[self.endpoint].relation_changed, self._emit_ingress_change_event
)
self.framework.observe(
self.charm.on[self.endpoint].relation_broken, self._emit_ingress_change_event
)
def _emit_ingress_change_event(self, event):
if isinstance(event, RelationBrokenEvent):
self.is_relation_broken = True
# Avoid spurious events, emit only when URL changes
new_url = self.url
if self._stored.current_url != new_url:
self._stored.current_url = new_url
self.on.ingress_changed.emit(self.relation)
def _complete_request(self, host: Optional[str], port: int):
if not host:
# TODO Make host mandatory?
host = "{app_name}.{model_name}.svc.cluster.local".format(
app_name=self.app.name,
model_name=self.model.name,
)
return {
self.app: {
"model": self.model.name,
"name": self.charm.unit.name,
"host": host,
"port": port,
}
}
def request(self, *, host: str = None, port: int):
"""Request ingress to this application.
Args:
host: Hostname to be used by the ingress provider to address the requirer; if
unspecified, the Kubernetes service address is used.
port: the port of the service (required)
"""
self.wrap(self.relation, self._complete_request(host, port))
@property
def relation(self):
"""The established Relation instance, or None."""
return self.relations[0] if self.relations else None
@property
def url(self):
"""The full ingress URL to reach the current unit.
May return None if the URL isn't available yet.
"""
if self.is_relation_broken or not self.is_ready():
return {}
data = self.unwrap(self.relation)
ingress = data[self.relation.app].get("ingress", {})
return ingress.get("url")

View File

@ -96,6 +96,7 @@ requires:
limit: 1
ingress:
interface: ingress
limit: 1
amqp:
interface: rabbitmq
identity-service: