Use juju secrets for fernet key rotation
Store the generated fernet keys as juju secrets and update the id on relation app data. The fernet keys can be distributed on peer relation based on secret id saved in app data. Use juju secret rotation policy to invoke an event periodically to rotate the fernet keys. Update the secret with new rotated keys which triggeres secret-changed event on non-leader units to update fernet keys locally on the unit. This patch supports for both fernet keys and credential keys. Removed config options fernet-max-active-keys, token-expiration, allow-expired-window. Modified token expiration to 1 hour and allow- expired-window to 47 hours and fernet-max-active-keys to 4 so that fernet secret rotation can be applied daily. This need to be revisited since 47 hours for allow-exired-window may not be sufficient in some cases, see bug [1] For credential keys, rotate them on a monthly basis. [1] https://bugs.launchpad.net/charm-cinder/+bug/1986886 Depends-On: https://review.opendev.org/c/openstack/charm-ops-sunbeam/+/866646 Change-Id: Idf78642601d8233f7e60f34ae392754041938690
This commit is contained in:
Hemanth Nakkina
parent
828f6451f9
commit
ef6be0e060
@ -47,32 +47,6 @@ options:
|
||||
description: Space delimited list of OpenStack regions
|
||||
type: string
|
||||
|
||||
fernet-max-active-keys:
|
||||
type: int
|
||||
default: 3
|
||||
description: |
|
||||
This is the maximum number of active keys. It has a minimum of 3, which includes the
|
||||
spare and staging keys. The key rotation time is calculated by:
|
||||
|
||||
rotation-time = (token-expiration + allow-expired-window) / (fernet-max-active-keys - 2)
|
||||
|
||||
Please see the charm documentation for further details about how to use the Fernet token
|
||||
parameters to achieve a key strategy appropriate for the system in question.
|
||||
NOTE: the minimum time between fernet key rotations is 5 minutes;
|
||||
token-expiration + allow-expired-window should not be less than this.
|
||||
token-expiration:
|
||||
type: int
|
||||
default: 3600 # 1 hour
|
||||
description: |
|
||||
Amount of time (in seconds) a token should remain valid.
|
||||
Default is 1 hour.
|
||||
allow-expired-window:
|
||||
type: int
|
||||
default: 172800 # 2 days
|
||||
description: |
|
||||
This controls the number of seconds that a token can be retrieved for beyond the built-in expiry time.
|
||||
This allows long running operations to succeed.
|
||||
Defaults to two days.
|
||||
catalog-cache-expiration:
|
||||
type: int
|
||||
default: 60
|
||||
|
||||
@ -13,4 +13,4 @@ ops
|
||||
pwgen
|
||||
git+https://opendev.org/openstack/charm-ops-sunbeam#egg=ops_sunbeam
|
||||
|
||||
python-keystoneclient # keystone-k8s
|
||||
python-keystoneclient # keystone-k8s
|
||||
|
||||
@ -25,16 +25,10 @@ develop a new k8s charm using the Operator Framework:
|
||||
https://discourse.charmhub.io/t/4208
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import (
|
||||
Callable,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
)
|
||||
|
||||
import charms.keystone_k8s.v0.cloud_credentials as sunbeam_cc_svc
|
||||
@ -48,24 +42,21 @@ import ops_sunbeam.guard as sunbeam_guard
|
||||
import ops_sunbeam.job_ctrl as sunbeam_job_ctrl
|
||||
import ops_sunbeam.relation_handlers as sunbeam_rhandlers
|
||||
import pwgen
|
||||
from ops import (
|
||||
model,
|
||||
)
|
||||
from ops.charm import (
|
||||
ActionEvent,
|
||||
CharmEvents,
|
||||
HookEvent,
|
||||
RelationChangedEvent,
|
||||
RelationEvent,
|
||||
)
|
||||
from ops.framework import (
|
||||
EventSource,
|
||||
Object,
|
||||
StoredState,
|
||||
)
|
||||
from ops.main import (
|
||||
main,
|
||||
)
|
||||
from ops.model import (
|
||||
MaintenanceStatus,
|
||||
SecretRotate,
|
||||
)
|
||||
from ops_sunbeam.interfaces import (
|
||||
OperatorPeers,
|
||||
)
|
||||
@ -77,8 +68,7 @@ from utils import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KEYSTONE_CONTAINER = "keystone"
|
||||
LAST_FERNET_KEY_ROTATION_KEY = "last_fernet_rotation"
|
||||
FERNET_KEYS_KEY = "fernet_keys"
|
||||
FERNET_KEYS_PREFIX = "fernet-"
|
||||
|
||||
|
||||
KEYSTONE_CONF = "/etc/keystone/keystone.conf"
|
||||
@ -124,13 +114,13 @@ class KeystoneConfigAdapter(sunbeam_contexts.ConfigContext):
|
||||
"default_domain_id": self.charm.default_domain_id,
|
||||
"public_port": self.charm.service_port,
|
||||
"debug": config["debug"],
|
||||
"token_expiration": config["token-expiration"],
|
||||
"allow_expired_window": config["allow-expired-window"],
|
||||
"token_expiration": 3600, # 1 hour
|
||||
"allow_expired_window": 169200, # 2 days - 1 hour
|
||||
"catalog_cache_expiration": config["catalog-cache-expiration"],
|
||||
"dogpile_cache_expiration": config["dogpile-cache-expiration"],
|
||||
"identity_backend": "sql",
|
||||
"token_provider": "fernet",
|
||||
"fernet_max_active_keys": config["fernet-max-active-keys"],
|
||||
"fernet_max_active_keys": 4, # adjusted to make rotation daily
|
||||
"public_endpoint": self.charm.public_endpoint,
|
||||
"admin_endpoint": self.charm.admin_endpoint,
|
||||
"domain_config_dir": "/etc/keystone/domains",
|
||||
@ -211,75 +201,6 @@ class CloudCredentialsProvidesHandler(sunbeam_rhandlers.RelationHandler):
|
||||
return True
|
||||
|
||||
|
||||
class FernetKeysUpdatedEvent(RelationEvent):
|
||||
"""This local event triggered if fernet keys were updated."""
|
||||
|
||||
def get_fernet_keys(self) -> Dict[str, str]:
|
||||
"""Retrieve the fernet keys from app data."""
|
||||
return json.loads(
|
||||
self.relation.data[self.relation.app].get(FERNET_KEYS_KEY, "{}")
|
||||
)
|
||||
|
||||
|
||||
class HeartbeatEvent(HookEvent):
|
||||
"""This local event triggered regularly as a wake up call."""
|
||||
|
||||
|
||||
class KeystoneEvents(CharmEvents):
|
||||
"""Custom local events."""
|
||||
|
||||
fernet_keys_updated = EventSource(FernetKeysUpdatedEvent)
|
||||
heartbeat = EventSource(HeartbeatEvent)
|
||||
|
||||
|
||||
class KeystoneInterface(Object):
|
||||
"""Define Keystone interface."""
|
||||
|
||||
def __init__(self, charm):
|
||||
"""Init KeystoneInterface class."""
|
||||
super().__init__(charm, "keystone-peers")
|
||||
self.charm = charm
|
||||
self.framework.observe(
|
||||
self.charm.on.peers_relation_changed, self._on_peer_data_changed
|
||||
)
|
||||
|
||||
def _on_peer_data_changed(self, event: RelationChangedEvent):
|
||||
"""Check the peer data updates for updated fernet keys.
|
||||
|
||||
Then we can pull the keys from the app data,
|
||||
and tell the local charm to write them to disk.
|
||||
"""
|
||||
old_data = event.relation.data[self.charm.unit].get(
|
||||
FERNET_KEYS_KEY, ""
|
||||
)
|
||||
data = self.charm.peers.get_app_data(FERNET_KEYS_KEY) or ""
|
||||
|
||||
# only launch the event if the data has changed
|
||||
# and there there are actually keys
|
||||
# (not just an empty dictionary string "{}")
|
||||
if data and data != old_data and json.loads(data):
|
||||
event.relation.data[self.charm.unit].update(
|
||||
{FERNET_KEYS_KEY: data}
|
||||
)
|
||||
# use an event here so we can defer it
|
||||
# if keystone isn't bootstrapped yet
|
||||
self.charm.on.fernet_keys_updated.emit(
|
||||
event.relation, app=event.app, unit=event.unit
|
||||
)
|
||||
|
||||
def distribute_fernet_keys(self, keys: Dict[str, str]):
|
||||
"""Trigger a fernet key distribution.
|
||||
|
||||
This is achieved by simply saving it to the app data here,
|
||||
which will trigger the peer data changed event across all the units.
|
||||
"""
|
||||
self.charm.peers.set_app_data(
|
||||
{
|
||||
FERNET_KEYS_KEY: json.dumps(keys),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class KeystonePasswordManager(Object):
|
||||
"""Helper for management of keystone credential passwords."""
|
||||
|
||||
@ -300,6 +221,7 @@ class KeystonePasswordManager(Object):
|
||||
"""Retrieve persisted password for provided username."""
|
||||
if not self.interface:
|
||||
return None
|
||||
|
||||
password = self.interface.get_app_data(f"password_{username}")
|
||||
return str(password) if password else None
|
||||
|
||||
@ -319,7 +241,6 @@ class KeystonePasswordManager(Object):
|
||||
class KeystoneOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
|
||||
"""Charm the service."""
|
||||
|
||||
on = KeystoneEvents()
|
||||
_state = StoredState()
|
||||
_authed = False
|
||||
service_name = "keystone"
|
||||
@ -348,13 +269,10 @@ class KeystoneOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
|
||||
self._state.set_default(admin_domain_id=None)
|
||||
self._state.set_default(default_domain_id=None)
|
||||
self._state.set_default(service_project_id=None)
|
||||
self.peer_interface = KeystoneInterface(self)
|
||||
|
||||
self.framework.observe(
|
||||
self.on.fernet_keys_updated, self._on_fernet_keys_updated
|
||||
self.on.peers_relation_changed, self._on_peer_data_changed
|
||||
)
|
||||
self.framework.observe(self.on.heartbeat, self._on_heartbeat)
|
||||
self._launch_heartbeat()
|
||||
|
||||
self.framework.observe(
|
||||
self.on.get_admin_password_action, self._get_admin_password_action
|
||||
@ -411,78 +329,143 @@ export OS_AUTH_VERSION=3
|
||||
}
|
||||
)
|
||||
|
||||
def _launch_heartbeat(self):
|
||||
"""Launch another process that will wake up the charm every 5 minutes.
|
||||
def _on_peer_data_changed(self, event: RelationChangedEvent):
|
||||
"""Check the peer data updates for updated fernet keys.
|
||||
|
||||
Used to auto schedule fernet key rotation.
|
||||
Then we can pull the keys from the app data,
|
||||
and tell the local charm to write them to disk.
|
||||
"""
|
||||
# check if already running
|
||||
if subprocess.call(["pgrep", "-f", "heartbeat"]) == 0:
|
||||
return
|
||||
|
||||
logger.debug("Launching the heartbeat")
|
||||
subprocess.Popen(
|
||||
["./src/heartbeat.sh"],
|
||||
cwd=os.environ["JUJU_CHARM_DIR"],
|
||||
)
|
||||
|
||||
def _on_fernet_keys_updated(self, event: FernetKeysUpdatedEvent):
|
||||
"""Respond to fernet_keys_updated event."""
|
||||
if not self.bootstrapped():
|
||||
logger.debug(
|
||||
"Deferring _on_peer_data_changed event as node is not bootstrapped yet"
|
||||
)
|
||||
event.defer()
|
||||
return
|
||||
|
||||
keys = event.get_fernet_keys()
|
||||
if keys:
|
||||
self.keystone_manager.write_fernet_keys(keys)
|
||||
fernet_secret_id = self.peers.get_app_data("fernet-secret-id")
|
||||
if fernet_secret_id:
|
||||
fernet_secret = self.model.get_secret(id=fernet_secret_id)
|
||||
keys = fernet_secret.get_content()
|
||||
|
||||
def _on_heartbeat(self, _event):
|
||||
"""Respond to heartbeat event.
|
||||
# Remove the prefix from keys retrieved from juju secrets
|
||||
# startswith can be replaced with removeprefix for python >= 3.9
|
||||
prefix_len = len(FERNET_KEYS_PREFIX)
|
||||
keys = {
|
||||
(k[prefix_len:] if k.startswith(FERNET_KEYS_PREFIX) else k): v
|
||||
for k, v in keys.items()
|
||||
}
|
||||
|
||||
This should be called regularly.
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/fernet-keys"
|
||||
)
|
||||
if keys and keys != existing_keys:
|
||||
logger.info("Updating fernet keys")
|
||||
self.keystone_manager.write_keys(
|
||||
key_repository="/etc/keystone/fernet-keys", keys=keys
|
||||
)
|
||||
|
||||
It will check if it's time to rotate the fernet keys,
|
||||
and perform the rotation and key distribution if it is time.
|
||||
"""
|
||||
# Only rotate and distribute keys from the leader unit.
|
||||
if not self.unit.is_leader():
|
||||
return
|
||||
|
||||
# if we're not set up, then don't try rotating keys
|
||||
if not self.bootstrapped():
|
||||
return
|
||||
|
||||
# minimum allowed for max_keys is 3
|
||||
max_keys = max(self.model.config["fernet-max-active-keys"], 3)
|
||||
exp = self.model.config["token-expiration"]
|
||||
exp_window = self.model.config["allow-expired-window"]
|
||||
rotation_seconds = (exp + exp_window) / (max_keys - 2)
|
||||
|
||||
# last time the fernet keys were rotated, in seconds since the epoch
|
||||
last_rotation: Optional[str] = self.peers.get_app_data(
|
||||
LAST_FERNET_KEY_ROTATION_KEY
|
||||
credential_keys_secret_id = self.peers.get_app_data(
|
||||
"credential-keys-secret-id"
|
||||
)
|
||||
now: int = int(time.time())
|
||||
if credential_keys_secret_id:
|
||||
credential_keys_secret = self.model.get_secret(
|
||||
id=credential_keys_secret_id
|
||||
)
|
||||
keys = credential_keys_secret.get_content()
|
||||
|
||||
# Remove the prefix from keys retrieved from juju secrets
|
||||
# startswith can be replaced with removeprefix for python >= 3.9
|
||||
prefix_len = len(FERNET_KEYS_PREFIX)
|
||||
keys = {
|
||||
(k[prefix_len:] if k.startswith(FERNET_KEYS_PREFIX) else k): v
|
||||
for k, v in keys.items()
|
||||
}
|
||||
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/credential-keys"
|
||||
)
|
||||
if keys and keys != existing_keys:
|
||||
logger.info("Updating credential keys")
|
||||
self.keystone_manager.write_keys(
|
||||
key_repository="/etc/keystone/credential-keys", keys=keys
|
||||
)
|
||||
|
||||
def _on_secret_changed(self, event: ops.charm.SecretChangedEvent):
|
||||
logger.info(f"secret-change triggered for label {event.secret.label}")
|
||||
if event.secret.label == "fernet-keys":
|
||||
keys = event.secret.get_content(refresh=True)
|
||||
prefix_len = len(FERNET_KEYS_PREFIX)
|
||||
keys = {
|
||||
(k[prefix_len:] if k.startswith(FERNET_KEYS_PREFIX) else k): v
|
||||
for k, v in keys.items()
|
||||
}
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/fernet-keys"
|
||||
)
|
||||
if keys and keys != existing_keys:
|
||||
logger.info("secret-change event: Updating the fernet keys")
|
||||
self.keystone_manager.write_keys(
|
||||
key_repository="/etc/keystone/fernet-keys", keys=keys
|
||||
)
|
||||
elif event.secret.label == "credential-keys":
|
||||
keys = event.secret.get_content(refresh=True)
|
||||
prefix_len = len(FERNET_KEYS_PREFIX)
|
||||
keys = {
|
||||
(k[prefix_len:] if k.startswith(FERNET_KEYS_PREFIX) else k): v
|
||||
for k, v in keys.items()
|
||||
}
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/credential-keys"
|
||||
)
|
||||
if keys and keys != existing_keys:
|
||||
logger.info(
|
||||
"secret-change event: Updating the credential keys"
|
||||
)
|
||||
self.keystone_manager.write_keys(
|
||||
key_repository="/etc/keystone/credential-keys", keys=keys
|
||||
)
|
||||
|
||||
def _on_secret_rotate(self, event: ops.charm.SecretRotateEvent):
|
||||
if not self.unit.is_leader():
|
||||
logger.warning("Not leader, not rotating the fernet keys")
|
||||
return
|
||||
|
||||
if event.secret.label == "fernet-keys":
|
||||
logger.info("secret-rotate event: Rotating the fernet keys")
|
||||
self.keystone_manager.rotate_fernet_keys()
|
||||
fernet_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/fernet-keys"
|
||||
)
|
||||
# Secret content keys should be at least 3 characters long,
|
||||
# no number to start, no dash to end
|
||||
# prepend fernet- to the key names
|
||||
fernet_keys_ = {
|
||||
f"{FERNET_KEYS_PREFIX}{k}": v for k, v in fernet_keys.items()
|
||||
}
|
||||
event.secret.set_content(fernet_keys_)
|
||||
elif event.secret.label == "credential-keys":
|
||||
logger.info("secret-rotate event: Rotating the credential keys")
|
||||
self.keystone_manager.rotate_credential_keys()
|
||||
fernet_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/credential-keys"
|
||||
)
|
||||
# Secret content keys should be at least 3 characters long,
|
||||
# no number to start, no dash to end
|
||||
# prepend fernet- to the key names
|
||||
fernet_keys_ = {
|
||||
f"{FERNET_KEYS_PREFIX}{k}": v for k, v in fernet_keys.items()
|
||||
}
|
||||
event.secret.set_content(fernet_keys_)
|
||||
|
||||
def _on_secret_remove(self, event: ops.charm.SecretRemoveEvent):
|
||||
logger.info(f"secret-remove triggered for label {event.secret.label}")
|
||||
if (
|
||||
last_rotation is None
|
||||
or now - int(last_rotation) >= rotation_seconds
|
||||
event.secret.label == "fernet-keys"
|
||||
or event.secret.label == "credential-keys"
|
||||
):
|
||||
self._rotate_fernet_keys()
|
||||
self.peers.set_app_data({LAST_FERNET_KEY_ROTATION_KEY: str(now)})
|
||||
|
||||
def _rotate_fernet_keys(self):
|
||||
"""Rotate fernet keys and trigger distribution.
|
||||
|
||||
If this is run on a non-leader unit, it's a noop.
|
||||
Keys should only ever be rotated and distributed from a single unit.
|
||||
"""
|
||||
if not self.unit.is_leader():
|
||||
return
|
||||
self.keystone_manager.rotate_fernet_keys()
|
||||
self.peer_interface.distribute_fernet_keys(
|
||||
self.keystone_manager.read_fernet_keys()
|
||||
)
|
||||
# TODO: Remove older revisions of the secret
|
||||
# event.secret.remove_revision(event.revision)
|
||||
pass
|
||||
|
||||
def get_relation_handlers(
|
||||
self, handlers=None
|
||||
@ -840,6 +823,101 @@ export OS_AUTH_VERSION=3
|
||||
"""Healthcheck HTTP URL for the service."""
|
||||
return f"http://localhost:{self.default_public_ingress_port}/v3"
|
||||
|
||||
def _create_fernet_secret(self) -> None:
|
||||
"""Create fernet juju secret.
|
||||
|
||||
Create a fernet juju secret if peer relation app data
|
||||
does not contain a fernet secret. This function might
|
||||
re-trigger until bootstrap is successful. So check if
|
||||
fernet secret is already created and update juju fernet
|
||||
secret with existing fernet keys on the unit.
|
||||
"""
|
||||
# max_keys = max(self.model.config['fernet-max-active-keys'], 3)
|
||||
# exp = self.model.config['token-expiration']
|
||||
# exp_window = self.model.config['allow-expired-window']
|
||||
# rotation_seconds = (exp + exp_window) / (max_keys - 2)
|
||||
|
||||
fernet_secret_id = self.peers.get_app_data("fernet-secret-id")
|
||||
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/fernet-keys"
|
||||
)
|
||||
# Secret content keys should be at least 3 characters long,
|
||||
# no number to start, no dash to end
|
||||
# prepend fernet- to the key names
|
||||
# existing_keys_ will be in format {'fernet-{filename}: data', ...}
|
||||
existing_keys_ = {f"fernet-{k}": v for k, v in existing_keys.items()}
|
||||
|
||||
# juju secret already created, update content with the fernet
|
||||
# keys on the unit if necessary.
|
||||
if fernet_secret_id:
|
||||
fernet_secret = self.model.get_secret(id=fernet_secret_id)
|
||||
keys = fernet_secret.get_content()
|
||||
if keys and keys != existing_keys_:
|
||||
logger.info("Updating Fernet juju secret")
|
||||
fernet_secret.set_content(existing_keys_)
|
||||
else:
|
||||
# If fernet secret does not exist in peer relation data,
|
||||
# create a new one
|
||||
# Fernet keys are rotated on daily basis considering 1 hour
|
||||
# as token expiration, 47 hours as allow-expired-window and
|
||||
# fernet-max-active-keys to 4.
|
||||
fernet_secret = self.model.app.add_secret(
|
||||
existing_keys_,
|
||||
label="fernet-keys",
|
||||
rotate=SecretRotate("daily"),
|
||||
)
|
||||
logger.info(f"Fernet keys secret created: {fernet_secret.id}")
|
||||
self.peers.set_app_data({"fernet-secret-id": fernet_secret.id})
|
||||
return
|
||||
|
||||
def _create_credential_keys_secret(self) -> None:
|
||||
"""Create credential_keys juju secret.
|
||||
|
||||
Create a credential_keys juju secret if peer relation app
|
||||
data does not contain a credential_keys secret. This function
|
||||
might re-trigger until bootstrap is successful. So check if
|
||||
the secret is already created and update juju credential_keys
|
||||
secret with existing fernet keys on the unit.
|
||||
"""
|
||||
credential_keys_secret_id = self.peers.get_app_data(
|
||||
"credential-keys-secret-id"
|
||||
)
|
||||
|
||||
existing_keys = self.keystone_manager.read_keys(
|
||||
key_repository="/etc/keystone/credential-keys"
|
||||
)
|
||||
# Secret content keys should be at least 3 characters long,
|
||||
# no number to start, no dash to end
|
||||
# prepend fernet- to the key names
|
||||
# existing_keys_ will be in format {'fernet-{filename}: data', ...}
|
||||
existing_keys_ = {f"fernet-{k}": v for k, v in existing_keys.items()}
|
||||
|
||||
# juju secret already created, update content with the fernet
|
||||
# keys on the unit if necessary.
|
||||
if credential_keys_secret_id:
|
||||
credential_keys_secret = self.model.get_secret(
|
||||
id=credential_keys_secret_id
|
||||
)
|
||||
keys = credential_keys_secret.get_content()
|
||||
if keys and keys != existing_keys_:
|
||||
logger.info("Updating Credential keys juju secret")
|
||||
credential_keys_secret.set_content(existing_keys_)
|
||||
else:
|
||||
# If credential_keys secret does not exist in peer relation data,
|
||||
# create a new one
|
||||
credential_keys_secret = self.model.app.add_secret(
|
||||
existing_keys_,
|
||||
label="credential-keys",
|
||||
rotate=SecretRotate("monthly"),
|
||||
)
|
||||
logger.info(
|
||||
f"Credential keys secret created: {credential_keys_secret.id}"
|
||||
)
|
||||
self.peers.set_app_data(
|
||||
{"credential-keys-secret-id": credential_keys_secret.id}
|
||||
)
|
||||
|
||||
@sunbeam_job_ctrl.run_once_per_unit("keystone_bootstrap")
|
||||
def keystone_bootstrap(self) -> bool:
|
||||
"""Starts the appropriate services in the order they are needed.
|
||||
@ -857,6 +935,15 @@ export OS_AUTH_VERSION=3
|
||||
"Failed to bootstrap"
|
||||
)
|
||||
|
||||
try:
|
||||
self._create_fernet_secret()
|
||||
self._create_credential_keys_secret()
|
||||
except (ops.pebble.ExecError, ops.pebble.ConnectionError) as error:
|
||||
logger.exception(error)
|
||||
raise sunbeam_guard.BlockedExceptionError(
|
||||
"Failed to create fernet keys"
|
||||
)
|
||||
|
||||
try:
|
||||
self.keystone_manager.setup_initial_projects_and_users()
|
||||
except Exception:
|
||||
@ -867,7 +954,7 @@ export OS_AUTH_VERSION=3
|
||||
raise sunbeam_guard.BlockedExceptionError(
|
||||
"Failed to setup projects and users"
|
||||
)
|
||||
self.unit.status = model.MaintenanceStatus("Starting Keystone")
|
||||
self.unit.status = MaintenanceStatus("Starting Keystone")
|
||||
|
||||
def configure_app_leader(self, event):
|
||||
"""Configure the lead unit."""
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright 2022 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# This dispatches a custom hook event
|
||||
# to the charm periodically.
|
||||
# This regular event is used by the charm
|
||||
# to schedule and run fernet key rotations.
|
||||
|
||||
# This script is designed to be launched by the charm on installation,
|
||||
# and remain running for the life of the container.
|
||||
|
||||
# juju-run will fail if the context id is set.
|
||||
unset JUJU_CONTEXT_ID
|
||||
|
||||
while true; do
|
||||
sleep $((60 * 5)) # 5 minutes
|
||||
juju-run -u "" JUJU_DISPATCH_PATH=hooks/heartbeat ./dispatch
|
||||
done
|
||||
@ -146,8 +146,6 @@ class KeystoneManager(framework.Object):
|
||||
def rotate_fernet_keys(self):
|
||||
"""Rotate the fernet keys.
|
||||
|
||||
And probably also distribute the keys to other units too.
|
||||
|
||||
See for more information:
|
||||
https://docs.openstack.org/keystone/latest/admin/fernet-token-faq.html
|
||||
|
||||
@ -170,22 +168,58 @@ class KeystoneManager(framework.Object):
|
||||
]
|
||||
)
|
||||
|
||||
def read_fernet_keys(self) -> typing.Mapping[str, str]:
|
||||
def rotate_credential_keys(self):
|
||||
"""Rotate the credential keys.
|
||||
|
||||
See for more information:
|
||||
https://docs.openstack.org/keystone/latest/admin/credential-encryption.html
|
||||
"""
|
||||
with sunbeam_guard.guard(self.charm, "Rotating credential keys"):
|
||||
self.run_cmd(
|
||||
[
|
||||
"sudo",
|
||||
"-u",
|
||||
"keystone",
|
||||
"keystone-manage",
|
||||
"credential_migrate",
|
||||
"--keystone-user",
|
||||
"keystone",
|
||||
"--keystone-group",
|
||||
"keystone",
|
||||
]
|
||||
)
|
||||
self.run_cmd(
|
||||
[
|
||||
"sudo",
|
||||
"-u",
|
||||
"keystone",
|
||||
"keystone-manage",
|
||||
"credential_rotate",
|
||||
"--keystone-user",
|
||||
"keystone",
|
||||
"--keystone-group",
|
||||
"keystone",
|
||||
]
|
||||
)
|
||||
|
||||
def read_keys(self, key_repository: str) -> typing.Mapping[str, str]:
|
||||
"""Pull the fernet keys from the on-disk repository."""
|
||||
container = self.charm.unit.get_container(self.container_name)
|
||||
files = container.list_files("/etc/keystone/fernet-keys")
|
||||
files = container.list_files(key_repository)
|
||||
return {file.name: container.pull(file.path).read() for file in files}
|
||||
|
||||
def write_fernet_keys(self, keys: typing.Mapping[str, str]):
|
||||
def write_keys(
|
||||
self, key_repository: str, keys: typing.Mapping[str, str]
|
||||
) -> None:
|
||||
"""Update the local fernet key repository with the provided keys."""
|
||||
container = self.charm.unit.get_container(self.container_name)
|
||||
|
||||
logger.debug("Writing updated fernet keys")
|
||||
logger.debug(f"Writing updated fernet keys at {key_repository}")
|
||||
|
||||
# write the keys
|
||||
for filename, contents in keys.items():
|
||||
container.push(
|
||||
f"/etc/keystone/fernet-keys/{filename}",
|
||||
f"{key_repository}/{filename}",
|
||||
contents,
|
||||
user="keystone",
|
||||
group="keystone",
|
||||
@ -193,7 +227,7 @@ class KeystoneManager(framework.Object):
|
||||
)
|
||||
|
||||
# remove old keys
|
||||
files = container.list_files("/etc/keystone/fernet-keys")
|
||||
files = container.list_files(key_repository)
|
||||
for file in files:
|
||||
if file.name not in keys:
|
||||
container.remove_path(file.path)
|
||||
@ -282,6 +316,10 @@ class KeystoneManager(framework.Object):
|
||||
"keystone",
|
||||
"keystone-manage",
|
||||
"credential_setup",
|
||||
"--keystone-user",
|
||||
"keystone",
|
||||
"--keystone-group",
|
||||
"keystone",
|
||||
]
|
||||
)
|
||||
except ops.pebble.ExecError:
|
||||
@ -652,7 +690,7 @@ class KeystoneManager(framework.Object):
|
||||
# TODO(wolsen) can we have more than one service with the same
|
||||
# service name? I don't think so, so we'll just handle the first
|
||||
# one for now.
|
||||
print("FOUND: {}".format(services))
|
||||
logger.debug(f"FOUND: {services}")
|
||||
for service in services:
|
||||
logger.debug(
|
||||
f"Service {name} already exists with "
|
||||
|
||||
@ -15,14 +15,6 @@ applications:
|
||||
scale: 1
|
||||
trust: true
|
||||
|
||||
traefik-public:
|
||||
charm: ch:traefik-k8s
|
||||
channel: 1.0/stable
|
||||
scale: 1
|
||||
trust: true
|
||||
options:
|
||||
kubernetes-service-annotations: metallb.universe.tf/address-pool=public
|
||||
|
||||
# required for glance
|
||||
rabbitmq:
|
||||
charm: ch:rabbitmq-k8s
|
||||
@ -58,12 +50,8 @@ relations:
|
||||
- glance:amqp
|
||||
|
||||
- - traefik:ingress
|
||||
- keystone:ingress-internal
|
||||
- - traefik-public:ingress
|
||||
- keystone:ingress-public
|
||||
- - traefik:ingress
|
||||
- glance:ingress-internal
|
||||
- - traefik-public:ingress
|
||||
- glance:ingress-public
|
||||
|
||||
- - mysql:database
|
||||
|
||||
@ -53,7 +53,6 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
|
||||
PATCHES = [
|
||||
"manager",
|
||||
"subprocess",
|
||||
"pwgen",
|
||||
]
|
||||
|
||||
@ -121,7 +120,7 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
km_mock.create_domain.return_value = service_domain_mock
|
||||
km_mock.create_user.return_value = service_user_mock
|
||||
km_mock.create_role.return_value = admin_role_mock
|
||||
km_mock.read_fernet_keys.return_value = {
|
||||
km_mock.read_keys.return_value = {
|
||||
"0": "Qf4vHdf6XC2dGKpEwtGapq7oDOqUWepcH2tKgQ0qOKc=",
|
||||
"3": "UK3qzLGvu-piYwau0BFyed8O3WP8lFKH_v1sXYulzhs=",
|
||||
"4": "YVYUJbQNASbVzzntqj2sG9rbDOV_QQfueDCz0PJEKKw=",
|
||||
@ -139,7 +138,6 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
# used by _launch_heartbeat.
|
||||
# value doesn't matter for tests because mocking
|
||||
os.environ["JUJU_CHARM_DIR"] = "/arbitrary/directory/"
|
||||
self.subprocess.call.return_value = 1
|
||||
self.pwgen.pwgen.return_value = "randonpassword"
|
||||
|
||||
self.km_mock = self.ks_manager_mock()
|
||||
@ -168,6 +166,16 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
self.addCleanup(self.harness.cleanup)
|
||||
self.harness.begin()
|
||||
|
||||
# This function need to be moved to operator
|
||||
def get_secret_by_label(self, label: str) -> str:
|
||||
"""Get secret by label from harness class."""
|
||||
print(self.harness._backend._secrets)
|
||||
for secret in self.harness._backend._secrets:
|
||||
if secret.label == label:
|
||||
return secret.id
|
||||
|
||||
return None
|
||||
|
||||
def test_pebble_ready_handler(self):
|
||||
"""Test pebble ready handler."""
|
||||
self.assertEqual(self.harness.charm.seen_events, [])
|
||||
@ -223,9 +231,16 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
peer_data = self.harness.get_relation_data(
|
||||
peer_rel_id, self.harness.charm.unit.app.name
|
||||
)
|
||||
fernet_secret_id = self.get_secret_by_label("fernet-keys")
|
||||
credential_secret_id = self.get_secret_by_label("credential-keys")
|
||||
self.assertEqual(
|
||||
peer_data,
|
||||
{"leader_ready": "true", "password_svc_cinder": "randonpassword"},
|
||||
{
|
||||
"leader_ready": "true",
|
||||
"fernet-secret-id": fernet_secret_id,
|
||||
"credential-keys-secret-id": credential_secret_id,
|
||||
"password_svc_cinder": "randonpassword",
|
||||
},
|
||||
)
|
||||
|
||||
def test_leader_bootstraps(self):
|
||||
@ -241,8 +256,50 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
self.km_mock.setup_keystone.assert_called_once_with()
|
||||
self.km_mock.setup_initial_projects_and_users.assert_called_once_with()
|
||||
|
||||
def test_leader_rotate_fernet_keys(self):
|
||||
"""Test leader fernet key rotation."""
|
||||
peer_data = self.harness.get_relation_data(
|
||||
rel_id, self.harness.charm.unit.app.name
|
||||
)
|
||||
fernet_secret_id = self.get_secret_by_label("fernet-keys")
|
||||
credential_secret_id = self.get_secret_by_label("credential-keys")
|
||||
self.assertEqual(
|
||||
peer_data,
|
||||
{
|
||||
"leader_ready": "true",
|
||||
"fernet-secret-id": fernet_secret_id,
|
||||
"credential-keys-secret-id": credential_secret_id,
|
||||
},
|
||||
)
|
||||
|
||||
def test_on_peer_data_changed_no_bootstrap(self):
|
||||
"""Test peer_relation_changed on no bootstrap."""
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
|
||||
event = MagicMock()
|
||||
self.harness.charm._on_peer_data_changed(event)
|
||||
self.assertTrue(event.defer.called)
|
||||
|
||||
def test_on_peer_data_changed_with_fernet_keys_and_fernet_secret_different(
|
||||
self,
|
||||
):
|
||||
"""Test peer_relation_changed when fernet keys and secret have different content."""
|
||||
updated_fernet_keys = {
|
||||
"0": "Qf4vHdf6XC2dGKpEwtGapq7oDOqUWepcH2tKgQ0qOKc=",
|
||||
"2": "UK3qzLGvu-piYwau0BFyed8O3WP8lFKH_v1sXYulzhs=",
|
||||
"3": "YVYUJbQNASbVzzntqj2sG9rbDOV_QQfueDCz0PJEKKw=",
|
||||
}
|
||||
secret_mock = mock.MagicMock()
|
||||
secret_mock.id = "test-secret-id"
|
||||
secret_mock.get_content.return_value = updated_fernet_keys
|
||||
|
||||
self.harness.model.app.add_secret = MagicMock()
|
||||
self.harness.model.app.add_secret.return_value = secret_mock
|
||||
self.harness.model.get_secret = MagicMock()
|
||||
self.harness.model.get_secret.return_value = secret_mock
|
||||
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
@ -251,23 +308,37 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
test_utils.add_db_relation_credentials(
|
||||
self.harness, test_utils.add_base_db_relation(self.harness)
|
||||
)
|
||||
self.harness.charm._rotate_fernet_keys()
|
||||
self.km_mock.rotate_fernet_keys.assert_called_once_with()
|
||||
|
||||
def test_not_leader_rotate_fernet_keys(self):
|
||||
"""Test non-leader fernet keys."""
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
test_utils.add_db_relation_credentials(
|
||||
self.harness, test_utils.add_base_db_relation(self.harness)
|
||||
event = MagicMock()
|
||||
self.harness.charm._on_peer_data_changed(event)
|
||||
self.assertTrue(self.harness.model.get_secret.called)
|
||||
self.assertTrue(self.km_mock.read_keys.called)
|
||||
self.assertEqual(self.km_mock.write_keys.call_count, 2)
|
||||
self.km_mock.write_keys.assert_has_calls(
|
||||
[
|
||||
mock.call(
|
||||
key_repository="/etc/keystone/fernet-keys",
|
||||
keys=updated_fernet_keys,
|
||||
),
|
||||
mock.call(
|
||||
key_repository="/etc/keystone/credential-keys",
|
||||
keys=updated_fernet_keys,
|
||||
),
|
||||
]
|
||||
)
|
||||
self.harness.charm._rotate_fernet_keys()
|
||||
self.km_mock.rotate_fernet_keys.assert_not_called()
|
||||
|
||||
def test_on_heartbeat(self):
|
||||
"""Test on_heartbeat calls."""
|
||||
def test_on_peer_data_changed_with_fernet_keys_and_fernet_secret_same(
|
||||
self,
|
||||
):
|
||||
"""Test peer_relation_changed when fernet keys and secret have same content."""
|
||||
secret_mock = mock.MagicMock()
|
||||
secret_mock.id = "test-secret-id"
|
||||
secret_mock.get_content.return_value = self.km_mock.read_keys()
|
||||
self.harness.model.app.add_secret = MagicMock()
|
||||
self.harness.model.app.add_secret.return_value = secret_mock
|
||||
self.harness.model.get_secret = MagicMock()
|
||||
self.harness.model.get_secret.return_value = secret_mock
|
||||
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
@ -276,27 +347,106 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
|
||||
test_utils.add_db_relation_credentials(
|
||||
self.harness, test_utils.add_base_db_relation(self.harness)
|
||||
)
|
||||
self.harness.charm._on_heartbeat(None)
|
||||
self.km_mock.rotate_fernet_keys.assert_called_once_with()
|
||||
|
||||
# run the heartbeat again immediately.
|
||||
# The keys should not be rotated again,
|
||||
# since by default the rotation interval will be > 2 days.
|
||||
self.harness.charm._on_heartbeat(None)
|
||||
self.km_mock.rotate_fernet_keys.assert_called_once_with()
|
||||
event = MagicMock()
|
||||
self.harness.charm._on_peer_data_changed(event)
|
||||
self.assertTrue(self.harness.model.get_secret.called)
|
||||
self.assertTrue(self.km_mock.read_keys.called)
|
||||
self.assertFalse(self.km_mock.write_keys.called)
|
||||
|
||||
def test_launching_heartbeat(self):
|
||||
"""Test launching a heartbeat."""
|
||||
# verify that the heartbeat script is launched during initialisation
|
||||
self.subprocess.Popen.assert_called_once_with(
|
||||
["./src/heartbeat.sh"],
|
||||
cwd="/arbitrary/directory/",
|
||||
)
|
||||
def _test_non_leader_on_secret_rotate(self, label: str):
|
||||
"""Test secert-rotate event on non leader unit."""
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
|
||||
# implementation detail, but probably good to double check
|
||||
self.subprocess.call.assert_called_once_with(
|
||||
["pgrep", "-f", "heartbeat"]
|
||||
)
|
||||
event = MagicMock()
|
||||
event.secret.label = label
|
||||
self.harness.charm._on_secret_rotate(event)
|
||||
if label == "fernet-keys":
|
||||
self.assertFalse(self.km_mock.rotate_fernet_keys.called)
|
||||
elif label == "credential-keys":
|
||||
self.assertFalse(self.km_mock.rotate_credential_keys.called)
|
||||
|
||||
def _test_leader_on_secret_rotate(self, label: str):
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
|
||||
event = MagicMock()
|
||||
event.secret.label = label
|
||||
self.harness.charm._on_secret_rotate(event)
|
||||
if label == "fernet-keys":
|
||||
fernet_keys_ = {
|
||||
f"fernet-{k}": v for k, v in self.km_mock.read_keys().items()
|
||||
}
|
||||
self.assertTrue(self.km_mock.rotate_fernet_keys.called)
|
||||
event.secret.set_content.assert_called_once_with(fernet_keys_)
|
||||
elif label == "credential-keys":
|
||||
fernet_keys_ = {
|
||||
f"fernet-{k}": v for k, v in self.km_mock.read_keys().items()
|
||||
}
|
||||
self.assertTrue(self.km_mock.rotate_credential_keys.called)
|
||||
event.secret.set_content.assert_called_once_with(fernet_keys_)
|
||||
|
||||
def test_leader_on_secret_rotate_for_label_fernet_keys(self):
|
||||
"""Test secret-rotate event for label fernet_keys on leader unit."""
|
||||
self._test_leader_on_secret_rotate(label="fernet-keys")
|
||||
|
||||
def test_leader_on_secret_rotate_for_label_credential_keys(self):
|
||||
"""Test secret-rotate event for label credential_keys on leader unit."""
|
||||
self._test_leader_on_secret_rotate(label="credential-keys")
|
||||
|
||||
def test_non_leader_on_secret_rotate_for_label_fernet_keys(self):
|
||||
"""Test secret-rotate event for label fernet_keys on non leader unit."""
|
||||
self._test_non_leader_on_secret_rotate(label="fernet-keys")
|
||||
|
||||
def test_non_leader_on_secret_rotate_for_label_credential_keys(self):
|
||||
"""Test secret-rotate event for label credential_keys on non leader unit."""
|
||||
self._test_non_leader_on_secret_rotate(label="credential-keys")
|
||||
|
||||
def test_on_secret_changed_with_fernet_keys_and_fernet_secret_same(self):
|
||||
"""Test secret change event when fernet keys and secret have same content."""
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
|
||||
event = MagicMock()
|
||||
event.secret.label = "fernet-keys"
|
||||
event.secret.get_content.return_value = self.km_mock.read_keys()
|
||||
self.harness.charm._on_secret_changed(event)
|
||||
|
||||
self.assertTrue(event.secret.get_content.called)
|
||||
self.assertTrue(self.km_mock.read_keys.called)
|
||||
self.assertFalse(self.km_mock.write_keys.called)
|
||||
|
||||
def test_on_secret_changed_with_fernet_keys_and_fernet_secret_different(
|
||||
self,
|
||||
):
|
||||
"""Test secret change event when fernet keys and secret have different content."""
|
||||
test_utils.add_complete_ingress_relation(self.harness)
|
||||
self.harness.set_leader()
|
||||
rel_id = self.harness.add_relation("peers", "keystone-k8s")
|
||||
self.harness.add_relation_unit(rel_id, "keystone-k8s/1")
|
||||
self.harness.container_pebble_ready("keystone")
|
||||
|
||||
event = MagicMock()
|
||||
event.secret.label = "fernet-keys"
|
||||
event.secret.get_content.return_value = {
|
||||
"0": "Qf4vHdf6XC2dGKpEwtGapq7oDOqUWepcH2tKgQ0qOKc=",
|
||||
"4": "UK3qzLGvu-piYwau0BFyed8O3WP8lFKH_v1sXYulzhs=",
|
||||
"5": "YVYUJbQNASbVzzntqj2sG9rbDOV_QQfueDCz0PJEKKw=",
|
||||
}
|
||||
self.harness.charm._on_secret_changed(event)
|
||||
|
||||
self.assertTrue(event.secret.get_content.called)
|
||||
self.assertTrue(self.km_mock.read_keys.called)
|
||||
self.assertTrue(self.km_mock.write_keys.called)
|
||||
|
||||
def test_non_leader_no_bootstraps(self):
|
||||
"""Test bootstraping on a non-leader."""
|
||||
|
||||
@ -24,6 +24,7 @@ setenv =
|
||||
PYTHONPATH = {toxinidir}:{[vars]lib_path}:{[vars]src_path}
|
||||
passenv =
|
||||
PYTHONPATH
|
||||
HOME
|
||||
install_command =
|
||||
pip install {opts} {packages}
|
||||
commands = stestr run --slowest {posargs}
|
||||
@ -126,7 +127,7 @@ commands =
|
||||
[testenv:func-smoke]
|
||||
basepython = python3
|
||||
setenv =
|
||||
TEST_MODEL_SETTINGS = automatically-retry-hooks=true
|
||||
TEST_MODEL_SETTINGS = automatically-retry-hooks=true;default-series=
|
||||
TEST_MAX_RESOLVE_COUNT = 5
|
||||
commands =
|
||||
functest-run-suite --keep-model --smoke
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user