openstack/charm-ceph-radosgw
Code Issues Proposed changes

Adds support for migration to multi-site system.

Browse Source 
1.) Currently multi-site can only be configured when system is being
deployed from scratch, migration works by renaming the existing
Zone/Zonegroups (Z/ZG) to Juju config values on primary site before
secondary site pulls the realm data and then rename and configure
secondary Zone accordingly.

During migration:
2.) If multiple Z/ZG not matching the config values are present at
primary site, the leader unit will block and prompt use of
'force-enable-multisite' which renames and configures selected Z/ZG
according to multisite config values.

3.) If the site being added as a secondary already contain Buckets,
the unit will block and prompt the operator to purge all such Buckets
before proceeding.

Closes-Bug: #1959837
Change-Id: I01a4c1c4551c797f0a32951dfbde8a1a4126c2d6
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/840
This commit is contained in:
utkarshbhatthere 2022-07-05 17:27:36 +05:30
parent 5c4cab3f82
commit 44fee84d4d
No known key found for this signature in database
GPG Key ID: 8AFC279E7CD87430
15 changed files with 1116 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
actions.yaml
View File

@ -10,3 +10,12 @@ readwrite:
description: Mark the zone associated with the local units as read/write (multi-site).
tidydefaults:
description: Delete default zone and zonegroup configuration (multi-site).
force-enable-multisite:
description: Reconfigure provided Zone and Zonegroup for migration to multisite.
params:
zone:
type: string
description: Existing Zone to be reconfigured as the 'zone' config value.
zonegroup:
type: string
description: Existing Zonegroup to be reconfigured as the 'zonegroup' config value.

108
actions/actions.py
View File

@ -17,6 +17,7 @@
import os
import subprocess
import sys
import uuid
sys.path.append('hooks/')
@ -25,12 +26,27 @@ import multisite
from charmhelpers.core.hookenv import (
action_fail,
config,
is_leader,
leader_set,
action_set,
action_get,
log,
ERROR,
DEBUG,
)
from charmhelpers.contrib.openstack.ip import (
canonical_url,
PUBLIC,
)
from utils import (
pause_unit_helper,
resume_unit_helper,
register_configs,
listen_port,
service_name,
)
from charmhelpers.core.host import (
service_restart,
)
@ -50,13 +66,19 @@ def resume(args):
def promote(args):
"""Promote zone associated with local RGW units to master/default"""
zone = config('zone')
zonegroup = config('zonegroup')
if not is_leader():
action_fail('This action can only be executed on leader unit.')
return
if not zone:
action_fail('No zone configuration set, not promoting')
return
try:
multisite.modify_zone(zone,
default=True, master=True)
multisite.update_period()
multisite.update_period(zonegroup=zonegroup, zone=zone)
leader_set(restart_nonce=str(uuid.uuid4()))
service_restart(service_name())
action_set(
values={'message': 'zone:{} promoted to '
'master/default'.format(zone)}
@ -122,6 +144,89 @@ def tidydefaults(args):
': {} - {}'.format(zone, cpe.output))
def force_enable_multisite(args):
"""Configure provided zone and zonegroup according to Multisite Config
In a situation when multiple zone or zonegroups are configured on the
primary site, the decision for which pair to use in multisite system
is taken through this action. It takes provided parameters (zone name
and zonegroup name) and rename/ modify them appropriately.
"""
public_url = '{}:{}'.format(
canonical_url(register_configs(), PUBLIC),
listen_port(),
)
current_zone = action_get("zone")
current_zonegroup = action_get("zonegroup")
endpoints = [public_url]
realm = config('realm')
new_zone = config('zone')
new_zonegroup = config('zonegroup')
log("zone:{}, zonegroup:{}, endpoints:{}, realm:{}, new_zone:{}, "
"new_zonegroup:{}".format(
current_zone, current_zonegroup, endpoints,
realm, new_zone, new_zonegroup
), level=DEBUG)
if not is_leader():
action_fail('This action can only be executed on leader unit.')
return
if not all((realm, new_zonegroup, new_zone)):
action_fail("Missing required charm configurations realm({}), "
"zonegroup({}) and zone({}).".format(
realm, new_zonegroup, new_zone
))
return
if current_zone not in multisite.list_zones():
action_fail('Provided zone {} does not exist.'.format(current_zone))
return
if current_zonegroup not in multisite.list_zonegroups():
action_fail('Provided zone {} does not exist.'
.format(current_zonegroup))
return
try:
# Rename chosen zonegroup/zone as per charm config value.
rename_result = multisite.rename_multisite_config(
[current_zonegroup],
new_zonegroup,
[current_zone], new_zone
)
if rename_result is None:
action_fail('Failed to rename zone {} or zonegroup {}.'
.format(current_zone, current_zonegroup))
return
# Configure zonegroup/zone as master for multisite.
modify_result = multisite.modify_multisite_config(
new_zone, new_zonegroup,
realm=realm,
endpoints=endpoints
)
if modify_result is None:
action_fail('Failed to configure zone {} or zonegroup {}.'
.format(new_zonegroup, new_zone))
return
leader_set(restart_nonce=str(uuid.uuid4()))
service_restart(service_name())
action_set(
values={
'message': 'Multisite Configuration Resolved'
}
)
except subprocess.CalledProcessError as cpe:
message = "Failed to configure zone ({}) and zonegroup ({})".format(
current_zone, current_zonegroup
)
log(message, level=ERROR)
action_fail(message + " : {}".format(cpe.output))
# A dictionary of all the defined actions to callables (which take
# parsed arguments).
ACTIONS = {
@ -131,6 +236,7 @@ ACTIONS = {
"readonly": readonly,
"readwrite": readwrite,
"tidydefaults": tidydefaults,
"force-enable-multisite": force_enable_multisite,
}

1
actions/force-enable-multisite Symbolic link
View File

@ -0,0 +1 @@
actions.py

2
hooks/ceph_radosgw_context.py
View File

@ -305,7 +305,7 @@ class MonContext(context.CephContext):
ctxt.update(user_provided)
if self.context_complete(ctxt):
# Multi-site Zone configuration is optional,
# Multi-site zone configuration is optional,
# so add after assessment
ctxt['rgw_zone'] = config('zone')
ctxt['rgw_zonegroup'] = config('zonegroup')

57
hooks/hooks.py
View File

@ -27,6 +27,7 @@ import charms_ceph.utils as ceph_utils
import multisite
from charmhelpers.core.hookenv import (
ERROR,
relation_get,
relation_id as ch_relation_id,
relation_ids,
@ -366,7 +367,7 @@ def mon_relation(rid=None, unit=None):
existing_zones = multisite.list_zones()
log('Existing zones {}'.format(existing_zones), level=DEBUG)
if zone not in existing_zones:
log("Zone '{}' doesn't exist, creating".format(zone))
log("zone '{}' doesn't exist, creating".format(zone))
try:
multisite.create_zone(zone,
endpoints=endpoints,
@ -377,7 +378,7 @@ def mon_relation(rid=None, unit=None):
# NOTE(lourot): may have been created in the
# background by the Rados Gateway daemon, see
# lp:1856106
log("Zone '{}' existed already after all".format(
log("zone '{}' existed already after all".format(
zone))
else:
raise
@ -741,8 +742,43 @@ def master_relation_joined(relation_id=None):
multisite.create_realm(realm, default=True)
mutation = True
# Migration if master site has buckets configured.
# Migration involves renaming existing zone/zongroups such that existing
# buckets and their objects can be preserved on the master site.
if multisite.check_cluster_has_buckets() is True:
log('Migrating to multisite with zone ({}) and zonegroup ({})'
.format(zone, zonegroup), level=DEBUG)
zones = multisite.list_zones()
zonegroups = multisite.list_zonegroups()
if (len(zonegroups) > 1) and (zonegroup not in zonegroups):
log('Multiple zonegroups found {}, aborting.'
.format(zonegroups), level=ERROR)
return
if (len(zones) > 1) and (zone not in zones):
log('Multiple zones found {}, aborting.'
.format(zones), level=ERROR)
return
rename_result = multisite.rename_multisite_config(
zonegroups, zonegroup,
zones, zone
)
if rename_result is None:
return
modify_result = multisite.modify_multisite_config(
zone, zonegroup,
endpoints=endpoints,
realm=realm
)
if modify_result is None:
return
mutation = True
if zonegroup not in multisite.list_zonegroups():
log('Zonegroup {} not found, creating now'.format(zonegroup))
log('zonegroup {} not found, creating now'.format(zonegroup))
multisite.create_zonegroup(zonegroup,
endpoints=endpoints,
default=True, master=True,
@ -750,7 +786,7 @@ def master_relation_joined(relation_id=None):
mutation = True
if zone not in multisite.list_zones():
log('Zone {} not found, creating now'.format(zone))
log('zone {} not found, creating now'.format(zone))
multisite.create_zone(zone,
endpoints=endpoints,
default=True, master=True,
@ -773,7 +809,7 @@ def master_relation_joined(relation_id=None):
log(
'Mutation detected. Restarting {}.'.format(service_name()),
'INFO')
multisite.update_period()
multisite.update_period(zonegroup=zonegroup, zone=zone)
service_restart(service_name())
leader_set(restart_nonce=str(uuid.uuid4()))
else:
@ -829,6 +865,13 @@ def slave_relation_changed(relation_id=None, unit=None):
mutation = False
# NOTE(utkarshbhatthere):
# A site with existing data can create inconsistencies when added as a
# secondary site for RGW. Hence it must be pristine.
if multisite.check_cluster_has_buckets():
log("Non-Pristine site can't be used as secondary", level=ERROR)
return
if realm not in multisite.list_realms():
log('Realm {} not found, pulling now'.format(realm))
multisite.pull_realm(url=master_data['url'],
@ -841,7 +884,7 @@ def slave_relation_changed(relation_id=None, unit=None):
mutation = True
if zone not in multisite.list_zones():
log('Zone {} not found, creating now'.format(zone))
log('zone {} not found, creating now'.format(zone))
multisite.create_zone(zone,
endpoints=endpoints,
default=False, master=False,
@ -854,7 +897,7 @@ def slave_relation_changed(relation_id=None, unit=None):
log(
'Mutation detected. Restarting {}.'.format(service_name()),
'INFO')
multisite.update_period()
multisite.update_period(zonegroup=zonegroup, zone=zone)
service_restart(service_name())
leader_set(restart_nonce=str(uuid.uuid4()))
else:

379
hooks/multisite.py
View File

@ -25,7 +25,7 @@ import charmhelpers.core.decorators as decorators
RGW_ADMIN = 'radosgw-admin'
@decorators.retry_on_exception(num_retries=5, base_delay=3,
@decorators.retry_on_exception(num_retries=10, base_delay=5,
exc_type=subprocess.CalledProcessError)
def _check_output(cmd):
"""Logging wrapper for subprocess.check_ouput"""
@ -105,6 +105,32 @@ list_zonegroups = functools.partial(_list, 'zonegroup')
list_users = functools.partial(_list, 'user')
def list_buckets(zone, zonegroup):
"""List Buckets served under the provided zone and zonegroup pair.
:param zonegroup: Parent zonegroup.
:type zonegroup: str
:param zone: Parent zone.
:type zone: str
:returns: List of buckets found
:rtype: list
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'bucket', 'list',
'--rgw-zone={}'.format(zone),
'--rgw-zonegroup={}'.format(zonegroup),
]
try:
return json.loads(_check_output(cmd))
except subprocess.CalledProcessError:
hookenv.log("Bucket queried for incorrect zone({})-zonegroup({}) "
"pair".format(zone, zonegroup), level=hookenv.ERROR)
return None
except TypeError:
return None
def create_realm(name, default=False):
"""
Create a new RADOS Gateway Realm.
@ -146,7 +172,7 @@ def set_default_realm(name):
def create_zonegroup(name, endpoints, default=False, master=False, realm=None):
"""
Create a new RADOS Gateway Zone Group
Create a new RADOS Gateway zone Group
:param name: name of zonegroup to create
:type name: str
@ -179,10 +205,49 @@ def create_zonegroup(name, endpoints, default=False, master=False, realm=None):
return None
def modify_zonegroup(name, endpoints=None, default=False,
master=False, realm=None):
"""Modify an existing RADOS Gateway zonegroup
An empty list of endpoints would cause NO-CHANGE in the configured
endpoints for the zonegroup.
:param name: name of zonegroup to modify
:type name: str
:param endpoints: list of URLs to endpoints for zonegroup
:type endpoints: list[str]
:param default: set zonegroup as the default zonegroup
:type default: boolean
:param master: set zonegroup as the master zonegroup
:type master: boolean
:param realm: realm name for provided zonegroup
:type realm: str
:return: zonegroup configuration
:rtype: dict
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'zonegroup', 'modify',
'--rgw-zonegroup={}'.format(name),
]
if realm:
cmd.append('--rgw-realm={}'.format(realm))
if endpoints:
cmd.append('--endpoints={}'.format(','.join(endpoints)))
if default:
cmd.append('--default')
if master:
cmd.append('--master')
try:
return json.loads(_check_output(cmd))
except TypeError:
return None
def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
access_key=None, secret=None, readonly=False):
"""
Create a new RADOS Gateway Zone
Create a new RADOS Gateway zone
:param name: name of zone to create
:type name: str
@ -226,9 +291,9 @@ def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
def modify_zone(name, endpoints=None, default=False, master=False,
access_key=None, secret=None, readonly=False):
"""
Modify an existing RADOS Gateway zone
access_key=None, secret=None, readonly=False,
realm=None, zonegroup=None):
"""Modify an existing RADOS Gateway zone
:param name: name of zone to create
:type name: str
@ -243,7 +308,11 @@ def modify_zone(name, endpoints=None, default=False, master=False,
:param secret: secret to use with access-key for the zone
:type secret: str
:param readonly: set zone as read only
:type: readonly: boolean
:type readonly: boolean
:param realm: realm to use for zone
:type realm: str
:param zonegroup: zonegroup to use for zone
:type zonegroup: str
:return: zone configuration
:rtype: dict
"""
@ -252,6 +321,10 @@ def modify_zone(name, endpoints=None, default=False, master=False,
'zone', 'modify',
'--rgw-zone={}'.format(name),
]
if realm:
cmd.append('--rgw-realm={}'.format(realm))
if zonegroup:
cmd.append('--rgw-zonegroup={}'.format(zonegroup))
if endpoints:
cmd.append('--endpoints={}'.format(','.join(endpoints)))
if access_key and secret:
@ -268,14 +341,24 @@ def modify_zone(name, endpoints=None, default=False, master=False,
return None
def update_period(fatal=True):
"""
Update RADOS Gateway configuration period
def update_period(fatal=True, zonegroup=None, zone=None):
"""Update RADOS Gateway configuration period
:param fatal: In failure case, whether CalledProcessError is to be raised.
:type fatal: boolean
:param zonegroup: zonegroup name
:type zonegroup: str
:param zone: zone name
:type zone: str
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'period', 'update', '--commit'
]
if zonegroup is not None:
cmd.append('--rgw-zonegroup={}'.format(zonegroup))
if zone is not None:
cmd.append('--rgw-zone={}'.format(zone))
if fatal:
_check_call(cmd)
else:
@ -439,3 +522,279 @@ def pull_period(url, access_key, secret):
return json.loads(_check_output(cmd))
except TypeError:
return None
def rename_zone(name, new_name, zonegroup):
"""Rename an existing RADOS Gateway zone
If the command execution succeeds, 0 is returned, otherwise
None is returned to the caller.
:param name: current name for the zone being renamed
:type name: str
:param new_name: new name for the zone being renamed
:type new_name: str
:rtype: int
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'zone', 'rename',
'--rgw-zone={}'.format(name),
'--zone-new-name={}'.format(new_name),
'--rgw-zonegroup={}'.format(zonegroup)
]
result = _call(cmd)
return 0 if result == 0 else None
def rename_zonegroup(name, new_name):
"""Rename an existing RADOS Gateway zonegroup
If the command execution succeeds, 0 is returned, otherwise
None is returned to the caller.
:param name: current name for the zonegroup being renamed
:type name: str
:param new_name: new name for the zonegroup being renamed
:type new_name: str
:rtype: int
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'zonegroup', 'rename',
'--rgw-zonegroup={}'.format(name),
'--zonegroup-new-name={}'.format(new_name),
]
result = _call(cmd)
return 0 if result == 0 else None
def get_zonegroup_info(zonegroup):
"""Fetch detailed info for the provided zonegroup
:param zonegroup: zonegroup Name for detailed query
:type zonegroup: str
:rtype: dict
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'zonegroup', 'get',
'--rgw-zonegroup={}'.format(zonegroup),
]
try:
return json.loads(_check_output(cmd))
except TypeError:
return None
def get_sync_status():
"""
Get sync status
:returns: Sync Status Report from radosgw-admin
:rtype: str
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'sync', 'status',
]
try:
return _check_output(cmd)
except subprocess.CalledProcessError:
hookenv.log("Failed to fetch sync status", level=hookenv.ERROR)
return None
def is_multisite_configured(zone, zonegroup):
"""Check if system is already multisite configured
Checks if zone and zonegroup are configured appropriately and
remote data sync source is detected in sync status
:rtype: Boolean
"""
if zone not in list_zones():
hookenv.log("No local zone found with name ({})".format(zonegroup),
level=hookenv.ERROR)
return False
if zonegroup not in list_zonegroups():
hookenv.log("No zonegroup found with name ({})".format(zonegroup),
level=hookenv.ERROR)
return False
sync_status = get_sync_status()
if sync_status is not None:
return ('data sync source:' in sync_status)
return False
def get_local_zone(zonegroup):
"""Get local zone to provided parent zonegroup.
In multisite systems, zonegroup contains both local and remote zone info
this method is used to fetch the zone local to querying site.
:param zonegroup: parent zonegroup name.
:type zonegroup: str
:returns: tuple with parent zonegroup and local zone name
:rtype: tuple
"""
local_zones = list_zones()
zonegroup_info = get_zonegroup_info(zonegroup)
if zonegroup_info is None:
hookenv.log("Failed to fetch zonegroup ({}) info".format(zonegroup),
level=hookenv.ERROR)
return None
# zonegroup info always contains self name and zones list so fetching
# directly is safe.
master_zonegroup = zonegroup_info['name']
for zone_info in zonegroup_info['zones']:
zone = zone_info['name']
if zone in local_zones:
return zone, master_zonegroup
hookenv.log(
"No local zone configured for zonegroup ({})".format(zonegroup),
level=hookenv.ERROR
)
return None
def rename_multisite_config(zonegroups, new_zonegroup_name,
zones, new_zone_name):
"""Rename zone and zonegroup to provided new names.
If zone list (zones) or zonegroup list (zonegroups) contain 1 element
rename the only element present in the list to provided (new_) value.
:param zonegroups: List of zonegroups available at site.
:type zonegroups: list[str]
:param new_zonegroup_name: Desired new name for master zonegroup.
:type new_zonegroup_name: str
:param zones: List of zones available at site.
:type zones: list[str]
:param new_zonegroup_name: Desired new name for master zone.
:type new_zonegroup_name: str
:return: Whether any of the zone or zonegroup is renamed.
:rtype: Boolean
"""
mutation = False
if (len(zonegroups) == 1) and (len(zones) == 1):
if new_zonegroup_name not in zonegroups:
result = rename_zonegroup(zonegroups[0], new_zonegroup_name)
if result is None:
hookenv.log(
"Failed renaming zonegroup from {} to {}"
.format(zonegroups[0], new_zonegroup_name),
level=hookenv.ERROR
)
return None
mutation = True
if new_zone_name not in zones:
result = rename_zone(zones[0], new_zone_name, new_zonegroup_name)
if result is None:
hookenv.log(
"Failed renaming zone from {} to {}"
.format(zones[0], new_zone_name), level=hookenv.ERROR
)
return None
mutation = True
if mutation:
hookenv.log("Renamed zonegroup {} to {}, and zone {} to {}".format(
zonegroups[0], new_zonegroup_name,
zones[0], new_zone_name))
return True
return False
def modify_multisite_config(zone, zonegroup, endpoints=None, realm=None):
"""Configure zone and zonegroup as master for multisite system.
:param zonegroup: zonegroup name being configured for multisite
:type zonegroup: str
:param zone: zone name being configured for multisite
:type zone: str
:param endpoints: list of URLs to RGW endpoints
:type endpoints: list[str]
:param realm: realm to use for multisite
:type realm: str
:rtype: Boolean
"""
if modify_zonegroup(zonegroup, endpoints=endpoints, default=True,
master=True, realm=realm) is None:
hookenv.log(
"Failed configuring zonegroup {}".format(zonegroup),
level=hookenv.ERROR
)
return None
if modify_zone(zone, endpoints=endpoints, default=True,
master=True, zonegroup=zonegroup, realm=realm) is None:
hookenv.log(
"Failed configuring zone {}".format(zone), level=hookenv.ERROR
)
return None
update_period(zonegroup=zonegroup, zone=zone)
hookenv.log("Configured zonegroup {}, and zone {} for multisite".format(
zonegroup, zone))
return True
def check_zone_has_buckets(zone, zonegroup):
"""Checks whether provided zone-zonegroup pair contains any bucket.
:param zone: zone name to query buckets in.
:type zone: str
:param zonegroup: Parent zonegroup of zone.
:type zonegroup: str
:rtype: Boolean
"""
buckets = list_buckets(zone, zonegroup)
if buckets is not None:
return (len(buckets) > 0)
hookenv.log(
"Failed to query buckets for zone {} zonegroup {}"
.format(zone, zonegroup),
level=hookenv.WARNING
)
return False
def check_zonegroup_has_buckets(zonegroup):
"""Checks whether any bucket exists in the master zone of a zonegroup.
:param zone: zonegroup name to query buckets.
:type zone: str
:rtype: Boolean
"""
# NOTE(utkarshbhatthere): sometimes querying against a particular
# zonegroup results in info of an entirely different zonegroup, thus to
# prevent a query against an incorrect pair in such cases, both zone and
# zonegroup names are taken from zonegroup info.
master_zone, master_zonegroup = get_local_zone(zonegroup)
# If master zone is not configured for zonegroup
if master_zone is None:
hookenv.log("No master zone configured for zonegroup {}"
.format(master_zonegroup), level=hookenv.WARNING)
return False
return check_zone_has_buckets(master_zone, master_zonegroup)
def check_cluster_has_buckets():
"""Iteratively check if ANY zonegroup has buckets on cluster.
:rtype: Boolean
"""
for zonegroup in list_zonegroups():
if check_zonegroup_has_buckets(zonegroup):
return True
return False

94
hooks/utils.py
View File

@ -20,6 +20,7 @@ from collections import OrderedDict
from copy import deepcopy
import ceph_radosgw_context
import multisite
from charmhelpers.core.hookenv import (
relation_get,
@ -184,6 +185,14 @@ def get_optional_interfaces():
return optional_interfaces
def get_zones_zonegroups():
"""Get a tuple with lists of zones and zonegroups existing on site
:rtype: tuple
"""
return multisite.list_zones(), multisite.list_zonegroups()
def check_optional_config_and_relations(configs):
"""Check that if we have a relation_id for high availability that we can
get the hacluster config. If we can't then we are blocked. This function
@ -201,41 +210,72 @@ def check_optional_config_and_relations(configs):
return ('blocked',
'hacluster missing configuration: '
'vip, vip_iface, vip_cidr')
# NOTE: misc multi-site relation and config checks
multisite_config = (config('realm'),
config('zonegroup'),
config('zone'))
if relation_ids('master') or relation_ids('slave'):
master_configured = (leader_get('access_key'),
leader_get('secret'),
leader_get('restart_nonce'))
# Any realm or zonegroup config is present, multisite checks can be done.
if (config('realm') or config('zonegroup')):
# All of Realm, zonegroup, and zone must be configured.
if not all(multisite_config):
return ('blocked',
'multi-site configuration incomplete '
'(realm={realm}, zonegroup={zonegroup}'
', zone={zone})'.format(**config()))
if (all(multisite_config) and not
(relation_ids('master') or relation_ids('slave'))):
return ('blocked',
'multi-site configuration but master/slave '
'relation missing')
if (all(multisite_config) and relation_ids('slave')):
multisite_ready = False
for rid in relation_ids('slave'):
for unit in related_units(rid):
if relation_get('url', unit=unit, rid=rid):
multisite_ready = True
continue
if not multisite_ready:
return ('waiting',
'multi-site master relation incomplete')
master_configured = (
leader_get('access_key'),
leader_get('secret'),
leader_get('restart_nonce'),
)
if (all(multisite_config) and
relation_ids('master') and
not all(master_configured)):
return ('waiting',
'waiting for configuration of master zone')
# Master/Slave Relation should be configured.
if not (relation_ids('master') or relation_ids('slave')):
return ('blocked',
'multi-site configuration but master/slave '
'relation missing')
# Primary site status check
if relation_ids('master'):
# Migration: The system is not multisite already.
if not multisite.is_multisite_configured(config('zone'),
config('zonegroup')):
if multisite.check_cluster_has_buckets():
zones, zonegroups = get_zones_zonegroups()
status_msg = "Multiple zone or zonegroup configured, " \
"use action 'config-multisite-values' to " \
"resolve."
if (len(zonegroups) > 1 and
config('zonegroup') not in zonegroups):
return('blocked', status_msg)
if len(zones) > 1 and config('zone') not in zones:
return('blocked', status_msg)
if not all(master_configured):
return ('blocked', "Failure in Multisite migration, "
"Refer to Logs.")
# Non-Migration scenario.
if not all(master_configured):
return ('waiting',
'waiting for configuration of master zone')
# Secondary site status check
if relation_ids('slave'):
# Migration: The system is not multisite already.
if not multisite.is_multisite_configured(config('zone'),
config('zonegroup')):
if multisite.check_cluster_has_buckets():
return ('blocked',
"Non-Pristine RGW site can't be used as secondary")
multisite_ready = False
for rid in relation_ids('slave'):
for unit in related_units(rid):
if relation_get('url', unit=unit, rid=rid):
multisite_ready = True
continue
if not multisite_ready:
return ('waiting',
'multi-site master relation incomplete')
# Check that provided Ceph BlueStoe configuration is valid.
try:

29
osci.yaml
View File

@ -4,12 +4,17 @@
- charm-unit-jobs-py39
check:
jobs:
- focal-xena-multisite
- vault-focal-xena_rgw
- vault-focal-xena-namespaced
- focal-yoga-multisite:
voting: false
- vault-focal-yoga_rgw:
voting: false
- vault-focal-yoga-namespaced:
voting: false
- jammy-yoga-multisite:
voting: false
- vault-jammy-yoga_rgw:
voting: false
- vault-jammy-yoga-namespaced:
@ -18,6 +23,16 @@
needs_charm_build: true
charm_build_name: ceph-radosgw
build_type: charmcraft
- job:
name: focal-xena-multisite
parent: func-target
dependencies:
- osci-lint
- charm-build
- tox-py38
- tox-py39
vars:
tox_extra_args: focal-xena-multisite
- job:
name: vault-focal-xena_rgw
parent: func-target
@ -38,6 +53,13 @@
vars:
tox_extra_args: vault:focal-xena-namespaced
- job:
name: jammy-yoga-multisite
parent: func-target
dependencies:
- focal-xena-multisite
vars:
tox_extra_args: jammy-yoga-multisite
- job:
name: vault-jammy-yoga_rgw
parent: func-target
@ -54,6 +76,13 @@
- vault-focal-xena-namespaced
vars:
tox_extra_args: vault:jammy-yoga-namespaced
- job:
name: focal-yoga-multisite
parent: func-target
dependencies:
- focal-xena-multisite
vars:
tox_extra_args: focal-yoga-multisite
- job:
name: vault-focal-yoga_rgw
parent: func-target

98
tests/bundles/focal-xena-multisite.yaml Normal file
View File

@ -0,0 +1,98 @@
options:
source: &source cloud:focal-xena
series: focal
comment:
- 'machines section to decide order of deployment. database sooner = faster'
machines:
'0':
'1':
'2':
'3':
'4':
'5':
'6':
'7':
'8':
'9':
applications:
ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '0'
secondary-ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '1'
ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '2'
- '6'
- '7'
channel: latest/edge
secondary-ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '3'
- '8'
- '9'
channel: latest/edge
ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '4'
channel: latest/edge
secondary-ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '5'
channel: latest/edge
relations:
- - 'ceph-osd:mon'
- 'ceph-mon:osd'
- - 'ceph-radosgw:mon'
- 'ceph-mon:radosgw'
- - 'secondary-ceph-osd:mon'
- 'secondary-ceph-mon:osd'
- - 'secondary-ceph-radosgw:mon'
- 'secondary-ceph-mon:radosgw'

99
tests/bundles/focal-yoga-multisite.yaml Normal file
View File

@ -0,0 +1,99 @@
options:
source: &source cloud:focal-yoga
series: focal
comment:
- 'machines section to decide order of deployment. database sooner = faster'
machines:
'0':
'1':
'2':
'3':
'4':
'5':
'6':
'7':
'8':
'9':
applications:
ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '0'
secondary-ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '1'
ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '2'
- '6'
- '7'
channel: latest/edge
secondary-ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '3'
- '8'
- '9'
channel: latest/edge
ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '4'
channel: latest/edge
secondary-ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '5'
channel: latest/edge
relations:
- - 'ceph-osd:mon'
- 'ceph-mon:osd'
- - 'ceph-radosgw:mon'
- 'ceph-mon:radosgw'
- - 'secondary-ceph-osd:mon'
- 'secondary-ceph-mon:osd'
- - 'secondary-ceph-radosgw:mon'
- 'secondary-ceph-mon:radosgw'

99
tests/bundles/jammy-yoga-multisite.yaml Normal file
View File

@ -0,0 +1,99 @@
options:
source: &source distro
series: jammy
comment:
- 'machines section to decide order of deployment. database sooner = faster'
machines:
'0':
'1':
'2':
'3':
'4':
'5':
'6':
'7':
'8':
'9':
applications:
ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '0'
secondary-ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '1'
ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '2'
- '6'
- '7'
channel: latest/edge
secondary-ceph-osd:
charm: ch:ceph-osd
num_units: 3
constraints: "mem=2048"
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '3'
- '8'
- '9'
channel: latest/edge
ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '4'
channel: latest/edge
secondary-ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '5'
channel: latest/edge
relations:
- - 'ceph-osd:mon'
- 'ceph-mon:osd'
- - 'ceph-radosgw:mon'
- 'ceph-mon:radosgw'
- - 'secondary-ceph-osd:mon'
- 'secondary-ceph-mon:osd'
- - 'secondary-ceph-radosgw:mon'
- 'secondary-ceph-mon:radosgw'

6
tests/tests.yaml
View File

@ -1,13 +1,17 @@
charm_name: ceph-radosgw
gate_bundles:
- focal-xena-multisite
- vault: focal-xena
- vault: focal-xena-namespaced
smoke_bundles:
- focal-xena-multisite
- vault: focal-xena
dev_bundles:
- focal-yoga-multisite
- jammy-yoga-multisite
- vault: focal-yoga
- vault: focal-yoga-namespaced
- vault: jammy-yoga
@ -16,7 +20,7 @@ dev_bundles:
target_deploy_status:
vault:
workload-status: blocked
workload-status-message: Vault needs to be initialized
workload-status-message-prefix: Vault needs to be initialized
configure:
- vault:

11
unit_tests/test_actions.py
View File

@ -85,6 +85,9 @@ class MultisiteActionsTestCase(CharmTestCase):
'action_set',
'multisite',
'config',
'is_leader',
'leader_set',
'service_name',
]
def setUp(self):
@ -93,17 +96,23 @@ class MultisiteActionsTestCase(CharmTestCase):
self.config.side_effect = self.test_config.get
def test_promote(self):
self.is_leader.return_value = True
self.test_config.set('zone', 'testzone')
self.test_config.set('zonegroup', 'testzonegroup')
actions.promote([])
self.multisite.modify_zone.assert_called_once_with(
'testzone',
default=True,
master=True,
)
self.multisite.update_period.assert_called_once_with()
self.multisite.update_period.assert_called_once_with(
zonegroup='testzonegroup', zone='testzone'
)
def test_promote_unconfigured(self):
self.is_leader.return_value = True
self.test_config.set('zone', None)
self.test_config.set('zonegroup', None)
actions.promote([])
self.action_fail.assert_called_once()

5
unit_tests/test_hooks.py
View File

@ -740,7 +740,7 @@ class MasterMultisiteTests(CephRadosMultisiteTests):
)
self.multisite.update_period.assert_has_calls([
call(fatal=False),
call(),
call(zonegroup='testzonegroup', zone='testzone'),
])
self.service_restart.assert_called_once_with('rgw@hostname')
self.leader_set.assert_has_calls([
@ -827,6 +827,7 @@ class SlaveMultisiteTests(CephRadosMultisiteTests):
self.relation_get.return_value = self._test_relation
self.multisite.list_realms.return_value = []
self.multisite.list_zones.return_value = []
self.multisite.check_cluster_has_buckets.return_value = False
ceph_hooks.slave_relation_changed('slave:1', 'rgw/0')
self.config.assert_has_calls([
call('realm'),
@ -857,7 +858,7 @@ class SlaveMultisiteTests(CephRadosMultisiteTests):
)
self.multisite.update_period.assert_has_calls([
call(fatal=False),
call(),
call(zonegroup='testzonegroup', zone='testzone2'),
])
self.service_restart.assert_called_once()
self.leader_set.assert_called_once_with(restart_nonce=ANY)

169
unit_tests/test_multisite.py
View File

@ -25,6 +25,19 @@ def whoami():
return inspect.stack()[1][3]
def get_zonegroup_stub():
# populate dummy zone info
zone = {}
zone['id'] = "test_zone_id"
zone['name'] = "test_zone"
# populate dummy zonegroup info
zonegroup = {}
zonegroup['name'] = "test_zonegroup"
zonegroup['zones'] = [zone]
return zonegroup
class TestMultisiteHelpers(CharmTestCase):
TO_PATCH = [
@ -285,3 +298,159 @@ class TestMultisiteHelpers(CharmTestCase):
'--url=http://master:80',
'--access-key=testkey', '--secret=testsecret',
], stderr=mock.ANY)
def test_list_buckets(self):
self.subprocess.CalledProcessError = BaseException
multisite.list_buckets('default', 'default')
self.subprocess.check_output.assert_called_once_with([
'radosgw-admin', '--id=rgw.testhost',
'bucket', 'list', '--rgw-zone=default',
'--rgw-zonegroup=default'
], stderr=mock.ANY)
def test_rename_zonegroup(self):
multisite.rename_zonegroup('default', 'test_zone_group')
self.subprocess.call.assert_called_once_with([
'radosgw-admin', '--id=rgw.testhost',
'zonegroup', 'rename', '--rgw-zonegroup=default',
'--zonegroup-new-name=test_zone_group'
])
def test_rename_zone(self):
multisite.rename_zone('default', 'test_zone', 'test_zone_group')
self.subprocess.call.assert_called_once_with([
'radosgw-admin', '--id=rgw.testhost',
'zone', 'rename', '--rgw-zone=default',
'--zone-new-name=test_zone',
'--rgw-zonegroup=test_zone_group'
])
def test_get_zonegroup(self):
multisite.get_zonegroup_info('test_zone')
self.subprocess.check_output.assert_called_once_with([
'radosgw-admin', '--id=rgw.testhost',
'zonegroup', 'get', '--rgw-zonegroup=test_zone'
], stderr=mock.ANY)
def test_modify_zonegroup_migrate(self):
multisite.modify_zonegroup('test_zonegroup',
endpoints=['http://localhost:80'],
default=True, master=True,
realm='test_realm')
self.subprocess.check_output.assert_called_once_with([
'radosgw-admin', '--id=rgw.testhost',
'zonegroup', 'modify',
'--rgw-zonegroup=test_zonegroup', '--rgw-realm=test_realm',
'--endpoints=http://localhost:80', '--default', '--master',
], stderr=mock.ANY)
def test_modify_zone_migrate(self):
multisite.modify_zone('test_zone', default=True, master=True,
endpoints=['http://localhost:80'],
zonegroup='test_zonegroup', realm='test_realm')
self.subprocess.check_output.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zone', 'modify',
'--rgw-zone=test_zone', '--rgw-realm=test_realm',
'--rgw-zonegroup=test_zonegroup',
'--endpoints=http://localhost:80',
'--master', '--default', '--read-only=0',
], stderr=mock.ANY)
@mock.patch.object(multisite, 'list_zones')
@mock.patch.object(multisite, 'get_zonegroup_info')
def test_get_local_zone(self, mock_get_zonegroup_info, mock_list_zones):
mock_get_zonegroup_info.return_value = get_zonegroup_stub()
mock_list_zones.return_value = ['test_zone']
zone, _zonegroup = multisite.get_local_zone('test_zonegroup')
self.assertEqual(
zone,
'test_zone'
)
def test_rename_multisite_config_zonegroup_fail(self):
self.assertEqual(
multisite.rename_multisite_config(
['default'], 'test_zonegroup',
['default'], 'test_zone'
),
None
)
self.subprocess.call.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zonegroup', 'rename', '--rgw-zonegroup=default',
'--zonegroup-new-name=test_zonegroup'
])
def test_modify_multisite_config_zonegroup_fail(self):
self.assertEqual(
multisite.modify_multisite_config(
'test_zone', 'test_zonegroup',
endpoints=['http://localhost:80'],
realm='test_realm'
),
None
)
self.subprocess.check_output.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zonegroup', 'modify', '--rgw-zonegroup=test_zonegroup',
'--rgw-realm=test_realm',
'--endpoints=http://localhost:80', '--default',
'--master',
], stderr=mock.ANY)
@mock.patch.object(multisite, 'modify_zonegroup')
def test_modify_multisite_config_zone_fail(self, mock_modify_zonegroup):
mock_modify_zonegroup.return_value = True
self.assertEqual(
multisite.modify_multisite_config(
'test_zone', 'test_zonegroup',
endpoints=['http://localhost:80'],
realm='test_realm'
),
None
)
self.subprocess.check_output.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zone', 'modify',
'--rgw-zone=test_zone',
'--rgw-realm=test_realm',
'--rgw-zonegroup=test_zonegroup',
'--endpoints=http://localhost:80',
'--master', '--default', '--read-only=0',
], stderr=mock.ANY)
@mock.patch.object(multisite, 'rename_zonegroup')
def test_rename_multisite_config_zone_fail(self, mock_rename_zonegroup):
mock_rename_zonegroup.return_value = True
self.assertEqual(
multisite.rename_multisite_config(
['default'], 'test_zonegroup',
['default'], 'test_zone'
),
None
)
self.subprocess.call.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zone', 'rename', '--rgw-zone=default',
'--zone-new-name=test_zone',
'--rgw-zonegroup=test_zonegroup',
])
@mock.patch.object(multisite, 'list_zonegroups')
@mock.patch.object(multisite, 'get_local_zone')
@mock.patch.object(multisite, 'list_buckets')
def test_check_zone_has_buckets(self, mock_list_zonegroups,
mock_get_local_zone,
mock_list_buckets):
mock_list_zonegroups.return_value = ['test_zonegroup']
mock_get_local_zone.return_value = 'test_zone', 'test_zonegroup'
mock_list_buckets.return_value = ['test_bucket_1', 'test_bucket_2']
self.assertEqual(
multisite.check_cluster_has_buckets(),
True
)