Wait until clustered before running client hooks
RabbitMQ takes some time to fully cluster. The charm was previously running amqp-relation-changed hooks whenever they were queued even if the cluster was not yet complete. This led to split brain scenarios. Client authentication to one or more nodes could fail. This change confirms the entire cluster is ready before running client amqp-relation-changed hooks. min-cluster-size can now be used to attempt to guarantee the cluster is ready with the expected number of nodes. If min-cluster-size is not set the charm will still determine based on the information available if all the cluster nodes are ready. Single node deployments are still possible. Partial-Bug: #1657245 Closes-Bug: #1657176 Change-Id: I870df71869c979e65a3a8764efdf35a746278507
This commit is contained in:
parent
5b0d68c2fd
commit
2472e1ca9f
21
README.md
21
README.md
@ -15,11 +15,32 @@ To deploy this charm:
|
||||
deploying multiple units will form a native RabbitMQ cluster:
|
||||
|
||||
juju deploy -n 3 rabbitmq-server
|
||||
juju config rabbitmq-server min-cluster-size=3
|
||||
|
||||
To make use of AMQP services, simply relate other charms that support the rabbitmq interface:
|
||||
|
||||
juju add-relation rabbitmq-server nova-cloud-controller
|
||||
|
||||
# Clustering
|
||||
|
||||
When more than one unit of the charm is deployed the charm will bring up a
|
||||
native RabbitMQ cluster. The process of clustering the units together takes
|
||||
some time. Due to the nature of asynchronous hook execution it is possible
|
||||
client relationship hooks may be executed before the cluster is complete.
|
||||
In some cases, this can lead to client charm errors.
|
||||
|
||||
To guarantee client relation hooks will not be executed until clustering is
|
||||
completed use the min-cluster-size configuration setting:
|
||||
|
||||
juju deploy -n 3 rabbitmq-server
|
||||
juju config rabbitmq-server min-cluster-size=3
|
||||
|
||||
When min-cluster-size is not set the charm will still cluster, however,
|
||||
there are no guarantees client relation hooks will not execute before it is
|
||||
complete.
|
||||
|
||||
Single unit deployments behave as expected.
|
||||
|
||||
# Configuration: SSL
|
||||
|
||||
Generate an unencrypted RSA private key for the servers and a certificate:
|
||||
|
@ -48,15 +48,19 @@ from charmhelpers.core.hookenv import (
|
||||
relation_ids,
|
||||
related_units,
|
||||
log, ERROR,
|
||||
WARNING,
|
||||
INFO, DEBUG,
|
||||
service_name,
|
||||
status_set,
|
||||
cached,
|
||||
unit_get,
|
||||
relation_set,
|
||||
relation_get,
|
||||
application_version_set,
|
||||
config,
|
||||
network_get_primary_address,
|
||||
is_leader,
|
||||
leader_get,
|
||||
)
|
||||
|
||||
from charmhelpers.core.host import (
|
||||
@ -258,7 +262,7 @@ def set_ha_mode(vhost, mode, params=None, sync_mode='automatic'):
|
||||
|
||||
if caching_cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
|
||||
log(("Mirroring queues cannot be enabled, only supported "
|
||||
"in rabbitmq-server >= 3.0"), level='WARN')
|
||||
"in rabbitmq-server >= 3.0"), level=WARNING)
|
||||
log(("More information at http://www.rabbitmq.com/blog/"
|
||||
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
|
||||
return
|
||||
@ -285,7 +289,7 @@ def clear_ha_mode(vhost, name='HA', force=False):
|
||||
"""
|
||||
if cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
|
||||
log(("Mirroring queues not supported "
|
||||
"in rabbitmq-server >= 3.0"), level='WARN')
|
||||
"in rabbitmq-server >= 3.0"), level=WARNING)
|
||||
log(("More information at http://www.rabbitmq.com/blog/"
|
||||
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
|
||||
return
|
||||
@ -305,7 +309,7 @@ def set_all_mirroring_queues(enable):
|
||||
"""
|
||||
if cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
|
||||
log(("Mirroring queues not supported "
|
||||
"in rabbitmq-server >= 3.0"), level='WARN')
|
||||
"in rabbitmq-server >= 3.0"), level=WARNING)
|
||||
log(("More information at http://www.rabbitmq.com/blog/"
|
||||
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
|
||||
return
|
||||
@ -410,7 +414,7 @@ def cluster_with():
|
||||
join_cluster(node)
|
||||
# NOTE: toggle the cluster relation to ensure that any peers
|
||||
# already clustered re-assess status correctly
|
||||
relation_set(clustered=get_unit_hostname())
|
||||
relation_set(clustered=get_unit_hostname(), timestamp=time.time())
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
status_set('blocked', 'Failed to cluster with %s. Exception: %s'
|
||||
@ -701,9 +705,9 @@ def get_node_hostname(ip_addr):
|
||||
nodename = get_hostname(ip_addr, fqdn=False)
|
||||
except:
|
||||
log('Cannot resolve hostname for %s using DNS servers' % ip_addr,
|
||||
level='WARNING')
|
||||
level=WARNING)
|
||||
log('Falling back to use socket.gethostname()',
|
||||
level='WARNING')
|
||||
level=WARNING)
|
||||
# If the private-address is not resolvable using DNS
|
||||
# then use the current hostname
|
||||
nodename = socket.gethostname()
|
||||
@ -734,8 +738,11 @@ def assess_cluster_status(*args):
|
||||
''' Assess the status for the current running unit '''
|
||||
# NOTE: ensure rabbitmq is actually installed before doing
|
||||
# any checks
|
||||
if os.path.exists(RABBITMQ_CTL):
|
||||
if rabbitmq_is_installed():
|
||||
# Clustering Check
|
||||
if not is_sufficient_peers():
|
||||
return 'waiting', ("Waiting for all {} peers to complete the "
|
||||
"cluster.".format(config('min-cluster-size')))
|
||||
peer_ids = relation_ids('cluster')
|
||||
if peer_ids and len(related_units(peer_ids[0])):
|
||||
if not clustered():
|
||||
@ -912,3 +919,128 @@ def get_unit_hostname():
|
||||
@returns hostname
|
||||
"""
|
||||
return socket.gethostname()
|
||||
|
||||
|
||||
def is_sufficient_peers():
|
||||
"""Sufficient number of expected peers to build a complete cluster
|
||||
|
||||
If min-cluster-size has been provided, check that we have sufficient
|
||||
number of peers as expected for a complete cluster.
|
||||
|
||||
If not defined assume a single unit.
|
||||
|
||||
@returns boolean
|
||||
"""
|
||||
min_size = config('min-cluster-size')
|
||||
if min_size:
|
||||
log("Checking for minimum of {} peer units".format(min_size),
|
||||
level=DEBUG)
|
||||
|
||||
# Include this unit
|
||||
units = 1
|
||||
for rid in relation_ids('cluster'):
|
||||
units += len(related_units(rid))
|
||||
|
||||
if units < min_size:
|
||||
log("Insufficient number of peer units to form cluster "
|
||||
"(expected=%s, got=%s)" % (min_size, units), level=INFO)
|
||||
return False
|
||||
else:
|
||||
log("Sufficient number of peer units to form cluster {}"
|
||||
"".format(min_size, level=DEBUG))
|
||||
return True
|
||||
else:
|
||||
log("min-cluster-size is not defined, race conditions may occur if "
|
||||
"this is not a single unit deployment.", level=WARNING)
|
||||
return True
|
||||
|
||||
|
||||
def rabbitmq_is_installed():
|
||||
"""Determine if rabbitmq is installed
|
||||
|
||||
@returns boolean
|
||||
"""
|
||||
return os.path.exists(RABBITMQ_CTL)
|
||||
|
||||
|
||||
def cluster_ready():
|
||||
"""Determine if each node in the cluster is ready and the cluster is
|
||||
complete with the expected number of peers.
|
||||
|
||||
Once cluster_ready returns True it is safe to execute client relation
|
||||
hooks. Having min-cluster-size set will guarantee cluster_ready will not
|
||||
return True until the expected number of peers are clustered and ready.
|
||||
|
||||
If min-cluster-size is not set it must assume the cluster is ready in order
|
||||
to allow for single unit deployments.
|
||||
|
||||
@returns boolean
|
||||
"""
|
||||
min_size = config('min-cluster-size')
|
||||
units = 1
|
||||
for relation_id in relation_ids('cluster'):
|
||||
units += len(related_units(relation_id))
|
||||
if not min_size:
|
||||
min_size = units
|
||||
|
||||
if not is_sufficient_peers():
|
||||
return False
|
||||
elif min_size > 1:
|
||||
if not clustered():
|
||||
return False
|
||||
clustered_units = 1
|
||||
for relation_id in relation_ids('cluster'):
|
||||
for remote_unit in related_units(relation_id):
|
||||
if not relation_get(attribute='clustered',
|
||||
rid=relation_id,
|
||||
unit=remote_unit):
|
||||
log("{} is not yet clustered".format(remote_unit),
|
||||
DEBUG)
|
||||
return False
|
||||
else:
|
||||
clustered_units += 1
|
||||
if clustered_units < min_size:
|
||||
log("Fewer than minimum cluster size:{} rabbit units reporting "
|
||||
"clustered".format(min_size),
|
||||
DEBUG)
|
||||
return False
|
||||
else:
|
||||
log("All {} rabbit units reporting clustered"
|
||||
"".format(min_size),
|
||||
DEBUG)
|
||||
return True
|
||||
|
||||
log("Must assume this is a single unit returning 'cluster' ready", DEBUG)
|
||||
return True
|
||||
|
||||
|
||||
def client_node_is_ready():
|
||||
"""Determine if the leader node has set amqp client data
|
||||
|
||||
@returns boolean
|
||||
"""
|
||||
# Bail if this unit is paused
|
||||
if is_unit_paused_set():
|
||||
return False
|
||||
for rid in relation_ids('amqp'):
|
||||
if leader_get(attribute='{}_password'.format(rid)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def leader_node_is_ready():
|
||||
"""Determine if the leader node is ready to handle client relationship
|
||||
hooks.
|
||||
|
||||
IFF rabbit is not paused, is installed, this is the leader node and the
|
||||
cluster is complete.
|
||||
|
||||
@returns boolean
|
||||
"""
|
||||
# Paused check must run before other checks
|
||||
# Bail if this unit is paused
|
||||
if is_unit_paused_set():
|
||||
return False
|
||||
return (rabbitmq_is_installed() and
|
||||
is_leader() and
|
||||
cluster_ready())
|
||||
|
@ -135,11 +135,24 @@ def configure_amqp(username, vhost, admin=False):
|
||||
return password
|
||||
|
||||
|
||||
def update_clients():
|
||||
"""Update amqp client relation hooks
|
||||
|
||||
IFF leader node is ready. Client nodes are considered ready once the leader
|
||||
has already run amqp_changed.
|
||||
"""
|
||||
if rabbit.leader_node_is_ready() or rabbit.client_node_is_ready():
|
||||
for rid in relation_ids('amqp'):
|
||||
for unit in related_units(rid):
|
||||
amqp_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
|
||||
@hooks.hook('amqp-relation-changed')
|
||||
def amqp_changed(relation_id=None, remote_unit=None):
|
||||
host_addr = rabbit.get_unit_ip()
|
||||
|
||||
if not is_elected_leader('res_rabbitmq_vip'):
|
||||
# TODO: Simplify what the non-leader needs to do
|
||||
if not is_leader() and rabbit.client_node_is_ready():
|
||||
# NOTE(jamespage) clear relation to deal with data being
|
||||
# removed from peer storage
|
||||
relation_clear(relation_id)
|
||||
@ -155,7 +168,7 @@ def amqp_changed(relation_id=None, remote_unit=None):
|
||||
relation_set(relation_id=rel_id, **peerdb_settings)
|
||||
|
||||
log('amqp_changed(): Deferring amqp_changed'
|
||||
' to is_elected_leader.')
|
||||
' to the leader.')
|
||||
|
||||
# NOTE: active/active case
|
||||
if config('prefer-ipv6'):
|
||||
@ -165,6 +178,10 @@ def amqp_changed(relation_id=None, remote_unit=None):
|
||||
|
||||
return
|
||||
|
||||
# Bail if not completely ready
|
||||
if not rabbit.leader_node_is_ready():
|
||||
return
|
||||
|
||||
relation_settings = {}
|
||||
settings = relation_get(rid=relation_id, unit=remote_unit)
|
||||
|
||||
@ -217,45 +234,6 @@ def amqp_changed(relation_id=None, remote_unit=None):
|
||||
relation_settings=relation_settings)
|
||||
|
||||
|
||||
def is_sufficient_peers():
|
||||
"""If min-cluster-size has been provided, check that we have sufficient
|
||||
number of peers to proceed with creating rabbitmq cluster.
|
||||
"""
|
||||
min_size = config('min-cluster-size')
|
||||
leader_election_available = True
|
||||
try:
|
||||
is_leader()
|
||||
except NotImplementedError:
|
||||
leader_election_available = False
|
||||
|
||||
if min_size:
|
||||
# Use min-cluster-size if we don't have Juju leader election.
|
||||
if not leader_election_available:
|
||||
log("Waiting for minimum of %d peer units since there's no Juju "
|
||||
"leader election" % (min_size))
|
||||
size = 0
|
||||
for rid in relation_ids('cluster'):
|
||||
size = len(related_units(rid))
|
||||
|
||||
# Include this unit
|
||||
size += 1
|
||||
if size < min_size:
|
||||
log("Insufficient number of peer units to form cluster "
|
||||
"(expected=%s, got=%s)" % (min_size, size), level=INFO)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
else:
|
||||
log("Ignoring min-cluster-size in favour of Juju leader election")
|
||||
return True
|
||||
elif leader_election_available:
|
||||
log("min-cluster-size is not defined, using juju leader-election.")
|
||||
else:
|
||||
log("min-cluster-size is not defined and juju leader election is not "
|
||||
"available!", level="WARNING")
|
||||
return True
|
||||
|
||||
|
||||
@hooks.hook('cluster-relation-joined')
|
||||
def cluster_joined(relation_id=None):
|
||||
relation_settings = {
|
||||
@ -286,10 +264,7 @@ def cluster_joined(relation_id=None):
|
||||
level=ERROR)
|
||||
return
|
||||
|
||||
if not is_sufficient_peers():
|
||||
return
|
||||
|
||||
if is_elected_leader('res_rabbitmq_vip'):
|
||||
if is_leader():
|
||||
log('Leader peer_storing cookie', level=INFO)
|
||||
cookie = open(rabbit.COOKIE_PATH, 'r').read().strip()
|
||||
peer_store('cookie', cookie)
|
||||
@ -298,9 +273,10 @@ def cluster_joined(relation_id=None):
|
||||
|
||||
|
||||
@hooks.hook('cluster-relation-changed')
|
||||
def cluster_changed():
|
||||
def cluster_changed(relation_id=None, remote_unit=None):
|
||||
# Future travelers beware ordering is significant
|
||||
rdata = relation_get()
|
||||
rdata = relation_get(rid=relation_id, unit=remote_unit)
|
||||
|
||||
# sync passwords
|
||||
blacklist = ['hostname', 'private-address', 'public-address']
|
||||
whitelist = [a for a in rdata.keys() if a not in blacklist]
|
||||
@ -308,10 +284,9 @@ def cluster_changed():
|
||||
|
||||
cookie = peer_retrieve('cookie')
|
||||
if not cookie:
|
||||
log('cluster_joined: cookie not yet set.', level=INFO)
|
||||
log('cluster_changed: cookie not yet set.', level=INFO)
|
||||
return
|
||||
|
||||
rdata = relation_get()
|
||||
if rdata:
|
||||
hostname = rdata.get('hostname', None)
|
||||
private_address = rdata.get('private-address', None)
|
||||
@ -319,11 +294,6 @@ def cluster_changed():
|
||||
if hostname and private_address:
|
||||
rabbit.update_hosts_file({private_address: hostname})
|
||||
|
||||
if not is_sufficient_peers():
|
||||
log('Not enough peers, waiting until leader is configured',
|
||||
level=INFO)
|
||||
return
|
||||
|
||||
# sync the cookie with peers if necessary
|
||||
update_cookie()
|
||||
|
||||
@ -334,20 +304,9 @@ def cluster_changed():
|
||||
return
|
||||
|
||||
# cluster with node?
|
||||
try:
|
||||
if not is_leader():
|
||||
rabbit.cluster_with()
|
||||
update_nrpe_checks()
|
||||
except NotImplementedError:
|
||||
if is_newer():
|
||||
rabbit.cluster_with()
|
||||
update_nrpe_checks()
|
||||
|
||||
# If cluster has changed peer db may have changed so run amqp_changed
|
||||
# to sync any changes
|
||||
for rid in relation_ids('amqp'):
|
||||
for unit in related_units(rid):
|
||||
amqp_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
|
||||
def update_cookie(leaders_cookie=None):
|
||||
@ -465,10 +424,6 @@ def ha_changed():
|
||||
log('ha_changed(): We are now HA clustered. '
|
||||
'Advertising our VIP (%s) to all AMQP clients.' %
|
||||
vip)
|
||||
# need to re-authenticate all clients since node-name changed.
|
||||
for rid in relation_ids('amqp'):
|
||||
for unit in related_units(rid):
|
||||
amqp_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
|
||||
@hooks.hook('ceph-relation-joined')
|
||||
@ -640,10 +595,15 @@ def config_changed():
|
||||
rabbit.disable_plugin(MAN_PLUGIN)
|
||||
close_port(55672)
|
||||
|
||||
rabbit.set_all_mirroring_queues(config('mirroring-queues'))
|
||||
rabbit.ConfigRenderer(
|
||||
rabbit.CONFIG_FILES).write_all()
|
||||
|
||||
# Only set values if this is the leader
|
||||
if not is_leader():
|
||||
return
|
||||
|
||||
rabbit.set_all_mirroring_queues(config('mirroring-queues'))
|
||||
|
||||
if is_relation_made("ha"):
|
||||
ha_is_active_active = config("ha-vip-only")
|
||||
|
||||
@ -658,12 +618,16 @@ def config_changed():
|
||||
else:
|
||||
update_nrpe_checks()
|
||||
|
||||
# NOTE(jamespage)
|
||||
# trigger amqp_changed to pickup and changes to network
|
||||
# configuration via the access-network config option.
|
||||
for rid in relation_ids('amqp'):
|
||||
# Update cluster in case min-cluster-size has changed
|
||||
for rid in relation_ids('cluster'):
|
||||
for unit in related_units(rid):
|
||||
amqp_changed(relation_id=rid, remote_unit=unit)
|
||||
cluster_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
|
||||
@hooks.hook('leader-elected')
|
||||
def leader_elected():
|
||||
status_set("maintenance", "{} is the elected leader".format(local_unit()))
|
||||
|
||||
|
||||
@hooks.hook('leader-settings-changed')
|
||||
def leader_settings_changed():
|
||||
@ -682,12 +646,6 @@ def leader_settings_changed():
|
||||
for rid in relation_ids('cluster'):
|
||||
relation_set(relation_id=rid, relation_settings={'cookie': cookie})
|
||||
|
||||
# If leader has changed and access credentials, ripple these
|
||||
# out from all units
|
||||
for rid in relation_ids('amqp'):
|
||||
for unit in related_units(rid):
|
||||
amqp_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
|
||||
def pre_install_hooks():
|
||||
for f in glob.glob('exec.d/*/charm-pre-install'):
|
||||
@ -706,4 +664,6 @@ if __name__ == '__main__':
|
||||
hooks.execute(sys.argv)
|
||||
except UnregisteredHookError as e:
|
||||
log('Unknown hook {} - skipping.'.format(e))
|
||||
# Gated client updates
|
||||
update_clients()
|
||||
rabbit.assess_status(rabbit.ConfigRenderer(rabbit.CONFIG_FILES))
|
||||
|
0
tests/gate-basic-yakkety-newton
Executable file → Normal file
0
tests/gate-basic-yakkety-newton
Executable file → Normal file
@ -14,12 +14,12 @@
|
||||
|
||||
import mock
|
||||
import os
|
||||
import unittest
|
||||
import tempfile
|
||||
import sys
|
||||
import collections
|
||||
from functools import wraps
|
||||
|
||||
from test_utils import CharmTestCase
|
||||
|
||||
with mock.patch('charmhelpers.core.hookenv.cached') as cached:
|
||||
def passthrough(func):
|
||||
@ -33,8 +33,20 @@ with mock.patch('charmhelpers.core.hookenv.cached') as cached:
|
||||
|
||||
sys.modules['MySQLdb'] = mock.Mock()
|
||||
|
||||
TO_PATCH = [
|
||||
# charmhelpers.core.hookenv
|
||||
'is_leader',
|
||||
'related_units',
|
||||
'relation_ids',
|
||||
'relation_get',
|
||||
'relation_set',
|
||||
'leader_get',
|
||||
'config',
|
||||
'is_unit_paused_set',
|
||||
]
|
||||
|
||||
class ConfigRendererTests(unittest.TestCase):
|
||||
|
||||
class ConfigRendererTests(CharmTestCase):
|
||||
|
||||
class FakeContext(object):
|
||||
def __call__(self, *a, **k):
|
||||
@ -49,7 +61,8 @@ class ConfigRendererTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
def setUp(self):
|
||||
super(ConfigRendererTests, self).setUp()
|
||||
super(ConfigRendererTests, self).setUp(rabbit_utils,
|
||||
TO_PATCH)
|
||||
self.renderer = rabbit_utils.ConfigRenderer(
|
||||
self.config_map)
|
||||
|
||||
@ -83,9 +96,10 @@ RABBITMQCTL_CLUSTERSTATUS_SOLO = """Cluster status of node 'rabbit@juju-devel3-m
|
||||
"""
|
||||
|
||||
|
||||
class UtilsTests(unittest.TestCase):
|
||||
class UtilsTests(CharmTestCase):
|
||||
def setUp(self):
|
||||
super(UtilsTests, self).setUp()
|
||||
super(UtilsTests, self).setUp(rabbit_utils,
|
||||
TO_PATCH)
|
||||
|
||||
@mock.patch("rabbit_utils.log")
|
||||
def test_update_empty_hosts_file(self, mock_log):
|
||||
@ -346,3 +360,159 @@ class UtilsTests(unittest.TestCase):
|
||||
check_call.return_value = 0
|
||||
local_nodename.return_value = 'rabbitmq-server-0'
|
||||
self.assertTrue(rabbit_utils.wait_app())
|
||||
|
||||
@mock.patch.object(rabbit_utils, 'is_leader')
|
||||
@mock.patch.object(rabbit_utils, 'related_units')
|
||||
@mock.patch.object(rabbit_utils, 'relation_ids')
|
||||
@mock.patch.object(rabbit_utils, 'config')
|
||||
def test_is_sufficient_peers(self, mock_config, mock_relation_ids,
|
||||
mock_related_units, mock_is_leader):
|
||||
# With leadership Election
|
||||
mock_is_leader.return_value = False
|
||||
_config = {'min-cluster-size': None}
|
||||
mock_config.side_effect = lambda key: _config.get(key)
|
||||
self.assertTrue(rabbit_utils.is_sufficient_peers())
|
||||
|
||||
mock_is_leader.return_value = False
|
||||
mock_relation_ids.return_value = ['cluster:0']
|
||||
mock_related_units.return_value = ['test/0']
|
||||
_config = {'min-cluster-size': 3}
|
||||
mock_config.side_effect = lambda key: _config.get(key)
|
||||
self.assertFalse(rabbit_utils.is_sufficient_peers())
|
||||
|
||||
mock_is_leader.return_value = False
|
||||
mock_related_units.return_value = ['test/0', 'test/1']
|
||||
_config = {'min-cluster-size': 3}
|
||||
mock_config.side_effect = lambda key: _config.get(key)
|
||||
self.assertTrue(rabbit_utils.is_sufficient_peers())
|
||||
|
||||
@mock.patch.object(rabbit_utils.os.path, 'exists')
|
||||
def test_rabbitmq_is_installed(self, mock_os_exists):
|
||||
mock_os_exists.return_value = True
|
||||
self.assertTrue(rabbit_utils.rabbitmq_is_installed())
|
||||
|
||||
mock_os_exists.return_value = False
|
||||
self.assertFalse(rabbit_utils.rabbitmq_is_installed())
|
||||
|
||||
@mock.patch.object(rabbit_utils, 'clustered')
|
||||
@mock.patch.object(rabbit_utils, 'rabbitmq_is_installed')
|
||||
@mock.patch.object(rabbit_utils, 'is_sufficient_peers')
|
||||
def test_cluster_ready(self, mock_is_sufficient_peers,
|
||||
mock_rabbitmq_is_installed, mock_clustered):
|
||||
|
||||
# Not sufficient number of peers
|
||||
mock_is_sufficient_peers.return_value = False
|
||||
self.assertFalse(rabbit_utils.cluster_ready())
|
||||
|
||||
# This unit not yet clustered
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = ['cluster:0']
|
||||
self.related_units.return_value = ['test/0', 'test/1']
|
||||
self.relation_get.return_value = 'teset/0'
|
||||
_config = {'min-cluster-size': 3}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = False
|
||||
self.assertFalse(rabbit_utils.cluster_ready())
|
||||
|
||||
# Not all cluster ready
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = ['cluster:0']
|
||||
self.related_units.return_value = ['test/0', 'test/1']
|
||||
self.relation_get.return_value = False
|
||||
_config = {'min-cluster-size': 3}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = True
|
||||
self.assertFalse(rabbit_utils.cluster_ready())
|
||||
|
||||
# All cluster ready
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = ['cluster:0']
|
||||
self.related_units.return_value = ['test/0', 'test/1']
|
||||
self.relation_get.return_value = 'teset/0'
|
||||
_config = {'min-cluster-size': 3}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = True
|
||||
self.assertTrue(rabbit_utils.cluster_ready())
|
||||
|
||||
# Not all cluster ready no min-cluster-size
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = ['cluster:0']
|
||||
self.related_units.return_value = ['test/0', 'test/1']
|
||||
self.relation_get.return_value = False
|
||||
_config = {'min-cluster-size': None}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = True
|
||||
self.assertFalse(rabbit_utils.cluster_ready())
|
||||
|
||||
# All cluster ready no min-cluster-size
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = ['cluster:0']
|
||||
self.related_units.return_value = ['test/0', 'test/1']
|
||||
self.relation_get.return_value = 'teset/0'
|
||||
_config = {'min-cluster-size': None}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = True
|
||||
self.assertTrue(rabbit_utils.cluster_ready())
|
||||
|
||||
# Assume single unit no-min-cluster-size
|
||||
mock_is_sufficient_peers.return_value = True
|
||||
self.relation_ids.return_value = []
|
||||
self.related_units.return_value = []
|
||||
self.relation_get.return_value = None
|
||||
_config = {'min-cluster-size': None}
|
||||
self.config.side_effect = lambda key: _config.get(key)
|
||||
mock_clustered.return_value = True
|
||||
self.assertTrue(rabbit_utils.cluster_ready())
|
||||
|
||||
def test_client_node_is_ready(self):
|
||||
# Paused
|
||||
self.is_unit_paused_set.return_value = True
|
||||
self.assertFalse(rabbit_utils.client_node_is_ready())
|
||||
|
||||
# Not ready
|
||||
self.is_unit_paused_set.return_value = False
|
||||
self.relation_ids.return_value = ['amqp:0']
|
||||
self.leader_get.return_value = {}
|
||||
self.assertFalse(rabbit_utils.client_node_is_ready())
|
||||
|
||||
# Ready
|
||||
self.is_unit_paused_set.return_value = False
|
||||
self.relation_ids.return_value = ['amqp:0']
|
||||
self.leader_get.return_value = {'amqp:0_password': 'password'}
|
||||
self.assertTrue(rabbit_utils.client_node_is_ready())
|
||||
|
||||
@mock.patch.object(rabbit_utils, 'cluster_ready')
|
||||
@mock.patch.object(rabbit_utils, 'rabbitmq_is_installed')
|
||||
def test_leader_node_is_ready(self, mock_rabbitmq_is_installed,
|
||||
mock_cluster_ready):
|
||||
# Paused
|
||||
self.is_unit_paused_set.return_value = True
|
||||
self.assertFalse(rabbit_utils.leader_node_is_ready())
|
||||
|
||||
# Not installed
|
||||
self.is_unit_paused_set.return_value = False
|
||||
mock_rabbitmq_is_installed.return_value = False
|
||||
self.is_leader.return_value = True
|
||||
mock_cluster_ready.return_value = True
|
||||
self.assertFalse(rabbit_utils.leader_node_is_ready())
|
||||
|
||||
# Not leader
|
||||
self.is_unit_paused_set.return_value = False
|
||||
mock_rabbitmq_is_installed.return_value = True
|
||||
self.is_leader.return_value = False
|
||||
mock_cluster_ready.return_value = True
|
||||
self.assertFalse(rabbit_utils.leader_node_is_ready())
|
||||
|
||||
# Not clustered
|
||||
self.is_unit_paused_set.return_value = False
|
||||
mock_rabbitmq_is_installed.return_value = True
|
||||
self.is_leader.return_value = True
|
||||
mock_cluster_ready.return_value = False
|
||||
self.assertFalse(rabbit_utils.leader_node_is_ready())
|
||||
|
||||
# Leader ready
|
||||
self.is_unit_paused_set.return_value = False
|
||||
mock_rabbitmq_is_installed.return_value = True
|
||||
self.is_leader.return_value = True
|
||||
mock_cluster_ready.return_value = True
|
||||
self.assertTrue(rabbit_utils.leader_node_is_ready())
|
||||
|
@ -15,7 +15,7 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from testtools import TestCase
|
||||
from test_utils import CharmTestCase
|
||||
from mock import patch, MagicMock
|
||||
|
||||
os.environ['JUJU_UNIT_NAME'] = 'UNIT_TEST/0' # noqa - needed for import
|
||||
@ -31,12 +31,21 @@ with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
|
||||
lambda *args, **kwargs: f(*args, **kwargs))
|
||||
import rabbitmq_server_relations
|
||||
|
||||
TO_PATCH = [
|
||||
# charmhelpers.core.hookenv
|
||||
'is_leader',
|
||||
'relation_ids',
|
||||
'related_units',
|
||||
]
|
||||
|
||||
class RelationUtil(TestCase):
|
||||
|
||||
class RelationUtil(CharmTestCase):
|
||||
def setUp(self):
|
||||
self.fake_repo = {}
|
||||
super(RelationUtil, self).setUp()
|
||||
super(RelationUtil, self).setUp(rabbitmq_server_relations,
|
||||
TO_PATCH)
|
||||
|
||||
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
|
||||
@patch('rabbitmq_server_relations.peer_store_and_set')
|
||||
@patch('rabbitmq_server_relations.config')
|
||||
@patch('rabbitmq_server_relations.relation_set')
|
||||
@ -56,7 +65,8 @@ class RelationUtil(TestCase):
|
||||
cmp_pkgrevno,
|
||||
relation_set,
|
||||
mock_config,
|
||||
mock_peer_store_and_set):
|
||||
mock_peer_store_and_set,
|
||||
mock_leader_node_is_ready):
|
||||
"""
|
||||
Compare version above and below 3.0.1.
|
||||
Make sure ha_queues is set correctly on each side.
|
||||
@ -68,6 +78,7 @@ class RelationUtil(TestCase):
|
||||
|
||||
return None
|
||||
|
||||
mock_leader_node_is_ready.return_value = True
|
||||
mock_config.side_effect = config
|
||||
host_addr = "10.1.2.3"
|
||||
get_unit_ip.return_value = host_addr
|
||||
@ -90,6 +101,7 @@ class RelationUtil(TestCase):
|
||||
'hostname': host_addr},
|
||||
relation_id=None)
|
||||
|
||||
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
|
||||
@patch('rabbitmq_server_relations.peer_store_and_set')
|
||||
@patch('rabbitmq_server_relations.config')
|
||||
@patch('rabbitmq_server_relations.relation_set')
|
||||
@ -109,7 +121,8 @@ class RelationUtil(TestCase):
|
||||
cmp_pkgrevno,
|
||||
relation_set,
|
||||
mock_config,
|
||||
mock_peer_store_and_set):
|
||||
mock_peer_store_and_set,
|
||||
mock_leader_node_is_ready):
|
||||
"""
|
||||
Compare version above and below 3.0.1.
|
||||
Make sure ha_queues is set correctly on each side.
|
||||
@ -121,6 +134,7 @@ class RelationUtil(TestCase):
|
||||
|
||||
return None
|
||||
|
||||
mock_leader_node_is_ready.return_value = True
|
||||
mock_config.side_effect = config
|
||||
ipv6_addr = "2001:db8:1:0:f816:3eff:fed6:c140"
|
||||
get_unit_ip.return_value = ipv6_addr
|
||||
@ -143,40 +157,41 @@ class RelationUtil(TestCase):
|
||||
'hostname': ipv6_addr},
|
||||
relation_id=None)
|
||||
|
||||
@patch.object(rabbitmq_server_relations, 'is_leader')
|
||||
@patch.object(rabbitmq_server_relations, 'related_units')
|
||||
@patch.object(rabbitmq_server_relations, 'relation_ids')
|
||||
@patch.object(rabbitmq_server_relations, 'config')
|
||||
def test_is_sufficient_peers(self, mock_config, mock_relation_ids,
|
||||
mock_related_units, mock_is_leader):
|
||||
# With leadership Election
|
||||
mock_is_leader.return_value = False
|
||||
_config = {'min-cluster-size': None}
|
||||
mock_config.side_effect = lambda key: _config.get(key)
|
||||
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
|
||||
@patch('rabbitmq_server_relations.amqp_changed')
|
||||
@patch('rabbitmq_server_relations.rabbit.client_node_is_ready')
|
||||
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
|
||||
def test_update_clients(self, mock_leader_node_is_ready,
|
||||
mock_client_node_is_ready,
|
||||
mock_amqp_changed):
|
||||
# Not ready
|
||||
mock_client_node_is_ready.return_value = False
|
||||
mock_leader_node_is_ready.return_value = False
|
||||
rabbitmq_server_relations.update_clients()
|
||||
self.assertFalse(mock_amqp_changed.called)
|
||||
|
||||
mock_is_leader.return_value = False
|
||||
mock_relation_ids.return_value = ['cluster:0']
|
||||
mock_related_units.return_value = ['test/0']
|
||||
_config = {'min-cluster-size': 3}
|
||||
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
|
||||
# Leader Ready
|
||||
self.relation_ids.return_value = ['amqp:0']
|
||||
self.related_units.return_value = ['client/0']
|
||||
mock_leader_node_is_ready.return_value = True
|
||||
mock_client_node_is_ready.return_value = False
|
||||
rabbitmq_server_relations.update_clients()
|
||||
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
|
||||
remote_unit='client/0')
|
||||
|
||||
mock_is_leader.return_value = False
|
||||
mock_related_units.return_value = ['test/0', 'test/1']
|
||||
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
|
||||
# Client Ready
|
||||
self.relation_ids.return_value = ['amqp:0']
|
||||
self.related_units.return_value = ['client/0']
|
||||
mock_leader_node_is_ready.return_value = False
|
||||
mock_client_node_is_ready.return_value = True
|
||||
rabbitmq_server_relations.update_clients()
|
||||
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
|
||||
remote_unit='client/0')
|
||||
|
||||
# Without leadership Election
|
||||
mock_is_leader.side_effect = NotImplementedError
|
||||
_config = {'min-cluster-size': None}
|
||||
mock_config.side_effect = lambda key: _config.get(key)
|
||||
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
|
||||
|
||||
mock_is_leader.side_effect = NotImplementedError
|
||||
mock_relation_ids.return_value = ['cluster:0']
|
||||
mock_related_units.return_value = ['test/0']
|
||||
_config = {'min-cluster-size': 3}
|
||||
self.assertFalse(rabbitmq_server_relations.is_sufficient_peers())
|
||||
|
||||
mock_is_leader.side_effect = NotImplementedError
|
||||
mock_related_units.return_value = ['test/0', 'test/1']
|
||||
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
|
||||
# Both Ready
|
||||
self.relation_ids.return_value = ['amqp:0']
|
||||
self.related_units.return_value = ['client/0']
|
||||
mock_leader_node_is_ready.return_value = True
|
||||
mock_client_node_is_ready.return_value = True
|
||||
rabbitmq_server_relations.update_clients()
|
||||
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
|
||||
remote_unit='client/0')
|
||||
|
Loading…
Reference in New Issue
Block a user