Merge "Notify peers bootstrap-uuid during upgrade-charm hook" into stable/17.02
This commit is contained in:
commit
260f37482f
|
@ -26,6 +26,7 @@ from charmhelpers.core.hookenv import (
|
||||||
network_get_primary_address,
|
network_get_primary_address,
|
||||||
charm_name,
|
charm_name,
|
||||||
leader_get,
|
leader_get,
|
||||||
|
status_set,
|
||||||
)
|
)
|
||||||
from charmhelpers.core.host import (
|
from charmhelpers.core.host import (
|
||||||
service_restart,
|
service_restart,
|
||||||
|
@ -96,6 +97,8 @@ from percona_utils import (
|
||||||
get_cluster_host_ip,
|
get_cluster_host_ip,
|
||||||
client_node_is_ready,
|
client_node_is_ready,
|
||||||
leader_node_is_ready,
|
leader_node_is_ready,
|
||||||
|
update_bootstrap_uuid,
|
||||||
|
LeaderNoBootstrapUUIDError,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -239,15 +242,28 @@ def update_shared_db_rels():
|
||||||
@harden()
|
@harden()
|
||||||
def upgrade():
|
def upgrade():
|
||||||
|
|
||||||
if leader_node_is_ready():
|
if is_leader():
|
||||||
# If this is the leader but we have not yet broadcast the cluster uuid
|
if is_unit_paused_set():
|
||||||
# then do so now.
|
log('Unit is paused, skiping upgrade', level=INFO)
|
||||||
|
return
|
||||||
|
|
||||||
|
# broadcast the bootstrap-uuid
|
||||||
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
|
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
|
||||||
if wsrep_ready.lower() in ['on', 'ready']:
|
if wsrep_ready.lower() in ['on', 'ready']:
|
||||||
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
|
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
|
||||||
if cluster_state_uuid:
|
if cluster_state_uuid:
|
||||||
mark_seeded()
|
mark_seeded()
|
||||||
notify_bootstrapped(cluster_uuid=cluster_state_uuid)
|
notify_bootstrapped(cluster_uuid=cluster_state_uuid)
|
||||||
|
else:
|
||||||
|
# Ensure all the peers have the bootstrap-uuid attribute set
|
||||||
|
# as this is all happening during the upgrade-charm hook is reasonable
|
||||||
|
# to expect the cluster is running.
|
||||||
|
|
||||||
|
# Wait until the leader has set the
|
||||||
|
try:
|
||||||
|
update_bootstrap_uuid()
|
||||||
|
except LeaderNoBootstrapUUIDError:
|
||||||
|
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
|
||||||
|
|
||||||
config_changed()
|
config_changed()
|
||||||
|
|
||||||
|
@ -671,6 +687,14 @@ def ha_relation_changed():
|
||||||
def leader_settings_changed():
|
def leader_settings_changed():
|
||||||
# Notify any changes to data in leader storage
|
# Notify any changes to data in leader storage
|
||||||
update_shared_db_rels()
|
update_shared_db_rels()
|
||||||
|
log('leader-settings-changed', level='DEBUG')
|
||||||
|
try:
|
||||||
|
update_bootstrap_uuid()
|
||||||
|
except LeaderNoBootstrapUUIDError:
|
||||||
|
# until the bootstrap-uuid attribute is not replicated cluster_ready()
|
||||||
|
# will evaluate to False, so it is necessary to feed back this info
|
||||||
|
# to the user.
|
||||||
|
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
|
||||||
|
|
||||||
|
|
||||||
@hooks.hook('nrpe-external-master-relation-joined',
|
@hooks.hook('nrpe-external-master-relation-joined',
|
||||||
|
|
|
@ -72,6 +72,33 @@ HOSTS_FILE = '/etc/hosts'
|
||||||
REQUIRED_INTERFACES = {}
|
REQUIRED_INTERFACES = {}
|
||||||
|
|
||||||
|
|
||||||
|
class LeaderNoBootstrapUUIDError(Exception):
|
||||||
|
"""Raised when the leader doesn't have set the bootstrap-uuid attribute"""
|
||||||
|
def __init__(self):
|
||||||
|
super(LeaderNoBootstrapUUIDError, self).__init__(
|
||||||
|
"the leader doesn't have set the bootstrap-uuid attribute")
|
||||||
|
|
||||||
|
|
||||||
|
class InconsistentUUIDError(Exception):
|
||||||
|
"""Raised when the leader and the unit have different UUIDs set"""
|
||||||
|
def __init__(self, leader_uuid, unit_uuid):
|
||||||
|
super(InconsistentUUIDError, self).__init__(
|
||||||
|
"Leader UUID ('%s') != Unit UUID ('%s')" % (leader_uuid,
|
||||||
|
unit_uuid))
|
||||||
|
|
||||||
|
|
||||||
|
class FakeOSConfigRenderer(object):
|
||||||
|
"""This class is to provide to register_configs() as a 'fake'
|
||||||
|
OSConfigRenderer object that has a complete_contexts method that returns
|
||||||
|
an empty list. This is so that the pause/resume framework can be used
|
||||||
|
from charmhelpers that requires configs to be able to run.
|
||||||
|
This is a bit of a hack, but via Python's duck-typing enables the function
|
||||||
|
to work.
|
||||||
|
"""
|
||||||
|
def complete_contexts(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def determine_packages():
|
def determine_packages():
|
||||||
if lsb_release()['DISTRIB_CODENAME'] >= 'wily':
|
if lsb_release()['DISTRIB_CODENAME'] >= 'wily':
|
||||||
# NOTE(beisner): Use recommended mysql-client package
|
# NOTE(beisner): Use recommended mysql-client package
|
||||||
|
@ -428,6 +455,48 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None):
|
||||||
leader_set(**{'bootstrap-uuid': cluster_uuid})
|
leader_set(**{'bootstrap-uuid': cluster_uuid})
|
||||||
|
|
||||||
|
|
||||||
|
def update_bootstrap_uuid():
|
||||||
|
"""This function verifies if the leader has set the bootstrap-uuid
|
||||||
|
attribute to then check it against the running cluster uuid, if the check
|
||||||
|
succeeds the bootstrap-uuid field is set in the cluster relation.
|
||||||
|
|
||||||
|
:returns: True if the cluster UUID was updated, False if the local UUID is
|
||||||
|
empty.
|
||||||
|
"""
|
||||||
|
|
||||||
|
lead_cluster_state_uuid = leader_get('bootstrap-uuid')
|
||||||
|
if not lead_cluster_state_uuid:
|
||||||
|
log('Leader has not set bootstrap-uuid', level=DEBUG)
|
||||||
|
raise LeaderNoBootstrapUUIDError()
|
||||||
|
|
||||||
|
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
|
||||||
|
log("wsrep_ready: '%s'" % wsrep_ready, DEBUG)
|
||||||
|
if wsrep_ready.lower() in ['on', 'ready']:
|
||||||
|
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
|
||||||
|
else:
|
||||||
|
cluster_state_uuid = None
|
||||||
|
|
||||||
|
if not cluster_state_uuid:
|
||||||
|
log("UUID is empty: '%s'" % cluster_state_uuid, level=DEBUG)
|
||||||
|
return False
|
||||||
|
elif lead_cluster_state_uuid != cluster_state_uuid:
|
||||||
|
# this may mean 2 things:
|
||||||
|
# 1) the units have diverged, which it's bad and we do stop.
|
||||||
|
# 2) cluster_state_uuid could not be retrieved because it
|
||||||
|
# hasn't been bootstrapped, mysqld is stopped, etc.
|
||||||
|
log('bootstrap uuid differs: %s != %s' % (lead_cluster_state_uuid,
|
||||||
|
cluster_state_uuid),
|
||||||
|
level=ERROR)
|
||||||
|
raise InconsistentUUIDError(lead_cluster_state_uuid,
|
||||||
|
cluster_state_uuid)
|
||||||
|
|
||||||
|
for rid in relation_ids('cluster'):
|
||||||
|
notify_bootstrapped(cluster_rid=rid,
|
||||||
|
cluster_uuid=cluster_state_uuid)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def cluster_in_sync():
|
def cluster_in_sync():
|
||||||
'''
|
'''
|
||||||
Determines whether the current unit is in sync
|
Determines whether the current unit is in sync
|
||||||
|
@ -479,18 +548,6 @@ def resolve_cnf_file():
|
||||||
return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf'
|
return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf'
|
||||||
|
|
||||||
|
|
||||||
class FakeOSConfigRenderer(object):
|
|
||||||
"""This class is to provide to register_configs() as a 'fake'
|
|
||||||
OSConfigRenderer object that has a complete_contexts method that returns
|
|
||||||
an empty list. This is so that the pause/resume framework can be used
|
|
||||||
from charmhelpers that requires configs to be able to run.
|
|
||||||
This is a bit of a hack, but via Python's duck-typing enables the function
|
|
||||||
to work.
|
|
||||||
"""
|
|
||||||
def complete_contexts(self):
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def register_configs():
|
def register_configs():
|
||||||
"""Return a OSConfigRenderer object.
|
"""Return a OSConfigRenderer object.
|
||||||
However, ceph-mon wasn't written using OSConfigRenderer objects to do the
|
However, ceph-mon wasn't written using OSConfigRenderer objects to do the
|
||||||
|
|
|
@ -113,6 +113,27 @@ class BasicDeployment(OpenStackAmuletDeployment):
|
||||||
for unit in self.d.sentry['percona-cluster']:
|
for unit in self.d.sentry['percona-cluster']:
|
||||||
assert self.is_mysqld_running(unit), 'mysql not running: %s' % unit
|
assert self.is_mysqld_running(unit), 'mysql not running: %s' % unit
|
||||||
|
|
||||||
|
self.test_bootstrap_uuid_set_in_the_relation()
|
||||||
|
|
||||||
|
def test_bootstrap_uuid_set_in_the_relation(self):
|
||||||
|
"""Verify that the bootstrap-uuid attribute was set by the leader and
|
||||||
|
all the peers where notified.
|
||||||
|
"""
|
||||||
|
(leader_uuid, code) = self.master_unit.run("leader-get bootstrap-uuid")
|
||||||
|
assert leader_uuid
|
||||||
|
|
||||||
|
cmd_rel_get = ("relation-get -r `relation-ids cluster` "
|
||||||
|
"bootstrap-uuid %s")
|
||||||
|
units = self.d.sentry['percona-cluster']
|
||||||
|
for unit in units:
|
||||||
|
for peer in units:
|
||||||
|
cmd = cmd_rel_get % peer.info['unit_name']
|
||||||
|
self.log.debug(cmd)
|
||||||
|
(output, code) = unit.run(cmd)
|
||||||
|
assert code == 0
|
||||||
|
assert output == leader_uuid, "%s != %s" % (output,
|
||||||
|
leader_uuid)
|
||||||
|
|
||||||
def find_master(self, ha=True):
|
def find_master(self, ha=True):
|
||||||
for unit in self.d.sentry['percona-cluster']:
|
for unit in self.d.sentry['percona-cluster']:
|
||||||
if not ha:
|
if not ha:
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
import sys
|
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
from test_utils import CharmTestCase
|
from test_utils import CharmTestCase
|
||||||
|
|
||||||
|
@ -14,6 +16,7 @@ with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
|
||||||
lambda *args, **kwargs: f(*args, **kwargs))
|
lambda *args, **kwargs: f(*args, **kwargs))
|
||||||
import percona_hooks as hooks
|
import percona_hooks as hooks
|
||||||
|
|
||||||
|
|
||||||
TO_PATCH = ['log', 'config',
|
TO_PATCH = ['log', 'config',
|
||||||
'get_db_helper',
|
'get_db_helper',
|
||||||
'relation_ids',
|
'relation_ids',
|
||||||
|
@ -206,3 +209,65 @@ class TestNRPERelation(CharmTestCase):
|
||||||
hooks.update_nrpe_config()
|
hooks.update_nrpe_config()
|
||||||
self.nrpe.add_init_service_checks.assert_called_once_with(
|
self.nrpe.add_init_service_checks.assert_called_once_with(
|
||||||
mock.ANY, ["mysql"], mock.ANY)
|
mock.ANY, ["mysql"], mock.ANY)
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpgradeCharm(CharmTestCase):
|
||||||
|
TO_PATCH = [
|
||||||
|
'config',
|
||||||
|
'log',
|
||||||
|
'is_leader',
|
||||||
|
'is_unit_paused_set',
|
||||||
|
'get_wsrep_value',
|
||||||
|
'config_changed',
|
||||||
|
]
|
||||||
|
|
||||||
|
def print_log(self, msg, level=None):
|
||||||
|
print('juju-log: %s: %s' % (level, msg))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
CharmTestCase.setUp(self, hooks, self.TO_PATCH)
|
||||||
|
self.config.side_effect = self.test_config.get
|
||||||
|
self.log.side_effect = self.print_log
|
||||||
|
self.tmpdir = tempfile.mkdtemp()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
CharmTestCase.tearDown(self)
|
||||||
|
try:
|
||||||
|
shutil.rmtree(self.tmpdir)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@mock.patch('percona_utils.is_leader')
|
||||||
|
@mock.patch('percona_utils.leader_set')
|
||||||
|
@mock.patch('percona_utils.relation_set')
|
||||||
|
@mock.patch('percona_utils.get_wsrep_value')
|
||||||
|
@mock.patch('percona_utils.relation_ids')
|
||||||
|
@mock.patch('percona_utils.resolve_data_dir')
|
||||||
|
def test_upgrade_charm(self, mock_data_dir, mock_rids, mock_wsrep,
|
||||||
|
mock_rset, mock_lset, mock_is_leader):
|
||||||
|
mock_rids.return_value = ['cluster:22']
|
||||||
|
mock_is_leader.return_value = True
|
||||||
|
self.is_leader.return_value = True
|
||||||
|
self.is_unit_paused_set.return_value = False
|
||||||
|
|
||||||
|
def c(k):
|
||||||
|
values = {'wsrep_ready': 'on',
|
||||||
|
'wsrep_cluster_state_uuid': '1234-abcd'}
|
||||||
|
return values[k]
|
||||||
|
|
||||||
|
self.get_wsrep_value.side_effect = c
|
||||||
|
mock_wsrep.side_effect = c
|
||||||
|
mock_data_dir.return_value = self.tmpdir
|
||||||
|
|
||||||
|
hooks.upgrade()
|
||||||
|
|
||||||
|
seeded_file = os.path.join(self.tmpdir, 'seeded')
|
||||||
|
self.assertTrue(os.path.isfile(seeded_file),
|
||||||
|
"%s is not file" % seeded_file)
|
||||||
|
with open(seeded_file) as f:
|
||||||
|
self.assertEqual(f.read(), 'done')
|
||||||
|
|
||||||
|
mock_rset.assert_called_with(relation_id='cluster:22',
|
||||||
|
**{'bootstrap-uuid': '1234-abcd'})
|
||||||
|
mock_lset.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
|
||||||
|
self.config_changed.assert_called_with()
|
||||||
|
|
|
@ -447,3 +447,74 @@ class UtilsTestsCTC(CharmTestCase):
|
||||||
self.is_leader.return_value = True
|
self.is_leader.return_value = True
|
||||||
mock_cluster_ready.return_value = True
|
mock_cluster_ready.return_value = True
|
||||||
self.assertTrue(percona_utils.leader_node_is_ready())
|
self.assertTrue(percona_utils.leader_node_is_ready())
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdateBootstrapUUID(CharmTestCase):
|
||||||
|
TO_PATCH = [
|
||||||
|
'log',
|
||||||
|
'leader_get',
|
||||||
|
'get_wsrep_value',
|
||||||
|
'relation_ids',
|
||||||
|
'relation_set',
|
||||||
|
'is_leader',
|
||||||
|
'leader_set',
|
||||||
|
]
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
CharmTestCase.setUp(self, percona_utils, self.TO_PATCH)
|
||||||
|
self.log.side_effect = self.juju_log
|
||||||
|
|
||||||
|
def juju_log(self, msg, level=None):
|
||||||
|
print('juju-log %s: %s' % (level, msg))
|
||||||
|
|
||||||
|
def test_no_bootstrap_uuid(self):
|
||||||
|
self.leader_get.return_value = None
|
||||||
|
self.assertRaises(percona_utils.LeaderNoBootstrapUUIDError,
|
||||||
|
percona_utils.update_bootstrap_uuid)
|
||||||
|
|
||||||
|
def test_bootstrap_uuid_already_set(self):
|
||||||
|
self.leader_get.return_value = '1234-abcd'
|
||||||
|
|
||||||
|
def fake_wsrep(k):
|
||||||
|
d = {'wsrep_ready': 'ON',
|
||||||
|
'wsrep_cluster_state_uuid': '1234-abcd'}
|
||||||
|
return d[k]
|
||||||
|
|
||||||
|
self.get_wsrep_value.side_effect = fake_wsrep
|
||||||
|
self.relation_ids.return_value = ['cluster:2']
|
||||||
|
self.is_leader.return_value = False
|
||||||
|
percona_utils.update_bootstrap_uuid()
|
||||||
|
self.relation_set.assert_called_with(relation_id='cluster:2',
|
||||||
|
**{'bootstrap-uuid': '1234-abcd'})
|
||||||
|
self.leader_set.assert_not_called()
|
||||||
|
|
||||||
|
self.is_leader.return_value = True
|
||||||
|
percona_utils.update_bootstrap_uuid()
|
||||||
|
self.relation_set.assert_called_with(relation_id='cluster:2',
|
||||||
|
**{'bootstrap-uuid': '1234-abcd'})
|
||||||
|
self.leader_set.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
|
||||||
|
|
||||||
|
@mock.patch.object(percona_utils, 'notify_bootstrapped')
|
||||||
|
def test_bootstrap_uuid_could_not_be_retrieved(self, mock_notify):
|
||||||
|
self.leader_get.return_value = '1234-abcd'
|
||||||
|
|
||||||
|
def fake_wsrep(k):
|
||||||
|
d = {'wsrep_ready': 'ON',
|
||||||
|
'wsrep_cluster_state_uuid': ''}
|
||||||
|
return d[k]
|
||||||
|
|
||||||
|
self.get_wsrep_value.side_effect = fake_wsrep
|
||||||
|
self.assertFalse(percona_utils.update_bootstrap_uuid())
|
||||||
|
mock_notify.assert_not_called()
|
||||||
|
|
||||||
|
def test_bootstrap_uuid_diffent_uuids(self):
|
||||||
|
self.leader_get.return_value = '1234-abcd'
|
||||||
|
|
||||||
|
def fake_wsrep(k):
|
||||||
|
d = {'wsrep_ready': 'ON',
|
||||||
|
'wsrep_cluster_state_uuid': '5678-dead-beef'}
|
||||||
|
return d[k]
|
||||||
|
|
||||||
|
self.get_wsrep_value.side_effect = fake_wsrep
|
||||||
|
self.assertRaises(percona_utils.InconsistentUUIDError,
|
||||||
|
percona_utils.update_bootstrap_uuid)
|
||||||
|
|
Loading…
Reference in New Issue