openstack/charm-ceph-mon
Code Issues Proposed changes

Add set require-osd-release command to osd hook.

Browse Source 
To access all ceph features for a new release,
 require-osd-release must be set to the current
 release. Else, features will not be available
 and ceph health gives a warning on luminous.
 Here, we check to see if an osd has upgraded its
 release and notified mon. If so, we run the
 post-upgrade steps when all osds have reached
 the new release. The one (and only) step
 is to set require-osd-release if and only if
 all osds (and mons) have been upgraded to the
 same version.

Get osd release information from ceph_release key in
relation dict.

Add call to set require-osd-release to current release.

Add execute post-upgrade steps func in osd-relations hook.

Add logic for determinig whether to run set
require-osd-release command.

Add logic for checking if all osds and mons have
converged to same release.

Create func to grab all unique osd releases on each unit.

Change-Id: Ia0bc15b3b6d7e8a21fda8e2343d70d9a0024a767
Closes-Bug: #1828630
This commit is contained in:
Zachary Zehring
2019-05-20 16:33:23 -04:00
parent 1457d302b8
commit 63b38bf5ce
3 changed files with 364 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hooks/ceph_hooks.py
View File

@@ -89,6 +89,8 @@ from utils import (
get_public_addr,
get_rbd_features,
has_rbd_mirrors,
get_ceph_osd_releases,
execute_post_osd_upgrade_steps
)
from charmhelpers.contrib.charmsupport import nrpe
@@ -586,6 +588,11 @@ def osd_relation(relid=None, unit=None):
relation_set(relation_id=relid,
relation_settings=data)
if is_leader():
ceph_osd_releases = get_ceph_osd_releases()
if len(ceph_osd_releases) == 1:
execute_post_osd_upgrade_steps(ceph_osd_releases[0])
# NOTE: radosgw key provision is gated on presence of OSD
# units so ensure that any deferred hooks are processed
notify_radosgws()

111
hooks/utils.py
View File

@@ -16,6 +16,7 @@ import json
import re
import socket
import subprocess
import errno
from charmhelpers.core.hookenv import (
DEBUG,
@@ -26,6 +27,7 @@ from charmhelpers.core.hookenv import (
network_get_primary_address,
related_units,
relation_ids,
relation_get,
status_set,
unit_get,
)
@@ -50,6 +52,11 @@ except ImportError:
import dns.resolver
class OsdPostUpgradeError(Exception):
"""Error class for OSD post-upgrade operations."""
pass
def enable_pocket(pocket):
apt_sources = "/etc/apt/sources.list"
with open(apt_sources, "r") as sources:
@@ -216,3 +223,107 @@ def get_rbd_features():
return int(rbd_feature_config)
elif has_rbd_mirrors():
return add_rbd_mirror_features(get_default_rbd_features())
def get_ceph_osd_releases():
ceph_osd_releases = set()
for r_id in relation_ids('osd'):
for unit in related_units(r_id):
ceph_osd_release = relation_get(
attribute='ceph_release',
unit=unit, rid=r_id
)
if ceph_osd_release is not None:
ceph_osd_releases.add(ceph_osd_release)
return list(ceph_osd_releases)
def execute_post_osd_upgrade_steps(ceph_osd_release):
"""Executes post-upgrade steps.
Allows execution of any steps that need to be taken after osd upgrades
have finished (often specified in ceph upgrade docs).
:param str ceph_osd_release: the new ceph-osd release.
"""
log('Executing post-ceph-osd upgrade commands.')
try:
if (_all_ceph_versions_same() and
not _is_required_osd_release(ceph_osd_release)):
log('Setting require_osd_release to {}.'.format(ceph_osd_release))
_set_require_osd_release(ceph_osd_release)
except OsdPostUpgradeError as upgrade_error:
msg = 'OSD post-upgrade steps failed: {}'.format(
upgrade_error)
log(message=msg, level='ERROR')
def _all_ceph_versions_same():
"""Checks that ceph-mon and ceph-osd have converged to the same version.
:return boolean: True if all same, false if not or command failed.
"""
try:
versions_command = 'ceph versions'
versions_str = subprocess.check_output(
versions_command.split()).decode('UTF-8')
except subprocess.CalledProcessError as call_error:
if call_error.returncode == errno.EINVAL:
log('Calling "ceph versions" failed. Command requires '
'luminous and above.', level='WARNING')
return False
else:
log('Calling "ceph versions" failed.', level='ERROR')
raise OsdPostUpgradeError(call_error)
versions_dict = json.loads(versions_str)
if len(versions_dict['overall']) > 1:
log('All upgrades of mon and osd have not completed.')
return False
if len(versions_dict['osd']) < 1:
log('Monitors have converged but no osd versions found.',
level='WARNING')
return False
return True
def _is_required_osd_release(release):
"""Checks to see if require_osd_release is set to input release.
Runs and parses the ceph osd dump command to determine if
require_osd_release is set to the input release. If so, return
True. Else, return False.
:param str release: the release to check against
:return bool: True if releases match, else False.
:raises: OsdPostUpgradeError
"""
try:
dump_command = 'ceph osd dump -f json'
osd_dump_str = subprocess.check_output(
dump_command.split()).decode('UTF-8')
osd_dump_dict = json.loads(osd_dump_str)
except subprocess.CalledProcessError as cmd_error:
log(message='Command {} failed.'.format(cmd_error.cmd),
level='ERROR')
raise OsdPostUpgradeError(cmd_error)
except json.JSONDecodeError as decode_error:
log(message='Failed to decode JSON.',
level='ERROR')
raise OsdPostUpgradeError(decode_error)
return osd_dump_dict.get('require_osd_release') == release
def _set_require_osd_release(release):
"""Attempts to set the required_osd_release osd config option.
:param str release: The release to set option to
:raises: OsdPostUpgradeError
"""
try:
command = 'ceph osd require-osd-release {} ' \
'--yes-i-really-mean-it'.format(release)
subprocess.check_call(command.split())
except subprocess.CalledProcessError as call_error:
msg = 'Unable to execute command <{}>'.format(call_error.cmd)
log(message=msg, level='ERROR')
raise OsdPostUpgradeError(call_error)

246
unit_tests/test_ceph_utils.py
View File

@@ -75,3 +75,249 @@ class CephUtilsTestCase(test_utils.CharmTestCase):
self.assertEquals(utils.get_rbd_features(), 125)
_has_rbd_mirrors.return_value = False
self.assertEquals(utils.get_rbd_features(), None)
@mock.patch.object(utils, '_is_required_osd_release')
@mock.patch.object(utils, '_all_ceph_versions_same')
@mock.patch.object(utils, '_set_require_osd_release')
@mock.patch.object(utils, 'log')
def test_execute_post_osd_upgrade_steps_executes(
self, log, _set_require_osd_release,
_all_ceph_versions_same, _is_required_osd_release):
release = 'luminous'
_all_ceph_versions_same.return_value = True
_is_required_osd_release.return_value = False
utils.execute_post_osd_upgrade_steps(release)
_set_require_osd_release.assert_called_once_with(release)
@mock.patch.object(utils, '_is_required_osd_release')
@mock.patch.object(utils, '_all_ceph_versions_same')
@mock.patch.object(utils, '_set_require_osd_release')
@mock.patch.object(utils, 'log')
def test_execute_post_osd_upgrade_steps_no_exec_already_set(
self, log, _set_require_osd_release,
_all_ceph_versions_same, _is_required_osd_release):
release = 'jewel'
_all_ceph_versions_same.return_value = True
_is_required_osd_release.return_value = True
utils.execute_post_osd_upgrade_steps(release)
_set_require_osd_release.assert_not_called()
@mock.patch.object(utils, '_is_required_osd_release')
@mock.patch.object(utils, '_all_ceph_versions_same')
@mock.patch.object(utils, '_set_require_osd_release')
@mock.patch.object(utils, 'log')
def test_execute_post_osd_upgrade_steps_handle_upgrade_error(
self, log, _set_require_osd_release,
_all_ceph_versions_same, _is_required_osd_release):
release = 'luminous'
_all_ceph_versions_same.side_effect = utils.OsdPostUpgradeError()
utils.execute_post_osd_upgrade_steps(release)
log.assert_called_with(message=mock.ANY, level='ERROR')
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
@mock.patch.object(utils, 'log')
def test_all_ceph_versions_same_one_overall_one_osd_true(
self, log, json_loads, subprocess_check_output):
mock_versions_dict = dict(
osd=dict(version_1=1),
overall=dict(version_1=2)
)
json_loads.return_value = mock_versions_dict
return_bool = utils._all_ceph_versions_same()
self.assertTrue(
return_bool,
msg='all_ceph_versions_same returned False but should be True')
log.assert_not_called()
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
@mock.patch.object(utils, 'log')
def test_all_ceph_versions_same_two_overall_returns_false(
self, log, json_loads, subprocess_check_output):
mock_versions_dict = dict(
osd=dict(version_1=1),
overall=dict(version_1=1, version_2=2)
)
json_loads.return_value = mock_versions_dict
return_bool = utils._all_ceph_versions_same()
self.assertFalse(
return_bool,
msg='all_ceph_versions_same returned True but should be False')
log.assert_called_once()
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
@mock.patch.object(utils, 'log')
def test_all_ceph_versions_same_one_overall_no_osd_returns_false(
self, log, json_loads, subprocess_check_output):
mock_versions_dict = dict(
osd=dict(),
overall=dict(version_1=1)
)
json_loads.return_value = mock_versions_dict
return_bool = utils._all_ceph_versions_same()
self.assertFalse(
return_bool,
msg='all_ceph_versions_same returned True but should be False')
log.assert_called_once()
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils, 'log')
def test_all_ceph_versions_same_cmd_not_found(
self, log, subprocess_check_output):
call_exception = utils.subprocess.CalledProcessError(
22, mock.MagicMock()
)
subprocess_check_output.side_effect = call_exception
return_bool = utils._all_ceph_versions_same()
self.assertFalse(return_bool)
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils, 'log')
def test_all_ceph_versions_same_raise_error_on_unknown_rc(
self, log, subprocess_check_output):
call_exception = utils.subprocess.CalledProcessError(
0, mock.MagicMock()
)
subprocess_check_output.side_effect = call_exception
with self.assertRaises(utils.OsdPostUpgradeError):
utils._all_ceph_versions_same()
@mock.patch.object(utils.subprocess, 'check_call')
@mock.patch.object(utils, 'log')
def test_set_require_osd_release_success(self, log, check_call):
release = 'luminous'
utils._set_require_osd_release(release)
expected_call = mock.call(
['ceph', 'osd', 'require-osd-release', release]
)
check_call.has_calls(expected_call)
@mock.patch.object(utils.subprocess, 'check_call')
@mock.patch.object(utils, 'log')
def test_set_require_osd_release_raise_call_error(self, log, check_call):
release = 'luminous'
check_call.side_effect = utils.subprocess.CalledProcessError(
0, mock.mock.MagicMock()
)
expected_call = mock.call(
['ceph', 'osd', 'require-osd-release', release]
)
with self.assertRaises(utils.OsdPostUpgradeError):
utils._set_require_osd_release(release)
check_call.has_calls(expected_call)
log.assert_called_once()
@mock.patch.object(utils, 'relation_ids')
@mock.patch.object(utils, 'related_units')
@mock.patch.object(utils, 'relation_get')
def test_get_ceph_osd_releases_one_release(
self, relation_get, related_units, relation_ids):
r_ids = ['a', 'b', 'c']
r_units = ['1']
ceph_release = 'mimic'
relation_ids.return_value = r_ids
related_units.return_value = r_units
relation_get.return_value = ceph_release
releases = utils.get_ceph_osd_releases()
self.assertEqual(len(releases), 1)
self.assertEqual(releases[0], ceph_release)
@mock.patch.object(utils, 'relation_ids')
@mock.patch.object(utils, 'related_units')
@mock.patch.object(utils, 'relation_get')
def test_get_ceph_osd_releases_two_releases(
self, relation_get, related_units, relation_ids):
r_ids = ['a', 'b']
r_units = ['1']
ceph_release_1 = 'luminous'
ceph_release_2 = 'mimic'
relation_ids.return_value = r_ids
related_units.return_value = r_units
relation_get.side_effect = [ceph_release_1, ceph_release_2]
releases = utils.get_ceph_osd_releases()
self.assertEqual(len(releases), 2)
self.assertEqual(releases[0], ceph_release_1)
self.assertEqual(releases[1], ceph_release_2)
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
def test_is_required_osd_release_not_set_return_false(
self, loads, check_output):
release = 'luminous'
previous_release = 'jewel'
osd_dump_dict = dict(require_osd_release=previous_release)
loads.return_value = osd_dump_dict
return_bool = utils._is_required_osd_release(release)
self.assertFalse(return_bool)
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
def test_is_required_osd_release_is_set_return_true(
self, loads, check_output):
release = 'luminous'
osd_dump_dict = dict(require_osd_release=release)
loads.return_value = osd_dump_dict
return_bool = utils._is_required_osd_release(release)
self.assertTrue(return_bool)
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
def test_is_required_osd_release_subprocess_error(self, loads,
check_output):
release = 'luminous'
call_exception = utils.subprocess.CalledProcessError(
0, mock.MagicMock()
)
check_output.side_effect = call_exception
with self.assertRaises(utils.OsdPostUpgradeError):
utils._is_required_osd_release(release)
@mock.patch.object(utils.subprocess, 'check_output')
@mock.patch.object(utils.json, 'loads')
def test_is_required_osd_release_json_loads_error(self, loads,
check_output):
release = 'luminous'
call_exception = utils.json.JSONDecodeError(
'', mock.MagicMock(), 0
)
loads.side_effect = call_exception
with self.assertRaises(utils.OsdPostUpgradeError):
utils._is_required_osd_release(release)