Restart mcollective on slave nodes after restore
Also, on the backup step a status of `mco ping` is serialized in an upgrade tarball and on the restore step it is compared with the actual status. All nodes that are not respond are logged. In additional, the status of `mco ping` is archived on the backup step and is compared on the restore step with the actual ones. Change-Id: Ibba81102214998d83614a42cdb21c21bebd8284a Related-Bug: #1561092
This commit is contained in:
parent
66e7a8c493
commit
f91ff40264
|
@ -18,6 +18,7 @@ from octane.handlers.backup_restore import cobbler
|
||||||
from octane.handlers.backup_restore import fuel_keys
|
from octane.handlers.backup_restore import fuel_keys
|
||||||
from octane.handlers.backup_restore import fuel_uuid
|
from octane.handlers.backup_restore import fuel_uuid
|
||||||
from octane.handlers.backup_restore import logs
|
from octane.handlers.backup_restore import logs
|
||||||
|
from octane.handlers.backup_restore import mcollective
|
||||||
from octane.handlers.backup_restore import mirrors
|
from octane.handlers.backup_restore import mirrors
|
||||||
from octane.handlers.backup_restore import nailgun_plugins
|
from octane.handlers.backup_restore import nailgun_plugins
|
||||||
from octane.handlers.backup_restore import postgres
|
from octane.handlers.backup_restore import postgres
|
||||||
|
@ -48,6 +49,7 @@ ARCHIVATORS = [
|
||||||
version.VersionArchivator,
|
version.VersionArchivator,
|
||||||
nailgun_plugins.NailgunPluginsArchivator,
|
nailgun_plugins.NailgunPluginsArchivator,
|
||||||
puppet.PuppetApplyTasks,
|
puppet.PuppetApplyTasks,
|
||||||
|
mcollective.McollectiveArchivator,
|
||||||
]
|
]
|
||||||
|
|
||||||
REPO_ARCHIVATORS = [
|
REPO_ARCHIVATORS = [
|
||||||
|
|
|
@ -0,0 +1,53 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import tarfile
|
||||||
|
|
||||||
|
from fuelclient import objects
|
||||||
|
|
||||||
|
from octane.handlers.backup_restore import base
|
||||||
|
from octane.util import fuel_client
|
||||||
|
from octane.util import mcollective
|
||||||
|
from octane.util import node as node_util
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class McollectiveArchivator(base.Base):
|
||||||
|
filename = "mco/ping.json"
|
||||||
|
|
||||||
|
def backup(self):
|
||||||
|
status = mcollective.get_mco_ping_status()
|
||||||
|
content = json.dumps(status)
|
||||||
|
info = tarfile.TarInfo(self.filename)
|
||||||
|
info.size = len(content)
|
||||||
|
fileobj = io.BytesIO(content)
|
||||||
|
self.archive.addfile(info, fileobj=fileobj)
|
||||||
|
|
||||||
|
def restore(self):
|
||||||
|
with fuel_client.set_auth_context(self.context):
|
||||||
|
nodes = objects.Node.get_all()
|
||||||
|
for node in nodes:
|
||||||
|
node_util.restart_mcollective(node)
|
||||||
|
content = self.archive.extractfile(self.filename)
|
||||||
|
if content is not None:
|
||||||
|
orig_status = json.load(content)
|
||||||
|
new_status = mcollective.get_mco_ping_status()
|
||||||
|
offline = mcollective.compair_mco_ping_statuses(orig_status,
|
||||||
|
new_status)
|
||||||
|
if offline:
|
||||||
|
LOG.warning("Some nodes went offline after the upgrade of the "
|
||||||
|
"master node (check them manually): %s",
|
||||||
|
", ".join(offline))
|
|
@ -20,6 +20,7 @@ from octane.handlers.backup_restore import base
|
||||||
from octane.handlers.backup_restore import cobbler
|
from octane.handlers.backup_restore import cobbler
|
||||||
from octane.handlers.backup_restore import fuel_keys
|
from octane.handlers.backup_restore import fuel_keys
|
||||||
from octane.handlers.backup_restore import fuel_uuid
|
from octane.handlers.backup_restore import fuel_uuid
|
||||||
|
from octane.handlers.backup_restore import mcollective
|
||||||
from octane.handlers.backup_restore import mirrors
|
from octane.handlers.backup_restore import mirrors
|
||||||
from octane.handlers.backup_restore import nailgun_plugins
|
from octane.handlers.backup_restore import nailgun_plugins
|
||||||
from octane.handlers.backup_restore import postgres
|
from octane.handlers.backup_restore import postgres
|
||||||
|
@ -271,3 +272,18 @@ def test_repos_backup(
|
||||||
def test_archivator_name(mocker, name, expected_name):
|
def test_archivator_name(mocker, name, expected_name):
|
||||||
|
|
||||||
assert expected_name == type(name, (base.Base, ), {})(None).archivator_name
|
assert expected_name == type(name, (base.Base, ), {})(None).archivator_name
|
||||||
|
|
||||||
|
|
||||||
|
def test_mcollective_backup(mocker):
|
||||||
|
archive = mock.Mock()
|
||||||
|
mocker.patch("octane.util.mcollective.get_mco_ping_status")
|
||||||
|
mock_json = mocker.patch("json.dumps")
|
||||||
|
mock_json.return_value = "{}"
|
||||||
|
mock_info = mocker.patch("tarfile.TarInfo")
|
||||||
|
mock_io = mocker.patch("io.BytesIO")
|
||||||
|
mcollective.McollectiveArchivator(archive).backup()
|
||||||
|
archive.addfile.assert_called_once_with(
|
||||||
|
mock_info.return_value, fileobj=mock_io.return_value)
|
||||||
|
mock_io.assert_called_once_with(mock_json.return_value)
|
||||||
|
mock_info.assert_called_once_with("mco/ping.json")
|
||||||
|
assert mock_info.return_value.size == len(mock_json.return_value)
|
||||||
|
|
|
@ -24,6 +24,7 @@ from octane.handlers.backup_restore import cobbler
|
||||||
from octane.handlers.backup_restore import fuel_keys
|
from octane.handlers.backup_restore import fuel_keys
|
||||||
from octane.handlers.backup_restore import fuel_uuid
|
from octane.handlers.backup_restore import fuel_uuid
|
||||||
from octane.handlers.backup_restore import logs
|
from octane.handlers.backup_restore import logs
|
||||||
|
from octane.handlers.backup_restore import mcollective
|
||||||
from octane.handlers.backup_restore import mirrors
|
from octane.handlers.backup_restore import mirrors
|
||||||
from octane.handlers.backup_restore import postgres
|
from octane.handlers.backup_restore import postgres
|
||||||
from octane.handlers.backup_restore import puppet
|
from octane.handlers.backup_restore import puppet
|
||||||
|
@ -677,3 +678,43 @@ def test_admin_network_restore(mocker, members, is_exist):
|
||||||
mock_puppet.assert_called_once_with('dhcp-ranges')
|
mock_puppet.assert_called_once_with('dhcp-ranges')
|
||||||
else:
|
else:
|
||||||
mock_puppet.assert_not_called()
|
mock_puppet.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("members", "check_status"), [
|
||||||
|
([TestMember("mco/ping.json", True, False)], True),
|
||||||
|
([], False),
|
||||||
|
])
|
||||||
|
def test_mcollective_restore(mocker, members, check_status):
|
||||||
|
nodes = [mock.Mock(), mock.Mock()]
|
||||||
|
mocker.patch("octane.util.fuel_client.set_auth_context")
|
||||||
|
mock_get = mocker.patch("fuelclient.objects.Node.get_all")
|
||||||
|
mock_get.return_value = nodes
|
||||||
|
mock_restart = mocker.patch("octane.util.node.restart_mcollective")
|
||||||
|
mock_json = mocker.patch("json.load")
|
||||||
|
mock_status = mocker.patch("octane.util.mcollective.get_mco_ping_status")
|
||||||
|
mock_cmp = mocker.patch(
|
||||||
|
"octane.util.mcollective.compair_mco_ping_statuses")
|
||||||
|
mock_cmp.return_value = set(["1"])
|
||||||
|
mock_log = mocker.patch(
|
||||||
|
"octane.handlers.backup_restore.mcollective.LOG.warning")
|
||||||
|
|
||||||
|
archive = TestArchive(members, mcollective.McollectiveArchivator)
|
||||||
|
mcollective.McollectiveArchivator(archive).restore()
|
||||||
|
assert mock_restart.call_args_list == [
|
||||||
|
mock.call(node) for node in nodes
|
||||||
|
]
|
||||||
|
if check_status:
|
||||||
|
effective = [
|
||||||
|
member
|
||||||
|
for member in members if member.name == "mco/ping.json"
|
||||||
|
][-1]
|
||||||
|
assert effective
|
||||||
|
mock_json.assert_called_once_with(effective)
|
||||||
|
mock_status.assert_called_once_with()
|
||||||
|
mock_cmp.assert_called_once_with(
|
||||||
|
mock_json.return_value, mock_status.return_value)
|
||||||
|
mock_log.assert_called_once_with(mock.ANY, "1")
|
||||||
|
else:
|
||||||
|
assert not mock_json.called
|
||||||
|
assert not mock_status.called
|
||||||
|
assert not mock_cmp.called
|
||||||
|
|
|
@ -0,0 +1,36 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from octane.util import mcollective
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("status", [
|
||||||
|
{"a": "b", "c": "d"},
|
||||||
|
])
|
||||||
|
def test_get_mco_ping_status(mocker, status):
|
||||||
|
stdout = io.BytesIO(json.dumps(status))
|
||||||
|
mock_popen = mocker.patch("octane.util.subprocess.popen")
|
||||||
|
mock_popen.return_value.__enter__.return_value.stdout = stdout
|
||||||
|
result = mcollective.get_mco_ping_status()
|
||||||
|
assert result == status
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("orig", "new", "offline"), [
|
||||||
|
([{"sender": 1}, {"sender": 2}], [{"sender": 1}], set([2])),
|
||||||
|
])
|
||||||
|
def test_compair_mco_ping_statuses(mocker, orig, new, offline):
|
||||||
|
assert mcollective.compair_mco_ping_statuses(orig, new) == offline
|
|
@ -242,3 +242,16 @@ def test_get_parameters(mocker, parameters, parameters_to_get, required,
|
||||||
mock_get.assert_called_once_with(
|
mock_get.assert_called_once_with(
|
||||||
mock_sftp.return_value.open.return_value.__enter__.return_value,
|
mock_sftp.return_value.open.return_value.__enter__.return_value,
|
||||||
parameters_to_get)
|
parameters_to_get)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("online", "result", "error"), [
|
||||||
|
(True, True, False),
|
||||||
|
(False, None, False),
|
||||||
|
(True, False, True),
|
||||||
|
])
|
||||||
|
def test_restart_mcollective(mocker, online, result, error):
|
||||||
|
node = mock.Mock(data={"online": online, "id": 123})
|
||||||
|
mock_ssh = mocker.patch("octane.util.ssh.call")
|
||||||
|
if error:
|
||||||
|
mock_ssh.side_effect = Exception()
|
||||||
|
assert node_util.restart_mcollective(node) == result
|
||||||
|
|
|
@ -0,0 +1,31 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from octane.util import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def get_mco_ping_status(node_id=None):
|
||||||
|
cmd = ["mco", "rpc", "rpcutil", "ping", "--json"]
|
||||||
|
if node_id is not None:
|
||||||
|
cmd.extend(["-I", str(node_id)])
|
||||||
|
with subprocess.popen(cmd, stdout=subprocess.PIPE) as proc:
|
||||||
|
return json.load(proc.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def compair_mco_ping_statuses(orig_status, new_status):
|
||||||
|
# NOTE(akcram): Statuses are present only for alive nodes.
|
||||||
|
orig_ids = {resp["sender"] for resp in orig_status}
|
||||||
|
new_ids = {resp["sender"] for resp in new_status}
|
||||||
|
offline = orig_ids - new_ids
|
||||||
|
return offline
|
|
@ -212,3 +212,21 @@ def get_parameters(node, filename, parameters_to_get, ensure=True):
|
||||||
raise AbsentParametersError(
|
raise AbsentParametersError(
|
||||||
node.data["id"], filename, flat_parameters)
|
node.data["id"], filename, flat_parameters)
|
||||||
return parameters
|
return parameters
|
||||||
|
|
||||||
|
|
||||||
|
def restart_mcollective(node):
|
||||||
|
node_id = node.data["id"]
|
||||||
|
if not node.data["online"]:
|
||||||
|
LOG.warning("Not possible to restart mcollective on the offline "
|
||||||
|
"node %s", node_id)
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
ssh.call(["service", "mcollective", "restart"], node=node)
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.warning("Failed to restart mcollective on the node %s: %s",
|
||||||
|
node_id, exc)
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
LOG.info("The mcollective service was successfully restarted on "
|
||||||
|
"the node %s", node_id)
|
||||||
|
return True
|
||||||
|
|
Loading…
Reference in New Issue