From c1ea3bdc2edbdc5683c277c67521619e7f0198cd Mon Sep 17 00:00:00 2001 From: Sergey Abramov Date: Tue, 2 Aug 2016 20:17:19 +0300 Subject: [PATCH] Set up repos for ceph upgrade in separate source * Set up repos for upgrade ceph on ceph_osd to separate source. * add preference file for ceph source on upgrade-osd step with highest Pin-Priority * add tests Change-Id: I1781eb4aa3e66b6e464256ab9b24e39f6a6d0b3d Closes-bug: 1585204 (cherry picked from commit 13043d0eb0cf21cc4252cb42e5e4f7532bc05c12) --- octane/commands/osd_upgrade.py | 86 +++++++---- octane/magic_consts.py | 24 +-- octane/tests/test_osd_upgrade.py | 254 ++++++++++++++++--------------- 3 files changed, 195 insertions(+), 169 deletions(-) diff --git a/octane/commands/osd_upgrade.py b/octane/commands/osd_upgrade.py index 7ae3150b..81a4d5d2 100644 --- a/octane/commands/osd_upgrade.py +++ b/octane/commands/osd_upgrade.py @@ -10,15 +10,17 @@ # License for the specific language governing permissions and limitations # under the License. +import contextlib import logging import os from cliff import command as cmd -from fuelclient.objects import node as node_obj +from fuelclient.objects import environment as env_obj from octane.handlers import backup_restore from octane import magic_consts +from octane.util import env from octane.util import fuel_client from octane.util import helpers from octane.util import ssh @@ -39,46 +41,66 @@ def _get_backup_path(path, node): node=node) +def write_content_to_tmp_file_on_node(node, content, directory, template): + tmp_name = ssh.call_output( + ["mktemp", "-p", directory, "-t", template], node=node).strip() + sftp = ssh.sftp(node) + with sftp.open(tmp_name, "w") as new: + new.write(content) + return tmp_name + + +@contextlib.contextmanager +def applied_repos(nodes, preference_priority): + admin_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]["ipaddress"] + packages = " ".join(magic_consts.OSD_UPGRADE_REQUIRED_PACKAGES) + preference_content = magic_consts.OSD_UPGADE_PREFERENCE_TEMPLATE.format( + packages=packages, priority=preference_priority) + source_content = magic_consts.OSD_UPGRADE_SOURCE_TEMPLATE.format( + admin_ip=admin_ip) + + node_file_to_clear_list = [] + try: + for node in nodes: + source = write_content_to_tmp_file_on_node( + node, source_content, + "/etc/apt/sources.list.d/", "mos.osd_XXX.list") + node_file_to_clear_list.append((node, source)) + preference = write_content_to_tmp_file_on_node( + node, preference_content, + "/etc/apt/preferences.d/", "mos.osd_XXX.pref") + node_file_to_clear_list.append((node, preference)) + yield + finally: + for node, file_name_to_remove in node_file_to_clear_list: + sftp = ssh.sftp(node) + sftp.unlink(file_name_to_remove) + + +def get_repo_highest_priority(orig_env): + editable = orig_env.get_attributes()['editable'] + repos = editable['repo_setup']['repos']['value'] + return max([i['priority'] for i in repos]) + + def upgrade_osd(env_id, user, password): with fuel_client.set_auth_context( backup_restore.NailgunCredentialsContext(user, password)): - nodes = [ - n for n in node_obj.Node.get_all() - if "ceph-osd" in n.data["roles"] and n.data["cluster"] == env_id] + orig_env = env_obj.Environment(env_id) + nodes = list(env.get_nodes(orig_env, ["ceph-osd"])) if not nodes: LOG.info("Nothing to upgrade") return - backup_val = [ - # (node, path, backup_path) - ] - admin_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]["ipaddress"] - try: - hostnames = [] - for node in nodes: - sftp = ssh.sftp(node) - for path, content in magic_consts.OSD_REPOS_UPDATE: - back_path = _get_backup_path(path, node) - ssh.call(["cp", path, back_path], node=node) - backup_val.append((node, path, back_path)) - with ssh.update_file(sftp, path) as (_, new): - new.write(content.format(admin_ip=admin_ip)) - hostnames.append(node.data["hostname"]) - ssh.call(["dpkg", "--configure", "-a"], node=node) + preference_priority = get_repo_highest_priority(orig_env) + hostnames = [n.data['hostname'] for n in nodes] + with applied_repos(nodes, preference_priority + 1): call_node = nodes[0] ssh.call(["ceph", "osd", "set", "noout"], node=call_node) ssh.call(['ceph-deploy', 'install', '--release', 'hammer'] + hostnames, - node=call_node, stdout=ssh.PIPE, stderr=ssh.PIPE) - for node in nodes: - ssh.call(["restart", "ceph-osd-all"], node=node) - ssh.call(["ceph", "osd", "unset", "noout"], node=call_node) - ssh.call(["ceph", "osd", "stat"], node=call_node) - finally: - nodes_to_revert = set() - for node, path, back_path in backup_val: - ssh.call(["mv", back_path, path], node=node) - nodes_to_revert.add(node) - for node in nodes_to_revert: - ssh.call(["dpkg", "--configure", "-a"], node=node) + node=call_node) + for node in nodes: + ssh.call(["restart", "ceph-osd-all"], node=node) + ssh.call(["ceph", "osd", "unset", "noout"], node=call_node) class UpgradeOSDCommand(cmd.Command): diff --git a/octane/magic_consts.py b/octane/magic_consts.py index eecd05a3..b7c58400 100644 --- a/octane/magic_consts.py +++ b/octane/magic_consts.py @@ -63,18 +63,20 @@ RUNNING_REQUIRED_CONTAINERS = [ OPENSTACK_FIXTURES = "/usr/share/fuel-openstack-metadata/openstack.yaml" -OSD_REPOS_UPDATE = [ - # ("path", "content") - ( - "/etc/apt/sources.list.d/mos.list", - "deb http://{admin_ip}:8080/liberty-8.0/ubuntu/x86_64 " - "mos8.0 main restricted" - ), - ( - "/etc/apt/sources.list.d/mos-updates.list", - 'deb http://{admin_ip}:8080/ubuntu/x86_64/ mos8.0 main restricted', - ), + +OSD_UPGRADE_REQUIRED_PACKAGES = [ + "libcephfs1", "librados2", "librbd1", "python-ceph", "python-cephfs", + "python-rados", "python-rbd", "ceph", "ceph-common", "ceph-fs-common", + "ceph-mds", ] +OSD_UPGRADE_SOURCE_TEMPLATE = \ + "deb http://{admin_ip}:8080/liberty-8.0/ubuntu/x86_64 " \ + "mos8.0 main restricted\n" \ + "deb http://{admin_ip}:8080/ubuntu/x86_64/ mos8.0 main restricted" + +OSD_UPGADE_PREFERENCE_TEMPLATE = "Package: {packages}\n" \ + "Pin: release a=mos8.0,n=mos8.0,l=mos8.0\n" \ + "Pin-Priority: {priority}" COBBLER_DROP_VERSION = "7.0" CEPH_UPSTART_VERSION = "7.0" diff --git a/octane/tests/test_osd_upgrade.py b/octane/tests/test_osd_upgrade.py index debad82f..dc77d56f 100644 --- a/octane/tests/test_osd_upgrade.py +++ b/octane/tests/test_osd_upgrade.py @@ -10,13 +10,10 @@ # License for the specific language governing permissions and limitations # under the License. -import contextlib import mock import pytest from octane.commands import osd_upgrade -from octane import magic_consts -from octane.util import ssh @pytest.mark.parametrize("env_id", [None, 1]) @@ -37,132 +34,137 @@ def test_osd_cmd_upgrade(mocker, octane_app, env_id, admin_pswd): assert not upgrade_osd_mock.called -@pytest.mark.parametrize("node_roles, exception_node", [ - ([('ceph-osd',)] * 10, None), - ([('ceph-osd', 'compute')] * 10, None), - ([('ceph-osd',), ('compute',)] * 10, None), - ([('ceph-osd',), ('compute',), ('controller',)] * 10, None), - ([], None), - ([('compute',)] * 10, None), - ([('ceph-osd',)] * 10, 0), - ([('ceph-osd',), ('compute',)] * 10, 9), -]) -@pytest.mark.parametrize("user", ["usr", "admin"]) -@pytest.mark.parametrize("password", ["admin", "pswd"]) -@pytest.mark.parametrize("env_id", [1, 2, 3]) -@pytest.mark.parametrize("master_ip", ["10.21.10.2", "10.20.1.2"]) -def test_upgrade_osd( - mocker, node_roles, user, password, exception_node, master_ip, env_id): - auth_mock_client = mocker.patch("octane.util.fuel_client.set_auth_context") - creds_mock = mocker.patch( - "octane.handlers.backup_restore.NailgunCredentialsContext") +@pytest.mark.parametrize("content", ["test_content"]) +@pytest.mark.parametrize("directory", ["/dir/path"]) +@pytest.mark.parametrize("template", ["templ"]) +@pytest.mark.parametrize("generated_name", ["gen_name", "\n\n gen_name\n\n"]) +def test_write_content_to_tmp_file_on_node( + mocker, content, directory, template, generated_name): + node = mock.MagicMock() + sftp_mock = mocker.patch("octane.util.ssh.sftp", return_value=node) + node.open.__enter__.return_value = node + short_gen_name = generated_name.strip() + ssh_mock = mocker.patch("octane.util.ssh.call_output", + return_value=generated_name) + assert short_gen_name == osd_upgrade.write_content_to_tmp_file_on_node( + node, content, directory, template) + node.open.assert_called_once_with(short_gen_name, "w") + node.write(content) + sftp_mock.assert_called_once_with(node) + ssh_mock.assert_called_once_with( + ["mktemp", "-p", directory, "-t", template], node=node) + + +@pytest.mark.parametrize("nodes_count", [0, 1, 2]) +@pytest.mark.parametrize("admin_ip,source_tmpl,source", [( + "10.10.0.1", "source {admin_ip}", "source 10.10.0.1" +)]) +@pytest.mark.parametrize("packages,priority,pref_tmpl,pref", [( + ["pack_1", "pack_2"], 1000, "pref {packages} {priority}", + "pref pack_1 pack_2 1000" +)]) +@pytest.mark.parametrize("error", [True, False]) +def test_applied_repos(mocker, nodes_count, admin_ip, source_tmpl, source, + packages, priority, pref_tmpl, pref, error): + mocker.patch("octane.magic_consts.OSD_UPGRADE_REQUIRED_PACKAGES", packages) mocker.patch( - "octane.commands.osd_upgrade._get_backup_path", - return_value="backup_path") - mocker.patch("octane.magic_consts.OSD_REPOS_UPDATE", - [("path", "{admin_ip}")]) - ssh_call_mock = mocker.patch("octane.util.ssh.call") - preinstall_calls = [] - rollbabk_calls = [] - dpkg_rollbabk_calls = [] - nodes = [] - osd_nodes = [] - hostnames = [] - osd_node_idx = 0 - call_node = None - - class TestException(Exception): - pass - - for roles in node_roles: - node = mocker.Mock() - hostname = "{0}_node.{1}".format("_".join(roles), osd_node_idx) - node.data = {"roles": roles, "hostname": hostname, "cluster": env_id} - nodes.append(node) - new_env_node = mocker.Mock() - new_env_node.data = { - "roles": roles, - "hostname": "{0}_env.{1}".format("_".join(roles), osd_node_idx), - "cluster": env_id + 1 - } - nodes.append(new_env_node) - if 'ceph-osd' not in roles: - continue - osd_nodes.append(node) - hostnames.append(hostname) - call_node = call_node or node - for path, _ in magic_consts.OSD_REPOS_UPDATE: - preinstall_calls.append(( - mock.call(["cp", path, "backup_path"], node=node), - exception_node == osd_node_idx, - )) - if exception_node == osd_node_idx: - break - rollbabk_calls.append( - (mock.call(["mv", "backup_path", path], node=node), False)) - if exception_node == osd_node_idx: - break - preinstall_calls.append( - (mock.call(["dpkg", "--configure", "-a"], node=node), False)) - dpkg_rollbabk_calls.append( - (mock.call(["dpkg", "--configure", "-a"], node=node), False)) - osd_node_idx += 1 - mocker.patch("fuelclient.objects.node.Node.get_all", return_value=nodes) - - file_mock = mock.Mock() - - @contextlib.contextmanager - def update_file(*args, **kwargs): - yield (None, file_mock) - - mocker.patch("octane.util.ssh.update_file", side_effect=update_file) - mocker.patch("octane.util.ssh.sftp") + "octane.magic_consts.OSD_UPGADE_PREFERENCE_TEMPLATE", pref_tmpl) mocker.patch( + "octane.magic_consts.OSD_UPGRADE_SOURCE_TEMPLATE", source_tmpl) + mock_get_astute = mocker.patch( "octane.util.helpers.get_astute_dict", - return_value={"ADMIN_NETWORK": {"ipaddress": master_ip}}) - update_calls = [] + return_value={"ADMIN_NETWORK": {"ipaddress": admin_ip}}) - if exception_node is None and osd_node_idx: - update_calls.append(( - mock.call(["ceph", "osd", "set", "noout"], node=call_node), False)) - update_calls.append(( + def mock_write_content_to_tmp_side_effect( + node, content, directory, template): + if directory == "/etc/apt/sources.list.d/": + return node.source + if directory == "/etc/apt/preferences.d/": + return node.preference + + mock_write_content_to_tmp = mocker.patch( + "octane.commands.osd_upgrade.write_content_to_tmp_file_on_node", + side_effect=mock_write_content_to_tmp_side_effect) + + sftp_mock = mocker.patch("octane.util.ssh.sftp") + + nodes = [mock.MagicMock() for _ in range(nodes_count)] + + with osd_upgrade.applied_repos(nodes, priority): + for node in nodes: + mock_write_content_to_tmp.assert_any_call( + node, source, "/etc/apt/sources.list.d/", "mos.osd_XXX.list") + mock_write_content_to_tmp.assert_any_call( + node, pref, "/etc/apt/preferences.d/", "mos.osd_XXX.pref") + assert not sftp_mock.called + + for node in nodes: + sftp_mock.assert_any_call(node) + sftp_mock.return_value.unlink.assert_any_call(node.source) + sftp_mock.return_value.unlink.assert_any_call(node.preference) + mock_get_astute.assert_called_once_with() + + +@pytest.mark.parametrize("priority_from, priority_to", [[100, 500]]) +def test_get_repo_highest_priority(mocker, priority_from, priority_to): + env = mock.MagicMock() + env.get_attributes.return_value = { + "editable": { + "repo_setup": { + "repos": { + "value": [{"priority": i} + for i in range(priority_from, priority_to + 1)] + } + } + } + } + assert priority_to == osd_upgrade.get_repo_highest_priority(env) + + +@pytest.mark.parametrize("env_id", [2]) +@pytest.mark.parametrize("user", ["user"]) +@pytest.mark.parametrize("password", ["password"]) +@pytest.mark.parametrize("nodes_count", [0, 10]) +@pytest.mark.parametrize("priority", [100, 500]) +def test_upgrade_osd(mocker, nodes_count, priority, user, password, env_id): + env = mock.Mock() + nodes = [] + hostnames = [] + restart_calls = [] + for idx in range(nodes_count): + hostname = "host_{0}".format(idx) + hostnames.append(hostname) + node = mock.Mock(data={'hostname': hostname}) + nodes.append(node) + restart_calls.append(mock.call(["restart", "ceph-osd-all"], node=node)) + env_get = mocker.patch("fuelclient.objects.environment.Environment", + return_value=env) + mocker.patch("octane.util.env.get_nodes", return_value=iter(nodes)) + mock_creds = mocker.patch( + "octane.handlers.backup_restore.NailgunCredentialsContext") + mock_auth_cntx = mocker.patch("octane.util.fuel_client.set_auth_context") + mock_applied = mocker.patch("octane.commands.osd_upgrade.applied_repos") + mock_get_priority = mocker.patch( + "octane.commands.osd_upgrade.get_repo_highest_priority", + return_value=priority) + ssh_call_mock = mocker.patch("octane.util.ssh.call") + + osd_upgrade.upgrade_osd(env_id, user, password) + + mock_creds.assert_called_once_with(user, password) + mock_auth_cntx.assert_called_once_with(mock_creds.return_value) + env_get.assert_called_once_with(env_id) + ssh_calls = [] + if nodes: + ssh_calls.append( + mock.call(["ceph", "osd", "set", "noout"], node=nodes[0])) + ssh_calls.append( mock.call( ['ceph-deploy', 'install', '--release', 'hammer'] + hostnames, - node=call_node, - stdout=ssh.PIPE, - stderr=ssh.PIPE, - ), - False - )) - for node in osd_nodes: - update_calls.append( - (mock.call(['restart', 'ceph-osd-all'], node=node), False)) - update_calls.append(( - mock.call(["ceph", "osd", "unset", "noout"], node=call_node), - False - )) - update_calls.append(( - mock.call(["ceph", "osd", "stat"], node=call_node), - False - )) - - calls = \ - preinstall_calls + \ - update_calls + \ - rollbabk_calls + \ - dpkg_rollbabk_calls - - ssh_calls = [i[0] for i in calls] - mock_calls = [TestException() if i[1] else mock.DEFAULT for i in calls] - ssh_call_mock.side_effect = mock_calls - if exception_node is not None: - with pytest.raises(TestException): - osd_upgrade.upgrade_osd(env_id, user, password) - else: - osd_upgrade.upgrade_osd(env_id, user, password) - ssh_call_mock.assert_has_calls(ssh_calls, any_order=True) - assert ssh_call_mock.call_count == len(ssh_calls) - auth_mock_client.assert_called_once_with(creds_mock.return_value) - creds_mock.assert_called_once_with(user, password) - if exception_node is not None and osd_node_idx: - file_mock.write.assert_called_with(master_ip) + node=nodes[0])) + ssh_calls.extend(restart_calls) + ssh_calls.append( + mock.call(["ceph", "osd", "unset", "noout"], node=nodes[0])) + mock_get_priority.assert_called_once_with(env) + mock_applied.assert_called_once_with(nodes, priority + 1) + assert ssh_calls == ssh_call_mock.mock_calls