Merge "Add ffwd-upgrade cli - overcloud ffwd-upgrade [prepare|run|converge]"

This commit is contained in:
Zuul 2018-04-19 20:01:14 +00:00 committed by Gerrit Code Review
commit 8d26f89ebd
11 changed files with 510 additions and 31 deletions

View File

@ -0,0 +1,13 @@
---
features:
- |
Adds a cli for fast forward upgrades, in particular the
openstack overcloud ffwd-upgrade prepare
openstack overcloud ffwd-upgrade run
openstack overcloud ffwd-upgrade converge
Which are meant to be the first, second and final step in the fast-forward
upgrade workflow. See the ffwd upgrade docs for more information on how to
use these cli commands, and the list of parameters is available with
openstack overcloud ffwd-upgrade [prepare,run,converge] --help

View File

@ -85,6 +85,9 @@ openstack.tripleoclient.v1 =
overcloud_upgrade_prepare = tripleoclient.v1.overcloud_upgrade:UpgradePrepare overcloud_upgrade_prepare = tripleoclient.v1.overcloud_upgrade:UpgradePrepare
overcloud_upgrade_run = tripleoclient.v1.overcloud_upgrade:UpgradeRun overcloud_upgrade_run = tripleoclient.v1.overcloud_upgrade:UpgradeRun
overcloud_upgrade_converge = tripleoclient.v1.overcloud_upgrade:UpgradeConvergeOvercloud overcloud_upgrade_converge = tripleoclient.v1.overcloud_upgrade:UpgradeConvergeOvercloud
overcloud_ffwd-upgrade_prepare = tripleoclient.v1.overcloud_ffwd_upgrade:FFWDUpgradePrepare
overcloud_ffwd-upgrade_run = tripleoclient.v1.overcloud_ffwd_upgrade:FFWDUpgradeRun
overcloud_ffwd-upgrade_converge = tripleoclient.v1.overcloud_ffwd_upgrade:FFWDUpgradeConverge
overcloud_execute = tripleoclient.v1.overcloud_execute:RemoteExecute overcloud_execute = tripleoclient.v1.overcloud_execute:RemoteExecute
overcloud_generate_fencing = tripleoclient.v1.overcloud_parameters:GenerateFencingParameters overcloud_generate_fencing = tripleoclient.v1.overcloud_parameters:GenerateFencingParameters
undercloud_deploy = tripleoclient.v1.undercloud_deploy:DeployUndercloud undercloud_deploy = tripleoclient.v1.undercloud_deploy:DeployUndercloud

View File

@ -39,8 +39,12 @@ PUPPET_BASE = "/etc/puppet/"
# Update Queue # Update Queue
UPDATE_QUEUE = 'update' UPDATE_QUEUE = 'update'
UPGRADE_QUEUE = 'upgrade' UPGRADE_QUEUE = 'upgrade'
FFWD_UPGRADE_QUEUE = 'ffwdupgrade'
STACK_TIMEOUT = 240 STACK_TIMEOUT = 240
# The default ffwd upgrade ansible playbooks generated from heat stack output
FFWD_UPGRADE_PLAYBOOK = "fast_forward_upgrade_playbook.yaml"
# The default minor update ansible playbooks generated from heat stack output # The default minor update ansible playbooks generated from heat stack output
MINOR_UPDATE_PLAYBOOKS = ['update_steps_playbook.yaml', MINOR_UPDATE_PLAYBOOKS = ['update_steps_playbook.yaml',
'deploy_steps_playbook.yaml'] 'deploy_steps_playbook.yaml']
@ -55,3 +59,5 @@ UPDATE_PREPARE_ENV = "environments/lifecycle/update-prepare.yaml"
UPDATE_CONVERGE_ENV = "environments/lifecycle/update-converge.yaml" UPDATE_CONVERGE_ENV = "environments/lifecycle/update-converge.yaml"
UPGRADE_PREPARE_ENV = "environments/lifecycle/upgrade-prepare.yaml" UPGRADE_PREPARE_ENV = "environments/lifecycle/upgrade-prepare.yaml"
UPGRADE_CONVERGE_ENV = "environments/lifecycle/upgrade-converge.yaml" UPGRADE_CONVERGE_ENV = "environments/lifecycle/upgrade-converge.yaml"
FFWD_UPGRADE_PREPARE_ENV = "environments/lifecycle/ffwd-upgrade-prepare.yaml"
FFWD_UPGRADE_CONVERGE_ENV = "environments/lifecycle/ffwd-upgrade-converge.yaml"

View File

@ -0,0 +1,73 @@
# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from osc_lib.tests import utils
from tripleoclient.tests import fakes
class FakeClientWrapper(object):
def __init__(self):
self._instance = mock.Mock()
self.object_store = FakeObjectClient()
def messaging_websocket(self):
return fakes.FakeWebSocket()
class FakeObjectClient(object):
def __init__(self):
self._instance = mock.Mock()
self.put_object = mock.Mock()
def get_object(self, *args):
return
class TestFFWDUpgradePrepare(utils.TestCommand):
def setUp(self):
super(TestFFWDUpgradePrepare, self).setUp()
self.app.client_manager.auth_ref = mock.Mock(auth_token="TOKEN")
self.app.client_manager.baremetal = mock.Mock()
self.app.client_manager.orchestration = mock.Mock()
self.app.client_manager.tripleoclient = FakeClientWrapper()
self.app.client_manager.workflow_engine = mock.Mock()
class TestFFWDUpgradeRun(utils.TestCommand):
def setUp(self):
super(TestFFWDUpgradeRun, self).setUp()
self.app.client_manager.auth_ref = mock.Mock(auth_token="TOKEN")
self.app.client_manager.tripleoclient = FakeClientWrapper()
self.app.client_manager.workflow_engine = mock.Mock()
class TestFFWDUpgradeConverge(utils.TestCommand):
def setUp(self):
super(TestFFWDUpgradeConverge, self).setUp()
self.app.client_manager.auth_ref = mock.Mock(auth_token="TOKEN")
self.app.client_manager.baremetal = mock.Mock()
self.app.client_manager.orchestration = mock.Mock()
self.app.client_manager.tripleoclient = FakeClientWrapper()
self.app.client_manager.workflow_engine = mock.Mock()

View File

@ -0,0 +1,203 @@
# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from osc_lib.tests.utils import ParserException
from tripleoclient import constants
from tripleoclient import exceptions
from tripleoclient.tests.v1.overcloud_ffwd_upgrade import fakes
from tripleoclient.v1 import overcloud_ffwd_upgrade
class TestFFWDUpgradePrepare(fakes.TestFFWDUpgradePrepare):
def setUp(self):
super(TestFFWDUpgradePrepare, self).setUp()
# Get the command object to test
app_args = mock.Mock()
app_args.verbose_level = 1
self.cmd = overcloud_ffwd_upgrade.FFWDUpgradePrepare(self.app,
app_args)
uuid4_patcher = mock.patch('uuid.uuid4', return_value="UUID4")
self.mock_uuid4 = uuid4_patcher.start()
self.addCleanup(self.mock_uuid4.stop)
@mock.patch('tripleoclient.utils.prepend_environment', autospec=True)
@mock.patch('tripleoclient.utils.get_stack',
autospec=True)
@mock.patch(
'tripleoclient.v1.overcloud_ffwd_upgrade.FFWDUpgradePrepare.log',
autospec=True)
@mock.patch('tripleoclient.workflows.package_update.update',
autospec=True)
@mock.patch('os.path.abspath')
@mock.patch('yaml.load')
@mock.patch('shutil.copytree', autospec=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_deploy_tripleo_heat_templates', autospec=True)
def test_ffwd_upgrade(self, mock_deploy, mock_open, mock_copy, mock_yaml,
mock_abspath, mock_ffwd_upgrade, mock_logger,
mock_get_stack, mock_prepend_env):
mock_stack = mock.Mock()
mock_stack.stack_name = 'mystack'
mock_get_stack.return_value = mock_stack
mock_abspath.return_value = '/home/fake/my-fake-registry.yaml'
mock_yaml.return_value = {'fake_container': 'fake_value'}
argslist = ['--stack', 'mystack', '--templates',
'--container-registry-file', 'my-fake-registry.yaml']
verifylist = [
('stack', 'mystack'),
('templates', constants.TRIPLEO_HEAT_TEMPLATES),
('container_registry_file', 'my-fake-registry.yaml')
]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
self.cmd.take_action(parsed_args)
mock_ffwd_upgrade.assert_called_once_with(
self.app.client_manager,
container='mystack',
container_registry={'fake_container': 'fake_value'},
ceph_ansible_playbook='/usr/share/ceph-ansible'
'/site-docker.yml.sample',
)
@mock.patch('tripleoclient.utils.prepend_environment', autospec=True)
@mock.patch('tripleoclient.workflows.package_update.update',
autospec=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('os.path.abspath')
@mock.patch('yaml.load')
@mock.patch('shutil.copytree', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_deploy_tripleo_heat_templates', autospec=True)
def test_ffwd_upgrade_failed(
self, mock_deploy, mock_copy, mock_yaml, mock_abspath, mock_open,
mock_ffwd_upgrade, mock_prepend_env):
mock_ffwd_upgrade.side_effect = exceptions.DeploymentError()
mock_abspath.return_value = '/home/fake/my-fake-registry.yaml'
mock_yaml.return_value = {'fake_container': 'fake_value'}
argslist = ['--stack', 'overcloud', '--templates',
'--container-registry-file', 'my-fake-registry.yaml']
verifylist = [
('stack', 'overcloud'),
('templates', constants.TRIPLEO_HEAT_TEMPLATES),
('container_registry_file', 'my-fake-registry.yaml')
]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
self.assertRaises(exceptions.DeploymentError,
self.cmd.take_action, parsed_args)
class TestFFWDUpgradeRun(fakes.TestFFWDUpgradeRun):
def setUp(self):
super(TestFFWDUpgradeRun, self).setUp()
# Get the command object to test
app_args = mock.Mock()
app_args.verbose_level = 1
self.cmd = overcloud_ffwd_upgrade.FFWDUpgradeRun(self.app, app_args)
uuid4_patcher = mock.patch('uuid.uuid4', return_value="UUID4")
self.mock_uuid4 = uuid4_patcher.start()
self.addCleanup(self.mock_uuid4.stop)
@mock.patch('tripleoclient.workflows.package_update.update_ansible',
autospec=True)
@mock.patch('os.path.expanduser')
@mock.patch('oslo_concurrency.processutils.execute')
@mock.patch('six.moves.builtins.open')
def test_ffwd_upgrade_playbook(
self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
mock_expanduser.return_value = '/home/fake/'
argslist = []
verifylist = []
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
with mock.patch('os.path.exists') as mock_exists:
mock_exists.return_value = True
self.cmd.take_action(parsed_args)
upgrade_ansible.assert_called_once_with(
self.app.client_manager,
inventory_file=mock_open().read(),
ansible_queue_name=constants.FFWD_UPGRADE_QUEUE,
nodes='',
playbook=constants.FFWD_UPGRADE_PLAYBOOK,
node_user='heat-admin',
skip_tags=''
)
@mock.patch('tripleoclient.workflows.package_update.update_ansible',
autospec=True)
@mock.patch('os.path.expanduser')
@mock.patch('oslo_concurrency.processutils.execute')
@mock.patch('six.moves.builtins.open')
def test_upgrade_no_nodes_or_roles(self, mock_open, mock_execute,
mock_expanduser, upgrade_ansible):
mock_expanduser.return_value = '/home/fake/'
argslist = ["--nodes", "controller-1", "--roles", "foo"]
verifylist = []
self.assertRaises(ParserException, lambda: self.check_parser(
self.cmd, argslist, verifylist))
class TestFFWDUpgradeConverge(fakes.TestFFWDUpgradeConverge):
def setUp(self):
super(TestFFWDUpgradeConverge, self).setUp()
# Get the command object to test
app_args = mock.Mock()
app_args.verbose_level = 1
self.cmd = overcloud_ffwd_upgrade.FFWDUpgradeConverge(self.app,
app_args)
uuid4_patcher = mock.patch('uuid.uuid4', return_value="UUID4")
self.mock_uuid4 = uuid4_patcher.start()
self.addCleanup(self.mock_uuid4.stop)
@mock.patch('tripleoclient.utils.prepend_environment', autospec=True)
@mock.patch('tripleoclient.utils.get_stack', autospec=True)
@mock.patch('tripleoclient.workflows.package_update.ffwd_converge_nodes',
autospec=True)
@mock.patch('os.path.expanduser')
@mock.patch('oslo_concurrency.processutils.execute')
@mock.patch('six.moves.builtins.open')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_deploy_tripleo_heat_templates_tmpdir', autospec=True)
def test_ffwd_upgrade_converge(
self, mock_deploy, mock_open, mock_execute, mock_expanduser,
ffwd_converge_nodes, mock_get_stack, mock_prepend_env):
mock_expanduser.return_value = '/home/fake/'
mock_stack = mock.Mock()
mock_stack.stack_name = 'le_overcloud'
mock_get_stack.return_value = mock_stack
argslist = ['--stack', 'le_overcloud', '--templates']
verifylist = [
('stack', 'le_overcloud'),
('templates', constants.TRIPLEO_HEAT_TEMPLATES),
]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
with mock.patch('os.path.exists') as mock_exists:
mock_exists.return_value = True
self.cmd.take_action(parsed_args)
ffwd_converge_nodes.assert_called_once_with(
self.app.client_manager,
queue_name=constants.FFWD_UPGRADE_QUEUE,
container='le_overcloud')

View File

@ -946,3 +946,18 @@ def prepend_environment(environment_files, templates_dir, environment):
% (environment, templates_dir)) % (environment, templates_dir))
return environment_files return environment_files
def load_container_registry(log, path):
registry = None
if path:
with open(os.path.abspath(path)) as content:
registry = yaml.load(content.read())
else:
log.warning(
"You have not provided a container registry file. Note "
"that none of the containers on your environment will be "
"updated. If you want to update your containers you have "
"to re-run this command and provide the registry file "
"with: --container-registry-file option.")
return registry

View File

@ -0,0 +1,172 @@
# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import logging
from osc_lib.i18n import _
from tripleoclient import command
from tripleoclient import constants
from tripleoclient import utils as oooutils
from tripleoclient.v1.overcloud_deploy import DeployOvercloud
from tripleoclient.workflows import package_update
class FFWDUpgradePrepare(DeployOvercloud):
"""Run heat stack update for overcloud nodes to refresh heat stack outputs.
The heat stack outputs are what we use later on to generate ansible
playbooks which deliver the ffwd upgrade workflow. This is used as the
first step for a fast forward upgrade of your overcloud.
"""
log = logging.getLogger(__name__ + ".FFWDUpgradePrepare")
def get_parser(self, prog_name):
parser = super(FFWDUpgradePrepare, self).get_parser(prog_name)
parser.add_argument('--container-registry-file',
dest='container_registry_file',
default=None,
help=_("File which contains the container "
"registry data for the upgrade"),
)
parser.add_argument('--ceph-ansible-playbook',
action="store",
default="/usr/share/ceph-ansible"
"/site-docker.yml.sample",
help=_('Path to switch the ceph-ansible playbook '
'used for upgrade. '),
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
clients = self.app.client_manager
stack = oooutils.get_stack(clients.orchestration,
parsed_args.stack)
stack_name = stack.stack_name
registry = oooutils.load_container_registry(
self.log, parsed_args.container_registry_file)
ceph_ansible_playbook = parsed_args.ceph_ansible_playbook
# In case of update and upgrade we need to force the
# update_plan_only. The heat stack update is done by the
# packag_update mistral action
parsed_args.update_plan_only = True
# Add the prepare environment into the args to unset noop etc
templates_dir = (parsed_args.templates or
constants.TRIPLEO_HEAT_TEMPLATES)
if not parsed_args.environment_files:
parsed_args.environment_files = []
parsed_args.environment_files = oooutils.prepend_environment(
parsed_args.environment_files, templates_dir,
constants.FFWD_UPGRADE_PREPARE_ENV)
super(FFWDUpgradePrepare, self).take_action(parsed_args)
package_update.update(clients, container=stack_name,
container_registry=registry,
ceph_ansible_playbook=ceph_ansible_playbook)
package_update.get_config(clients, container=stack_name)
print("FFWD Upgrade Prepare on stack {0} complete.".format(
parsed_args.stack))
class FFWDUpgradeRun(command.Command):
"""Run fast forward upgrade ansible playbooks on Overcloud nodes
This will run the fast_forward_upgrade_playbook.yaml ansible playbook.
This playbook was generated when you ran the 'ffwd-upgrade prepare'
command. Running 'ffwd-upgrade run ' is the second step in the ffwd
upgrade workflow.
"""
log = logging.getLogger(__name__ + ".FFWDUpgradeRun")
def get_parser(self, prog_name):
parser = super(FFWDUpgradeRun, self).get_parser(prog_name)
parser.add_argument('--static-inventory',
dest='static_inventory',
action="store",
default=None,
help=_('Path to an existing ansible inventory to '
'use. If not specified, one will be '
'generated in '
'~/tripleo-ansible-inventory.yaml')
)
parser.add_argument("--ssh-user",
dest="ssh_user",
action="store",
default="heat-admin",
help=_("The ssh user name for connecting to "
"the overcloud nodes.")
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
clients = self.app.client_manager
# Run ansible:
inventory = oooutils.get_tripleo_ansible_inventory(
parsed_args.static_inventory)
# Don't expost limit_hosts. We need this on the whole overcloud.
limit_hosts = ''
oooutils.run_update_ansible_action(
self.log, clients, limit_hosts, inventory,
constants.FFWD_UPGRADE_PLAYBOOK, constants.FFWD_UPGRADE_QUEUE,
[], package_update, parsed_args.ssh_user)
class FFWDUpgradeConverge(DeployOvercloud):
"""Converge the fast-forward upgrade on Overcloud Nodes
This is the last step for completion of a fast forward upgrade.
There is no heat stack update performed here. The main task is updating
the plan to unblock future stack updates. For the ffwd upgrade workflow
we have set and used the config-download Software/Structured Deployment
for the OS::TripleO and OS::Heat resources. This unsets those back
to their default values, in the swift stored plan.
"""
log = logging.getLogger(__name__ + ".FFWDUpgradeConverge")
def get_parser(self, prog_name):
parser = super(FFWDUpgradeConverge, self).get_parser(prog_name)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
clients = self.app.client_manager
stack = oooutils.get_stack(clients.orchestration,
parsed_args.stack)
stack_name = stack.stack_name
parsed_args.update_plan_only = True
# Add the converge environment into the args to unset noop etc
templates_dir = (parsed_args.templates or
constants.TRIPLEO_HEAT_TEMPLATES)
if not parsed_args.environment_files:
parsed_args.environment_files = []
parsed_args.environment_files = oooutils.prepend_environment(
parsed_args.environment_files, templates_dir,
constants.FFWD_UPGRADE_CONVERGE_ENV)
super(FFWDUpgradeConverge, self).take_action(parsed_args)
# Run converge steps
package_update.ffwd_converge_nodes(
clients, container=stack_name,
queue_name=constants.FFWD_UPGRADE_QUEUE)
print("FFWD Upgrade Converge on stack {0} complete.".format(
parsed_args.stack))

View File

@ -14,8 +14,6 @@
# #
import logging import logging
import os
import yaml
from osc_lib.i18n import _ from osc_lib.i18n import _
@ -61,20 +59,9 @@ class UpdatePrepare(DeployOvercloud):
parsed_args.stack) parsed_args.stack)
stack_name = stack.stack_name stack_name = stack.stack_name
container_registry = parsed_args.container_registry_file registry = oooutils.load_container_registry(
self.log, parsed_args.container_registry_file)
# Update the container registry:
if container_registry:
with open(os.path.abspath(container_registry)) as content:
registry = yaml.load(content.read())
else:
self.log.warning(
"You have not provided a container registry file. Note "
"that none of the containers on your environement will be "
"updated. If you want to update your container you have "
"to re-run this command and provide the registry file "
"with: --container-registry-file option.")
registry = None
# Run update # Run update
ceph_ansible_playbook = parsed_args.ceph_ansible_playbook ceph_ansible_playbook = parsed_args.ceph_ansible_playbook
# Run Overcloud deploy (stack update) # Run Overcloud deploy (stack update)

View File

@ -13,8 +13,6 @@
# under the License. # under the License.
# #
import logging import logging
import os
import yaml
from osc_lib.i18n import _ from osc_lib.i18n import _
@ -61,23 +59,11 @@ class UpgradePrepare(DeployOvercloud):
parsed_args.stack) parsed_args.stack)
stack_name = stack.stack_name stack_name = stack.stack_name
container_registry = parsed_args.container_registry_file registry = oooutils.load_container_registry(
self.log, parsed_args.container_registry_file)
# Update the container registry:
if container_registry:
with open(os.path.abspath(container_registry)) as content:
registry = yaml.load(content.read())
else:
self.log.warning(
"You have not provided a container registry file. Note "
"that none of the containers on your environement will be "
"updated. If you want to update your container you have "
"to re-run this command and provide the registry file "
"with: --container-registry-file option.")
registry = None
# Run update # Run update
ceph_ansible_playbook = parsed_args.ceph_ansible_playbook ceph_ansible_playbook = parsed_args.ceph_ansible_playbook
# Run Overcloud deploy (stack update)
# In case of update and upgrade we need to force the # In case of update and upgrade we need to force the
# update_plan_only. The heat stack update is done by the # update_plan_only. The heat stack update is done by the
# packag_update mistral action # packag_update mistral action

View File

@ -157,3 +157,24 @@ def converge_nodes(clients, **workflow_input):
if not create_result: if not create_result:
shell.OpenStackShell().run(["stack", "failures", "list", plan_name]) shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
raise exceptions.DeploymentError("Heat Stack update failed.") raise exceptions.DeploymentError("Heat Stack update failed.")
def ffwd_converge_nodes(clients, **workflow_input):
workflow_client = clients.workflow_engine
tripleoclients = clients.tripleoclient
with tripleoclients.messaging_websocket(
workflow_input['queue_name']) as ws:
execution = base.start_workflow(
workflow_client,
'tripleo.package_update.v1.ffwd_upgrade_converge_plan',
workflow_input=workflow_input
)
for payload in base.wait_for_messages(workflow_client, ws, execution):
assert payload['status'] == "SUCCESS", pprint.pformat(payload)
if payload['status'] == 'SUCCESS':
print('Success')
else:
raise RuntimeError('ffwd upgrade converge failed: {}'.format(payload))