Update the 'overcloud deploy --templates' command to use Mistral

This is a backwards compatible version of the deploy command
which uses the new Mistral workflows to power the deploy.

Depends-On: I7a61bf6fb71cc4a26aaf4322f2215683aafe20f3
Change-Id: I292958b277abf5b23f8e2e14330eca97f656effa

tripleoclient/tests/fakes.py

@@ -39,9 +39,24 @@ class FakeClientManager(object):
         self.tripleoclient = None
         self.auth_ref = None
         self.tripleoclient = FakeClientWrapper()
+        self.workflow_engine = mock.Mock()
+
+
+class FakeWebSocket(object):
+
+    def wait_for_message(self, execution_id):
+        return {
+            'status': 'SUCCESS'
+        }
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return
 
 
 class FakeClientWrapper(object):
 
-    def messaging_websocket(self, queue_name='tripleo'):
+    def messaging_websocket(self, queue_name):
-        return mock.MagicMock()
+        return FakeWebSocket()

tripleoclient/tests/v1/overcloud_deploy/fakes.py

@@ -16,6 +16,8 @@
 import mock
 from openstackclient.tests import utils
 
+from tripleoclient.tests import fakes
+
 
 FAKE_STACK = {
     'parameters': {
@@ -105,6 +107,10 @@ class FakeClientWrapper(object):
 
     def __init__(self):
         self._instance = mock.Mock()
+        self.object_store = mock.Mock()
+
+    def messaging_websocket(self, queue_name):
+        return fakes.FakeWebSocket()
 
 
 class TestDeployOvercloud(utils.TestCommand):
@@ -119,4 +125,5 @@ class TestDeployOvercloud(utils.TestCommand):
         self.app.client_manager.image = mock.Mock()
         self.app.client_manager.network = mock.Mock()
         self.app.client_manager.orchestration = mock.Mock()
+        self.app.client_manager.workflow_engine = mock.Mock()
         self.app.client_manager.tripleoclient = FakeClientWrapper()

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -16,6 +16,7 @@
 import fixtures
 import json
 import os
+import shutil
 import six
 import tempfile
 import yaml
@@ -58,16 +59,20 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         super(TestDeployOvercloud, self).tearDown()
         os.unlink(self.parameter_defaults_env_file)
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch("heatclient.common.event_utils.get_events")
-    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env')
+    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env',
+                autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_create_parameters_env')
+                '_create_parameters_env', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_deploy_postconfig')
+                '_deploy_postconfig', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
-    @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
+    @mock.patch('tripleoclient.utils.generate_overcloud_passwords',
-    @mock.patch('tripleoclient.utils.create_overcloudrc')
+                autospec=True)
+    @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
     @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True)
     @mock.patch('time.sleep', return_value=None)
     @mock.patch('os_cloud_config.keystone.initialize', autospec=True)
@@ -100,7 +105,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                        mock_deploy_postconfig,
                        mock_create_parameters_env,
                        mock_breakpoints_cleanupm,
-                       mock_events):
+                       mock_events, mock_tarball):
 
         arglist = ['--templates', '--ceph-storage-scale', '3']
         verifylist = [
@@ -119,6 +124,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_event = mock.Mock()
         mock_event.id = '1234'
         mock_events.return_value = [mock_events]
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
 
         mock_check_hypervisor_stats.return_value = {
             'count': 4,
@@ -187,28 +197,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             'StackAction': 'UPDATE',
         }
 
-        def _custom_create_params_env(parameters):
+        testcase = self
+
+        def _custom_create_params_env(self, parameters):
             for key, value in six.iteritems(parameters):
-                self.assertEqual(value, expected_parameters[key])
+                testcase.assertEqual(value, expected_parameters[key])
             parameter_defaults = {"parameter_defaults": parameters}
-            with open(self.parameter_defaults_env_file, 'w') as temp_file:
+            with open(testcase.parameter_defaults_env_file, 'w') as temp_file:
                 temp_file.write(json.dumps(parameter_defaults))
-            return [self.parameter_defaults_env_file]
+            return [testcase.parameter_defaults_env_file]
 
         mock_create_parameters_env.side_effect = _custom_create_params_env
 
         self.cmd.take_action(parsed_args)
 
-        args, kwargs = orchestration_client.stacks.update.call_args
+        self.assertFalse(orchestration_client.stacks.update.called)
-
-        self.assertEqual(args, (orchestration_client.stacks.get().id, ))
-
-        self.assertEqual(kwargs['files'], {})
-        self.assertEqual(kwargs['template'], 'template')
-        self.assertEqual(kwargs['environment'], mock_env)
-        self.assertEqual(kwargs['stack_name'], 'overcloud')
-        self.assertEqual(kwargs['clear_parameters'],
-                         mock_env['parameter_defaults'].keys())
 
         mock_get_templte_contents.assert_called_with(
             '/usr/share/openstack-tripleo-heat-templates/' +
@@ -218,15 +221,22 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_process_multiple_env.assert_called_with(
             [self.parameter_defaults_env_file])
 
-    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env')
+    @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_deploy_postconfig', autospec=True)
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
+    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env',
+                autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_validate_args')
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_create_parameters_env')
+                '_create_parameters_env', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
-    @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
+    @mock.patch('tripleoclient.utils.generate_overcloud_passwords',
-    @mock.patch('tripleoclient.utils.create_overcloudrc')
+                autospec=True)
+    @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
     @mock.patch('os_cloud_config.utils.clients.get_nova_bm_client',
                 autospec=True)
     @mock.patch('os_cloud_config.utils.clients.get_keystone_client',
@@ -262,7 +272,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                         mock_generate_overcloud_passwords,
                         mock_create_tempest_deployer_input,
                         mock_create_parameters_env, mock_validate_args,
-                        mock_breakpoints_cleanup):
+                        mock_breakpoints_cleanup, mock_tarball,
+                        mock_postconfig, mock_get_overcloud_endpoint):
 
         arglist = ['--templates', '--ceph-storage-scale', '3']
         verifylist = [
@@ -278,7 +289,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         clients = self.app.client_manager
         orchestration_client = clients.orchestration
         mock_stack = fakes.create_tht_stack()
-        orchestration_client.stacks.get.return_value = None
+        orchestration_client.stacks.get.side_effect = [None, mock.Mock()]
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
 
         def _orch_clt_create(**kwargs):
             orchestration_client.stacks.get.return_value = mock_stack
@@ -354,26 +370,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             'StackAction': 'CREATE',
         }
 
-        def _custom_create_params_env(parameters):
+        testcase = self
+
+        def _custom_create_params_env(self, parameters):
             for key, value in six.iteritems(parameters):
-                self.assertEqual(value, expected_parameters[key])
+                testcase.assertEqual(value, expected_parameters[key])
             parameter_defaults = {"parameter_defaults": parameters}
-            with open(self.parameter_defaults_env_file, 'w') as temp_file:
+            with open(testcase.parameter_defaults_env_file, 'w') as temp_file:
                 temp_file.write(json.dumps(parameter_defaults))
-            return [self.parameter_defaults_env_file]
+            return [testcase.parameter_defaults_env_file]
 
         mock_create_parameters_env.side_effect = _custom_create_params_env
 
         self.cmd.take_action(parsed_args)
 
-        args, kwargs = orchestration_client.stacks.create.call_args
+        self.assertFalse(orchestration_client.stacks.create.called)
-
-        self.assertEqual(kwargs['files'], {})
-        self.assertEqual(kwargs['template'], 'template')
-        self.assertEqual(kwargs['environment'], mock_env)
-        self.assertEqual(kwargs['stack_name'], 'overcloud')
-        self.assertEqual(kwargs['clear_parameters'],
-                         mock_env['parameter_defaults'].keys())
 
         mock_get_templte_contents.assert_called_with(
             '/usr/share/openstack-tripleo-heat-templates/' +
@@ -381,25 +392,27 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_create_tempest_deployer_input.assert_called_with()
         mock_process_multiple_env.assert_called_with(
-            ['/usr/share/openstack-tripleo-heat-templates/overcloud-resource-'
+            ['/fake/path', self.parameter_defaults_env_file])
-             'registry-puppet.yaml', '/fake/path',
-             self.parameter_defaults_env_file])
 
         mock_validate_args.assert_called_once_with(parsed_args)
 
-        mock_remove_known_hosts.assert_called_once_with('0.0.0.0')
+        mock_tarball.create_tarball.assert_called_with(
-        mock_get_keystone_client.assert_called_once_with('admin', 'password',
+            '/usr/share/openstack-tripleo-heat-templates/', mock.ANY)
-                                                         'admin',
+        mock_tarball.tarball_extract_to_swift_container.assert_called_with(
-                                                         'http://0.0.0.0:8000')
+            clients.tripleoclient.object_store, mock.ANY, 'overcloud')
 
-    @mock.patch("heatclient.common.event_utils.get_events")
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
-    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env')
+                autospec=True)
+    @mock.patch("heatclient.common.event_utils.get_events", autospec=True)
+    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env',
+                autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_deploy_postconfig')
+                '_deploy_postconfig', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
-    @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
+    @mock.patch('tripleoclient.utils.generate_overcloud_passwords',
-    @mock.patch('tripleoclient.utils.create_overcloudrc')
+                autospec=True)
+    @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
     @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True)
     @mock.patch('time.sleep', return_value=None)
     @mock.patch('os_cloud_config.keystone.initialize', autospec=True)
@@ -431,7 +444,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                      mock_create_tempest_deployer_input,
                                      mock_deploy_postconfig,
                                      mock_breakpoints_cleanup,
-                                     mock_events):
+                                     mock_events, mock_tarball):
 
         arglist = ['--templates', '/home/stack/tripleo-heat-templates']
         verifylist = [
@@ -465,22 +478,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         baremetal = clients.baremetal
         baremetal.node.list.return_value = range(10)
 
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
+
         with mock.patch('tempfile.mkstemp') as mkstemp:
             mkstemp.return_value = (os.open(self.parameter_defaults_env_file,
                                             os.O_RDWR),
                                     self.parameter_defaults_env_file)
             self.cmd.take_action(parsed_args)
 
-        args, kwargs = orchestration_client.stacks.update.call_args
+        self.assertFalse(orchestration_client.stacks.update.called)
-
-        self.assertEqual(args, (orchestration_client.stacks.get().id, ))
-
-        self.assertEqual(kwargs['files'], {})
-        self.assertEqual(kwargs['template'], 'template')
-        self.assertEqual(kwargs['environment'], mock_env)
-        self.assertEqual(kwargs['stack_name'], 'overcloud')
-        self.assertEqual(kwargs['clear_parameters'],
-                         mock_env['parameter_defaults'].keys())
 
         mock_get_templte_contents.assert_called_with(
             '/home/stack/tripleo-heat-templates/' +
@@ -513,6 +523,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           parsed_args)
         self.assertFalse(mock_deploy_tht.called)
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
     @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
@@ -527,13 +539,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     def test_environment_dirs(self, mock_deploy_heat,
                               mock_update_parameters, mock_post_config,
                               mock_utils_check_nodes, mock_utils_endpoint,
-                              mock_utils_createrc, mock_utils_tempest):
+                              mock_utils_createrc, mock_utils_tempest,
+                              mock_tarball):
+
+        clients = self.app.client_manager
+        workflow_client = clients.workflow_engine
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
 
         mock_update_parameters.return_value = {}
         mock_utils_endpoint.return_value = 'foo.bar'
 
         tmp_dir = self.useFixture(fixtures.TempDir())
-        test_env = os.path.join(tmp_dir.path, 'foo.yaml')
+        test_env = os.path.join(tmp_dir.path, 'foo1.yaml')
 
         env_dirs = [os.path.join(os.environ.get('HOME', ''), '.tripleo',
                     'environments'), tmp_dir.path]
@@ -548,7 +566,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         ]
 
         def _fake_heat_deploy(self, stack, stack_name, template_path,
-                              parameters, environments, timeout):
+                              parameters, environments, timeout, tht_root):
             assert test_env in environments
 
         mock_deploy_heat.side_effect = _fake_heat_deploy
@@ -556,6 +574,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
         self.cmd.take_action(parsed_args)
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
     @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
@@ -570,15 +590,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     def test_environment_dirs_env(self, mock_deploy_heat,
                                   mock_update_parameters, mock_post_config,
                                   mock_utils_check_nodes, mock_utils_endpoint,
-                                  mock_utils_createrc, mock_utils_tempest):
+                                  mock_utils_createrc, mock_utils_tempest,
+                                  mock_tarball):
+
+        clients = self.app.client_manager
+        workflow_client = clients.workflow_engine
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
 
         mock_update_parameters.return_value = {}
         mock_utils_endpoint.return_value = 'foo.bar'
 
-        tmp_dir = tempfile.NamedTemporaryFile(mode='w', delete=False).name
+        tmp_dir = tempfile.mkdtemp()
-        os.unlink(tmp_dir)
+        test_env = os.path.join(tmp_dir, 'foo2.yaml')
-        os.mkdir(tmp_dir)
+        self.addCleanup(shutil.rmtree, tmp_dir)
-        test_env = os.path.join(tmp_dir, 'foo.yaml')
+
         with open(test_env, 'w') as temp_file:
             temp_file.write('#just a comment')
 
@@ -589,15 +615,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = tmp_dir
 
         def _fake_heat_deploy(self, stack, stack_name, template_path,
-                              parameters, environments, timeout):
+                              parameters, environments, timeout, tht_root):
             assert test_env in environments
 
         mock_deploy_heat.side_effect = _fake_heat_deploy
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
         self.cmd.take_action(parsed_args)
-        os.unlink(test_env)
-        os.rmdir(tmp_dir)
 
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
@@ -634,14 +658,18 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_create_tempest_deployer_input.assert_called_with()
 
-    @mock.patch("heatclient.common.event_utils.get_events")
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
-    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env')
+                autospec=True)
+    @mock.patch("heatclient.common.event_utils.get_events", autospec=True)
+    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env',
+                autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_deploy_postconfig')
+                '_deploy_postconfig', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
-    @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
+    @mock.patch('tripleoclient.utils.generate_overcloud_passwords',
-    @mock.patch('tripleoclient.utils.create_overcloudrc')
+                autospec=True)
+    @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True)
     @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True)
     @mock.patch('time.sleep', return_value=None)
     @mock.patch('os_cloud_config.keystone.initialize', autospec=True)
@@ -673,7 +701,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                              mock_create_tempest_deployer_input,
                              mock_deploy_postconfig,
                              mock_breakpoints_cleanup,
-                             mock_events):
+                             mock_events, mock_tarball):
 
         arglist = ['--templates', '--rhel-reg',
                    '--reg-sat-url', 'https://example.com',
@@ -697,6 +725,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         orchestration_client = clients.orchestration
         orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
         mock_events.return_value = []
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
 
         mock_check_hypervisor_stats.return_value = {
             'count': 4,
@@ -851,7 +884,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_heat_deploy_func.assert_called_once_with(
             self.cmd, {}, 'overcloud',
             '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[0], {},
-            ['~/overcloud-env.json'], 1)
+            ['~/overcloud-env.json'], 1, '/fake/path')
 
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_heat_deploy')
@@ -866,10 +899,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_heat_deploy_func.assert_has_calls(
             [mock.call({}, 'overcloud',
                        '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[0], {},
-                       ['~/overcloud-env.json'], 1),
+                       ['~/overcloud-env.json'], 1, '/fake/path'),
              mock.call({}, 'overcloud',
                        '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[1], {},
-                       ['~/overcloud-env.json'], 1)])
+                       ['~/overcloud-env.json'], 1, '/fake/path')])
 
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_heat_deploy', autospec=True)
@@ -924,6 +957,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertFalse(mock_create_ocrc.called)
         self.assertFalse(mock_create_tempest_deployer_input.called)
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch('tripleoclient.utils.check_nodes_count',
                 autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
@@ -940,8 +975,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           mock_oc_endpoint,
                           mock_create_ocrc,
                           mock_create_tempest_deployer_input,
-                          mock_check_nodes_count):
+                          mock_check_nodes_count, mock_tarball):
         clients = self.app.client_manager
+
+        workflow_client = clients.workflow_engine
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
+
         network_client = clients.network
         network_client.stacks.get.return_value = None
         net = network_client.api.find_attr('networks', 'ctlplane')
@@ -951,7 +991,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             with open('/tmp/environment.yaml', "w+t") as environmentfile:
                 yaml.dump(
                     {'templates': '/dev/null',
-                     'environments': ['/tmp/foo.yaml']
+                     'environments': ['/tmp/foo3.yaml']
                      },
                     answerfile
                 )
@@ -978,9 +1018,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         call_args = mock_heat_deploy.call_args[0]
         self.assertEqual(call_args[3],
                          '/dev/null/overcloud-without-mergepy.yaml')
-        self.assertIn('/tmp/foo.yaml', call_args[5])
+        self.assertIn('/tmp/foo3.yaml', call_args[5])
         self.assertIn('/tmp/environment.yaml', call_args[5])
-        foo_index = call_args[5].index('/tmp/foo.yaml')
+        foo_index = call_args[5].index('/tmp/foo3.yaml')
         env_index = call_args[5].index('/tmp/environment.yaml')
         self.assertGreater(env_index, foo_index)
 
@@ -1071,6 +1111,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             stack, parameters, parsed_args)
         self.assertEqual(1, self.cmd.predeploy_errors)
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_create_parameters_env')
     @mock.patch('tripleoclient.utils.generate_overcloud_passwords')
@@ -1088,7 +1130,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                   mock_process_multiple_env,
                                   mock_create_overcloudrc,
                                   mock_generate_overcloud_passwords,
-                                  mock_create_parameters_env):
+                                  mock_create_parameters_env,
+                                  mock_tarball):
 
         arglist = ['--templates', '--control-scale', '3']
         verifylist = [
@@ -1098,6 +1141,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_get_key.return_value = "PASSWORD"
 
+        clients = self.app.client_manager
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
+
         mock_generate_overcloud_passwords.return_value = self._get_passwords()
 
         mock_create_env.return_value = "/fake/path"
@@ -1111,6 +1161,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           self.cmd.take_action,
                           parsed_args)
 
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_deploy_postconfig', autospec=True)
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
     @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env')
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_validate_args')
@@ -1160,7 +1214,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                  mock_create_tempest_deployer_input,
                                  mock_create_parameters_env,
                                  mock_validate_args,
-                                 mock_breakpoints_cleanup):
+                                 mock_breakpoints_cleanup,
+                                 mock_tarball,
+                                 mock_deploy_post_config):
 
         arglist = ['--templates', '--ceph-storage-scale', '3',
                    '--control-scale', '3']
@@ -1178,13 +1234,22 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         clients = self.app.client_manager
         orchestration_client = clients.orchestration
         mock_stack = fakes.create_tht_stack()
-        orchestration_client.stacks.get.return_value = None
+        orchestration_client.stacks.get.side_effect = [
+            None,
+            mock.MagicMock()
+        ]
 
         def _orch_clt_create(**kwargs):
             orchestration_client.stacks.get.return_value = mock_stack
 
         orchestration_client.stacks.create.side_effect = _orch_clt_create
 
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
+
         mock_check_hypervisor_stats.return_value = {
             'count': 4,
             'memory_mb': 4096,
@@ -1266,28 +1331,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         self.cmd.take_action(parsed_args)
 
-        args, kwargs = orchestration_client.stacks.create.call_args
-
-        self.assertEqual(kwargs['files'], {})
-        self.assertEqual(kwargs['template'], 'template')
-        self.assertEqual(kwargs['environment'], mock_env)
-        self.assertEqual(kwargs['stack_name'], 'overcloud')
-        self.assertEqual(kwargs['clear_parameters'],
-                         mock_env['parameter_defaults'].keys())
-
         mock_get_templte_contents.assert_called_with(
             '/usr/share/openstack-tripleo-heat-templates/' +
             constants.OVERCLOUD_YAML_NAMES[0])
 
         mock_create_tempest_deployer_input.assert_called_with()
         mock_process_multiple_env.assert_called_with(
-            ['/usr/share/openstack-tripleo-heat-templates/overcloud-resource-'
+            ['/fake/path', self.parameter_defaults_env_file])
-             'registry-puppet.yaml', '/fake/path',
-             self.parameter_defaults_env_file])
 
         mock_validate_args.assert_called_once_with(parsed_args)
-
-        mock_remove_known_hosts.assert_called_once_with('0.0.0.0')
-        mock_get_keystone_client.assert_called_once_with('admin', 'password',
-                                                         'admin',
-                                                         'http://0.0.0.0:8000')

tripleoclient/v1/overcloud_deploy.py

@@ -16,9 +16,11 @@ from __future__ import print_function
 
 import argparse
 import glob
+import hashlib
 import json
 import logging
 import os
+import os.path
 import re
 import six
 import tempfile
@@ -41,6 +43,9 @@ from tripleo_common import update
 from tripleoclient import constants
 from tripleoclient import exceptions
 from tripleoclient import utils
+from tripleoclient.workflows import deployment
+from tripleoclient.workflows import parameters
+from tripleoclient.workflows import plan_management
 
 
 class DeployOvercloud(command.Command):
@@ -228,10 +233,13 @@ class DeployOvercloud(command.Command):
         return [parameter_defaults_env_file]
 
     def _heat_deploy(self, stack, stack_name, template_path, parameters,
-                     environments, timeout):
+                     environments, timeout, tht_root):
         """Verify the Baremetal nodes are available and do a stack update"""
 
-        self.log.debug("Processing environment files")
+        clients = self.app.client_manager
+        workflow_client = clients.workflow_engine
+
+        self.log.debug("Processing environment files %s" % environments)
         env_files, env = (
             template_utils.process_multiple_environments_and_files(
                 environments))
@@ -253,47 +261,42 @@ class DeployOvercloud(command.Command):
                     '(with HA).')
 
         clients = self.app.client_manager
+        objectclient = clients.tripleoclient.object_store
+
+        moved_files = self._upload_missing_files(
+            stack_name, objectclient, files, tht_root)
+        self._process_and_upload_environment(
+            stack_name, objectclient, env, moved_files, tht_root,
+            workflow_client)
+
+        self._start_mistral_deploy(clients, stack, stack_name)
+
+    def _start_mistral_deploy(self, clients, stack, plan_name):
+
+        deployment.deploy(clients, container=plan_name,
+                          queue_name=str(uuid.uuid4()))
+
         orchestration_client = clients.orchestration
 
-        self.log.debug("Deploying stack: %s", stack_name)
-        self.log.debug("Deploying template: %s", template)
-        self.log.debug("Deploying parameters: %s", parameters)
-        self.log.debug("Deploying environment: %s", env)
-        self.log.debug("Deploying files: %s", files)
-
-        stack_args = {
-            'stack_name': stack_name,
-            'template': template,
-            'environment': env,
-            'files': files,
-            'clear_parameters': env['parameter_defaults'].keys(),
-        }
-
-        if timeout:
-            stack_args['timeout_mins'] = timeout
-
         if stack is None:
             self.log.info("Performing Heat stack create")
             action = 'CREATE'
             marker = None
-            orchestration_client.stacks.create(**stack_args)
         else:
             self.log.info("Performing Heat stack update")
             # Make sure existing parameters for stack are reused
-            stack_args['existing'] = 'true'
             # Find the last top-level event to use for the first marker
             events = event_utils.get_events(orchestration_client,
-                                            stack_id=stack_name,
+                                            stack_id=plan_name,
                                             event_args={'sort_dir': 'desc',
                                                         'limit': 1})
             marker = events[0].id if events else None
             action = 'UPDATE'
 
-            orchestration_client.stacks.update(stack.id, **stack_args)
+        time.sleep(10)
-
         verbose_events = self.app_args.verbose_level > 0
         create_result = utils.wait_for_stack_ready(
-            orchestration_client, stack_name, marker, action, verbose_events)
+            orchestration_client, plan_name, marker, action, verbose_events)
         if not create_result:
             if stack is None:
                 raise exceptions.DeploymentError("Heat Stack create failed.")
@@ -314,16 +317,107 @@ class DeployOvercloud(command.Command):
                         environments.append(f)
         return environments
 
+    def _process_and_upload_environment(self, container_name, swift_client,
+                                        env, moved_files, tht_root, mistral):
+        """Process the environment and upload to Swift
+
+        The environment at this point should be the result of the merged
+        custom user environments. We need to look at the paths in the
+        environment and update any that changed when they were uploaded to
+        swift.
+        """
+
+        file_prefix = "file://"
+
+        if 'resource_registry' in env:
+            for name, path in env['resource_registry'].items():
+                if not isinstance(path, six.string_types):
+                    continue
+                if path in moved_files:
+                    new_path = moved_files[path]
+                    env['resource_registry'][name] = new_path
+                elif path.startswith(file_prefix):
+                    path = path[len(file_prefix):]
+                    if path.startswith(tht_root):
+                        path = path[len(tht_root):]
+                    # We want to make sure all the paths are relative.
+                    if path.startswith("/"):
+                        path = path[1:]
+                    env['resource_registry'][name] = path
+
+        # Parameters are removed from the environment and sent to the update
+        # parameters action, this stores them in the Mistral environment and
+        # means the UI can find them.
+        if 'parameter_defaults' in env:
+            params = env.pop('parameter_defaults')
+            parameters.update_parameters(mistral, container=container_name,
+                                         parameters=params)
+
+        contents = yaml.safe_dump(env)
+
+        swift_path = "user-environment.yaml"
+        swift_client.put_object(container_name, swift_path, contents)
+
+        mistral_env = mistral.environments.get(container_name)
+        mistral_env.variables['environments'].append({'path': swift_path})
+        mistral.environments.update(
+            name=container_name,
+            variables=mistral_env.variables
+        )
+
+    def _upload_missing_files(self, container_name, swift_client, files_dict,
+                              tht_root):
+        """Find the files referenced in custom environments and upload them
+
+        Heat environments can be passed to be included in the deployment, these
+        files can include references to other files anywhere on the local
+        file system. These need to be discovered and uploaded to Swift. When
+        they have been uploaded to Swift the path to them will be different,
+        the new paths are store din the file_relocation dict, which is returned
+        and used by _process_and_upload_environment which will merge the
+        environment and update paths to the relative Swift path.
+        """
+
+        file_relocation = {}
+        file_prefix = "file://"
+
+        for fullpath, contents in files_dict.items():
+
+            if not fullpath.startswith(file_prefix):
+                continue
+
+            path = fullpath[len(file_prefix):]
+
+            if path.startswith(tht_root):
+                # This should already be uploaded.
+                continue
+
+            filename = os.path.basename(path)
+            checksum = hashlib.md5()
+            checksum.update(filename)
+            digest = checksum.hexdigest()
+            swift_path = "user-files/{}-{}".format(digest, filename)
+            swift_client.put_object(container_name, swift_path, contents)
+            file_relocation[fullpath] = swift_path
+
+        return file_relocation
+
     def _deploy_tripleo_heat_templates(self, stack, parsed_args):
         """Deploy the fixed templates in TripleO Heat Templates"""
         clients = self.app.client_manager
         network_client = clients.network
+        workflow_client = clients.workflow_engine
 
         parameters = self._update_parameters(
             parsed_args, network_client, stack)
 
         tht_root = parsed_args.templates
 
+        plans = plan_management.list_deployment_plans(workflow_client)
+        if parsed_args.stack not in plans:
+            plan_management.create_plan_from_templates(
+                clients, parsed_args.stack, tht_root)
+
         print("Deploying templates in the directory {0}".format(
             os.path.abspath(tht_root)))
 
@@ -333,13 +427,11 @@ class DeployOvercloud(command.Command):
         # parameters and keystone cert is generated on create only
         env_path = utils.create_environment_file()
         environments = []
-        add_registry = False
 
         if stack is None:
             self.log.debug("Creating Keystone certificates")
             keystone_pki.generate_certs_into_json(env_path, False)
             environments.append(env_path)
-            add_registry = True
 
         if parsed_args.environment_directories:
             environments.extend(self._load_environment_directories(
@@ -349,19 +441,8 @@ class DeployOvercloud(command.Command):
         if parsed_args.rhel_reg:
             reg_env = self._create_registration_env(parsed_args)
             environments.extend(reg_env)
-            add_registry = True
         if parsed_args.environment_files:
             environments.extend(parsed_args.environment_files)
-            add_registry = True
-
-        if add_registry:
-            # default resource registry file should be passed only
-            # when creating a new stack, or when custom environments are
-            # specified, otherwise it might overwrite
-            # resource_registries in existing stack
-            resource_registry_path = os.path.join(
-                tht_root, constants.RESOURCE_REGISTRY_NAME)
-            environments.insert(0, resource_registry_path)
 
         self._try_overcloud_deploy_with_compat_yaml(
             tht_root, stack, parsed_args.stack, parameters, environments,
@@ -375,7 +456,7 @@ class DeployOvercloud(command.Command):
             overcloud_yaml = os.path.join(tht_root, overcloud_yaml_name)
             try:
                 self._heat_deploy(stack, stack_name, overcloud_yaml,
-                                  parameters, environments, timeout)
+                                  parameters, environments, timeout, tht_root)
             except six.moves.urllib.error.URLError as e:
                 messages.append(str(e.reason))
             else:

tripleoclient/workflows/deployment.py

@@ -0,0 +1,30 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from __future__ import print_function
+
+import pprint
+
+
+def deploy(clients, **workflow_input):
+
+    workflow_client = clients.workflow_engine
+    tripleoclients = clients.tripleoclient
+    queue_name = workflow_input['queue_name']
+
+    execution = workflow_client.executions.create(
+        'tripleo.deployment.v1.deploy_plan',
+        workflow_input=workflow_input
+    )
+
+    with tripleoclients.messaging_websocket(queue_name) as ws:
+        message = ws.wait_for_message(execution.id)
+        assert message['status'] == "SUCCESS", pprint.pformat(message)

tripleoclient/workflows/parameters.py

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tripleoclient.workflows import base
+
+
+def update_parameters(workflow_client, **input_):
+    return base.call_action(workflow_client, 'tripleo.update_parameters',
+                            **input_)