Convert enable-ssh-admin.sh to python

Instead of using the script from the templates, use python for the
enable-ssh-admin logic. This will allow for more properly handling
failures.

This also fixes a race condition where sshd has not already started on
some of the nodes before we try and connect via ssh. A timeout is added
where we wait for the port to come up. If the timeout has passed and the
port is still not up, then an exception is raised.

Change-Id: I3431d2ec724a880baf0de8f586490d145bedf870
Closes-Bug: #1769230

tripleoclient/constants.py

@@ -63,3 +63,7 @@ UPGRADE_CONVERGE_ENV = "environments/lifecycle/upgrade-converge.yaml"
 FFWD_UPGRADE_PREPARE_ENV = "environments/lifecycle/ffwd-upgrade-prepare.yaml"
 FFWD_UPGRADE_CONVERGE_ENV = "environments/lifecycle/ffwd-upgrade-converge.yaml"
 CEPH_UPGRADE_PREPARE_ENV = "environments/lifecycle/ceph-upgrade-prepare.yaml"
+
+ENABLE_SSH_ADMIN_TIMEOUT = 300
+ENABLE_SSH_ADMIN_STATUS_INTERVAL = 5
+ENABLE_SSH_ADMIN_SSH_PORT_TIMEOUT = 300

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -70,6 +70,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.cmd._download_missing_files_from_plan = self.real_download_missing
         shutil.rmtree = self.real_shutil
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -109,7 +111,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                        mock_events, mock_tarball,
                        mock_get_horizon_url,
                        mock_list_plans,
-                       mock_config_download):
+                       mock_config_download,
+                       mock_enable_ssh_admin,
+                       mock_get_overcloud_hosts):
 
         arglist = ['--templates', '--ceph-storage-scale', '3']
         verifylist = [
@@ -182,6 +186,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_create_tempest_deployer_input.assert_called_with()
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -227,7 +233,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                         mock_invoke_plan_env_wf,
                         mock_get_horizon_url,
                         mock_list_plans,
-                        mock_config_download):
+                        mock_config_download,
+                        mock_enable_ssh_admin,
+                        mock_get_overcloud_hosts):
 
         arglist = ['--templates', '--ceph-storage-scale', '3',
                    '--control-flavor', 'oooq_control', '--no-cleanup']
@@ -324,6 +332,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertEqual(env_map.get('parameter_defaults'),
                          parameters_env.get('parameter_defaults'))
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -360,16 +370,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('shutil.copytree', autospec=True)
     @mock.patch('tempfile.mkdtemp', autospec=True)
     def test_tht_deploy_with_plan_environment_file(
-        self, mock_tmpdir, mock_copy, mock_time, mock_sleep, mock_uuid1,
+            self, mock_tmpdir, mock_copy, mock_time, mock_sleep, mock_uuid1,
-        mock_get_template_contents, wait_for_stack_ready_mock,
+            mock_get_template_contents, wait_for_stack_ready_mock,
-        mock_remove_known_hosts, mock_overcloudrc, mock_write_overcloudrc,
+            mock_remove_known_hosts, mock_overcloudrc, mock_write_overcloudrc,
-        mock_create_tempest_deployer, mock_create_parameters_env,
+            mock_create_tempest_deployer, mock_create_parameters_env,
-        mock_validate_args,
+            mock_validate_args,
-        mock_breakpoints_cleanup,
+            mock_breakpoints_cleanup,
-        mock_tarball, mock_postconfig,
+            mock_tarball, mock_postconfig,
-        mock_get_overcloud_endpoint, mock_shutil_rmtree,
+            mock_get_overcloud_endpoint, mock_shutil_rmtree,
-        mock_invoke_plan_env_wf, mock_get_horizon_url,
+            mock_invoke_plan_env_wf, mock_get_horizon_url,
-            mock_list_plans, mock_config_download):
+            mock_list_plans, mock_config_download,
+            mock_enable_ssh_admin,
+            mock_get_overcloud_hosts):
+
         arglist = ['--templates', '-p', 'the-plan-environment.yaml']
         verifylist = [
             ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
@@ -468,6 +481,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         clients.tripleoclient.object_store.put_object.assert_called()
         self.assertTrue(mock_invoke_plan_env_wf.called)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -514,7 +529,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             mock_breakpoints_cleanup, mock_tarball,
             mock_postconfig, mock_get_overcloud_endpoint,
             mock_deprecated_params, mock_get_horizon_url,
-            mock_list_plans, mock_config_downlad):
+            mock_list_plans, mock_config_downlad,
+            mock_enable_ssh_admin,
+            mock_get_overcloud_hosts):
 
         arglist = ['--templates', '--skip-deploy-identifier']
         verifylist = [
@@ -574,6 +591,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         deploy_plan_call_input = deploy_plan_call[1]['workflow_input']
         self.assertTrue(deploy_plan_call_input['skip_deploy_identifier'])
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -608,7 +627,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                      mock_events, mock_tarball,
                                      mock_get_horizon_url,
                                      mock_list_plans,
-                                     mock_config_download):
+                                     mock_config_download,
+                                     mock_enable_ssh_admin,
+                                     mock_get_overcloud_hosts):
 
         arglist = ['--templates', '/home/stack/tripleo-heat-templates']
         verifylist = [
@@ -693,6 +714,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             self.cmd.take_action, parsed_args)
         self.assertFalse(mock_deploy_tht.called)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -717,7 +740,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                               mock_utils_endpoint, mock_utils_createrc,
                               mock_utils_tempest, mock_tarball,
                               mock_get_horizon_url, mock_list_plans,
-                              mock_config_download):
+                              mock_config_download,
+                              mock_enable_ssh_admin,
+                              mock_get_overcloud_hosts):
 
         clients = self.app.client_manager
         mock_list_plans.return_value = []
@@ -937,6 +962,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                   parsed_args)
         self.assertIn('/tmp/notthere', str(error))
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch('tripleoclient.workflows.deployment.get_horizon_url',
                 autospec=True)
@@ -952,7 +979,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                       mock_create_ocrc,
                                       mock_create_tempest_deployer_input,
                                       mock_get_horizon_url,
-                                      mock_config_download):
+                                      mock_config_download,
+                                      mock_enable_ssh_admin,
+                                      mock_get_overcloud_hosts):
 
         clients = self.app.client_manager
         workflow_client = clients.workflow_engine
@@ -979,6 +1008,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_create_tempest_deployer_input.assert_called_with()
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -1019,7 +1050,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                              mock_events, mock_tarball,
                              mock_get_horizon_url,
                              mock_list_plans,
-                             mock_config_download):
+                             mock_config_download,
+                             mock_enable_ssh_admin,
+                             mock_get_overcloud_hosts):
 
         arglist = ['--templates', '--rhel-reg',
                    '--reg-sat-url', 'https://example.com',
@@ -1230,6 +1263,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertFalse(mock_create_ocrc.called)
         self.assertFalse(mock_create_tempest_deployer_input.called)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -1256,7 +1291,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           mock_create_tempest_deployer_input,
                           mock_tarball, mock_get_horizon_url,
                           mock_list_plans,
-                          mock_config_download):
+                          mock_config_download,
+                          mock_enable_ssh_admin,
+                          mock_get_overcloud_hosts):
         clients = self.app.client_manager
 
         mock_list_plans.return_value = []
@@ -1420,6 +1457,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           self.cmd.take_action,
                           parsed_args)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch(
         'tripleoclient.workflows.plan_management.list_deployment_plans',
@@ -1464,7 +1503,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                                  mock_deploy_post_config,
                                  mock_get_horizon_url,
                                  mock_list_plans,
-                                 mock_config_download):
+                                 mock_config_download,
+                                 mock_enable_ssh_admin,
+                                 mock_get_overcloud_hosts):
 
         arglist = ['--templates', '--ceph-storage-scale', '3',
                    '--control-scale', '3', '--ntp-server', 'ntp']
@@ -1616,6 +1657,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertRaises(exceptions.StackInProgress,
                           self.cmd.take_action, parsed_args)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch('tripleoclient.workflows.deployment.get_horizon_url',
                 autospec=True)
@@ -1633,7 +1676,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             mock_overcloud_endpoint,
             mock_create_tempest_deployer_input,
             mock_get_horizon_url,
-            mock_config_download):
+            mock_config_download,
+            mock_enable_ssh_admin,
+            mock_get_overcloud_hosts):
         clients = self.app.client_manager
         orchestration_client = clients.orchestration
         orchestration_client.stacks.get.return_value = mock.Mock()
@@ -1648,6 +1693,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertNotCalled(self.cmd._predeploy_verify_capabilities)
         mock_create_tempest_deployer_input.assert_called_with()
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch('tripleoclient.workflows.deployment.get_horizon_url',
                 autospec=True)
@@ -1665,7 +1712,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             mock_overcloud_endpoint,
             mock_create_tempest_deployer_input,
             mock_get_horizon_url,
-            mock_config_download):
+            mock_config_download,
+            mock_enable_ssh_admin,
+            mock_get_overcloud_hosts):
         clients = self.app.client_manager
         orchestration_client = clients.orchestration
         orchestration_client.stacks.get.return_value = mock.Mock()
@@ -1707,6 +1756,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         self.assertRaises(exceptions.InvalidConfiguration,
                           self.cmd.take_action, parsed_args)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.config_download')
     @mock.patch('tripleoclient.workflows.deployment.get_horizon_url',
                 autospec=True)
@@ -1724,7 +1775,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                              mock_get_overcloud_endpoint,
                              mock_provision, mock_tempest_deploy_input,
                              mock_get_horizon_url,
-                             mock_config_download):
+                             mock_config_download,
+                             mock_enable_ssh_admin,
+                             mock_get_overcloud_hosts):
         arglist = ['--templates', '--deployed-server', '--disable-validations']
         verifylist = [
             ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
@@ -1764,6 +1817,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                           parsed_args)
         self.assertFalse(mock_deploy_tmpdir.called)
 
+    @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts')
+    @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin')
     @mock.patch('tripleoclient.workflows.deployment.get_horizon_url',
                 autospec=True)
     @mock.patch('tripleoclient.workflows.deployment.config_download')
@@ -1780,7 +1835,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             mock_overcloudrc, mock_write_overcloudrc,
             mock_overcloud_endpoint,
             mock_create_tempest_deployer_input,
-            mock_config_download, mock_get_horizon_url):
+            mock_config_download, mock_get_horizon_url,
+            mock_enable_ssh_admin,
+            mock_get_overcloud_hosts):
         clients = self.app.client_manager
         orchestration_client = clients.orchestration
         orchestration_client.stacks.get.return_value = mock.Mock()
@@ -1794,6 +1851,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         self.cmd.take_action(parsed_args)
         self.assertTrue(mock_deploy_tmpdir.called)
+        self.assertTrue(mock_enable_ssh_admin.called)
+        self.assertTrue(mock_get_overcloud_hosts.called)
         self.assertTrue(mock_config_download.called)
 
     def test_download_missing_files_from_plan(self):

tripleoclient/tests/workflows/test_deployment.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osc_lib.tests import utils
+
+from tripleoclient.workflows import deployment
+
+
+class TestDeploymentWorkflows(utils.TestCommand):
+
+    def setUp(self):
+        super(TestDeploymentWorkflows, self).setUp()
+
+        self.app.client_manager.workflow_engine = self.workflow = mock.Mock()
+        self.tripleoclient = mock.Mock()
+        self.websocket = mock.Mock()
+        self.websocket.__enter__ = lambda s: self.websocket
+        self.websocket.__exit__ = lambda s, *exc: None
+        self.tripleoclient.messaging_websocket.return_value = self.websocket
+        self.app.client_manager.tripleoclient = self.tripleoclient
+
+        self.message_success = iter([{
+            "execution": {"id": "IDID"},
+            "status": "SUCCESS",
+            "message": "Success.",
+            "registered_nodes": [],
+        }])
+        self.message_failed = iter([{
+            "execution": {"id": "IDID"},
+            "status": "FAIL",
+            "message": "Fail.",
+        }])
+
+    @mock.patch('tripleoclient.workflows.deployment.wait_for_ssh_port')
+    @mock.patch('tripleoclient.workflows.deployment.time.sleep')
+    @mock.patch('tripleoclient.workflows.deployment.shutil.rmtree')
+    @mock.patch('tripleoclient.workflows.deployment.open')
+    @mock.patch('tripleoclient.workflows.deployment.tempfile')
+    @mock.patch('tripleoclient.workflows.deployment.subprocess.check_call')
+    def test_enable_ssh_admin(self, mock_check_call, mock_tempfile,
+                              mock_open, mock_rmtree, mock_sleep,
+                              mock_wait_for_ssh_port):
+        log = mock.Mock()
+        hosts = 'a', 'b', 'c'
+        ssh_user = 'test-user'
+        ssh_key = 'test-key'
+
+        mock_tempfile.mkdtemp.return_value = '/foo'
+        mock_read = mock.Mock()
+        mock_read.read.return_value = 'key'
+        mock_open.return_value = mock_read
+        mock_state = mock.Mock()
+        mock_state.state = 'SUCCESS'
+        self.workflow.executions.get.return_value = mock_state
+        deployment.enable_ssh_admin(log, self.app.client_manager,
+                                    hosts, ssh_user, ssh_key)
+
+        # once for ssh-keygen, then twice per host
+        self.assertEqual(7, mock_check_call.call_count)
+
+        # execution ran
+        self.assertEqual(1, self.workflow.executions.create.call_count)
+        call_args = self.workflow.executions.create.call_args
+        self.assertEqual('tripleo.access.v1.enable_ssh_admin', call_args[0][0])
+        self.assertEqual(('a', 'b', 'c'),
+                         call_args[1]['workflow_input']['ssh_servers'])
+        self.assertEqual('test-user',
+                         call_args[1]['workflow_input']['ssh_user'])
+        self.assertEqual('key',
+                         call_args[1]['workflow_input']['ssh_private_key'])
+
+        # tmpdir should be cleaned up
+        self.assertEqual(1, mock_rmtree.call_count)
+        self.assertEqual('/foo', mock_rmtree.call_args[0][0])

tripleoclient/v1/overcloud_deploy.py

@@ -625,6 +625,8 @@ class DeployOvercloud(command.Command):
         )
         parser.add_argument(
             '--overcloud-ssh-key',
+            default=os.path.join(
+                os.path.expanduser('~'), '.ssh', 'id_rsa'),
             help=_('Key path for ssh access to overcloud nodes.')
         )
         parser.add_argument(
@@ -899,9 +901,13 @@ class DeployOvercloud(command.Command):
         if parsed_args.config_download:
             print("Deploying overcloud configuration")
 
+            hosts = deployment.get_overcloud_hosts(self.clients, stack)
+            deployment.enable_ssh_admin(self.log, self.clients,
+                                        hosts,
+                                        parsed_args.overcloud_ssh_user,
+                                        parsed_args.overcloud_ssh_key)
             deployment.config_download(self.log, self.clients, stack,
                                        parsed_args.templates,
-                                       parsed_args.deployed_server,
                                        parsed_args.overcloud_ssh_user,
                                        parsed_args.overcloud_ssh_key,
                                        parsed_args.output_dir,

tripleoclient/workflows/deployment.py

@@ -14,12 +14,16 @@ from __future__ import print_function
 import os
 import pprint
 import re
+import shutil
+import socket
 import subprocess
+import tempfile
 import time
 
 from heatclient.common import event_utils
 from openstackclient import shell
 
+from tripleoclient import constants
 from tripleoclient import exceptions
 from tripleoclient import utils
 
@@ -104,46 +108,129 @@ def overcloudrc(workflow_client, **input_):
                             **input_)
 
 
-def config_download(log, clients, stack, templates, deployed_server,
+def get_overcloud_hosts(clients, stack):
-                    ssh_user, ssh_key, output_dir, verbosity=1):
     role_net_hostname_map = utils.get_role_net_hostname_map(stack)
     hostnames = []
     for role in role_net_hostname_map:
         hostnames.extend(role_net_hostname_map[role].get('ctlplane', []))
 
-    ips = []
+    hosts = []
     hosts_entry = utils.get_hosts_entry(stack)
     for hostname in hostnames:
         for line in hosts_entry.split('\n'):
             match = re.search('\s*%s\s*' % hostname, line)
             if match:
-                ips.append(line.split(' ')[0])
+                hosts.append(line.split(' ')[0])
 
-    script_path = os.path.join(templates,
+    return hosts
-                               'deployed-server',
-                               'scripts',
-                               'enable-ssh-admin.sh')
 
-    env = os.environ.copy()
-    env.update(dict(OVERCLOUD_HOSTS=' '.join(ips),
-                    OVERCLOUD_SSH_USER=ssh_user))
-
-    if ssh_key:
-        env['OVERCLOUD_SSH_KEY'] = ssh_key
-
-    proc = subprocess.Popen([script_path], env=env, shell=True,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.STDOUT)
 
+def wait_for_ssh_port(host):
+    start = int(time.time())
     while True:
-        line = proc.stdout.readline().decode('utf-8')
+        now = int(time.time())
-        if line:
+        if (now - start) > constants.ENABLE_SSH_ADMIN_SSH_PORT_TIMEOUT:
-            log.info(line.rstrip())
+            raise exceptions.DeploymentError(
-        if line == '' and proc.poll() is not None:
+                "Timed out waiting for port 22 from %s" % host)
-            break
-    if proc.returncode != 0:
-        raise RuntimeError('%s failed.' % script_path)
 
+        try:
+            socket.socket().connect((host, 22))
+            return
+        except socket.error:
+            pass
+
+        time.sleep(1)
+
+
+def enable_ssh_admin(log, clients, hosts, ssh_user, ssh_key):
+    print("Enabling ssh admin (tripleo-admin) for hosts:")
+    print(" ".join(hosts))
+    print("Using ssh user %s for initial connection." % ssh_user)
+    print("Using ssh key at %s for initial connection." % ssh_key)
+
+    ssh_options = ("-o ConnectionAttempts=6 "
+                   "-o ConnectTimeout=30 "
+                   "-o StrictHostKeyChecking=no "
+                   "-o UserKnownHostsFile=/dev/null")
+    tmp_key_dir = tempfile.mkdtemp()
+    tmp_key_private = os.path.join(tmp_key_dir, 'id_rsa')
+    tmp_key_public = os.path.join(tmp_key_dir, 'id_rsa.pub')
+    tmp_key_comment = "TripleO split stack short term key"
+
+    try:
+        tmp_key_command = ["ssh-keygen", "-N", "", "-t", "rsa", "-b", "4096",
+                           "-f", tmp_key_private, "-C", tmp_key_comment]
+        subprocess.check_call(tmp_key_command, stderr=subprocess.STDOUT)
+        tmp_key_public_contents = open(tmp_key_public).read()
+
+        for host in hosts:
+            wait_for_ssh_port(host)
+            copy_tmp_key_command = ["ssh"] + ssh_options.split()
+            copy_tmp_key_command += \
+                ["-o", "StrictHostKeyChecking=no",
+                 "-i", ssh_key, "-l", ssh_user, host,
+                 "echo -e '\n%s' >> $HOME/.ssh/authorized_keys" %
+                 tmp_key_public_contents]
+            print("Inserting TripleO short term key for %s" % host)
+            subprocess.check_call(copy_tmp_key_command,
+                                  stderr=subprocess.STDOUT)
+
+        print("Starting ssh admin enablement workflow")
+
+        workflow_client = clients.workflow_engine
+
+        workflow_input = {
+            "ssh_user": ssh_user,
+            "ssh_servers": hosts,
+            "ssh_private_key": open(tmp_key_private).read(),
+        }
+
+        execution = base.start_workflow(
+            workflow_client,
+            'tripleo.access.v1.enable_ssh_admin',
+            workflow_input=workflow_input
+        )
+
+        start = int(time.time())
+        while True:
+            now = int(time.time())
+            if (now - start) > constants.ENABLE_SSH_ADMIN_TIMEOUT:
+                raise exceptions.DeploymentError(
+                    "ssh admin enablement workflow - TIMED OUT.")
+
+            time.sleep(1)
+            execution = workflow_client.executions.get(execution.id)
+            state = execution.state
+
+            if state == 'RUNNING':
+                if (now - start) % constants.ENABLE_SSH_ADMIN_STATUS_INTERVAL\
+                        == 0:
+                    print("ssh admin enablement workflow - RUNNING.")
+                continue
+            elif state == 'SUCCESS':
+                print("ssh admin enablement workflow - COMPLETE.")
+                break
+            elif state == 'FAILED':
+                raise exceptions.DeploymentError(
+                    "ssh admin enablement workflow - FAILED.")
+
+        for host in hosts:
+            rm_tmp_key_command = ["ssh"] + ssh_options.split()
+            rm_tmp_key_command += \
+                ["-l", ssh_user, host,
+                 "sed -i -e '/%s/d' $HOME/.ssh/authorized_keys" %
+                 tmp_key_comment]
+            print("Removing TripleO short term key from %s" % host)
+            subprocess.check_call(rm_tmp_key_command, stderr=subprocess.STDOUT)
+    finally:
+        print("Removing short term keys locally")
+        shutil.rmtree(tmp_key_dir)
+
+    print("Enabling ssh admin - COMPLETE.")
+
+
+def config_download(log, clients, stack, templates,
+                    ssh_user, ssh_key, output_dir, verbosity=1):
     workflow_client = clients.workflow_engine
     tripleoclients = clients.tripleoclient