Dynamic defaults for roles, networks, vips and bm

When running 'overcloud deploy' command look for dynamic
defaults file for these options:
  --roles-file, --network-file, --vip-file and
  --baremetal-deployment

When the option is set by the user, use the user provided
file and make sure a copy is created in the working
directory. If the argument is not set look in the working
directory for the file used previously and use that file.

overclod node, and overcloud network commands require the
user input. But will place a copy in the working_dir for
overcloud deploy.

The depends-on creates these "defaults" by running the
different nova-less/network-v2 export commands when
upgrading the undercloud. With this change the next
'overcloud deploy' after the undercloud upgrade will use
the correct files (unless the user set the args ...)

Depends-On: https://review.opendev.org/795773
Change-Id: I53ba631dc80428c6f1fe71c2bbfb0b5a36dd8f01

tripleoclient/constants.py

@@ -279,3 +279,13 @@ DEFAULT_PARTITION_IMAGE = 'overcloud-full.qcow2'
 DEFAULT_WHOLE_DISK_IMAGE = 'overcloud-hardened-uefi-full.qcow2'
 
 FIPS_COMPLIANT_HASHES = {'sha1', 'sha224', 'sha256', 'sha384', 'sha512'}
+
+# Work-Dir default file names
+WD_DEFAULT_ROLES_FILE_NAME = 'tripleo-{}-roles-data.yaml'
+WD_DEFAULT_NETWORKS_FILE_NAME = 'tripleo-{}-network-data.yaml'
+WD_DEFAULT_VIP_FILE_NAME = 'tripleo-{}-virtual-ips.yaml'
+WD_DEFAULT_BAREMETAL_FILE_NAME = 'tripleo-{}-baremetal-deployment.yaml'
+KIND_TEMPLATES = {'roles': WD_DEFAULT_ROLES_FILE_NAME,
+                  'networks': WD_DEFAULT_NETWORKS_FILE_NAME,
+                  'baremetal': WD_DEFAULT_BAREMETAL_FILE_NAME,
+                  'vips': WD_DEFAULT_VIP_FILE_NAME}

tripleoclient/tests/test_utils.py

@@ -2369,3 +2369,74 @@ class TestParseContainerImagePrepare(TestCase):
                 utils.parse_container_image_prepare(key, keys,
                                                     cfgfile.name)
         self.assertEqual(reg_actual, reg_expected)
+
+
+class TestWorkingDirDefaults(base.TestCase):
+
+    def setUp(self):
+        super(TestWorkingDirDefaults, self).setUp()
+        self.working_dir = tempfile.mkdtemp()
+        self.stack = 'overcloud'
+        self.wd_roles_file = os.path.join(
+            self.working_dir,
+            utils.constants.WD_DEFAULT_ROLES_FILE_NAME.format(self.stack))
+        self.wd_networks_file = os.path.join(
+            self.working_dir,
+            utils.constants.WD_DEFAULT_NETWORKS_FILE_NAME.format(self.stack))
+        self.wd_vip_file = os.path.join(
+            self.working_dir,
+            utils.constants.WD_DEFAULT_VIP_FILE_NAME.format(self.stack))
+        self.wd_barametal_file = os.path.join(
+            self.working_dir,
+            utils.constants.WD_DEFAULT_BAREMETAL_FILE_NAME.format(self.stack))
+
+    def tearDown(self):
+        super(TestWorkingDirDefaults, self).tearDown()
+        shutil.rmtree(self.working_dir)
+
+    @mock.patch.object(utils, 'rewrite_ansible_playbook_paths', autospec=True)
+    @mock.patch.object(shutil, 'copy', autospec=True)
+    def test_update_working_dir_defaults(self, mock_shutil_copy,
+                                         mock_rewrite_ansible_playbook_paths):
+        args = mock.Mock()
+        args.stack = self.stack
+        args.templates = '/tht_root'
+        args.roles_file = '/dir/roles_file.yaml'
+        args.networks_file = '/dir/networks_file.yaml'
+        args.vip_file = '/dir/vip_file.yaml'
+        args.baremetal_deployment = '/dir/baremetal_deployment.yaml'
+
+        utils.update_working_dir_defaults(self.working_dir, args)
+
+        mock_shutil_copy.assert_has_calls(
+            [mock.call(args.baremetal_deployment, self.wd_barametal_file),
+             mock.call(args.roles_file, self.wd_roles_file),
+             mock.call(args.networks_file, self.wd_networks_file),
+             mock.call(args.vip_file, self.wd_vip_file)])
+
+    def test_rewrite_ansible_playbook_paths(self):
+        src = '/rel/path/baremetal.yaml'
+        dest = self.wd_barametal_file
+        roles = '''
+        - name: Controller
+          ansible_playbooks:
+          - playbook: controller-playbook.yaml
+          - playbook: /abs/path/controller-playbook.yaml
+        - name: Compute
+          ansible_playbooks:
+          - playbook: compute-playbook.yaml
+          - playbook: /abs/path/compute-playbook.yaml
+        '''
+        with open(dest, 'w') as f:
+            f.write(roles)
+        utils.rewrite_ansible_playbook_paths(src, dest)
+        with open(dest, 'r') as f:
+            data = yaml.safe_load(f.read())
+        self.assertEqual(data[0]['ansible_playbooks'][0]['playbook'],
+                         '/rel/path/controller-playbook.yaml')
+        self.assertEqual(data[0]['ansible_playbooks'][1]['playbook'],
+                         '/abs/path/controller-playbook.yaml')
+        self.assertEqual(data[1]['ansible_playbooks'][0]['playbook'],
+                         '/rel/path/compute-playbook.yaml')
+        self.assertEqual(data[1]['ansible_playbooks'][1]['playbook'],
+                         '/abs/path/compute-playbook.yaml')

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -120,6 +120,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         horizon_url.return_value = 'fake://url:12345'
         self.addCleanup(horizon_url.stop)
 
+        # Mock copy to working dir
+        mock_copy_to_wd = mock.patch(
+            'tripleoclient.utils.copy_to_wd', autospec=True)
+        mock_copy_to_wd.start()
+        self.addCleanup(mock_copy_to_wd.stop)
+
     def tearDown(self):
         super(TestDeployOvercloud, self).tearDown()
         os.unlink(self.parameter_defaults_env_file)
@@ -359,7 +365,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         utils_overcloud_fixture.mock_deploy_tht.assert_called_with(
             output_dir=self.cmd.working_dir)
 
-        mock_validate_args.assert_called_once_with(parsed_args)
+        mock_validate_args.assert_called_once_with(parsed_args,
+                                                   self.cmd.working_dir)
         self.assertFalse(mock_invoke_plan_env_wf.called)
 
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
@@ -700,10 +707,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         ]
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
-
+        working_dir = self.tmp_dir.join('working_dir')
         self.assertRaises(oscexc.CommandError,
                           overcloud_deploy._validate_args,
-                          parsed_args)
+                          parsed_args, working_dir)
 
     @mock.patch('os.path.isfile', autospec=True)
     def test_validate_args_missing_rendered_files(self, mock_isfile):
@@ -718,8 +725,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         mock_isfile.side_effect = [False, True]
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        working_dir = self.tmp_dir.join('working_dir')
 
-        overcloud_deploy._validate_args(parsed_args)
+        overcloud_deploy._validate_args(parsed_args, working_dir)
         calls = [mock.call(env_path),
                  mock.call(env_path.replace(".yaml", ".j2.yaml"))]
         mock_isfile.assert_has_calls(calls)
@@ -1042,7 +1050,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_create_tempest_deployer_input.assert_called_with(
             output_dir=self.cmd.working_dir)
 
-        mock_validate_args.assert_called_once_with(parsed_args)
+        mock_validate_args.assert_called_once_with(parsed_args,
+                                                   self.cmd.working_dir)
         mock_copy.assert_called_once()
 
     @mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@@ -1481,11 +1490,15 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             utils_fixture2.mock_run_ansible_playbook.mock_calls)
 
     def test_provision_baremetal(self):
+        self.cmd.working_dir = self.tmp_dir.join('working_dir')
+        os.mkdir(self.cmd.working_dir)
+        bm_deploy_path = os.path.join(
+            self.cmd.working_dir,
+            'tripleo-overcloud-baremetal-deployment.yaml')
         baremetal_deployed = {
             'parameter_defaults': {'foo': 'bar'}
         }
 
-        bm_deploy_path = self.tmp_dir.join('bm_deploy.yaml')
         deploy_data = [
             {'name': 'Compute', 'count': 10},
             {'name': 'Controller', 'count': 3},
@@ -1570,8 +1583,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         ])
         self.mock_role_playbooks.assert_called_once_with(
             self.cmd,
-            self.tmp_dir.join('working_dir'),
+            self.cmd.working_dir,
-            self.tmp_dir.path,
+            self.cmd.working_dir,
             [
                 {'count': 10, 'name': 'Compute'},
                 {'count': 3, 'name': 'Controller'}
@@ -1580,7 +1593,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         )
 
     def test__provision_networks(self):
-        networks_file_path = self.tmp_dir.join('networks.yaml')
+        self.cmd.working_dir = self.tmp_dir.join('working_dir')
+        os.mkdir(self.cmd.working_dir)
+        networks_file_path = os.path.join(
+            self.cmd.working_dir, 'tripleo-overcloud-network-data.yaml')
         fake_network_data = [{'name': 'Network', 'name_lower': 'network'}]
         fake_deployed_env = {
             'parameter_defaults':
@@ -1624,13 +1640,17 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             workdir=mock.ANY)
 
     def test__provision_virtual_ips(self):
-        networks_file_path = self.tmp_dir.join('networks.yaml')
+        self.cmd.working_dir = self.tmp_dir.join('working_dir')
+        os.mkdir(self.cmd.working_dir)
+        networks_file_path = os.path.join(
+            self.cmd.working_dir, 'tripleo-overcloud-network-data.yaml')
         network_data = [
             {'name': 'Network', 'name_lower': 'network', 'subnets': {}}
         ]
         with open(networks_file_path, 'w') as temp_file:
             yaml.safe_dump(network_data, temp_file)
-        vips_file_path = self.tmp_dir.join('virtual_ips.yaml')
+        vips_file_path = os.path.join(
+            self.cmd.working_dir, 'tripleo-overcloud-virtual-ips.yaml')
         vip_data = [
             {'network': 'internal_api', 'subnet': 'internal_api_subnet'}
         ]

tripleoclient/tests/v2/overcloud_network/test_overcloud_network.py

@@ -80,6 +80,12 @@ class TestOvercloudNetworkProvision(fakes.FakePlaybookExecution):
         self.cmd = overcloud_network.OvercloudNetworkProvision(self.app, None)
         self.cmd.app_args = mock.Mock(verbose_level=1)
 
+        # Mock copy to working dir
+        mock_copy_to_wd = mock.patch(
+            'tripleoclient.utils.copy_to_wd', autospec=True)
+        mock_copy_to_wd.start()
+        self.addCleanup(mock_copy_to_wd.stop)
+
     @mock.patch('tripleoclient.utils.TempDirs', autospect=True)
     @mock.patch('os.path.abspath', autospect=True)
     @mock.patch('os.path.exists', autospect=True)

tripleoclient/tests/v2/overcloud_node/test_overcloud_node.py

@@ -313,6 +313,12 @@ class TestProvisionNode(fakes.TestOvercloudNode):
         self.cmd = overcloud_node.ProvisionNode(self.app, None)
         self.cmd.app_args = mock.Mock(verbose_level=1)
 
+        # Mock copy to working dir
+        mock_copy_to_wd = mock.patch(
+            'tripleoclient.utils.copy_to_wd', autospec=True)
+        mock_copy_to_wd.start()
+        self.addCleanup(mock_copy_to_wd.stop)
+
     @mock.patch('tripleoclient.utils.run_ansible_playbook',
                 autospec=True)
     @mock.patch('tripleoclient.utils.run_role_playbooks',

tripleoclient/utils.py

@@ -1735,22 +1735,99 @@ def cleanup_tripleo_ansible_inventory_file(path):
         processutils.execute('/usr/bin/rm', '-f', path)
 
 
-def get_roles_file_path(roles_file, tht_root):
+def get_roles_file_path(working_dir, stack_name):
-    roles_file = roles_file or os.path.join(
+    roles_file = os.path.join(
-        tht_root, constants.OVERCLOUD_ROLES_FILE)
+        working_dir,
-    return os.path.abspath(roles_file)
+        constants.WD_DEFAULT_ROLES_FILE_NAME.format(stack_name))
+
+    return roles_file
 
 
-def get_networks_file_path(networks_file, tht_root):
+def get_networks_file_path(working_dir, stack_name):
-    networks_file = networks_file or os.path.join(
+    networks_file = os.path.join(
-        tht_root, constants.OVERCLOUD_NETWORKS_FILE)
+        working_dir,
-    return os.path.abspath(networks_file)
+        constants.WD_DEFAULT_NETWORKS_FILE_NAME.format(stack_name))
+
+    return networks_file
 
 
-def get_vip_file_path(vip_file, tht_root):
+def get_baremetal_file_path(working_dir, stack_name):
-    vip_file = vip_file or os.path.join(
+    baremetal_file_name = os.path.join(
-        tht_root, constants.OVERCLOUD_VIP_FILE)
+        working_dir,
-    return os.path.abspath(vip_file)
+        constants.WD_DEFAULT_BAREMETAL_FILE_NAME.format(stack_name))
+    baremetal_file = (baremetal_file_name
+                      if os.path.exists(baremetal_file_name) else None)
+
+    return baremetal_file
+
+
+def get_vip_file_path(working_dir, stack_name):
+    vip_file = os.path.join(
+        working_dir,
+        constants.WD_DEFAULT_VIP_FILE_NAME.format(stack_name))
+
+    return vip_file
+
+
+def rewrite_ansible_playbook_paths(src, dest):
+    """Rewrite relative paths to playbooks in the dest roles file, so that
+    the path is the absolute path relative to the src roles file
+    """
+    with open(dest, 'r') as f:
+        wd_roles = yaml.safe_load(f.read())
+    for role_idx, role in enumerate(wd_roles):
+        for pb_idx, pb_def in enumerate(role.get('ansible_playbooks', [])):
+            path = rel_or_abs_path_role_playbook(os.path.dirname(src),
+                                                 pb_def['playbook'])
+            wd_roles[role_idx]['ansible_playbooks'][pb_idx][
+                'playbook'] = path
+    with open(dest, 'w') as f:
+        f.write(yaml.safe_dump(wd_roles))
+
+
+def copy_to_wd(working_dir, file, stack, kind):
+    src = os.path.abspath(file)
+    dest = os.path.join(working_dir,
+                        constants.KIND_TEMPLATES[kind].format(stack))
+    shutil.copy(src, dest)
+    if kind == 'baremetal':
+        rewrite_ansible_playbook_paths(src, dest)
+
+
+def update_working_dir_defaults(working_dir, args):
+    stack_name = args.stack
+    tht_root = os.path.abspath(args.templates)
+
+    if args.baremetal_deployment:
+        copy_to_wd(working_dir, args.baremetal_deployment, stack_name,
+                   'baremetal')
+
+    if args.roles_file:
+        copy_to_wd(working_dir, args.roles_file, stack_name, 'roles')
+    elif not os.path.exists(
+            os.path.join(
+                working_dir,
+                constants.WD_DEFAULT_ROLES_FILE_NAME.format(stack_name))):
+        file = os.path.join(tht_root, constants.OVERCLOUD_ROLES_FILE)
+        copy_to_wd(working_dir, file, stack_name, 'roles')
+
+    if args.networks_file:
+        copy_to_wd(working_dir, args.networks_file, args.stack, 'networks')
+    elif not os.path.exists(
+            os.path.join(
+                working_dir,
+                constants.WD_DEFAULT_NETWORKS_FILE_NAME.format(stack_name))):
+        file = os.path.join(tht_root, constants.OVERCLOUD_NETWORKS_FILE)
+        copy_to_wd(working_dir, file, stack_name, 'networks')
+
+    if args.vip_file:
+        copy_to_wd(working_dir, args.vip_file, args.stack, 'vips')
+    elif not os.path.exists(
+            os.path.join(
+                working_dir,
+                constants.WD_DEFAULT_VIP_FILE_NAME.format(stack_name))):
+        file = os.path.join(tht_root, constants.OVERCLOUD_VIP_FILE)
+        copy_to_wd(working_dir, file, stack_name, 'vips')
 
 
 def build_stack_data(clients, stack_name, template,
@@ -1815,12 +1892,8 @@ def jinja_render_files(log, templates, working_dir,
     process_templates = os.path.join(
         templates, 'tools/process-templates.py')
     args = [python_cmd, process_templates]
-
+    args.extend(['--roles-data', roles_file])
-    roles_file_path = get_roles_file_path(roles_file, base_path)
+    args.extend(['--network-data', networks_file])
-    args.extend(['--roles-data', roles_file_path])
-
-    networks_file_path = get_networks_file_path(networks_file, base_path)
-    args.extend(['--network-data', networks_file_path])
 
     if base_path:
         args.extend(['-p', base_path])
@@ -2790,8 +2863,9 @@ def get_undercloud_host_entry():
 
 
 def build_enabled_sevices_image_params(env_files, parsed_args,
-                                       new_tht_root, user_tht_root):
+                                       new_tht_root, user_tht_root,
-    params = {}
+                                       working_dir):
+    params = dict()
     if parsed_args.environment_directories:
         env_files.extend(load_environment_directories(
             parsed_args.environment_directories))
@@ -2802,8 +2876,7 @@ def build_enabled_sevices_image_params(env_files, parsed_args,
         env_files, new_tht_root, user_tht_root,
         cleanup=(not parsed_args.no_cleanup))
 
-    roles_data = roles.get_roles_data(
+    roles_data = roles.get_roles_data(working_dir, parsed_args.stack)
-        parsed_args.roles_file, new_tht_root)
 
     params.update(kolla_builder.get_enabled_services(env, roles_data))
     params.update(plan_utils.default_image_params())

tripleoclient/v1/overcloud_deploy.py

@@ -69,7 +69,7 @@ def _update_args_from_answers_file(parsed_args):
         parsed_args.environment_files = answers['environments']
 
 
-def _validate_args(parsed_args):
+def _validate_args(parsed_args, working_dir):
     if parsed_args.templates is None and parsed_args.answers_file is None:
         raise oscexc.CommandError(
             "You must specify either --templates or --answers-file")
@@ -126,8 +126,8 @@ def _validate_args(parsed_args):
 
     if parsed_args.vip_file:
         # Check vip_file only used with network data v2
-        networks_file_path = utils.get_networks_file_path(
+        networks_file_path = utils.get_networks_file_path(working_dir,
-            parsed_args.networks_file, parsed_args.templates)
+                                                          parsed_args.stack)
         if not utils.is_network_data_v2(networks_file_path):
             raise oscexc.CommandError(
                 'The --vip-file option can only be used in combination with a '
@@ -232,8 +232,12 @@ class DeployOvercloud(command.Command):
         files = dict(list(template_files.items()) + list(env_files.items()))
 
         workflow_params.check_deprecated_parameters(
-            self.clients, stack_name, tht_root, template,
+            self.clients,
-            roles_file, files, env_files_tracker)
+            stack_name,
+            template,
+            files,
+            env_files_tracker,
+            self.working_dir)
 
         self.log.info("Deploying templates in the directory {0}".format(
             os.path.abspath(tht_root)))
@@ -247,13 +251,18 @@ class DeployOvercloud(command.Command):
         new_tht_root = "%s/tripleo-heat-templates" % self.working_dir
         self.log.debug("Creating working templates tree in %s"
                        % new_tht_root)
+        roles_file_path = utils.get_roles_file_path(self.working_dir,
+                                                    parsed_args.stack)
+        networks_file_path = utils.get_networks_file_path(self.working_dir,
+                                                          parsed_args.stack)
         shutil.rmtree(new_tht_root, ignore_errors=True)
         shutil.copytree(tht_root, new_tht_root, symlinks=True)
-        utils.jinja_render_files(self.log, parsed_args.templates,
+        utils.jinja_render_files(self.log,
-                                 new_tht_root,
+                                 templates=parsed_args.templates,
-                                 parsed_args.roles_file,
+                                 working_dir=new_tht_root,
-                                 parsed_args.networks_file,
+                                 roles_file=roles_file_path,
-                                 new_tht_root)
+                                 networks_file=networks_file_path,
+                                 base_path=new_tht_root)
         return new_tht_root, tht_root
 
     def create_env_files(self, stack, parsed_args,
@@ -266,7 +275,8 @@ class DeployOvercloud(command.Command):
             os.path.join(new_tht_root, constants.DEFAULT_RESOURCE_REGISTRY)]
 
         parameters = utils.build_enabled_sevices_image_params(
-            created_env_files, parsed_args, new_tht_root, user_tht_root)
+            created_env_files, parsed_args, new_tht_root, user_tht_root,
+            self.working_dir)
 
         self._update_parameters(
             parsed_args, parameters, new_tht_root, user_tht_root)
@@ -348,9 +358,9 @@ class DeployOvercloud(command.Command):
                 'derived_parameters.yaml', new_tht_root)
             workflow_params.build_derived_params_environment(
                 self.clients, parsed_args.stack, new_tht_root, env_files,
-                env_files_tracker, parsed_args.roles_file,
+                env_files_tracker, parsed_args.plan_environment_file,
-                parsed_args.plan_environment_file,
+                output_path, utils.playbook_verbosity(self=self),
-                output_path, utils.playbook_verbosity(self=self))
+                self.working_dir)
 
             created_env_files.append(output_path)
             env_files_tracker = []
@@ -447,15 +457,17 @@ class DeployOvercloud(command.Command):
         utils.remove_known_hosts(overcloud_ip_or_fqdn)
 
     def _provision_baremetal(self, parsed_args, tht_root, protected_overrides):
-        if not parsed_args.baremetal_deployment:
+
+        baremetal_file = utils.get_baremetal_file_path(self.working_dir,
+                                                       parsed_args.stack)
+        if not baremetal_file:
             return []
 
-        roles_file_path = os.path.abspath(parsed_args.baremetal_deployment)
+        baremetal_file_dir = os.path.dirname(baremetal_file)
-        roles_file_dir = os.path.dirname(roles_file_path)
+        with open(baremetal_file, 'r') as fp:
-        with open(parsed_args.baremetal_deployment, 'r') as fp:
             roles = yaml.safe_load(fp)
 
-        utils.validate_roles_playbooks(roles_file_dir, roles)
+        utils.validate_roles_playbooks(baremetal_file_dir, roles)
 
         key = self.get_key_pair(parsed_args)
         with open('{}.pub'.format(key), 'rt') as fp:
@@ -487,7 +499,7 @@ class DeployOvercloud(command.Command):
                 verbosity=utils.playbook_verbosity(self=self),
                 extra_vars=extra_vars,
             )
-        utils.run_role_playbooks(self, self.working_dir, roles_file_dir,
+        utils.run_role_playbooks(self, self.working_dir, baremetal_file_dir,
                                  roles, parsed_args.network_config)
 
         utils.extend_protected_overrides(protected_overrides, output_path)
@@ -496,10 +508,12 @@ class DeployOvercloud(command.Command):
 
     def _unprovision_baremetal(self, parsed_args):
 
-        if not parsed_args.baremetal_deployment:
+        baremetal_file = utils.get_baremetal_file_path(self.working_dir,
+                                                       parsed_args.stack)
+        if not baremetal_file:
             return
 
-        with open(parsed_args.baremetal_deployment, 'r') as fp:
+        with open(baremetal_file, 'r') as fp:
             roles = yaml.safe_load(fp)
 
         with utils.TempDirs() as tmp:
@@ -522,7 +536,8 @@ class DeployOvercloud(command.Command):
         # 'ipv6_subnet' keys this is not a network-v2 format file. In this
         # case do nothing.
         networks_file_path = utils.get_networks_file_path(
-            parsed_args.networks_file, parsed_args.templates)
+            self.working_dir, parsed_args.stack)
+
         if not utils.is_network_data_v2(networks_file_path):
             return []
 
@@ -552,13 +567,14 @@ class DeployOvercloud(command.Command):
 
     def _provision_virtual_ips(self, parsed_args, tht_root,
                                protected_overrides):
-        networks_file_path = utils.get_networks_file_path(
+        networks_file_path = utils.get_networks_file_path(self.working_dir,
-            parsed_args.networks_file, parsed_args.templates)
+                                                          parsed_args.stack)
         if not utils.is_network_data_v2(networks_file_path):
             return []
 
-        vip_file_path = utils.get_vip_file_path(parsed_args.vip_file,
+        vip_file_path = utils.get_vip_file_path(self.working_dir,
-                                                parsed_args.templates)
+                                                parsed_args.stack)
+
         output_path = utils.build_user_env_path(
             'virtual-ips-deployed.yaml',
             tht_root)
@@ -1056,7 +1072,12 @@ class DeployOvercloud(command.Command):
         sc_logger = logging.getLogger("swiftclient")
         sc_logger.setLevel(logging.CRITICAL)
 
-        _validate_args(parsed_args)
+        _update_args_from_answers_file(parsed_args)
+
+        # Make a copy of the files provided on command line in the working dir
+        # If the command is re-run without providing the argument the "backup"
+        # from the previous run in the working dir is used.
+        utils.update_working_dir_defaults(self.working_dir, parsed_args)
 
         # Throw warning if deprecated service is enabled and
         # ask user if deployment should still be continued.
@@ -1064,7 +1085,7 @@ class DeployOvercloud(command.Command):
             utils.check_deprecated_service_is_enabled(
                 parsed_args.environment_files)
 
-        _update_args_from_answers_file(parsed_args)
+        _validate_args(parsed_args, self.working_dir)
 
         if parsed_args.dry_run:
             self.log.info("Validation Finished")

tripleoclient/v1/tripleo_deploy.py

@@ -598,7 +598,8 @@ class Deploy(command.Command):
 
         # generate jinja templates by its work dir location
         self.log.debug(_("Using roles file %s") % roles_file_path)
-        utils.jinja_render_files(self.log, parsed_args.templates,
+        utils.jinja_render_files(self.log,
+                                 templates=parsed_args.templates,
                                  working_dir=self.tht_render,
                                  roles_file=roles_file_path,
                                  networks_file=networks_file_path,

tripleoclient/v2/overcloud_network.py

@@ -100,6 +100,18 @@ class OvercloudNetworkProvision(command.Command):
                             help=_("The directory containing the Heat "
                                    "templates to deploy"),
                             default=constants.TRIPLEO_HEAT_TEMPLATES)
+        parser.add_argument('--stack', dest='stack',
+                            help=_('Name or ID of heat stack, when set the '
+                                   'networks file will be copied to the '
+                                   'working dir.'),
+                            default=utils.env('OVERCLOUD_STACK_NAME',
+                                              default=None))
+        parser.add_argument(
+            '--working-dir', action='store',
+            help=_('The working directory for the deployment where all '
+                   'input, output, and generated files will be stored.\n'
+                   'Defaults to "$HOME/overcloud-deploy-<stack>"')
+        )
 
         return parser
 
@@ -141,6 +153,17 @@ class OvercloudNetworkProvision(command.Command):
                 extra_vars=extra_vars,
             )
 
+        if parsed_args.stack:
+            if not parsed_args.working_dir:
+                working_dir = oooutils.get_default_working_dir(
+                    parsed_args.stack)
+            else:
+                working_dir = os.path.abspath(parsed_args.working_dir)
+            oooutils.makedirs(working_dir)
+
+            oooutils.copy_to_wd(working_dir, networks_file_path,
+                                parsed_args.stack, 'networks')
+
 
 class OvercloudVirtualIPsExtract(command.Command):
 
@@ -223,12 +246,25 @@ class OvercloudVirtualIPsProvision(command.Command):
                             help=_("The directory containing the Heat "
                                    "templates to deploy"),
                             default=constants.TRIPLEO_HEAT_TEMPLATES)
+        parser.add_argument(
+            '--working-dir', action='store',
+            help=_('The working directory for the deployment where all '
+                   'input, output, and generated files will be stored.\n'
+                   'Defaults to "$HOME/overcloud-deploy-<stack>"')
+        )
 
         return parser
 
     def take_action(self, parsed_args):
         self.log.debug("take_action(%s)" % parsed_args)
 
+        if not parsed_args.working_dir:
+            working_dir = oooutils.get_default_working_dir(
+                parsed_args.stack)
+        else:
+            working_dir = os.path.abspath(parsed_args.working_dir)
+        oooutils.makedirs(working_dir)
+
         vip_file_path = os.path.abspath(parsed_args.vip_file)
         output_path = os.path.abspath(parsed_args.output)
 
@@ -265,6 +301,9 @@ class OvercloudVirtualIPsProvision(command.Command):
                 extra_vars=extra_vars,
             )
 
+        oooutils.copy_to_wd(working_dir, vip_file_path, parsed_args.stack,
+                            'vips')
+
 
 class OvercloudNetworkUnprovision(command.Command):
 

tripleoclient/v2/overcloud_node.py

@@ -340,6 +340,9 @@ class ProvisionNode(command.Command):
         oooutils.run_role_playbooks(self, working_dir, roles_file_dir,
                                     roles, parsed_args.network_config)
 
+        oooutils.copy_to_wd(working_dir, roles_file_path, parsed_args.stack,
+                            'baremetal')
+
         print('Nodes deployed successfully, add %s to your deployment '
               'environment' % parsed_args.output)
 

tripleoclient/workflows/parameters.py

@@ -80,13 +80,9 @@ def invoke_plan_env_workflows(clients, stack_name, plan_env_file,
             )
 
 
-def build_derived_params_environment(clients, stack_name,
+def build_derived_params_environment(clients, stack_name, tht_root, env_files,
-                                     tht_root, env_files,
+                                     env_files_tracker, plan_env_file,
-                                     env_files_tracker,
+                                     derived_env_file, verbosity, working_dir):
-                                     roles_file,
-                                     plan_env_file,
-                                     derived_env_file,
-                                     verbosity):
     template_path = os.path.join(tht_root, OVERCLOUD_YAML_NAME)
     template_files, template = template_utils.get_template_contents(
         template_file=template_path)
@@ -98,10 +94,9 @@ def build_derived_params_environment(clients, stack_name,
         files, env_files_tracker)
 
     # Get role list
-    role_list = roles.get_roles(
+    role_list = roles.get_roles(clients, stack_name, template, files,
-        clients, roles_file, tht_root, stack_name,
+                                env_files_tracker, working_dir, detail=False,
-        template, files, env_files_tracker,
+                                valid=True)
-        detail=False, valid=True)
 
     invoke_plan_env_workflows(
             clients,
@@ -114,22 +109,29 @@ def build_derived_params_environment(clients, stack_name,
         )
 
 
-def check_deprecated_parameters(clients, stack_name, tht_root, template,
+def check_deprecated_parameters(clients, stack_name, template, files,
-                                roles_file, files, env_files_tracker):
+                                env_files_tracker, working_dir):
     """Checks for deprecated parameters and adds warning if present.
 
     :param clients: application client object.
     :type clients: Object
-
+    :param stack_neme: Heat stack name
-    :param container: Name of the stack container.
+    :type stack_name: String
-    :type container: String
+    :param template:
+    :type template: String
+    :param files:
+    :type files:
+    :param env_files_tracker:
+    :type env_files_tracker:
+    :param working_dir: Tripleo working directory
+    :type working_dir: String
     """
 
     # Get role list
-    role_list = roles.get_roles(
+    role_list = roles.get_roles(clients, stack_name, template, files,
-        clients, roles_file, tht_root, stack_name,
+                                env_files_tracker, working_dir,
-        template, files, env_files_tracker,
+                                detail=False,
-        detail=False, valid=True)
+                                valid=True)
 
     # Build stack_data
     stack_data = utils.build_stack_data(

tripleoclient/workflows/roles.py

@@ -19,22 +19,22 @@ from tripleoclient import utils
 LOG = logging.getLogger(__name__)
 
 
-def get_roles_data(roles_file, tht_root):
+def get_roles_data(working_dir, stack_name):
-    abs_roles_file = utils.get_roles_file_path(
+    abs_roles_file = utils.get_roles_file_path(working_dir, stack_name)
-        roles_file, tht_root)
-    roles_data = None
     with open(abs_roles_file, 'r') as fp:
         roles_data = yaml.safe_load(fp)
+
     return roles_data
 
 
-def get_roles(clients, roles_file, tht_root,
+def get_roles(clients,
               stack_name,
               template,
               files,
               env_files,
+              working_dir,
               detail=False, valid=False):
-    roles_data = get_roles_data(roles_file, tht_root)
+    roles_data = get_roles_data(working_dir, stack_name)
 
     if detail:
         return roles_data