Optional override of the plan environment file

This change is to add optional override of the
plan-environment.yaml file with custom plan environment
file.
Implements: blueprint tripleo-derive-parameters

Change-Id: I45e8103826fdee76a8ec40aebd95cb5551cc5fed

releasenotes/notes/override-plan-env-e3df45a51bda717e.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Adds optional override of the plan environment file with custom
+    plan environment file.

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -291,6 +291,142 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_tarball.tarball_extract_to_swift_container.assert_called_with(
             clients.tripleoclient.object_store, mock.ANY, 'overcloud')
 
+    @mock.patch('shutil.rmtree', autospec=True)
+    @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_deploy_postconfig', autospec=True)
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
+    @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env',
+                autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_validate_args')
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_create_parameters_env', autospec=True)
+    @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
+                autospec=True)
+    @mock.patch('tripleoclient.workflows.deployment.overcloudrc',
+                autospec=True)
+    @mock.patch('tripleoclient.utils.write_overcloudrc', autospec=True)
+    @mock.patch('tripleoclient.utils.remove_known_hosts', autospec=True)
+    @mock.patch('tripleoclient.utils.wait_for_stack_ready',
+                autospec=True)
+    @mock.patch('heatclient.common.template_utils.get_template_contents',
+                autospec=True)
+    @mock.patch('tripleoclient.utils.check_hypervisor_stats',
+                autospec=True)
+    @mock.patch('uuid.uuid1', autospec=True)
+    @mock.patch('time.time', autospec=True)
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('tempfile.mkdtemp', autospec=True)
+    def test_tht_deploy_with_plan_environment_file(
+        self, mock_tmpdir, mock_copy, mock_time, mock_uuid1,
+        mock_check_hypervisor_stats, mock_get_template_contents,
+        wait_for_stack_ready_mock, mock_remove_known_hosts,
+        mock_overcloudrc, mock_write_overcloudrc,
+        mock_create_tempest_deployer, mock_create_parameters_env,
+        mock_validate_args, mock_breakpoints_cleanup,
+        mock_tarball, mock_postconfig,
+            mock_get_overcloud_endpoint, mock_shutil_rmtree):
+        arglist = ['--templates', '-p', 'the-plan-environment.yaml']
+        verifylist = [
+            ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
+            ('plan_environment_file', 'the-plan-environment.yaml')
+        ]
+
+        mock_tmpdir.return_value = "/tmp/tht"
+        mock_uuid1.return_value = "uuid"
+        mock_time.return_value = 123456789
+
+        clients = self.app.client_manager
+        orchestration_client = clients.orchestration
+        mock_stack = fakes.create_tht_stack()
+        orchestration_client.stacks.get.side_effect = [None, mock.Mock()]
+        workflow_client = clients.workflow_engine
+        workflow_client.environments.get.return_value = mock.MagicMock(
+            variables={'environments': []})
+        workflow_client.action_executions.create.return_value = mock.MagicMock(
+            output='{"result":[]}')
+
+        def _orch_clt_create(**kwargs):
+            orchestration_client.stacks.get.return_value = mock_stack
+
+        orchestration_client.stacks.create.side_effect = _orch_clt_create
+
+        object_client = clients.tripleoclient.object_store
+        object_client.get_object = mock.Mock()
+        mock_env = yaml.safe_dump({'environments': []})
+        object_client.get_object.return_value = ({}, mock_env)
+
+        mock_check_hypervisor_stats.return_value = {
+            'count': 4,
+            'memory_mb': 4096,
+            'vcpus': 8,
+        }
+        clients.network.api.find_attr.return_value = {
+            "id": "network id"
+        }
+        mock_get_template_contents.return_value = [{}, "template"]
+        wait_for_stack_ready_mock.return_value = True
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        baremetal = clients.baremetal
+        baremetal.node.list.return_value = range(10)
+
+        expected_parameters = {
+            'CephClusterFSID': 'uuid',
+            'CephStorageCount': 3,
+            'ExtraConfig': '{}',
+            'HypervisorNeutronPhysicalBridge': 'br-ex',
+            'HypervisorNeutronPublicInterface': 'nic1',
+            'NeutronDnsmasqOptions': 'dhcp-option-force=26,1400',
+            'NeutronFlatNetworks': 'datacentre',
+            'NeutronNetworkType': 'gre',
+            'NeutronPublicInterface': 'nic1',
+            'NeutronTunnelTypes': 'gre',
+            'NtpServer': '',
+            'SnmpdReadonlyUserPassword': 'PASSWORD',
+            'DeployIdentifier': 123456789,
+            'UpdateIdentifier': '',
+            'StackAction': 'CREATE',
+        }
+
+        testcase = self
+
+        def _custom_create_params_env(self, parameters):
+            for key, value in six.iteritems(parameters):
+                testcase.assertEqual(value, expected_parameters[key])
+            parameter_defaults = {"parameter_defaults": parameters}
+            return parameter_defaults
+
+        mock_create_parameters_env.side_effect = _custom_create_params_env
+
+        mock_open_context = mock.mock_open()
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            self.cmd.take_action(parsed_args)
+
+        self.assertFalse(orchestration_client.stacks.create.called)
+
+        mock_get_template_contents.assert_called_with(
+            object_request=mock.ANY,
+            template_object=constants.OVERCLOUD_YAML_NAME)
+
+        mock_create_tempest_deployer.assert_called_with()
+        mock_validate_args.assert_called_once_with(parsed_args)
+
+        mock_tarball.create_tarball.assert_called_with(
+            '/tmp/tht/tripleo-heat-templates', mock.ANY)
+        mock_tarball.tarball_extract_to_swift_container.assert_called_with(
+            clients.tripleoclient.object_store, mock.ANY, 'overcloud')
+
+        workflow_client.action_executions.create.assert_called()
+        workflow_client.executions.create.assert_called()
+
+        mock_open_context.assert_has_calls(
+            [mock.call('the-plan-environment.yaml')])
+        clients.tripleoclient.object_store.put_object.assert_called()
+
     @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_deploy_postconfig', autospec=True)

tripleoclient/tests/v1/test_overcloud_plan.py

@@ -273,6 +273,50 @@ class TestOvercloudCreatePlan(utils.TestCommand):
             mock.call('overcast', u'all-nodes-validation.yaml'),
         ], any_order=True)
 
+    @mock.patch("tripleoclient.workflows.plan_management.tarball")
+    def test_create_custom_plan_plan_environment_file(self,
+                                                      mock_tarball):
+        # Setup
+        arglist = ['overcast', '--templates', '/fake/path',
+                   '-p', 'the_plan_environment.yaml']
+        verifylist = [
+            ('name', 'overcast'),
+            ('templates', '/fake/path'),
+            ('plan_environment_file', 'the_plan_environment.yaml')
+        ]
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.websocket.wait_for_messages.return_value = iter([{
+            "execution": {"id": "IDID"},
+            "status": "SUCCESS"
+        }])
+        mock_result = mock.Mock(output='{"result": null}')
+        self.workflow.action_executions.create.return_value = mock_result
+
+        mock_open_context = mock.mock_open()
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            self.cmd.take_action(parsed_args)
+
+        # Verify
+        self.workflow.action_executions.create.assert_called_once_with(
+            'tripleo.plan.create_container', {"container": "overcast"},
+            run_sync=True, save_result=True
+        )
+
+        self.workflow.executions.create.assert_called_once_with(
+            'tripleo.plan_management.v1.create_deployment_plan',
+            workflow_input={
+                'container': 'overcast',
+                'queue_name': 'UUID4',
+                'generate_passwords': True
+            })
+
+        mock_open_context.assert_has_calls(
+            [mock.call('the_plan_environment.yaml')])
+
+        self.tripleoclient.object_store.put_object.assert_called_once_with(
+            'overcast', 'plan-environment.yaml', mock_open_context())
+
     def test_create_default_plan_with_password_gen_disabled(self):
 
         # Setup

tripleoclient/tests/workflows/test_plan_management.py

@@ -118,6 +118,38 @@ class TestPlanCreationWorkflows(utils.TestCommand):
         self.tripleoclient.object_store.put_object.assert_called_once_with(
             'test-overcloud', 'roles_data.yaml', mock_open_context())
 
+    @mock.patch('tripleoclient.workflows.plan_management.tarball',
+                autospec=True)
+    def test_create_plan_from_templates_plan_env_data(self, mock_tarball):
+        output = mock.Mock(output='{"result": ""}')
+        self.workflow.action_executions.create.return_value = output
+        self.websocket.wait_for_messages.return_value = self.message_success
+
+        mock_open_context = mock.mock_open()
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            plan_management.create_plan_from_templates(
+                self.app.client_manager,
+                'test-overcloud',
+                '/tht-root/',
+                plan_env_file='the-plan-environment.yaml')
+
+        self.workflow.action_executions.create.assert_called_once_with(
+            'tripleo.plan.create_container',
+            {'container': 'test-overcloud'},
+            run_sync=True, save_result=True)
+
+        self.workflow.executions.create.assert_called_once_with(
+            'tripleo.plan_management.v1.create_deployment_plan',
+            workflow_input={'queue_name': 'UUID4',
+                            'container': 'test-overcloud',
+                            'generate_passwords': True})
+
+        mock_open_context.assert_has_calls(
+            [mock.call('the-plan-environment.yaml')])
+
+        self.tripleoclient.object_store.put_object.assert_called_once_with(
+            'test-overcloud', 'plan-environment.yaml', mock_open_context())
+
     def test_delete_plan(self):
         self.workflow.action_executions.create.return_value = (
             mock.Mock(output='{"result": null}'))

tripleoclient/v1/overcloud_deploy.py

@@ -388,11 +388,13 @@ class DeployOvercloud(command.Command):
             # templates.
             plan_management.update_plan_from_templates(
                 self.clients, parsed_args.stack, tht_root,
-                parsed_args.roles_file, generate_passwords)
+                parsed_args.roles_file, generate_passwords,
+                parsed_args.plan_environment_file)
         else:
             plan_management.create_plan_from_templates(
                 self.clients, parsed_args.stack, tht_root,
-                parsed_args.roles_file, generate_passwords)
+                parsed_args.roles_file, generate_passwords,
+                parsed_args.plan_environment_file)
 
         # Get any missing (e.g j2 rendered) files from the plan to tht_root
         self._download_missing_files_from_plan(
@@ -685,6 +687,11 @@ class DeployOvercloud(command.Command):
             help=_('Roles file, overrides the default %s in the --templates '
                    'directory') % constants.OVERCLOUD_ROLES_FILE
         )
+        parser.add_argument(
+            '--plan-environment-file', '-p',
+            help=_('Plan Environment file, overrides the default %s in the '
+                   '--templates directory') % constants.PLAN_ENVIRONMENT
+        )
         parser.add_argument(
             '--no-cleanup', action='store_true',
             help=_('Don\'t cleanup temporary files, just log their location')

tripleoclient/v1/overcloud_plan.py

@@ -19,6 +19,7 @@ from osc_lib.command import command
 from osc_lib.i18n import _
 from six.moves.urllib import request
 
+from tripleoclient import constants
 from tripleoclient import exceptions
 from tripleoclient import utils
 from tripleoclient.workflows import deployment
@@ -94,6 +95,11 @@ class CreatePlan(command.Command):
                    'If this or --source_url isn\'t provided, the templates '
                    'packaged on the Undercloud will be used.'),
         )
+        parser.add_argument(
+            '--plan-environment-file', '-p',
+            help=_('Plan Environment file, overrides the default %s in the '
+                   '--templates directory') % constants.PLAN_ENVIRONMENT
+        )
         parser.add_argument(
             '--disable-password-generation',
             action='store_true',
@@ -125,7 +131,8 @@ class CreatePlan(command.Command):
         if parsed_args.templates:
             plan_management.create_plan_from_templates(
                 clients, name, parsed_args.templates,
-                generate_passwords=generate_passwords)
+                generate_passwords=generate_passwords,
+                plan_env_file=parsed_args.plan_environment_file)
         else:
             plan_management.create_deployment_plan(
                 clients, container=name, queue_name=str(uuid.uuid4()),

tripleoclient/workflows/plan_management.py

@@ -30,7 +30,8 @@ LOG = logging.getLogger(__name__)
 _WORKFLOW_TIMEOUT = 360  # 6 * 60 seconds
 
 
-def _upload_templates(swift_client, container_name, tht_root, roles_file=None):
+def _upload_templates(swift_client, container_name, tht_root, roles_file=None,
+                      plan_env_file=None):
     """tarball up a given directory and upload it to Swift to be extracted"""
 
     with tempfile.NamedTemporaryFile() as tmp_tarball:
@@ -44,6 +45,15 @@ def _upload_templates(swift_client, container_name, tht_root, roles_file=None):
             swift_client.put_object(container_name,
                                     constants.OVERCLOUD_ROLES_FILE,
                                     rf)
+    # Optional override of the plan-environment.yaml file
+    if plan_env_file:
+        # TODO(jpalanis): Instead of overriding default file,
+        # merging the user override plan-environment with default
+        # plan-environment file will avoid explict merging issues.
+        with open(plan_env_file) as pf:
+            swift_client.put_object(container_name,
+                                    constants.PLAN_ENVIRONMENT,
+                                    pf)
 
 
 def create_default_plan(clients, **workflow_input):
@@ -139,7 +149,7 @@ def create_container(workflow_client, **input_):
 
 
 def create_plan_from_templates(clients, name, tht_root, roles_file=None,
-                               generate_passwords=True):
+                               generate_passwords=True, plan_env_file=None):
     workflow_client = clients.workflow_engine
     swift_client = clients.tripleoclient.object_store
 
@@ -152,7 +162,7 @@ def create_plan_from_templates(clients, name, tht_root, roles_file=None,
             "Unable to create plan. {}".format(result))
 
     print("Creating plan from template files in: {}".format(tht_root))
-    _upload_templates(swift_client, name, tht_root, roles_file)
+    _upload_templates(swift_client, name, tht_root, roles_file, plan_env_file)
 
     try:
         create_deployment_plan(clients, container=name,
@@ -164,7 +174,7 @@ def create_plan_from_templates(clients, name, tht_root, roles_file=None,
 
 
 def update_plan_from_templates(clients, name, tht_root, roles_file=None,
-                               generate_passwords=True):
+                               generate_passwords=True, plan_env_file=None):
     swift_client = clients.tripleoclient.object_store
 
     # If the plan environment was migrated to Swift, save the generated
@@ -195,7 +205,7 @@ def update_plan_from_templates(clients, name, tht_root, roles_file=None,
     # need to special-case plan-environment.yaml to avoid this.
 
     print("Uploading new plan files")
-    _upload_templates(swift_client, name, tht_root, roles_file)
+    _upload_templates(swift_client, name, tht_root, roles_file, plan_env_file)
     _update_passwords(swift_client, name, passwords)
     update_deployment_plan(clients, container=name,
                            queue_name=str(uuid.uuid4()),