Wire-in hieradata_override for UC heat-install

Maintain undercloud.conf feature parity with instack.

Allow to override hiera data for UndercloudExtraConfig in tht.
Ensure higher data precedence over anything that comes from
custom_env_files.

Partial-bug: #1766579
Change-Id: Idf595498e46eeb88acfd4c3a21094d22197a6d7b
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

tripleoclient/tests/v1/undercloud/test_undercloud_deploy.py

@@ -248,6 +248,9 @@ class TestDeployUndercloud(TestPluginV1):
                 autospec=True)
     @mock.patch('tripleoclient.utils.'
                 'process_multiple_environments', autospec=True)
+    @mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
+                '_process_hieradata_overrides', return_value='foo.yaml',
+                autospec=True)
     @mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
                 '_update_passwords_env', autospec=True)
     @mock.patch('subprocess.check_call', autospec=True)
@@ -258,6 +261,7 @@ class TestDeployUndercloud(TestPluginV1):
                                      mock_mktemp,
                                      mock_exec,
                                      mock_update_pass_env,
+                                     mock_process_hiera,
                                      mock_process_multiple_environments,
                                      mock_hc_get_templ_cont,
                                      mock_hc_process):
@@ -266,6 +270,7 @@ class TestDeployUndercloud(TestPluginV1):
                                         ['--local-ip', '127.0.0.1/8',
                                          '--templates', '/tmp/thtroot',
                                          '--output-dir', '/my',
+                                         '--hieradata-override', 'legacy.yaml',
                                          '-e', '/tmp/thtroot/puppet/foo.yaml',
                                          '-e', '/tmp/thtroot//docker/bar.yaml',
                                          '-e', '/tmp/thtroot42/notouch.yaml',
@@ -284,7 +289,7 @@ class TestDeployUndercloud(TestPluginV1):
             '~/custom.yaml',
             'something.yaml',
             '../../../outside.yaml',
-            mock.ANY]
+            mock.ANY, 'foo.yaml']
 
         environment = self.cmd._setup_heat_environments(parsed_args)
 

tripleoclient/v1/undercloud_config.py

@@ -58,6 +58,8 @@ SUBNET_PARAMETER_MAPPING = {
 THT_HOME = os.environ.get('THT_HOME',
                           "/usr/share/openstack-tripleo-heat-templates/")
 
+USER_HOME = os.environ.get('HOME', '')
+
 TELEMETRY_DOCKER_ENV_YAML = [
     'environments/services/undercloud-gnocchi.yaml',
     'environments/services/undercloud-aodh.yaml',
@@ -224,11 +226,15 @@ _opts = [
                ),
     cfg.StrOpt('hieradata_override',
                default='',
-               help=('Path to hieradata override file. If set, the file will '
-                     'be copied under /etc/puppet/hieradata and set as the '
-                     'first file in the hiera hierarchy. This can be used '
-                     'to custom configure services beyond what '
-                     'undercloud.conf provides')
+               help=('Path to hieradata override file. Relative paths get '
+                     'computed inside of $HOME. When it points to a heat '
+                     'env file, it is passed in t-h-t via "-e <file>", as is. '
+                     'When the file contains legacy instack data, '
+                     'it is wrapped with UndercloudExtraConfig and also '
+                     'passed in for t-h-t as a temp file created in '
+                     'output_dir. Note, instack hiera data may be '
+                     'not t-h-t compatible and will highly likely require a '
+                     'manual revision.')
                ),
     cfg.StrOpt('net_config_override',
                default='',
@@ -619,7 +625,7 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False,
 
     # Set the undercloud home dir parameter so that stackrc is produced in
     # the users home directory.
-    env_data['UndercloudHomeDir'] = os.environ.get('HOME', '')
+    env_data['UndercloudHomeDir'] = USER_HOME
 
     for param_key, param_value in PARAMETER_MAPPING.items():
         if param_key in CONF.keys():
@@ -815,6 +821,18 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False,
         for custom_file in CONF['custom_env_files']:
             deploy_args += ['-e', custom_file]
 
+    if CONF.get('hieradata_override', None):
+        data_file = CONF['hieradata_override']
+        if os.path.abspath(data_file) != data_file:
+            data_file = os.path.join(USER_HOME, data_file)
+
+        if not os.path.exists(data_file):
+            msg = "Could not find hieradata_override file '%s'" % data_file
+            LOG.error(msg)
+            raise RuntimeError(msg)
+
+        deploy_args += ['--hieradata-override=%s' % data_file]
+
     if CONF.get('enable_validations') and not no_validations:
         undercloud_preflight.check()
 

tripleoclient/v1/undercloud_deploy.py

@@ -382,6 +382,9 @@ class DeployUndercloud(command.Command):
                                default_flow_style=False)
             environments.append(self.tmp_env_file_name)
 
+        if parsed_args.hieradata_override:
+            environments.append(self._process_hieradata_overrides(parsed_args))
+
         return environments
 
     def _prepare_container_images(self, env, roles_file):
@@ -593,8 +596,55 @@ class DeployUndercloud(command.Command):
             action='store_true', default=False,
             help=_('Cleanup temporary files')
         )
+        parser.add_argument(
+            '--hieradata-override', nargs='?',
+            help=_('Path to hieradata override file. When it points to a heat '
+                   'env file, it is passed in t-h-t via --environment-file. '
+                   'When the file contains legacy instack data, '
+                   'it is wrapped with UndercloudExtraConfig and also '
+                   'passed in for t-h-t as a temp file created in '
+                   '--output-dir. Note, instack hiera data may be '
+                   'not t-h-t compatible and will highly likely require a '
+                   'manual revision.')
+        )
         return parser
 
+    def _process_hieradata_overrides(self, parsed_args):
+        """Count in hiera data overrides including legacy formats
+
+        Return a file name that points to processed hiera data overrides file
+        """
+        target = parsed_args.hieradata_override
+        data = open(target, 'r').read()
+        hiera_data = yaml.safe_load(data)
+        if not hiera_data:
+            msg = 'Unsupported data format in hieradata override %s' % target
+            self.log.error(msg)
+            raise exceptions.DeploymentError(msg)
+
+        # NOTE(bogdando): In t-h-t, hiera data should come in wrapped as
+        # {parameter_defaults: {UndercloudExtraConfig: ... }}
+        if ('UndercloudExtraConfig' not in
+           hiera_data.get('parameter_defaults', {})):
+            with tempfile.NamedTemporaryFile(
+                dir=parsed_args.output_dir,
+                delete=parsed_args.cleanup,
+                prefix='tripleoclient-',
+                suffix=os.path.splitext(
+                    os.path.basename(target))[0]) as f:
+                self.log.info('Converting hiera overrides for t-h-t from '
+                              'legacy format into a tempfile %s' % f.name)
+                with open(f.name, 'w') as tht_data:
+                    yaml.safe_dump(
+                        {'parameter_defaults': {
+                            'UndercloudExtraConfig': hiera_data}},
+                        tht_data,
+                        default_flow_style=False)
+
+                target = f.name
+
+        return target
+
     def take_action(self, parsed_args):
         self.log.debug("take_action(%s)" % parsed_args)