Cleanup UndercloudHostsEntries

When we update the UndercloudHostsEnries we basically
take the output of 'getent hosts "$(hostname -s).ctlplane"'
and push it into a parameter, so tripleo-ansible can
make sure it adds the hostentry for the undercloud on
the whole overcloud.

The problem is that getent hosts can return multiple
entries, which will then be injected into the parameter
and then written into /etc/hosts. This contstantly adds the
undercloud.ctlplane string and we end up adding it at
everydeploy making it grow quadratically.

This eventually leads to a too large json file and the deploys
start failing with:
    heat.common.exception.RequestLimitExceeded:
      Request limit exceeded: JSON body size (4396634 bytes)
        exceeds maximum allowed size (4194304 bytes).

Tested this by deploying an environment and then running
a few redeploys to make sure that the undercloud entries
in /etc/hosts on the undercloud itself do not grow at
each redeploy.

Change-Id: I37d75600825f48be9e15470cacba7b3a0371a3e2
Closes-Bug: #1887165
(cherry picked from commit 2873dd4df6)

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -1708,12 +1708,20 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('subprocess.Popen', autospec=True)
     def test__get_undercloud_host_entry(self, mock_popen):
         mock_process = mock.Mock()
-        mock_process.communicate.return_value = (
-            'fd12::1 uc.ctlplane.localdomain uc.ctlplane', '')
-        mock_process.returncode = 0
-        mock_popen.return_value = mock_process
-        expected = ('fd12::1 uc.ctlplane.localdomain uc.ctlplane')
-        self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
+        mock_hosts = {
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane\n'
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            '1.2.3.4 uc.ctlplane foo uc.ctlplane bar uc.ctlplane':
+                '1.2.3.4 uc.ctlplane foo bar'
+        }
+        for value, expected in mock_hosts.items():
+            mock_process.communicate.return_value = (value, '')
+            mock_process.returncode = 0
+            mock_popen.return_value = mock_process
+            self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
 
     def test_check_limit_warning(self):
         mock_warning = mock.MagicMock()

tripleoclient/v1/overcloud_deploy.py

@@ -15,6 +15,7 @@
 from __future__ import print_function
 
 import argparse
+from collections import OrderedDict
 import logging
 import os
 import os.path
@@ -102,6 +103,20 @@ class DeployOvercloud(command.Command):
 
         return parameters
 
+    def _cleanup_host_entry(self, entry):
+        # remove any tab or space excess
+        entry_stripped = re.sub('[ \t]+', ' ', str(entry).rstrip())
+        # removes any duplicate identical lines
+        unique_lines = list(set(entry_stripped.splitlines()))
+        ret = ''
+        for line in unique_lines:
+            # remove any duplicate word
+            hosts_unique = (' '.join(
+                OrderedDict((w, w) for w in line.split()).keys()))
+            if hosts_unique != '':
+                ret += hosts_unique + '\n'
+        return ret.rstrip('\n')
+
     def _get_undercloud_host_entry(self):
         """Get hosts entry for undercloud ctlplane network
 
@@ -115,8 +130,7 @@ class DeployOvercloud(command.Command):
         if process.returncode != 0:
             raise exceptions.DeploymentError('No entry for %s in /etc/hosts'
                                              % ctlplane_hostname)
-
-        return re.sub(' +', ' ', str(out).rstrip())
+        return self._cleanup_host_entry(out)
 
     def _create_breakpoint_cleanup_env(self, tht_root, container_name):
         bp_env = {}