From 2873dd4df62f104bef6a9ce36bdf5385b75b96b5 Mon Sep 17 00:00:00 2001
From: Michele Baldessari <michele@acksyn.org>
Date: Fri, 10 Jul 2020 15:19:45 +0200
Subject: [PATCH] Cleanup UndercloudHostsEntries

When we update the UndercloudHostsEnries we basically
take the output of 'getent hosts "$(hostname -s).ctlplane"'
and push it into a parameter, so tripleo-ansible can
make sure it adds the hostentry for the undercloud on
the whole overcloud.

The problem is that getent hosts can return multiple
entries, which will then be injected into the parameter
and then written into /etc/hosts. This contstantly adds the
undercloud.ctlplane string and we end up adding it at
everydeploy making it grow quadratically.

This eventually leads to a too large json file and the deploys
start failing with:
    heat.common.exception.RequestLimitExceeded:
      Request limit exceeded: JSON body size (4396634 bytes)
        exceeds maximum allowed size (4194304 bytes).

Tested this by deploying an environment and then running
a few redeploys to make sure that the undercloud entries
in /etc/hosts on the undercloud itself do not grow at
each redeploy.

Change-Id: I37d75600825f48be9e15470cacba7b3a0371a3e2
Closes-Bug: #1887165
---
 .../overcloud_deploy/test_overcloud_deploy.py | 20 +++++++++++++------
 tripleoclient/v1/overcloud_deploy.py          | 18 +++++++++++++++--
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
index 278704af2..9b637895b 100644
--- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
+++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
@@ -1703,12 +1703,20 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('subprocess.Popen', autospec=True)
     def test__get_undercloud_host_entry(self, mock_popen):
         mock_process = mock.Mock()
-        mock_process.communicate.return_value = (
-            'fd12::1 uc.ctlplane.localdomain uc.ctlplane', '')
-        mock_process.returncode = 0
-        mock_popen.return_value = mock_process
-        expected = ('fd12::1 uc.ctlplane.localdomain uc.ctlplane')
-        self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
+        mock_hosts = {
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane\n'
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            '1.2.3.4 uc.ctlplane foo uc.ctlplane bar uc.ctlplane':
+                '1.2.3.4 uc.ctlplane foo bar'
+        }
+        for value, expected in mock_hosts.items():
+            mock_process.communicate.return_value = (value, '')
+            mock_process.returncode = 0
+            mock_popen.return_value = mock_process
+            self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
 
 
 class TestArgumentValidation(fakes.TestDeployOvercloud):
diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py
index f4cad86f3..f026d3971 100644
--- a/tripleoclient/v1/overcloud_deploy.py
+++ b/tripleoclient/v1/overcloud_deploy.py
@@ -14,6 +14,7 @@
 #
 
 import argparse
+from collections import OrderedDict
 import logging
 import os
 import os.path
@@ -101,6 +102,20 @@ class DeployOvercloud(command.Command):
 
         return parameters
 
+    def _cleanup_host_entry(self, entry):
+        # remove any tab or space excess
+        entry_stripped = re.sub('[ \t]+', ' ', str(entry).rstrip())
+        # removes any duplicate identical lines
+        unique_lines = list(set(entry_stripped.splitlines()))
+        ret = ''
+        for line in unique_lines:
+            # remove any duplicate word
+            hosts_unique = (' '.join(
+                OrderedDict((w, w) for w in line.split()).keys()))
+            if hosts_unique != '':
+                ret += hosts_unique + '\n'
+        return ret.rstrip('\n')
+
     def _get_undercloud_host_entry(self):
         """Get hosts entry for undercloud ctlplane network
 
@@ -114,8 +129,7 @@ class DeployOvercloud(command.Command):
         if process.returncode != 0:
             raise exceptions.DeploymentError('No entry for %s in /etc/hosts'
                                              % ctlplane_hostname)
-
-        return re.sub(' +', ' ', str(out).rstrip())
+        return self._cleanup_host_entry(out)
 
     def _create_breakpoint_cleanup_env(self, tht_root, container_name):
         bp_env = {}