From 29686a6564b333c8814e081233418228eb234a86 Mon Sep 17 00:00:00 2001
From: Michele Baldessari <michele@acksyn.org>
Date: Fri, 10 Jul 2020 15:19:45 +0200
Subject: [PATCH] Cleanup UndercloudHostsEntries

When we update the UndercloudHostsEnries we basically
take the output of 'getent hosts "$(hostname -s).ctlplane"'
and push it into a parameter, so tripleo-ansible can
make sure it adds the hostentry for the undercloud on
the whole overcloud.

The problem is that getent hosts can return multiple
entries, which will then be injected into the parameter
and then written into /etc/hosts. This contstantly adds the
undercloud.ctlplane string and we end up adding it at
everydeploy making it grow quadratically.

This eventually leads to a too large json file and the deploys
start failing with:
    heat.common.exception.RequestLimitExceeded:
      Request limit exceeded: JSON body size (4396634 bytes)
        exceeds maximum allowed size (4194304 bytes).

Tested this by deploying an environment and then running
a few redeploys to make sure that the undercloud entries
in /etc/hosts on the undercloud itself do not grow at
each redeploy.

NB: Non 100% clean backport due to different context around imports

Change-Id: I37d75600825f48be9e15470cacba7b3a0371a3e2
Closes-Bug: #1887165
(cherry picked from commit 2873dd4df62f104bef6a9ce36bdf5385b75b96b5)
(cherry picked from commit a06c7194c0027a144fce0551870d85f27c6848bd)
---
 .../overcloud_deploy/test_overcloud_deploy.py | 20 +++++++++++++------
 tripleoclient/v1/overcloud_deploy.py          | 18 +++++++++++++++--
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
index ad8bbdbf4..180fd8f0d 100644
--- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
+++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
@@ -1686,12 +1686,20 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('subprocess.Popen', autospec=True)
     def test__get_undercloud_host_entry(self, mock_popen):
         mock_process = mock.Mock()
-        mock_process.communicate.return_value = (
-            'fd12::1 uc.ctlplane.localdomain uc.ctlplane', '')
-        mock_process.returncode = 0
-        mock_popen.return_value = mock_process
-        expected = ('fd12::1 uc.ctlplane.localdomain uc.ctlplane')
-        self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
+        mock_hosts = {
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane\n'
+            'fd12::1 uc.ctlplane.localdomain uc.ctlplane':
+                'fd12::1 uc.ctlplane.localdomain uc.ctlplane',
+            '1.2.3.4 uc.ctlplane foo uc.ctlplane bar uc.ctlplane':
+                '1.2.3.4 uc.ctlplane foo bar'
+        }
+        for value, expected in mock_hosts.items():
+            mock_process.communicate.return_value = (value, '')
+            mock_process.returncode = 0
+            mock_popen.return_value = mock_process
+            self.assertEqual(expected, self.cmd._get_undercloud_host_entry())
 
     def test_check_limit_warning(self):
         mock_warning = mock.MagicMock()
diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py
index 0eed514fb..c76486c97 100644
--- a/tripleoclient/v1/overcloud_deploy.py
+++ b/tripleoclient/v1/overcloud_deploy.py
@@ -15,6 +15,7 @@
 from __future__ import print_function
 
 import argparse
+from collections import OrderedDict
 import json
 import logging
 import os
@@ -115,6 +116,20 @@ class DeployOvercloud(command.Command):
 
         return parameters
 
+    def _cleanup_host_entry(self, entry):
+        # remove any tab or space excess
+        entry_stripped = re.sub('[ \t]+', ' ', str(entry).rstrip())
+        # removes any duplicate identical lines
+        unique_lines = list(set(entry_stripped.splitlines()))
+        ret = ''
+        for line in unique_lines:
+            # remove any duplicate word
+            hosts_unique = (' '.join(
+                OrderedDict((w, w) for w in line.split()).keys()))
+            if hosts_unique != '':
+                ret += hosts_unique + '\n'
+        return ret.rstrip('\n')
+
     def _get_undercloud_host_entry(self):
         """Get hosts entry for undercloud ctlplane network
 
@@ -128,8 +143,7 @@ class DeployOvercloud(command.Command):
         if process.returncode != 0:
             raise exceptions.DeploymentError('No entry for %s in /etc/hosts'
                                              % ctlplane_hostname)
-
-        return re.sub(' +', ' ', str(out).rstrip())
+        return self._cleanup_host_entry(out)
 
     def _create_breakpoint_cleanup_env(self, tht_root, container_name):
         bp_env = {}