Fixing replacement of /etc/hosts timeouts

During the restore operation, there is a span of time where we end up
without the /etc/hosts file and, when executing the playbooks remotely,
we reach the 12s timeout for command. This commit fixes it by ensuring
that /etc/hosts is available and with the bare minimum entries at all
times and also add a rescue operation to recover the previous /etc/hosts
file in case any task fails.

TEST PLAN
PASS remote play of the restore playbook
PASS local play of the restore playbook
PASS bootstrap newly installed system

Closes-Bug: #1986693
Signed-off-by: Thiago Brito <thiago.brito@windriver.com>
Change-Id: Iff8e56478339f660ec66e2d4f7cd8ad000b4d306

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/refresh_local_dns.yml

@@ -10,27 +10,33 @@
 
 # Check host connectivity, change password if provided
 
-- name: Remove existing /etc/hosts
-  file:
-    path: /etc/hosts
-    state: absent
+- name: Refresh local DNS
+  block:
 
-- block:
+    - name: Moving existing /etc/hosts to backup and creating a new one with bare minimum entries
+      copy:
+        dest: /etc/hosts
+        backup: yes
+        # If this is not done, sudo operations might timeout. See bug #1986693
+        content: |
+          {{ localhost_name_ip_mapping }}
+          {{ derived_network_params.controller_0_address }}    controller-0
+          {{ derived_network_params.controller_1_address }}    controller-1
+      register: etc_hosts
+
+    - block:
       - name: Populate /etc/hosts
         lineinfile:
           path: /etc/hosts
           line: "{{ item }}"
           create: yes
         with_items:
-      - "{{ localhost_name_ip_mapping }}"
           - "{{ controller_floating_address }}\tcontroller"
           # May not need this entry
           - "{{ controller_0_cluster_host }}\tcontroller-0-infra"
           - "{{ controller_pxeboot_floating_address }}\tpxecontroller"
           - "{{ external_oam_floating_address }}\toamcontroller"
           - "{{ derived_network_params.nfs_management_address_1 }}\tcontroller-platform-nfs"
-      - "{{ derived_network_params.controller_1_address }}\tcontroller-1"
-      - "{{ derived_network_params.controller_0_address }}\tcontroller-0"
           # May not need this entry
           - "{{ controller_1_cluster_host }}\tcontroller-1-infra"
           - "{{ derived_network_params.nfs_management_address_2 }}\tcontroller-nfs"
@@ -64,9 +70,9 @@
 
       when: mode == 'bootstrap'
 
-- block:
+    - block:
       - name: Restore /etc/hosts file
-    command: tar -C /etc -xpf {{ restore_data_file }} --transform='s,.*/,,' etc/hosts
+        command: tar -C /etc --overwrite -xpf {{ restore_data_file }} --transform='s,.*/,,' etc/hosts
         args:
           warn: false
 
@@ -88,5 +94,16 @@
 
       when: mode == 'restore'
 
+  rescue:
+    - name: Restore backed up /etc/hosts on failure
+      copy:
+        dest: /etc/hosts
+        src: etc_hosts.backup_file
+        remote_src: true
+
+    - name: Force a failure of playbook on /etc/hosts update failure
+      command: /bin/false
+
+
 - name: Update name service caching server
   command: nscd -i hosts