Fix neutron_sanity_check for ML2/OVS overcloud

Previously validator failed running on neutron_api as the
neutron-sanity-check binary requires additional capabilities, causing it
to alway fail [0]. In addition, in some cases a race condition in
oslo_privsep can cause the validator to hang indefinitely [1].

This fixes it if the overcloud uses ML2/OVS, using the neutron_ovs_agent
container as we already did on the undercloud, and skipping the check if
it cannot be found (ML2/OVN deployments typically).

Additional fixes to the validator:
* skip gather_facts step
* fix output parsing
* call neutron-sanity-check only once, with all configuration files
  passed to it
* drop redundant default values in playbook itself

[0] https://bugzilla.redhat.com/show_bug.cgi?id=1783195
[1] https://bugzilla.redhat.com/show_bug.cgi?id=1862364

Change-Id: Ifacbbd6a493019e606105f45f0302bf1a88bed62
Co-Authored-By: Cédric Jeanneret <cjeanner@redhat.com>

playbooks/neutron-sanity-check.yaml

@@ -1,5 +1,6 @@
 ---
 - hosts: Controller
+  gather_facts: false
   vars:
     metadata:
       name: Neutron Sanity Check
@@ -13,15 +14,5 @@
       groups:
         - post-deployment
 
-    # The list of Neutron configuration files and directories that
-    # will be passed to the Neutron services. The order is important
-    # here: the values in later files take precedence.
-    configs:
-      - /etc/neutron/neutron.conf
-      - /usr/share/neutron/neutron-dist.conf
-      - /etc/neutron/metadata_agent.ini
-      - /etc/neutron/dhcp_agent.ini
-      - /etc/neutron/l3_agent.ini
-
   roles:
     - neutron_sanity_check

roles/neutron_sanity_check/defaults/main.yml

@@ -1,4 +1,7 @@
 ---
+# The list of Neutron configuration files and directories that
+# will be passed to the Neutron services. The order is important
+# here: the values in later files take precedence.
 configs:
   - /etc/neutron/neutron.conf
   - /usr/share/neutron/neutron-dist.conf

roles/neutron_sanity_check/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Set oc_container_cli and container_name for the Controller
   set_fact:
     oc_container_cli: "{{ hostvars[inventory_hostname].container_cli | default('podman', true) }}"
-    container_name: "neutron_api"
+    container_name: "neutron_ovs_agent"
   when: "'Controller' in group_names"
 
 - when: "'Undercloud' in group_names"
@@ -26,42 +26,49 @@
         uc_container_cli: "{{ container_cli.value|default('podman', true) }}"
         container_name: "neutron_ovs_agent"
 
-- name: Run neutron-sanity-check
+- name: Check if wanted container exists
   command: >
     {% if oc_container_cli is defined %}{{ oc_container_cli }}{% else %}{{ uc_container_cli }}{% endif %}
-    exec -u root {{ container_name }}
-    /bin/bash -c 'neutron-sanity-check --config-file {{ item }}'
-  with_items: "{{ configs }}"
+    ps --filter name={{ container_name }} -q
   become: true
-  register: nsc_return
+  register: container_exists
   ignore_errors: true
-  changed_when: false
 
-- name: Detect errors
-  set_fact:
-    has_errors: "{{ nsc_return.results
-                    | sum(attribute='stderr_lines', start=[])
-                    | select('search', '(ERROR)')
-                    | list | length | int > 0 }}"
+- name: Run sanity check only if container exists
+  when: container_exists.stdout != ''
+  block:
+    - name: Run neutron-sanity-check
+      command: >
+        {% if oc_container_cli is defined %}{{ oc_container_cli }}{% else %}{{ uc_container_cli }}{% endif %}
+        exec -u root {{ container_name }}
+        /bin/bash -c 'neutron-sanity-check --config-file {{ " --config-file ".join(configs) }} -d'
+      become: true
+      register: nsc_return
+      ignore_errors: true
+      changed_when: false
 
-- name: Detect warnings
-  set_fact:
-    has_warnings: "{{ nsc_return.results
-                    | sum(attribute='stderr_lines', start=[])
-                    | select('search', '(WARNING)')
-                    | list | length | int > 0 }}"
+    - name: Detect errors
+      set_fact:
+        has_errors: "{{ nsc_return.stderr_lines
+                        | select('search', '(ERROR)')
+                        | list | length | int > 0 }}"
 
-- name: Create output
-  set_fact:
-    output_msg: "{{ nsc_return.results
-                    | sum(attribute='stderr_lines', start=[])
-                    | select('search', '(ERROR|WARNING)')
-                    | list }}"
+    - name: Detect warnings
+      set_fact:
+        has_warnings: "{{ nsc_return.stderr_lines
+                        | select('search', '(WARNING)')
+                        | list | length | int > 0 }}"
 
-- name: Output warning
-  warn: msg="{{ output_msg | join('\n') }}"
-  when: has_warnings and not has_errors
+    - name: Create output
+      set_fact:
+        output_msg: "{{ nsc_return.stderr_lines
+                        | select('search', '(ERROR|WARNING)')
+                        | list }}"
 
-- name: Fail
-  fail: msg="{{ output_msg | join('\n') }}"
-  when: has_errors
+    - name: Output warning
+      warn: msg="{{ output_msg | join('\n') }}"
+      when: has_warnings and not has_errors
+
+    - name: Fail
+      fail: msg="{{ output_msg | join('\n') }}"
+      when: has_errors