Add basic minion validation

The minion doesn't actually have any api services so we validate it by
checking the service status on the undercloud. This change adds a role
to validate the heat engine and ironic conductor status on the
undercloud includes additional hosts (undercloud + minion) and there are
no failed services.

Change-Id: I41859f93980dd8cd723ea4ad0f4e6592a78d0082
Related-Blueprint: undercloud-minion

playbooks/multinode-undercloud-minion.yml

@@ -28,6 +28,10 @@
 
 - name: Setup the undercloud minion
   hosts: overcloud
+  vars:
+    # these are the defaults but we should be explicit
+    minion_enable_heat_engine: true
+    minion_enable_ironic_conductor: true
   tasks:
     - name: Include atop setup
       include_role:
@@ -57,3 +61,13 @@
     - undercloud-minion-deploy
   tags:
     - undercloud-setup
+
+- name: Validate minion
+  hosts: undercloud
+  vars:
+    validate_minion_heat_engine: true
+    validate_minion_ironic_conductor: true
+  roles:
+    - validate-minion
+  tags:
+    - overcloud-validate

roles/validate-minion/README.md

@@ -0,0 +1,39 @@
+validate-minion
+===============
+
+A basic role to validate that a minion is correctly reporting into the undercloud.
+
+Requirements
+------------
+
+N/A
+
+Role Variables
+--------------
+
+- `validate_minion_heat_engine`: <'true'> -- Should the heat-engine service be checked
+- `validate_minion_ironic_conductor`: <'false'> -- Should the ironic-conductor service be checked
+- `validate_minion_simple_script`: <'validate_minion_simple.sh.j2'> -- Simple validation script source
+- `validate_minion_simple_log`: <'False'> -- Log file for the validations
+- `undercloud_user`: <'stack'> -- User that the undercloud was installed under.
+
+Dependencies
+------------
+
+N/A
+
+Example Playbook
+----------------
+
+    - name: Validate minion
+      hosts: undercloud
+      vars:
+        validate_minion_heat_engine: true
+        validate_minion_ironic_conductor: true
+      roles:
+        - validate-minion
+
+License
+-------
+
+Apache

roles/validate-minion/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+validate_minion_heat_engine: true
+validate_minion_ironic_conductor: false
+validate_minion_simple_script: validate_minion_simple.sh.j2
+validate_minion_simple_log: validate_minion_simple.log
+undercloud_user: stack

roles/validate-minion/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for validate-minion

roles/validate-minion/meta/main.yml

@@ -0,0 +1 @@
+---

roles/validate-minion/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Create the tripleo-minion simple validate script
+  template:
+    src: "{{ validate_minion_simple_script }}"
+    dest: "{{ local_working_dir }}/validate_minion_simple.sh"
+    mode: 0744
+
+- name: Execute the simple minion validation
+  shell: >
+    set -o pipefail &&
+    {{ local_working_dir }}/validate_minion_simple.sh
+    | tee {{ validate_minion_simple_log }}

roles/validate-minion/templates/validate_minion_simple.sh.j2

@@ -0,0 +1,58 @@
+#!/bin/bash
+#   Copyright 2019 Red Hat, Inc. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+# simple script to check if the minion is has successfully reported into the
+# undercloud.
+
+set -eux
+VALIDATE_HEAT_ENGINE="${VALIDATE_HEAT_ENGINE:-{{ validate_minion_heat_engine }}}"
+VALIDATE_IRONIC_CONDUCTOR="${VALIDATE_IRONIC_CONDUCTOR:-{{ validate_minion_ironic_conductor }}}"
+RC=0
+
+set +x
+source "/home/{{ undercloud_user }}/stackrc"
+set -x
+
+if [[ ${VALIDATE_HEAT_ENGINE,,} == "true" ]]; then
+    openstack orchestration service list
+    HOST_COUNT=$(openstack orchestration service list -c Host -f value | sort | uniq | wc -l)
+    if [ "$HOST_COUNT" -le 1 ]; then
+        # count should be > 1 if we have an undercloud + minion
+        echo "ERROR: Heat Engine host count is 1 or less."
+        RC=1
+    fi
+
+    DOWN_STATUS=$(openstack orchestration service list -c Status -f value | grep -v up | wc -l)
+    if [ "$DOWN_STATUS" -ge 1 ]; then
+        echo "ERROR: Heat Engine status contains a service that is not up."
+        RC=1
+    fi
+fi
+if [[ ${VALIDATE_IRONIC_CONDUCTOR,,} == "true" ]]; then
+    openstack baremetal conductor list
+    HOST_COUNT=$(openstack baremetal conductor list -c Hostname -f value | sort | uniq | wc -l)
+    if [ "$HOST_COUNT" -le 1 ]; then
+        # count should be > 1 if we have an undercloud + minion
+        echo "ERROR: Ironic Conductor host count is 1 or less."
+        RC=1
+    fi
+    DOWN_STATUS=$(openstack baremetal conductor list -c Alive -f value | grep -v True | wc -l)
+    if [ "$DOWN_STATUS" -ge 1 ]; then
+        echo "ERROR: Ironic Conductor status contains a service that is not up."
+        RC=1
+    fi
+fi
+
+exit $RC

roles/validate-minion/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for validate-minion