Introducing the overcloud-service-status role

This role will hit the overcloud API for nova and cinder, retrieve the
services and will trig a failure if one of these services are either
down or deprecated. The original intent was to validate that
nova-consoleauth was deleted after an update to RHOSP16.

Related: https://bugzilla.redhat.com/1921115
Change-Id: I057349fdac90a093c67aeb0b2f0a825c4c915e0b
(cherry picked from commit c94f38798d)
(cherry picked from commit febdfdefab)

doc/source/roles/role-overcloud_service_status.rst

@@ -0,0 +1,7 @@
+========================
+overcloud_service_status
+========================
+
+.. ansibleautoplugin::
+   :role: roles/overcloud_service_status
+

playbooks/overcloud-service-status.yaml

@@ -0,0 +1,18 @@
+---
+- hosts: Undercloud
+  gather_facts: false
+  vars:
+    metadata:
+      name: Verify overcloud services state after running a deployment or an update
+      description: |
+        An Ansible role to verify the Overcloud services states after a deployment
+        or an update.  It checks the API /os-services and looks for deprecated
+        services (nova-consoleauth) or any down services.
+      groups:
+        - post-deployment
+        - pre-upgrade
+        - post-upgrade
+        - post-overcloud-upgrade
+        - post-overcloud-converge
+  roles:
+    - overcloud_service_status

releasenotes/notes/add-overcloud-service-status-128bf74993d035fa.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Introducing the overcloud_service_status role. This role will hit the
+    overcloud API for nova and cinder, retrieve the services and will trig a
+    failure if one of these services are either down or deprecated.
+    The original intent was to validate that nova-consoleauth was deleted after
+    an update to RHOSP16.

roles/overcloud_service_status/README.md

@@ -0,0 +1,47 @@
+Overcloud-service-status
+=========================
+
+An Ansible role to verify the Overcloud services states after a deployment or an update.
+It checks the API /os-services and looks for deprecated services (nova-consoleauth) or
+any down services.
+
+Requirements
+------------
+
+This role needs to be run on an Undercloud with a deployed Overcloud.
+
+Role Variables
+--------------
+
+- overcloud_service_status_debug: Wether or not to log the token request
+- overcloud_deprecated_services: A list of services that shouldn't be registered any more
+- overcloud_service_api: overcloud API to validate against
+
+These variables are normally set as host variables for the undercloud when generating
+the inventory with tripleo-ansible-inventory:
+- overcloud_keystone_url
+- overcloud_admin_password
+
+
+Dependencies
+------------
+
+No dependencies.
+
+Example Playbook
+----------------
+
+
+    - hosts: undercloud
+      roles:
+         - { role: overcloud_service_status }
+
+License
+-------
+
+Apache
+
+Author Information
+------------------
+
+Red Hat Nova Deployment Squad Team.

roles/overcloud_service_status/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# All variables intended for modification should place placed in this file.
+
+# All variables within this role should have a prefix of "overcloud_service_status"
+overcloud_service_status_debug: false
+overcloud_service_api:
+  - nova
+  - cinderv3
+overcloud_deprecated_services:
+  nova:
+    - nova-consoleauth

roles/overcloud_service_status/molecule/default/Dockerfile

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/overcloud_service_status/molecule/default/molecule.yml

@@ -0,0 +1,54 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    privileged: true
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../../resources/playbooks/prepare.yml
+    converge: ../../resources/playbooks/converge.yml
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+            overcloud_keystone_url: http://127.0.0.1:8080
+            overcloud_admin_password: hello
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra

roles/overcloud_service_status/molecule/deprecated_services/Dockerfile

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/overcloud_service_status/molecule/deprecated_services/molecule.yml

@@ -0,0 +1,54 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    privileged: true
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../../resources/playbooks/prepare.yml
+    converge: ../../resources/playbooks/converge.yml
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+            overcloud_keystone_url: http://127.0.0.1:8080
+            overcloud_admin_password: hello
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra

roles/overcloud_service_status/molecule/down_services/Dockerfile

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/overcloud_service_status/molecule/down_services/molecule.yml

@@ -0,0 +1,54 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    privileged: true
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../../resources/playbooks/prepare.yml
+    converge: ../../resources/playbooks/converge.yml
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+            overcloud_keystone_url: http://127.0.0.1:8080
+            overcloud_admin_password: hello
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra

roles/overcloud_service_status/resources/README.md

@@ -0,0 +1 @@
+Shared resources used for molecule unit testing

roles/overcloud_service_status/resources/playbooks/converge.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  tasks:
+    - block:
+        - name: "Include overcloud_service_status role"
+          include_role:
+            name: "overcloud_service_status"
+      rescue:
+        - fail:
+            msg: "Default test failed"
+          when: molecule_yml.scenario.name == "default"
+        - set_fact:
+            output_var: "{{ lookup('vars',  molecule_yml.scenario.name + '_output')}}"
+        - fail:
+            msg: "No {{ molecule_yml.scenario.name }} found"
+          when: "'failed' not in output_var"

roles/overcloud_service_status/resources/playbooks/prepare.yml

@@ -0,0 +1,12 @@
+---
+- name: Prepare
+  hosts: all
+
+  tasks:
+  - name: Copy http_server.py
+    copy:
+      src: ../scripts/http_server.py
+      dest: /
+
+  - name: "Start http_server.py scenario {{ molecule_yml.scenario.name }}"
+    shell: cd /; nohup python3 /http_server.py --scenario {{ molecule_yml.scenario.name }} > http_server.log 2>&1 &

roles/overcloud_service_status/resources/scripts/http_server.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""
+Simple http server to mock a keystone token and service list.
+If arguments are passed, they will either set the services as down
+or create additionnal services.
+
+Example:
+./http_server.py --scenario default
+- will return services based on the 'services' dict below.
+
+./http_server.py --scenario deprecated_services
+- will return services based on the 'services' dict below, as well
+  as a nova-consoleauth service.
+
+./http_server.py --scenario down_services
+- will return services based on the 'services' dict below, as well
+  as marking one of the services as down.
+"""
+import argparse
+from datetime import datetime
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import json
+
+server_port = 8080
+server_url = "http://127.0.0.1"
+
+# List of services to mock
+# Controllers are going to be created 3 times, computes and hostgroups once
+services = {
+    "nova": {
+        "controller": ["nova-scheduler", "nova-conductor"],
+        "compute": ["nova-compute"],
+    },
+    "cinder": {
+        "controller": ["cinder-scheduler"],
+        "hostgroup": ["cinder-volume"],
+    },
+}
+
+parser = argparse.ArgumentParser(description="mocking keystone and os-service calls")
+parser.add_argument(
+    "--scenario",
+    action="store",
+    default="default",
+    help="Scenario to reproduce",
+)
+args = parser.parse_args()
+
+
+class S(BaseHTTPRequestHandler):
+    def _set_response(self, code=200, **kwargs):
+        self.send_response(code)
+        self.send_header("Content-type", "application/json; charset=utf-8")
+        for key, val in kwargs.items():
+            self.send_header(key, val)
+        self.end_headers()
+
+    def _write_body(self, text):
+        self.wfile.write(text.encode("utf-8"))
+
+    def do_GET(self):
+        self._set_response()
+        path_split = self.path.split("/")
+        self._write_body(self._generate_services(path_split[1]))
+
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        self._set_response(201, x_subject_token=123)
+        self._write_body(self._generate_token())
+
+    def _generate_services(self, service):
+        data = {"services": []}
+        svc = services[service]
+        for key, binaries in svc.items():
+            number_of_nodes = 3 if key == "controller" else 1
+            for i in range(number_of_nodes):
+                for binary in binaries:
+                    data["services"].append(
+                        self._generate_service(binary, f"{key}-{i}.redhat.local")
+                    )
+        # NOTE(dvd): yeah this is ugly and won't work if we remove nova-consoleauth
+        #            from overcloud_deprecated_services. We should probably just
+        #            pass the overcloud_deprecated_services list as an argument to
+        #            to make this future proof
+        if service == "nova" and args.scenario == "deprecated_services":
+            data["services"].extend(
+                [
+                    self._generate_service(
+                        "nova-consoleauth", "controller-0.redhat.local"
+                    ),
+                    self._generate_service(
+                        "nova-consoleauth", "controller-1.redhat.local", "disabled"
+                    ),
+                    self._generate_service(
+                        "nova-consoleauth",
+                        "controller-2.redhat.local",
+                        "enabled",
+                        "down",
+                    ),
+                ]
+            )
+        if args.scenario == "down_services":
+            data["services"][0]["state"] = "down"
+
+        return json.dumps(data)
+
+    def _generate_service(self, binary, host, status="enabled", state="up"):
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        return {
+            "binary": binary,
+            "host": host,
+            "status": status,
+            "state": state,
+            "updated_at": now,
+        }
+
+    def _generate_token(self):
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        data = {
+            "token": {
+                "catalog": [
+                    {
+                        "endpoints": [
+                            {
+                                "url": f"{server_url}:{server_port}/cinder",
+                                "interface": "admin",
+                            },
+                            {
+                                "url": f"{server_url}:{server_port}/cinder",
+                                "interface": "admin",
+                            },
+                            {
+                                "url": f"{server_url}:{server_port}/cinder",
+                                "interface": "admin",
+                            },
+                        ],
+                        "name": "cinderv3",
+                    },
+                    {
+                        "endpoints": [
+                            {
+                                "url": f"{server_url}:{server_port}/nova",
+                                "interface": "admin",
+                            },
+                            {
+                                "url": f"{server_url}:{server_port}/nova",
+                                "interface": "admin",
+                            },
+                            {
+                                "url": f"{server_url}:{server_port}/nova",
+                                "interface": "admin",
+                            },
+                        ],
+                        "name": "nova",
+                    },
+                ],
+            }
+        }
+        return json.dumps(data)
+
+
+def run(server_class=HTTPServer, handler_class=S, port=server_port):
+    server_address = ("", port)
+    httpd = server_class(server_address, handler_class)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+
+
+if __name__ == "__main__":
+    run()

roles/overcloud_service_status/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Geting a system scoped Keystone token
+  no_log: "{{ not overcloud_service_status_debug | bool }}"
+  uri:
+    url: "{{ overcloud_keystone_url | urlsplit('scheme') }}://{{ overcloud_keystone_url | urlsplit('netloc') }}/v3/auth/tokens"
+    method: POST
+    body_format: json
+    body:
+      auth:
+        identity:
+          methods:
+            - password
+          password:
+            user:
+              password: "{{ overcloud_admin_password }}"
+              name: admin
+              domain:
+                id: default
+        scope:
+          project:
+            name: admin
+            domain:
+              name: Default
+    return_content: true
+    status_code: 201
+  register: auth_token
+  when: overcloud_keystone_url|default('')
+
+- name: Checking openstack services
+  include_tasks: tasks/os_service.yml
+  loop: "{{ overcloud_service_api }}"
+  loop_control:
+    loop_var: os_service

roles/overcloud_service_status/tasks/os_service.yml

@@ -0,0 +1,82 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Extracting the endpoint url
+  set_fact:
+    endpoint: "{{ catalog.endpoints|selectattr('interface', 'eq', 'admin')|first }}"
+  loop: "{{ auth_token.json.token.catalog }}"
+  loop_control:
+    loop_var: catalog
+  when: catalog.name == os_service
+
+- fail:
+    msg: "No endpoint found for {{ os_service }} interface admin in catalog"
+  when: endpoint is not defined
+
+- name: Get services
+  uri:
+    url: "{{ endpoint.url }}/os-services"
+    method: GET
+    headers:
+      Accept: application/json
+      X-Auth-Token: "{{ auth_token.x_subject_token }}"
+    return_content: true
+    status_code: 200
+  register: os_services
+
+- name: Verifying deprecated services are absent
+  assert:
+    that:
+      - service[0].binary != service[1]
+    fail_msg: "{{ service[0].binary }} should be removed on {{ service[0].host }}"
+  loop: "{{ os_services.json.services | product(overcloud_deprecated_services[os_service]) | list }}"
+  loop_control:
+    loop_var: service
+  when: os_service in overcloud_deprecated_services
+  register: deprecated_services_output
+  ignore_errors: true
+
+- name: Verifying all services are up
+  assert:
+    that:
+      - (service.state == "up" and service.status == "enabled") or service.status == "disabled"
+    fail_msg: "{{ service.binary }} on {{ service.host }} is problematic (service state is {{ service.state }} while it's {{ service.status }})"
+  loop: "{{ os_services.json.services }}"
+  loop_control:
+    loop_var: service
+  register: down_services_output
+  ignore_errors: true
+
+- name: Asserted failure
+  fail:
+    msg: |
+        At least one of the assertion failed.
+        {% if 'failed' in deprecated_services_output %}
+          {% for service in deprecated_services_output.results %}
+            {% if service.failed %}
+        {{ service.msg }}
+            {% endif %}
+          {% endfor %}
+        {% endif %}
+        {% if 'failed' in down_services_output %}
+          {% for service in down_services_output.results %}
+            {% if service.failed %}
+        {{ service.msg }}
+            {% endif %}
+          {% endfor %}
+        {% endif %}
+
+  when: "'failed' in deprecated_services_output or 'failed' in down_services_output"

roles/overcloud_service_status/vars/main.yml

@@ -0,0 +1,32 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# While options found within the vars/ path can be overridden using extra
+# vars, items within this path are considered part of the role and not
+# intended to be modified.
+metadata:
+  name: Verify overcloud services state after running a deployment or an update
+  description: >
+    An Ansible role to verify the Overcloud services states after a deployment
+    or an update.  It checks the API /os-services and looks for deprecated
+    services (nova-consoleauth) or any down services.
+  groups:
+    - post-deployment
+    - pre-upgrade
+    - post-upgrade
+    - post-overcloud-upgrade
+    - post-overcloud-converge

zuul.d/molecule.yaml

@@ -321,3 +321,10 @@
     parent: tripleo-validations-centos-8-base
     vars:
       tripleo_validations_role_name: system_encoding
+- job:
+    files:
+      - ^roles/overcloud_service_status/.*
+    name: tripleo-validations-centos-8-molecule-overcloud_service_status
+    parent: tripleo-validations-centos-8-base
+    vars:
+      tripleo_validations_role_name: overcloud_service_status