diff --git a/.zuul.yaml b/.zuul.yaml index a8e958321..afc995988 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -391,3 +391,10 @@ parent: tripleo-validations-centos-8-base vars: tripleo_validations_role_name: check_undercloud_conf +- job: + files: + - ^roles/overcloud_service_status/.* + name: tripleo-validations-centos-8-molecule-overcloud_service_status + parent: tripleo-validations-centos-8-base + vars: + tripleo_validations_role_name: overcloud_service_status diff --git a/doc/source/roles/role-overcloud_service_status.rst b/doc/source/roles/role-overcloud_service_status.rst new file mode 100644 index 000000000..4ab8d6cce --- /dev/null +++ b/doc/source/roles/role-overcloud_service_status.rst @@ -0,0 +1,7 @@ +======================== +overcloud_service_status +======================== + +.. ansibleautoplugin:: + :role: roles/overcloud_service_status + diff --git a/playbooks/overcloud-service-status.yaml b/playbooks/overcloud-service-status.yaml new file mode 100644 index 000000000..7cf2c340e --- /dev/null +++ b/playbooks/overcloud-service-status.yaml @@ -0,0 +1,18 @@ +--- +- hosts: Undercloud + gather_facts: false + vars: + metadata: + name: Verify overcloud services state after running a deployment or an update + description: | + An Ansible role to verify the Overcloud services states after a deployment + or an update. It checks the API /os-services and looks for deprecated + services (nova-consoleauth) or any down services. + groups: + - post-deployment + - pre-upgrade + - post-upgrade + - post-overcloud-upgrade + - post-overcloud-converge + roles: + - overcloud_service_status diff --git a/releasenotes/notes/add-overcloud-service-status-128bf74993d035fa.yaml b/releasenotes/notes/add-overcloud-service-status-128bf74993d035fa.yaml new file mode 100644 index 000000000..a5fb770da --- /dev/null +++ b/releasenotes/notes/add-overcloud-service-status-128bf74993d035fa.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Introducing the overcloud_service_status role. This role will hit the + overcloud API for nova and cinder, retrieve the services and will trig a + failure if one of these services are either down or deprecated. + The original intent was to validate that nova-consoleauth was deleted after + an update to RHOSP16. diff --git a/roles/overcloud_service_status/README.md b/roles/overcloud_service_status/README.md new file mode 100644 index 000000000..342bed99e --- /dev/null +++ b/roles/overcloud_service_status/README.md @@ -0,0 +1,47 @@ +Overcloud-service-status +========================= + +An Ansible role to verify the Overcloud services states after a deployment or an update. +It checks the API /os-services and looks for deprecated services (nova-consoleauth) or +any down services. + +Requirements +------------ + +This role needs to be run on an Undercloud with a deployed Overcloud. + +Role Variables +-------------- + +- overcloud_service_status_debug: Wether or not to log the token request +- overcloud_deprecated_services: A list of services that shouldn't be registered any more +- overcloud_service_api: overcloud API to validate against + +These variables are normally set as host variables for the undercloud when generating +the inventory with tripleo-ansible-inventory: +- overcloud_keystone_url +- overcloud_admin_password + + +Dependencies +------------ + +No dependencies. + +Example Playbook +---------------- + + + - hosts: undercloud + roles: + - { role: overcloud_service_status } + +License +------- + +Apache + +Author Information +------------------ + +Red Hat Nova Deployment Squad Team. diff --git a/roles/overcloud_service_status/defaults/main.yml b/roles/overcloud_service_status/defaults/main.yml new file mode 100644 index 000000000..f83fc07b2 --- /dev/null +++ b/roles/overcloud_service_status/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# All variables intended for modification should place placed in this file. + +# All variables within this role should have a prefix of "overcloud_service_status" +overcloud_service_status_debug: false +overcloud_service_api: + - nova + - cinderv3 +overcloud_deprecated_services: + nova: + - nova-consoleauth diff --git a/roles/overcloud_service_status/molecule/default/Dockerfile b/roles/overcloud_service_status/molecule/default/Dockerfile new file mode 100644 index 000000000..90646fff5 --- /dev/null +++ b/roles/overcloud_service_status/molecule/default/Dockerfile @@ -0,0 +1,37 @@ +# Molecule managed +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +{% for pkg in item.easy_install | default([]) %} +# install pip for centos where there is no python-pip rpm in default repos +RUN easy_install {{ pkg }} +{% endfor %} + + +CMD ["sh", "-c", "while true; do sleep 10000; done"] diff --git a/roles/overcloud_service_status/molecule/default/molecule.yml b/roles/overcloud_service_status/molecule/default/molecule.yml new file mode 100644 index 000000000..f0f5c40ca --- /dev/null +++ b/roles/overcloud_service_status/molecule/default/molecule.yml @@ -0,0 +1,54 @@ +--- +driver: + name: podman + +log: true + +platforms: + - name: ubi8 + hostname: ubi8 + image: ubi8/ubi-init + registry: + url: registry.access.redhat.com + dockerfile: Dockerfile + pkg_extras: python*-setuptools + privileged: true + volumes: + - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro + - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg + - /opt/yum.repos.d:/etc/yum.repos.d:rw + environment: &env + http_proxy: "{{ lookup('env', 'http_proxy') }}" + https_proxy: "{{ lookup('env', 'https_proxy') }}" + ulimits: &ulimit + - host + +provisioner: + name: ansible + playbooks: + prepare: ../../resources/playbooks/prepare.yml + converge: ../../resources/playbooks/converge.yml + inventory: + hosts: + all: + hosts: + ubi8: + ansible_python_interpreter: /usr/bin/python3 + overcloud_keystone_url: http://127.0.0.1:8080 + overcloud_admin_password: hello + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}" + +scenario: + test_sequence: + - destroy + - create + - prepare + - converge + - verify + - destroy + +verifier: + name: testinfra diff --git a/roles/overcloud_service_status/molecule/deprecated_services/Dockerfile b/roles/overcloud_service_status/molecule/deprecated_services/Dockerfile new file mode 100644 index 000000000..90646fff5 --- /dev/null +++ b/roles/overcloud_service_status/molecule/deprecated_services/Dockerfile @@ -0,0 +1,37 @@ +# Molecule managed +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +{% for pkg in item.easy_install | default([]) %} +# install pip for centos where there is no python-pip rpm in default repos +RUN easy_install {{ pkg }} +{% endfor %} + + +CMD ["sh", "-c", "while true; do sleep 10000; done"] diff --git a/roles/overcloud_service_status/molecule/deprecated_services/molecule.yml b/roles/overcloud_service_status/molecule/deprecated_services/molecule.yml new file mode 100644 index 000000000..f0f5c40ca --- /dev/null +++ b/roles/overcloud_service_status/molecule/deprecated_services/molecule.yml @@ -0,0 +1,54 @@ +--- +driver: + name: podman + +log: true + +platforms: + - name: ubi8 + hostname: ubi8 + image: ubi8/ubi-init + registry: + url: registry.access.redhat.com + dockerfile: Dockerfile + pkg_extras: python*-setuptools + privileged: true + volumes: + - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro + - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg + - /opt/yum.repos.d:/etc/yum.repos.d:rw + environment: &env + http_proxy: "{{ lookup('env', 'http_proxy') }}" + https_proxy: "{{ lookup('env', 'https_proxy') }}" + ulimits: &ulimit + - host + +provisioner: + name: ansible + playbooks: + prepare: ../../resources/playbooks/prepare.yml + converge: ../../resources/playbooks/converge.yml + inventory: + hosts: + all: + hosts: + ubi8: + ansible_python_interpreter: /usr/bin/python3 + overcloud_keystone_url: http://127.0.0.1:8080 + overcloud_admin_password: hello + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}" + +scenario: + test_sequence: + - destroy + - create + - prepare + - converge + - verify + - destroy + +verifier: + name: testinfra diff --git a/roles/overcloud_service_status/molecule/down_services/Dockerfile b/roles/overcloud_service_status/molecule/down_services/Dockerfile new file mode 100644 index 000000000..90646fff5 --- /dev/null +++ b/roles/overcloud_service_status/molecule/down_services/Dockerfile @@ -0,0 +1,37 @@ +# Molecule managed +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +{% for pkg in item.easy_install | default([]) %} +# install pip for centos where there is no python-pip rpm in default repos +RUN easy_install {{ pkg }} +{% endfor %} + + +CMD ["sh", "-c", "while true; do sleep 10000; done"] diff --git a/roles/overcloud_service_status/molecule/down_services/molecule.yml b/roles/overcloud_service_status/molecule/down_services/molecule.yml new file mode 100644 index 000000000..f0f5c40ca --- /dev/null +++ b/roles/overcloud_service_status/molecule/down_services/molecule.yml @@ -0,0 +1,54 @@ +--- +driver: + name: podman + +log: true + +platforms: + - name: ubi8 + hostname: ubi8 + image: ubi8/ubi-init + registry: + url: registry.access.redhat.com + dockerfile: Dockerfile + pkg_extras: python*-setuptools + privileged: true + volumes: + - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro + - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg + - /opt/yum.repos.d:/etc/yum.repos.d:rw + environment: &env + http_proxy: "{{ lookup('env', 'http_proxy') }}" + https_proxy: "{{ lookup('env', 'https_proxy') }}" + ulimits: &ulimit + - host + +provisioner: + name: ansible + playbooks: + prepare: ../../resources/playbooks/prepare.yml + converge: ../../resources/playbooks/converge.yml + inventory: + hosts: + all: + hosts: + ubi8: + ansible_python_interpreter: /usr/bin/python3 + overcloud_keystone_url: http://127.0.0.1:8080 + overcloud_admin_password: hello + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}" + +scenario: + test_sequence: + - destroy + - create + - prepare + - converge + - verify + - destroy + +verifier: + name: testinfra diff --git a/roles/overcloud_service_status/resources/README.md b/roles/overcloud_service_status/resources/README.md new file mode 100644 index 000000000..be369befc --- /dev/null +++ b/roles/overcloud_service_status/resources/README.md @@ -0,0 +1 @@ +Shared resources used for molecule unit testing diff --git a/roles/overcloud_service_status/resources/playbooks/converge.yml b/roles/overcloud_service_status/resources/playbooks/converge.yml new file mode 100644 index 000000000..8257351f9 --- /dev/null +++ b/roles/overcloud_service_status/resources/playbooks/converge.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Converge + hosts: all + tasks: + - block: + - name: "Include overcloud_service_status role" + include_role: + name: "overcloud_service_status" + rescue: + - fail: + msg: "Default test failed" + when: molecule_yml.scenario.name == "default" + - set_fact: + output_var: "{{ lookup('vars', molecule_yml.scenario.name + '_output')}}" + - fail: + msg: "No {{ molecule_yml.scenario.name }} found" + when: "'failed' not in output_var" diff --git a/roles/overcloud_service_status/resources/playbooks/prepare.yml b/roles/overcloud_service_status/resources/playbooks/prepare.yml new file mode 100644 index 000000000..c538eea76 --- /dev/null +++ b/roles/overcloud_service_status/resources/playbooks/prepare.yml @@ -0,0 +1,12 @@ +--- +- name: Prepare + hosts: all + + tasks: + - name: Copy http_server.py + copy: + src: ../scripts/http_server.py + dest: / + + - name: "Start http_server.py scenario {{ molecule_yml.scenario.name }}" + shell: cd /; nohup python3 /http_server.py --scenario {{ molecule_yml.scenario.name }} > http_server.log 2>&1 & diff --git a/roles/overcloud_service_status/resources/scripts/http_server.py b/roles/overcloud_service_status/resources/scripts/http_server.py new file mode 100644 index 000000000..1e460cc9f --- /dev/null +++ b/roles/overcloud_service_status/resources/scripts/http_server.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Simple http server to mock a keystone token and service list. +If arguments are passed, they will either set the services as down +or create additionnal services. + +Example: +./http_server.py --scenario default +- will return services based on the 'services' dict below. + +./http_server.py --scenario deprecated_services +- will return services based on the 'services' dict below, as well + as a nova-consoleauth service. + +./http_server.py --scenario down_services +- will return services based on the 'services' dict below, as well + as marking one of the services as down. +""" +import argparse +from datetime import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer +import json + +server_port = 8080 +server_url = "http://127.0.0.1" + +# List of services to mock +# Controllers are going to be created 3 times, computes and hostgroups once +services = { + "nova": { + "controller": ["nova-scheduler", "nova-conductor"], + "compute": ["nova-compute"], + }, + "cinder": { + "controller": ["cinder-scheduler"], + "hostgroup": ["cinder-volume"], + }, +} + +parser = argparse.ArgumentParser(description="mocking keystone and os-service calls") +parser.add_argument( + "--scenario", + action="store", + default="default", + help="Scenario to reproduce", +) +args = parser.parse_args() + + +class S(BaseHTTPRequestHandler): + def _set_response(self, code=200, **kwargs): + self.send_response(code) + self.send_header("Content-type", "application/json; charset=utf-8") + for key, val in kwargs.items(): + self.send_header(key, val) + self.end_headers() + + def _write_body(self, text): + self.wfile.write(text.encode("utf-8")) + + def do_GET(self): + self._set_response() + path_split = self.path.split("/") + self._write_body(self._generate_services(path_split[1])) + + def do_POST(self): + content_length = int(self.headers["Content-Length"]) + self._set_response(201, x_subject_token=123) + self._write_body(self._generate_token()) + + def _generate_services(self, service): + data = {"services": []} + svc = services[service] + for key, binaries in svc.items(): + number_of_nodes = 3 if key == "controller" else 1 + for i in range(number_of_nodes): + for binary in binaries: + data["services"].append( + self._generate_service(binary, f"{key}-{i}.redhat.local") + ) + # NOTE(dvd): yeah this is ugly and won't work if we remove nova-consoleauth + # from overcloud_deprecated_services. We should probably just + # pass the overcloud_deprecated_services list as an argument to + # to make this future proof + if service == "nova" and args.scenario == "deprecated_services": + data["services"].extend( + [ + self._generate_service( + "nova-consoleauth", "controller-0.redhat.local" + ), + self._generate_service( + "nova-consoleauth", "controller-1.redhat.local", "disabled" + ), + self._generate_service( + "nova-consoleauth", + "controller-2.redhat.local", + "enabled", + "down", + ), + ] + ) + if args.scenario == "down_services": + data["services"][0]["state"] = "down" + + return json.dumps(data) + + def _generate_service(self, binary, host, status="enabled", state="up"): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return { + "binary": binary, + "host": host, + "status": status, + "state": state, + "updated_at": now, + } + + def _generate_token(self): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + data = { + "token": { + "catalog": [ + { + "endpoints": [ + { + "url": f"{server_url}:{server_port}/cinder", + "interface": "admin", + }, + { + "url": f"{server_url}:{server_port}/cinder", + "interface": "admin", + }, + { + "url": f"{server_url}:{server_port}/cinder", + "interface": "admin", + }, + ], + "name": "cinderv3", + }, + { + "endpoints": [ + { + "url": f"{server_url}:{server_port}/nova", + "interface": "admin", + }, + { + "url": f"{server_url}:{server_port}/nova", + "interface": "admin", + }, + { + "url": f"{server_url}:{server_port}/nova", + "interface": "admin", + }, + ], + "name": "nova", + }, + ], + } + } + return json.dumps(data) + + +def run(server_class=HTTPServer, handler_class=S, port=server_port): + server_address = ("", port) + httpd = server_class(server_address, handler_class) + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + httpd.server_close() + + +if __name__ == "__main__": + run() diff --git a/roles/overcloud_service_status/tasks/main.yml b/roles/overcloud_service_status/tasks/main.yml new file mode 100644 index 000000000..10dfdf7f6 --- /dev/null +++ b/roles/overcloud_service_status/tasks/main.yml @@ -0,0 +1,48 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Geting a system scoped Keystone token + no_log: "{{ not overcloud_service_status_debug | bool }}" + uri: + url: "{{ overcloud_keystone_url | urlsplit('scheme') }}://{{ overcloud_keystone_url | urlsplit('netloc') }}/v3/auth/tokens" + method: POST + body_format: json + body: + auth: + identity: + methods: + - password + password: + user: + password: "{{ overcloud_admin_password }}" + name: admin + domain: + id: default + scope: + project: + name: admin + domain: + name: Default + return_content: true + status_code: 201 + register: auth_token + when: overcloud_keystone_url|default('') + +- name: Checking openstack services + include_tasks: tasks/os_service.yml + loop: "{{ overcloud_service_api }}" + loop_control: + loop_var: os_service diff --git a/roles/overcloud_service_status/tasks/os_service.yml b/roles/overcloud_service_status/tasks/os_service.yml new file mode 100644 index 000000000..8fbd58089 --- /dev/null +++ b/roles/overcloud_service_status/tasks/os_service.yml @@ -0,0 +1,82 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Extracting the endpoint url + set_fact: + endpoint: "{{ catalog.endpoints|selectattr('interface', 'eq', 'admin')|first }}" + loop: "{{ auth_token.json.token.catalog }}" + loop_control: + loop_var: catalog + when: catalog.name == os_service + +- fail: + msg: "No endpoint found for {{ os_service }} interface admin in catalog" + when: endpoint is not defined + +- name: Get services + uri: + url: "{{ endpoint.url }}/os-services" + method: GET + headers: + Accept: application/json + X-Auth-Token: "{{ auth_token.x_subject_token }}" + return_content: true + status_code: 200 + register: os_services + +- name: Verifying deprecated services are absent + assert: + that: + - service[0].binary != service[1] + fail_msg: "{{ service[0].binary }} should be removed on {{ service[0].host }}" + loop: "{{ os_services.json.services | product(overcloud_deprecated_services[os_service]) | list }}" + loop_control: + loop_var: service + when: os_service in overcloud_deprecated_services + register: deprecated_services_output + ignore_errors: true + +- name: Verifying all services are up + assert: + that: + - (service.state == "up" and service.status == "enabled") or service.status == "disabled" + fail_msg: "{{ service.binary }} on {{ service.host }} is problematic (service state is {{ service.state }} while it's {{ service.status }})" + loop: "{{ os_services.json.services }}" + loop_control: + loop_var: service + register: down_services_output + ignore_errors: true + +- name: Asserted failure + fail: + msg: | + At least one of the assertion failed. + {% if 'failed' in deprecated_services_output %} + {% for service in deprecated_services_output.results %} + {% if service.failed %} + {{ service.msg }} + {% endif %} + {% endfor %} + {% endif %} + {% if 'failed' in down_services_output %} + {% for service in down_services_output.results %} + {% if service.failed %} + {{ service.msg }} + {% endif %} + {% endfor %} + {% endif %} + + when: "'failed' in deprecated_services_output or 'failed' in down_services_output" diff --git a/roles/overcloud_service_status/vars/main.yml b/roles/overcloud_service_status/vars/main.yml new file mode 100644 index 000000000..644ac3720 --- /dev/null +++ b/roles/overcloud_service_status/vars/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# While options found within the vars/ path can be overridden using extra +# vars, items within this path are considered part of the role and not +# intended to be modified. +metadata: + name: Verify overcloud services state after running a deployment or an update + description: > + An Ansible role to verify the Overcloud services states after a deployment + or an update. It checks the API /os-services and looks for deprecated + services (nova-consoleauth) or any down services. + groups: + - post-deployment + - pre-upgrade + - post-upgrade + - post-overcloud-upgrade + - post-overcloud-converge