Add post-deployment frr_status validation

The frr_status is a post-deployment validation that checks for non-operational FRR daemons. Change-Id: I78089c0203593fd45b887c9ebc7558bc8b5e3162
2 years ago · 463ebb77cf
13 changed files with 406 additions and 0 deletions
--- a/doc/source/roles/role-frr_status.rst
+++ b/doc/source/roles/role-frr_status.rst
@ -0,0 +1,7 @@
+===========
+frr_status
+===========
+
+.. ansibleautoplugin::
+   :role: roles/frr_status
+
--- a/playbooks/frr-status.yaml
+++ b/playbooks/frr-status.yaml
@ -0,0 +1,14 @@
+---
+- hosts: all
+  vars:
+    metadata:
+      name: FRR Daemons Status Check
+      description: |
+        Runs 'show watchfrr' and checks for any non-operational daemon.
+
+        A failed status post-deployment indicates at least one enabled FRR
+        daemon is not operational.
+      groups:
+        - post-deployment
+  roles:
+    - frr_status
--- a/roles/frr_status/molecule/default/Dockerfile
+++ b/roles/frr_status/molecule/default/Dockerfile
@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]
--- a/roles/frr_status/molecule/default/converge.yml
+++ b/roles/frr_status/molecule/default/converge.yml
@ -0,0 +1,28 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: run validation
+      include_role:
+        name: frr_status
+      vars:
+        enabled_services:
+          - frr
--- a/roles/frr_status/molecule/default/molecule.yml
+++ b/roles/frr_status/molecule/default/molecule.yml
@ -0,0 +1,49 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    privileged: true
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra
--- a/roles/frr_status/molecule/default/prepare.yml
+++ b/roles/frr_status/molecule/default/prepare.yml
@ -0,0 +1,44 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: Populate successful podman CLI
+      copy:
+        dest: /usr/bin/podman
+        mode: 0755
+        content: |
+          #!/bin/bash
+          action="$1"
+          shift
+          container="$3"
+          shift
+          command="$@"
+
+          case $container in
+            'frr')
+              echo 'watchfrr global phase: Idle'
+              echo '  zebra                Up'
+              echo '  bgpd                 Up'
+              ;;
+            *)
+              echo "Unknown container ${container}"
+              ;;
+          esac
--- a/roles/frr_status/molecule/faulty/Dockerfile
+++ b/roles/frr_status/molecule/faulty/Dockerfile
@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]
--- a/roles/frr_status/molecule/faulty/converge.yml
+++ b/roles/frr_status/molecule/faulty/converge.yml
@ -0,0 +1,48 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: make validation fail
+      block:
+        - name: run validation
+          include_role:
+            name: frr_status
+          vars:
+            enabled_services:
+              - frr
+
+      rescue:
+        - name: Clear host errors
+          meta: clear_host_errors
+
+        - name: check at least one FRR daemon is down
+          assert:
+            that:
+              - has_errors is defined
+
+        - name: End play
+          meta: end_play
+
+    - name: Fail playbook if reached
+      fail:
+        msg: |
+          The frr_status validation didn't properly detect non-operational FRR
+          daemons!
--- a/roles/frr_status/molecule/faulty/molecule.yml
+++ b/roles/frr_status/molecule/faulty/molecule.yml
@ -0,0 +1,49 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    privileged: true
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra
--- a/roles/frr_status/molecule/faulty/prepare.yml
+++ b/roles/frr_status/molecule/faulty/prepare.yml
@ -0,0 +1,44 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: Populate successful podman CLI
+      copy:
+        dest: /usr/bin/podman
+        mode: 0755
+        content: |
+          #!/bin/bash
+          action="$1"
+          shift
+          container="$3"
+          shift
+          command="$@"
+
+          case $container in
+            'frr')
+              echo 'watchfrr global phase: Idle'
+              echo '  zebra                Up'
+              echo '  bgpd                 Down'
+              ;;
+            *)
+              echo "Unknown container ${container}"
+              ;;
+          esac
--- a/roles/frr_status/tasks/main.yml
+++ b/roles/frr_status/tasks/main.yml
@ -0,0 +1,33 @@
+---
+- name: Ensure we get needed facts
+  setup:
+    gather_subset:
+      - '!all'
+      - '!any'
+      - '!min'
+      - env
+
+- name: Block for FRR daemon status checks
+  when: "'frr' in enabled_services"
+  block:
+    - name: Set container_cli fact from the inventory
+      set_fact:
+        container_cli: "{{ hostvars[inventory_hostname].container_cli |default('podman', true) }}"
+      when: container_cli is not defined
+
+    - name: Check FRR daemon statuses
+      become: true
+      command: "{{ container_cli }} exec -u root frr vtysh -c 'show watchfrr'"
+      changed_when: false
+      register: watchfrr_return
+
+    - name: Detect down FRR daemons
+      set_fact:
+        has_errors: "{{ watchfrr_return.stdout_lines
+                        | select('search', '(Down)')
+                        | list | length | int > 0 }}"
+
+    - name: Fail when at least one enabled FRR daemon is down
+      fail:
+        msg: "{{ watchfrr_return.stdout_lines }}"
+      when: has_errors
--- a/roles/frr_status/vars/main.yml
+++ b/roles/frr_status/vars/main.yml
@ -0,0 +1,7 @@
+---
+metadata:
+  name: FRR status check
+  description: >
+    Runs 'show watchfrr' and checks for any non-operational daemon.
+  groups:
+    - post-deployment
--- a/zuul.d/molecule.yaml
+++ b/zuul.d/molecule.yaml
@ -10,6 +10,7 @@
      - tripleo-validations-centos-8-molecule-controller_token
      - tripleo-validations-centos-8-molecule-controller_ulimits
      - tripleo-validations-centos-8-molecule-ctlplane_ip_range
+      - tripleo-validations-centos-8-molecule-frr_status
      - tripleo-validations-centos-8-molecule-image_serve
      - tripleo-validations-centos-8-molecule-nova_status
      - tripleo-validations-centos-8-molecule-nova_svirt
@ -32,6 +33,7 @@
      - tripleo-validations-centos-8-molecule-controller_token
      - tripleo-validations-centos-8-molecule-controller_ulimits
      - tripleo-validations-centos-8-molecule-ctlplane_ip_range
+      - tripleo-validations-centos-8-molecule-frr_status
      - tripleo-validations-centos-8-molecule-nova_status
      - tripleo-validations-centos-8-molecule-nova_svirt
      - tripleo-validations-centos-8-molecule-package_version
@ -217,6 +219,13 @@
    parent: tripleo-validations-centos-8-base
    vars:
      tripleo_validations_role_name: ctlplane_ip_range
+- job:
+    files:
+    - ^roles/frra_status/.*
+    name: tripleo-validations-centos-8-molecule-frr_status
+    parent: tripleo-validations-centos-8-base
+    vars:
+      tripleo_validations_role_name: frr_status
 - job:
    files:
    - ^roles/repos/.*