NFV OVS-DPDK Counter statistics validation

This change is to perform counter statistics validation for nfv ovsdpdk
interfaces. The counters validated here are ovs_tx_failure_drops,
ovs_tx_mtu_exceeded_drops, ovs_rx_qos_drops, ovs_tx_qos_drops,
ovs_tx_retries.

Change-Id: I997385f9e36dc74d60fdf933c980a2b080158ddb

doc/source/roles/role-check_nfv_ovsdpdk_counter_stat_validation.rst

@@ -0,0 +1,45 @@
+=========================================
+check_nfv_ovsdpdk_counter_stat_validation
+=========================================
+
+--------------
+About the role
+--------------
+
+This role validates the NFV OvS DPDK interface counter statistics.
+
+Requirements
+============
+
+- Gathers information about all the existing interfaces.
+- Validates ovs_tx_failure_drops, ovs_tx_mtu_exceeded_drops, ovs_rx_qos_drops, ovs_tx_qos_drops, ovs_tx_retries metrics for every interface.
+- Checks if any of the listed counters are increasing and outputs its related configuration.
+
+Dependencies
+============
+
+Example Playbook
+================
+
+.. code-block:: yaml
+
+    - hosts: servers
+      roles:
+
+        - { role: check_nfv_ovsdpdk_counter_stat_validation }
+
+License
+=======
+Apache
+
+Author Information
+==================
+
+**Red Hat TripleO DFG:NFV Integration**
+
+----------------
+Full Description
+----------------
+
+.. ansibleautoplugin::
+   :role: roles/check_nfv_ovsdpdk_counter_stat_validation

playbooks/nfv-ovsdpdk-counter-stat-validation.yaml

@@ -0,0 +1,18 @@
+---
+- hosts: "{{ compute_ovsdpdk_rolename | default('ComputeOvsDpdk') }}"
+  vars:
+    metadata:
+      name: NFV OvS DPDK Counter Stat Validations
+      description: |
+        Run `check-nfv-ovsdpdk-counter-stat-validation` on the compute ovsdpdk nodes
+        to find out the issues with NFV OvS Dpdk interface statistics.
+      groups:
+        - post-deployment
+      categories:
+        - networking
+        - compute
+        - nfv
+      products:
+        - tripleo
+  roles:
+    - check_nfv_ovsdpdk_counter_stat_validation

roles/check_nfv_ovsdpdk_counter_stat_validation/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# counters under validation
+counters: ["ovs_tx_failure_drops", "ovs_tx_mtu_exceeded_drops", "ovs_rx_qos_drops", "ovs_tx_qos_drops", "ovs_tx_retries"]
+# default value of maximum tx_retries
+default_tx_retries_max: 8

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/datacollection.yml

@@ -0,0 +1,37 @@
+---
+- name: Reset statistics
+  set_fact:
+    stats: {}
+
+- name: Get counter statistics
+  vars:
+    port: "{{ dpdk_port }}"
+  include_tasks: read_counter_stats.yml
+  loop: "{{ ovs_dpdk_ports }}"
+  loop_control:
+    loop_var: dpdk_port
+
+- name: First attempt counter statistics
+  set_fact:
+    initial: "{{ stats | default({}) }}"
+
+- name: Sleep for 15 seconds
+  wait_for:
+    timeout: 15
+  delegate_to: localhost
+
+- name: Reset statistics
+  set_fact:
+    stats: {}
+
+- name: Get counter statistics
+  vars:
+    port: "{{ dpdk_port }}"
+  include_tasks: read_counter_stats.yml
+  loop: "{{ ovs_dpdk_ports }}"
+  loop_control:
+    loop_var: dpdk_port
+
+- name: Second attempt counter statistics
+  set_fact:
+    final: "{{ stats | default({}) }}"

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Get list of dpdk ports
+  include_tasks: ovs-dpdk-ports.yml
+
+- name: Initialize validation message list
+  set_fact:
+    validation_msg: []
+
+- name: Collect data
+  include_tasks: datacollection.yml
+
+- name: validate data of ovs_dpdk_ports
+  vars:
+    interface: "{{ port }}"
+  include_tasks: validate.yml
+  loop: "{{ ovs_dpdk_ports }}"
+  loop_control:
+    loop_var: port
+
+- name: Validation errors
+  fail:
+    msg:
+      - "Failed NFV counter stat validation rules:"
+      - "{{ validation_msg | join('\n') }}"
+  when:
+    - validation_msg is defined
+    - validation_msg | length > 0

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs-dpdk-ports.yml

@@ -0,0 +1,12 @@
+---
+- name: Get list of dpdk ports
+  become: true
+  shell: "ovs-appctl dpctl/show | grep 'dpdk: configured\\|vhostuser'"
+  register: ports
+
+- name: Get list of dpdk ports name
+  set_fact:
+    ovs_dpdk_ports: "{{ ovs_dpdk_ports | default([]) + [dpdk_port_line.split(': ')[1].split(' ')[0]] }}"
+  loop: "{{ ports.stdout.split('\n') }}"
+  loop_control:
+    loop_var: dpdk_port_line

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_rx_qos_drops.yml

@@ -0,0 +1,17 @@
+---
+- name: configured ingress policy for interface {{ interface }}
+  become: true
+  shell: ovs-vsctl list interface {{interface}} | grep -w "ingress_policing_rate"
+  register: ingress_policing_rate
+
+- name: Append port name to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}"
+
+- name: Append drops message to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ ['ovs_rx_qos_drops are increasing. Below is the related configuration '] }}"
+
+- name: Append port ingress policy configuration to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [ingress_policing_rate.stdout] }}"

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_failure_drops.yml

@@ -0,0 +1,25 @@
+---
+- name: Get port configurations
+  become: true
+  shell: "ovs-appctl dpctl/show | grep {{port}}"
+  register: stats
+
+- name: Get {{ port }} queues configuration from stats
+  set_fact:
+    stats: "{{stats.stdout.split(':')[2].split(',')}}"
+
+- name: Append port name to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}"
+
+- name: Append drops message to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_failure_drops are increasing. Below is the related configuration'] }}"
+
+- name: Loop all collected stats and check for configured tx and rx queues
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [stat] }}"
+  when: "'configured_rx_queues' in '{{stat}}' or 'configured_tx_queues' in '{{stat}}'"
+  loop: "{{ stats }}"
+  loop_control:
+    loop_var: stat

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_mtu_exceeded_drops.yml

@@ -0,0 +1,17 @@
+---
+- name: configured MTU for interface {{ interface }}
+  become: true
+  shell: ovs-vsctl list interface {{interface}} | grep -w "mtu"
+  register: mtu_conf
+
+- name: Append port name to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}"
+
+- name: Append drops message to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_mtu_exceeded_drops are increasing. Below is the related configuration '] }} "
+
+- name: Append port MTU configuration to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [mtu_conf.stdout] }}"

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_qos_drops.yml

@@ -0,0 +1,17 @@
+---
+- name: configured Qos/Egress policy for interface {{ interface }}
+  become: true
+  shell: ovs-appctl -t ovs-vswitchd qos/show {{interface}}
+  register: tx_qos_conf
+
+- name: Append port name to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}"
+
+- name: Append drops message to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_qos_drops are increasing. Below is the related configuration'] }}"
+
+- name: Append port TX QOS configuration to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [tx_qos_conf.stdout] }}"

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_retries.yml

@@ -0,0 +1,30 @@
+---
+- name: Get Interface options
+  shell: ovs-vsctl get interface '{{port}}' options
+  register: retries_conf
+
+- name: Split the retries_conf to search for retries configuration
+  set_fact:
+    data: "{{retries_conf.stdout.split(',')}}"
+
+- name: Mark the flag retries_found False
+  set_fact:
+    retries_found: false
+
+- name: Append port name to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}"
+
+- name: Check for max-retries configuration in options and append to validation_msg
+  set_fact:
+    validation_msg: "{{ validation_msg + ['ovs_tx_retries are increasing. Below is the related configuration'] + [data_item[1:]]}}"
+    retries_found: true
+  when: "'tx-retries-max' in '{{data_item}}'"
+  loop: "{{data}}"
+  loop_control:
+    loop_var: data_item
+
+- name: Append default max-retries to validation_msg when retries conf not found in options
+  set_fact:
+    validation_msg: "{{validation_msg + ['ovs_tx_retries are increasing. Related default configuration is max default_tx_retries'] + [default_tx_retries_max]}}"
+  when: " {{ retries_found }} == false"

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/read_counter_stats.yml

@@ -0,0 +1,21 @@
+---
+- name: Get interface statistics
+  become: true
+  shell: "ovs-vsctl get interface {{ port }} statistics"
+  register: if_stat
+
+- name: Parse if_stat
+  set_fact:
+    if_stat: "{{if_stat.stdout[0:-1].split(',')}}"
+
+- name: convert into dictionary
+  set_fact:
+    modified_if_stat: "{{ modified_if_stat | default({}) | combine({ stat[1:].split('=')[0] : stat[1:].split('=')[1]|int })}}"
+  loop: "{{ if_stat }}"
+  loop_control:
+    loop_var: stat
+
+- name: update dictionary stats
+  set_fact:
+    stats: "{{ (stats | default({})) | combine({ port : modified_if_stat})}}"
+  when: if_stat is defined

roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/validate.yml

@@ -0,0 +1,5 @@
+---
+- name: print if there are drops
+  include_tasks: '{{ item }}.yml'
+  when: "'{{item}}' in initial[port] and initial[port]['{{item}}'] < final[port]['{{item}}']"
+  loop: "{{ counters }}"