diff --git a/doc/source/roles/role-check_nfv_ovsdpdk_counter_stat_validation.rst b/doc/source/roles/role-check_nfv_ovsdpdk_counter_stat_validation.rst new file mode 100644 index 000000000..56a26cef3 --- /dev/null +++ b/doc/source/roles/role-check_nfv_ovsdpdk_counter_stat_validation.rst @@ -0,0 +1,45 @@ +========================================= +check_nfv_ovsdpdk_counter_stat_validation +========================================= + +-------------- +About the role +-------------- + +This role validates the NFV OvS DPDK interface counter statistics. + +Requirements +============ + +- Gathers information about all the existing interfaces. +- Validates ovs_tx_failure_drops, ovs_tx_mtu_exceeded_drops, ovs_rx_qos_drops, ovs_tx_qos_drops, ovs_tx_retries metrics for every interface. +- Checks if any of the listed counters are increasing and outputs its related configuration. + +Dependencies +============ + +Example Playbook +================ + +.. code-block:: yaml + + - hosts: servers + roles: + + - { role: check_nfv_ovsdpdk_counter_stat_validation } + +License +======= +Apache + +Author Information +================== + +**Red Hat TripleO DFG:NFV Integration** + +---------------- +Full Description +---------------- + +.. ansibleautoplugin:: + :role: roles/check_nfv_ovsdpdk_counter_stat_validation diff --git a/playbooks/nfv-ovsdpdk-counter-stat-validation.yaml b/playbooks/nfv-ovsdpdk-counter-stat-validation.yaml new file mode 100644 index 000000000..9d746649a --- /dev/null +++ b/playbooks/nfv-ovsdpdk-counter-stat-validation.yaml @@ -0,0 +1,18 @@ +--- +- hosts: "{{ compute_ovsdpdk_rolename | default('ComputeOvsDpdk') }}" + vars: + metadata: + name: NFV OvS DPDK Counter Stat Validations + description: | + Run `check-nfv-ovsdpdk-counter-stat-validation` on the compute ovsdpdk nodes + to find out the issues with NFV OvS Dpdk interface statistics. + groups: + - post-deployment + categories: + - networking + - compute + - nfv + products: + - tripleo + roles: + - check_nfv_ovsdpdk_counter_stat_validation diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/defaults/main.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/defaults/main.yml new file mode 100644 index 000000000..fd5de11f2 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# counters under validation +counters: ["ovs_tx_failure_drops", "ovs_tx_mtu_exceeded_drops", "ovs_rx_qos_drops", "ovs_tx_qos_drops", "ovs_tx_retries"] +# default value of maximum tx_retries +default_tx_retries_max: 8 diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/datacollection.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/datacollection.yml new file mode 100644 index 000000000..83d873904 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/datacollection.yml @@ -0,0 +1,37 @@ +--- +- name: Reset statistics + set_fact: + stats: {} + +- name: Get counter statistics + vars: + port: "{{ dpdk_port }}" + include_tasks: read_counter_stats.yml + loop: "{{ ovs_dpdk_ports }}" + loop_control: + loop_var: dpdk_port + +- name: First attempt counter statistics + set_fact: + initial: "{{ stats | default({}) }}" + +- name: Sleep for 15 seconds + wait_for: + timeout: 15 + delegate_to: localhost + +- name: Reset statistics + set_fact: + stats: {} + +- name: Get counter statistics + vars: + port: "{{ dpdk_port }}" + include_tasks: read_counter_stats.yml + loop: "{{ ovs_dpdk_ports }}" + loop_control: + loop_var: dpdk_port + +- name: Second attempt counter statistics + set_fact: + final: "{{ stats | default({}) }}" diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/main.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/main.yml new file mode 100644 index 000000000..d5d8c64f5 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Get list of dpdk ports + include_tasks: ovs-dpdk-ports.yml + +- name: Initialize validation message list + set_fact: + validation_msg: [] + +- name: Collect data + include_tasks: datacollection.yml + +- name: validate data of ovs_dpdk_ports + vars: + interface: "{{ port }}" + include_tasks: validate.yml + loop: "{{ ovs_dpdk_ports }}" + loop_control: + loop_var: port + +- name: Validation errors + fail: + msg: + - "Failed NFV counter stat validation rules:" + - "{{ validation_msg | join('\n') }}" + when: + - validation_msg is defined + - validation_msg | length > 0 diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs-dpdk-ports.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs-dpdk-ports.yml new file mode 100644 index 000000000..cdafc8684 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs-dpdk-ports.yml @@ -0,0 +1,12 @@ +--- +- name: Get list of dpdk ports + become: true + shell: "ovs-appctl dpctl/show | grep 'dpdk: configured\\|vhostuser'" + register: ports + +- name: Get list of dpdk ports name + set_fact: + ovs_dpdk_ports: "{{ ovs_dpdk_ports | default([]) + [dpdk_port_line.split(': ')[1].split(' ')[0]] }}" + loop: "{{ ports.stdout.split('\n') }}" + loop_control: + loop_var: dpdk_port_line diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_rx_qos_drops.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_rx_qos_drops.yml new file mode 100644 index 000000000..b51fe192f --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_rx_qos_drops.yml @@ -0,0 +1,17 @@ +--- +- name: configured ingress policy for interface {{ interface }} + become: true + shell: ovs-vsctl list interface {{interface}} | grep -w "ingress_policing_rate" + register: ingress_policing_rate + +- name: Append port name to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}" + +- name: Append drops message to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ ['ovs_rx_qos_drops are increasing. Below is the related configuration '] }}" + +- name: Append port ingress policy configuration to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [ingress_policing_rate.stdout] }}" diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_failure_drops.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_failure_drops.yml new file mode 100644 index 000000000..3a0fe0d96 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_failure_drops.yml @@ -0,0 +1,25 @@ +--- +- name: Get port configurations + become: true + shell: "ovs-appctl dpctl/show | grep {{port}}" + register: stats + +- name: Get {{ port }} queues configuration from stats + set_fact: + stats: "{{stats.stdout.split(':')[2].split(',')}}" + +- name: Append port name to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}" + +- name: Append drops message to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_failure_drops are increasing. Below is the related configuration'] }}" + +- name: Loop all collected stats and check for configured tx and rx queues + set_fact: + validation_msg: "{{ validation_msg }} + {{ [stat] }}" + when: "'configured_rx_queues' in '{{stat}}' or 'configured_tx_queues' in '{{stat}}'" + loop: "{{ stats }}" + loop_control: + loop_var: stat diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_mtu_exceeded_drops.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_mtu_exceeded_drops.yml new file mode 100644 index 000000000..389e8e744 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_mtu_exceeded_drops.yml @@ -0,0 +1,17 @@ +--- +- name: configured MTU for interface {{ interface }} + become: true + shell: ovs-vsctl list interface {{interface}} | grep -w "mtu" + register: mtu_conf + +- name: Append port name to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}" + +- name: Append drops message to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_mtu_exceeded_drops are increasing. Below is the related configuration '] }} " + +- name: Append port MTU configuration to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [mtu_conf.stdout] }}" diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_qos_drops.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_qos_drops.yml new file mode 100644 index 000000000..9ba26bd72 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_qos_drops.yml @@ -0,0 +1,17 @@ +--- +- name: configured Qos/Egress policy for interface {{ interface }} + become: true + shell: ovs-appctl -t ovs-vswitchd qos/show {{interface}} + register: tx_qos_conf + +- name: Append port name to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}" + +- name: Append drops message to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ ['ovs_tx_qos_drops are increasing. Below is the related configuration'] }}" + +- name: Append port TX QOS configuration to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [tx_qos_conf.stdout] }}" diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_retries.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_retries.yml new file mode 100644 index 000000000..be7d2b129 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/ovs_tx_retries.yml @@ -0,0 +1,30 @@ +--- +- name: Get Interface options + shell: ovs-vsctl get interface '{{port}}' options + register: retries_conf + +- name: Split the retries_conf to search for retries configuration + set_fact: + data: "{{retries_conf.stdout.split(',')}}" + +- name: Mark the flag retries_found False + set_fact: + retries_found: false + +- name: Append port name to validation_msg + set_fact: + validation_msg: "{{ validation_msg }} + {{ [port] }} + {{ ['\r\n'] }}" + +- name: Check for max-retries configuration in options and append to validation_msg + set_fact: + validation_msg: "{{ validation_msg + ['ovs_tx_retries are increasing. Below is the related configuration'] + [data_item[1:]]}}" + retries_found: true + when: "'tx-retries-max' in '{{data_item}}'" + loop: "{{data}}" + loop_control: + loop_var: data_item + +- name: Append default max-retries to validation_msg when retries conf not found in options + set_fact: + validation_msg: "{{validation_msg + ['ovs_tx_retries are increasing. Related default configuration is max default_tx_retries'] + [default_tx_retries_max]}}" + when: " {{ retries_found }} == false" diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/read_counter_stats.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/read_counter_stats.yml new file mode 100644 index 000000000..04508077b --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/read_counter_stats.yml @@ -0,0 +1,21 @@ +--- +- name: Get interface statistics + become: true + shell: "ovs-vsctl get interface {{ port }} statistics" + register: if_stat + +- name: Parse if_stat + set_fact: + if_stat: "{{if_stat.stdout[0:-1].split(',')}}" + +- name: convert into dictionary + set_fact: + modified_if_stat: "{{ modified_if_stat | default({}) | combine({ stat[1:].split('=')[0] : stat[1:].split('=')[1]|int })}}" + loop: "{{ if_stat }}" + loop_control: + loop_var: stat + +- name: update dictionary stats + set_fact: + stats: "{{ (stats | default({})) | combine({ port : modified_if_stat})}}" + when: if_stat is defined diff --git a/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/validate.yml b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/validate.yml new file mode 100644 index 000000000..7739b38d8 --- /dev/null +++ b/roles/check_nfv_ovsdpdk_counter_stat_validation/tasks/validate.yml @@ -0,0 +1,5 @@ +--- +- name: print if there are drops + include_tasks: '{{ item }}.yml' + when: "'{{item}}' in initial[port] and initial[port]['{{item}}'] < final[port]['{{item}}']" + loop: "{{ counters }}"