Coverage change job

Zuul check for changes in unit test coverage.
With this running we can ensure that no more code is
merged without tests.

At this stage, I'm proposing is a non-voting job.
After a grace period, for example one sprint or two, it would become voting.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: If2e61c9bbf59e91163357023d16620934dc7d3a0
This commit is contained in:
Jiri Podivin 2021-06-17 15:56:40 +02:00
parent 4d1df7b0e7
commit 3eff44eac8
2 changed files with 99 additions and 0 deletions

View File

@ -27,6 +27,18 @@
files:
- ^requirements.txt$
- job:
name: validations-libs-coverchange
nodeset: centos-8
parent: base
run: playbooks/coverchange.yaml
timeout: 1600
voting: false
required-projects:
- openstack/validations-libs
files:
- ^validations_libs/.*
- job:
name: tripleo-ci-centos-8-standalone-validation-libs
parent: tripleo-ci-centos-8-standalone
@ -42,6 +54,7 @@
check:
jobs:
- validations-libs-reqcheck
- validations-libs-coverchange
- openstack-tox-linters
- openstack-tox-cover
- openstack-tox-py36

View File

@ -0,0 +1,86 @@
---
- hosts: all
name: Coverage change
roles:
- ensure-virtualenv
- ensure-pip
- role: ensure-tox
vars:
ensure_global_symlinks: true
tasks:
- name: Environment setup
shell:
cmd: |
set -e
virtualenv --system-site-packages {{ ansible_user_dir }}/.venv
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
- name: Measure coverage with the submitted change
shell:
cmd: |
set -e
source {{ ansible_user_dir }}/.venv/bin/activate
tox -e cover | awk '/^[[:alpha:]].*\.py/{if ($3 > 0){ print $1 ":" 1-$3/$2; }}'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: coverage_after
- name: Transform to dict
set_fact:
coverage_after_dict: "{{ coverage_after_dict | default({}) | combine({ item.split(':')[0]: item.split(':')[1] }) }}"
loop: "{{ coverage_after.stdout_lines }}"
- name: Record total after change
shell:
cmd: |
set -e
source {{ ansible_user_dir }}/.venv/bin/activate
tox -e cover | awk '/TOTAL/{ print 1-$3/$2 }'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: coverage_after_total
- name: Checkout previous commit
shell:
cmd: |
set -e
source {{ ansible_user_dir }}/.venv/bin/activate
git checkout HEAD^
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
- name: Measure coverage before submitted change
shell:
cmd: |
set -e
source {{ ansible_user_dir }}/.venv/bin/activate
tox -e cover | awk '/^[[:alpha:]].*\.py/{if ($3 > 0){ print $1 ":" 1-$3/$2; }}'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: coverage_before
- name: Transform to dict
set_fact:
coverage_before_dict: "{{ coverage_before_dict | default({}) | combine({ item.split(':')[0]: item.split(':')[1] }) }}"
loop: "{{ coverage_before.stdout_lines }}"
- name: Record total before change
shell:
cmd: |
set -e
source {{ ansible_user_dir }}/.venv/bin/activate
tox -e cover | awk '/TOTAL/{ print 1-$3/$2 }'
args:
chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
register: coverage_before_total
- name: Coverage comparison per file - Check
assert:
that: "{{coverage_before_dict[item] | float}} <= {{ coverage_after_dict[item] | float}}"
fail_msg: |
Before the change {{ (coverage_before_dict[item] | float)*100 }}% of the lines were covered.
Now it's {{ (coverage_after_dict[item] | float )*100 }}%. Did you write your unit tests?
success_msg: |
File by file code coverage check successful.
loop: "{{ coverage_after_dict.keys() | intersect(coverage_before_dict.keys()) | sort }}"
- name: Coverage comparison Total - Check
assert:
that: "{{ (coverage_after_total.stdout | float) }} >= {{ (coverage_before_total.stdout | float) }}"
fail_msg: |
Before the change {{ (coverage_before.stdout | float)*100 }}% of the lines were covered.
Now it's {{ (coverage_after_total.stdout | float )*100 }}%. Did you write your unit tests?
success_msg: |
Code coverage check successful, {{ (coverage_before_total.stdout | float) * 100 }}% of code is now covered.