From b706d7c7c92eecd0aac5664288e43fb347eae782 Mon Sep 17 00:00:00 2001 From: Mathieu Bultel Date: Fri, 24 Jul 2020 13:03:46 +0200 Subject: [PATCH] Switch legacy Zuul jobs to native Zuul v3 jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://governance.openstack.org/tc/goals/selected/victoria/native-zuulv3-jobs.html for more details. Co-Authored-By: Cédric Jeanneret Change-Id: I94eadcecbae58ad322cc3028b315eab795711027 --- .zuul.yaml | 30 +++++ playbooks/post.yaml | 10 ++ playbooks/validations-common-functional.yaml | 14 +++ roles/fetch_validations/defaults/main.yaml | 4 + roles/fetch_validations/tasks/main.yaml | 25 ++++ roles/validations/defaults/main.yaml | 29 +++++ roles/validations/tasks/main.yaml | 30 +++++ roles/validations/tasks/validations.yaml | 26 +++++ zuul.d/base.yaml | 15 --- zuul.d/layout.yaml | 13 --- zuul.d/molecule.yaml | 115 ------------------- zuul.d/playbooks/pre.yml | 28 ----- zuul.d/playbooks/run-local.yml | 13 --- zuul.d/playbooks/run.yml | 20 ---- 14 files changed, 168 insertions(+), 204 deletions(-) create mode 100644 .zuul.yaml create mode 100644 playbooks/post.yaml create mode 100644 playbooks/validations-common-functional.yaml create mode 100644 roles/fetch_validations/defaults/main.yaml create mode 100644 roles/fetch_validations/tasks/main.yaml create mode 100644 roles/validations/defaults/main.yaml create mode 100644 roles/validations/tasks/main.yaml create mode 100644 roles/validations/tasks/validations.yaml delete mode 100644 zuul.d/base.yaml delete mode 100644 zuul.d/layout.yaml delete mode 100644 zuul.d/molecule.yaml delete mode 100644 zuul.d/playbooks/pre.yml delete mode 100644 zuul.d/playbooks/run-local.yml delete mode 100644 zuul.d/playbooks/run.yml diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..047ca7e --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,30 @@ +--- +- job: + name: validations-common-functional + parent: devstack + run: playbooks/validations-common-functional.yaml + post-run: playbooks/post.yaml + timeout: 7200 + required-projects: + - openstack/validations-libs + - openstack/validations-common + vars: + devstack_localrc: + USE_PYTHON3: true + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ + +- project: + templates: + - check-requirements + - openstack-lower-constraints-jobs + - openstack-python3-victoria-jobs + check: + jobs: + - validations-common-functional + gate: + jobs: + - openstack-tox-linters + - validations-common-functional diff --git a/playbooks/post.yaml b/playbooks/post.yaml new file mode 100644 index 0000000..397c7c4 --- /dev/null +++ b/playbooks/post.yaml @@ -0,0 +1,10 @@ +--- +- hosts: all + vars: + tox_envlist: functional + roles: + - fetch-subunit-output + - fetch-devstack-log-dir + - fetch-output + - fetch_validations + - fetch-python-sdist-output diff --git a/playbooks/validations-common-functional.yaml b/playbooks/validations-common-functional.yaml new file mode 100644 index 0000000..3ad124a --- /dev/null +++ b/playbooks/validations-common-functional.yaml @@ -0,0 +1,14 @@ +--- +- hosts: tempest + name: validations-common-functional + roles: + - ensure-tox + - ensure-pip + - ensure-virtualenv + - role: ensure-if-python + vars: + zuul_work_dir: "src/opendev.org/openstack/validations-libs" + - role: ensure-if-python + vars: + zuul_work_dir: "src/opendev.org/openstack/validations-common" + - role: validations diff --git a/roles/fetch_validations/defaults/main.yaml b/roles/fetch_validations/defaults/main.yaml new file mode 100644 index 0000000..9ae8482 --- /dev/null +++ b/roles/fetch_validations/defaults/main.yaml @@ -0,0 +1,4 @@ +--- +src_dir: "{{ zuul.project.src_dir }}" +output_dir: "{{ ansible_user_dir }}" +log_path: "{{ zuul.executor.log_root }}" diff --git a/roles/fetch_validations/tasks/main.yaml b/roles/fetch_validations/tasks/main.yaml new file mode 100644 index 0000000..5b82851 --- /dev/null +++ b/roles/fetch_validations/tasks/main.yaml @@ -0,0 +1,25 @@ +--- +- name: Ensure local output dirs + delegate_to: localhost + file: + path: "{{ vf_output_dir }}" + state: directory + with_items: + - "{{ log_path }}/validations-logs" + - "{{ zuul.executor.work_root }}/artifacts" + loop_control: + loop_var: vf_output_dir + +- name: Collect logs and artifacts + synchronize: + dest: "{{ vf_output.dest }}/" + mode: pull + src: "{{ output_dir }}/{{ vf_output.src }}/" + verify_host: true + owner: false + group: false + loop: + - dest: "{{ log_path }}/validations-logs" + src: "logs" + loop_control: + loop_var: vf_output diff --git a/roles/validations/defaults/main.yaml b/roles/validations/defaults/main.yaml new file mode 100644 index 0000000..becc2e7 --- /dev/null +++ b/roles/validations/defaults/main.yaml @@ -0,0 +1,29 @@ +--- +log_dir: "{{ ansible_user_dir }}/logs" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +ansible_dir: "{{ zuul_work_virtualenv }}/share/validations-common" +validation_dir: "{{ zuul_work_virtualenv }}/share/validations-common/playbooks" +vf_log_dir: "/var/log/validations" +val_exec: "source {{ zuul_work_virtualenv }}/bin/activate; validation.py" +command: + - output: "{{ log_dir }}/run.log" + command: >- + {{ val_exec }} run --validation check-ftype,512e + --validation-dir {{ validation_dir }} + --ansible-base-dir {{ ansible_dir }} + --output-log {{ log_dir }}/run.log + - output: "{{ log_dir }}/run-group.log" + command: >- + {{ val_exec }} run --group prep --validation-dir {{ validation_dir }} + --ansible-base-dir {{ ansible_dir }} + --output-log {{ log_dir }}/run-group.log + - output: "{{ log_dir }}/list.log" + command: >- + {{ val_exec }} list --validation-dir {{ validation_dir }} + --ansible-base-dir {{ ansible_dir }} + --output-log {{ log_dir }}/list.log + - output: "{{ log_dir }}/show.log" + command: >- + {{ val_exec }} show --validation-dir {{ validation_dir }} + --ansible-base-dir {{ ansible_dir }} + --output-log {{ log_dir }}/show.log diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml new file mode 100644 index 0000000..f46d096 --- /dev/null +++ b/roles/validations/tasks/main.yaml @@ -0,0 +1,30 @@ +--- +- name: Make sure Validations Log dir exists + become: true + file: + path: "{{ vf_log_dir }}" + state: directory + mode: "0755" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + recurse: true + +# @matbu: TODO make the /var/log/validations as a parameter +# We should be able to write validation logs outside of the /var/log path +# This statement will have to be remove then. +- name: Create Validations log symlink + file: + src: "{{ vf_log_dir }}" + dest: "{{ log_dir }}" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + state: link + +# @matbu: TODO make ansible-runner depends on Ansible +- name: Install Ansible + pip: + name: ansible + virtualenv: "{{ zuul_work_virtualenv }}" + +- include: validations.yaml validation_command="{{ item.command }}" val_output="{{ item.output }}" + loop: "{{command}}" diff --git a/roles/validations/tasks/validations.yaml b/roles/validations/tasks/validations.yaml new file mode 100644 index 0000000..8484cf1 --- /dev/null +++ b/roles/validations/tasks/validations.yaml @@ -0,0 +1,26 @@ +--- +- name: Execute validation commands + shell: + cmd: "{{ validation_command }}" + executable: /bin/bash + +- name: Get run results + register: result + shell: + cmd: "cat {{ val_output }}" + executable: /bin/bash + +- name: Get json data + set_fact: + jsondata: "{{ result.stdout | from_json }}" + +- name: Get Validations Status + set_fact: + status: "{{ jsondata | json_query(jsonres) }}" + vars: + jsonres: 'results[*].Status' + +- fail: + msg: "Validation failed: some of the validations has failed." + when: item != "PASSED" + loop: "{{ status }}" diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml deleted file mode 100644 index 478f97e..0000000 --- a/zuul.d/base.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- job: - description: Base validations-common job - name: validations-common-centos-8-base - nodeset: centos-8 - parent: base - success-url: "reports.html" - failure-url: "reports.html" - pre-run: - - tests/prepare-test-host.yml - - zuul.d/playbooks/pre.yml - run: - - zuul.d/playbooks/run.yml - timeout: 1800 - voting: true diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml deleted file mode 100644 index 57be71c..0000000 --- a/zuul.d/layout.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- project: - templates: - - check-requirements - - openstack-python3-victoria-jobs - - openstack-lower-constraints-jobs - - release-notes-jobs-python3 - - validations-common-molecule-jobs - check: - jobs: - - openstack-tox-linters - gate: - jobs: - - openstack-tox-linters diff --git a/zuul.d/molecule.yaml b/zuul.d/molecule.yaml deleted file mode 100644 index 372f331..0000000 --- a/zuul.d/molecule.yaml +++ /dev/null @@ -1,115 +0,0 @@ ---- -- project-template: - check: - jobs: - - validations-common-centos-8-molecule-advanced_format_512e_support - - validations-common-centos-8-molecule-check_latest_packages_version - - validations-common-centos-8-molecule-dns - - validations-common-centos-8-molecule-haproxy - - validations-common-centos-8-molecule-ntp - - validations-common-centos-8-molecule-service_status - - validations-common-centos-8-molecule-check_cpu - - validations-common-centos-8-molecule-check_disk_space - - validations-common-centos-8-molecule-check_ram - - validations-common-centos-8-molecule-check_selinux_mode - - validations-common-centos-8-molecule-validate_selinux - - validations-common-centos-8-molecule-xfs_check_ftype - gate: - jobs: - - validations-common-centos-8-molecule-check_latest_packages_version - - validations-common-centos-8-molecule-dns - - validations-common-centos-8-molecule-haproxy - - validations-common-centos-8-molecule-check_cpu - - validations-common-centos-8-molecule-check_disk_space - - validations-common-centos-8-molecule-check_ram - - validations-common-centos-8-molecule-validate_selinux - name: validations-common-molecule-jobs -- job: - files: - - ^validations_common/roles/check_selinux_mode/.* - name: validations-common-centos-8-molecule-check_selinux_mode - parent: validations-common-centos-8-base - vars: - validations_common_role_name: check_selinux_mode - voting: false -- job: - files: - - ^validations_common/roles/ntp/.* - name: validations-common-centos-8-molecule-ntp - parent: validations-common-centos-8-base - vars: - validations_common_role_name: ntp - voting: false -- job: - files: - - ^validations_common/roles/service_status/.* - name: validations-common-centos-8-molecule-service_status - parent: validations-common-centos-8-base - vars: - validations_common_role_name: service_status - voting: false -- job: - files: - - ^validations_common/roles/advanced_format_512e_support/.* - name: validations-common-centos-8-molecule-advanced_format_512e_support - parent: validations-common-centos-8-base - vars: - validations_common_role_name: advanced_format_512e_support - voting: false -- job: - files: - - ^validations_common/roles/dns/.* - name: validations-common-centos-8-molecule-dns - parent: validations-common-centos-8-base - vars: - validations_common_role_name: dns -- job: - files: - - ^validations_common/roles/haproxy/.* - name: validations-common-centos-8-molecule-haproxy - parent: validations-common-centos-8-base - vars: - validations_common_role_name: haproxy -- job: - files: - - ^validations_common/roles/check_cpu/.* - name: validations-common-centos-8-molecule-check_cpu - parent: validations-common-centos-8-base - vars: - validations_common_role_name: check_cpu -- job: - files: - - ^validations_common/roles/check_ram/.* - name: validations-common-centos-8-molecule-check_ram - parent: validations-common-centos-8-base - vars: - validations_common_role_name: check_ram -- job: - files: - - ^validations_common/roles/check_disk_space/.* - name: validations-common-centos-8-molecule-check_disk_space - parent: validations-common-centos-8-base - vars: - validations_common_role_name: check_disk_space -- job: - files: - - ^validations_common/roles/xfs_check_ftype/.* - name: validations-common-centos-8-molecule-xfs_check_ftype - parent: validations-common-centos-8-base - vars: - validations_common_role_name: xfs_check_ftype - voting: false -- job: - files: - - ^validations_common/roles/check_latest_packages_version/.* - name: validations-common-centos-8-molecule-check_latest_packages_version - parent: validations-common-centos-8-base - vars: - validations_common_role_name: check_latest_packages_version -- job: - files: - - ^validations_common/roles/validate_selinux/.* - name: validations-common-centos-8-molecule-validate_selinux - parent: validations-common-centos-8-base - vars: - validations_common_role_name: validate_selinux diff --git a/zuul.d/playbooks/pre.yml b/zuul.d/playbooks/pre.yml deleted file mode 100644 index 01a97e4..0000000 --- a/zuul.d/playbooks/pre.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- hosts: all - pre_tasks: - - name: Ensure output dirs - file: - path: "{{ ansible_user_dir }}/zuul-output/logs" - state: directory - - - name: Setup bindep - pip: - name: "bindep" - virtualenv: "{{ ansible_user_dir }}/test-python" - virtualenv_site_packages: true - - - name: Run bindep - shell: |- - . {{ ansible_user_dir }}/test-python/bin/activate - {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/scripts/bindep-install - become: true - changed_when: false - - - name: Setup test-python - pip: - requirements: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/molecule-requirements.txt" - virtualenv: "{{ ansible_user_dir }}/test-python" - virtualenv_site_packages: true - roles: - - role: ensure-docker diff --git a/zuul.d/playbooks/run-local.yml b/zuul.d/playbooks/run-local.yml deleted file mode 100644 index cd0f83e..0000000 --- a/zuul.d/playbooks/run-local.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- hosts: all - tasks: - - name: set basic zuul fact - set_fact: - zuul: - project: - src_dir: "{{ tripleo_src }}" - ansible_connection: ssh - -- import_playbook: pre.yml - -- import_playbook: run.yml diff --git a/zuul.d/playbooks/run.yml b/zuul.d/playbooks/run.yml deleted file mode 100644 index c9f2c81..0000000 --- a/zuul.d/playbooks/run.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- - -- hosts: all - environment: - ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/zuul-output/logs/ansible-execution.log" - tasks: - - name: Run role test job - shell: |- - . {{ ansible_user_dir }}/test-python/bin/activate - . {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/ansible-test-env.rc - pytest --color=no \ - --html={{ ansible_user_dir }}/zuul-output/logs/reports.html \ - --self-contained-html \ - --ansible-args='{{ tripleo_job_ansible_args | default("") }}' \ - {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/test_molecule.py - args: - chdir: - "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/validations_common/roles/{{ validations_common_role_name }}" - executable: /bin/bash - changed_when: false