From f686e205f5225e52454aebdaa7b9d9b58285ae92 Mon Sep 17 00:00:00 2001 From: Mathieu Bultel Date: Wed, 15 Jul 2020 15:45:16 +0200 Subject: [PATCH] Switch legacy Zuul jobs to native Zuul v3 jobs See https://governance.openstack.org/tc/goals/selected/victoria/native-zuulv3-jobs.html for more details. Change-Id: I7b9251caecc0ef5f4c07814ef17c7a025631ab85 --- .zuul.yaml | 16 ++-- .../validations-dsvm-functional/post.yaml | 96 ------------------- .../validations-dsvm-functional/run.yaml | 37 ------- .../validations-dsvm-functional/setup.yaml | 60 ------------ playbooks/post.yaml | 9 ++ playbooks/validations-libs-functional.yaml | 14 +++ roles/fetch-validations/defaults/main.yaml | 3 + roles/fetch-validations/tasks/main.yaml | 24 +++++ roles/validations/defaults/main.yaml | 23 +++++ roles/validations/tasks/main.yaml | 29 ++++++ .../validations/tasks}/validations.yaml | 6 +- 11 files changed, 115 insertions(+), 202 deletions(-) delete mode 100644 playbooks/legacy/validations-dsvm-functional/post.yaml delete mode 100644 playbooks/legacy/validations-dsvm-functional/run.yaml delete mode 100644 playbooks/legacy/validations-dsvm-functional/setup.yaml create mode 100644 playbooks/post.yaml create mode 100644 playbooks/validations-libs-functional.yaml create mode 100644 roles/fetch-validations/defaults/main.yaml create mode 100644 roles/fetch-validations/tasks/main.yaml create mode 100644 roles/validations/defaults/main.yaml create mode 100644 roles/validations/tasks/main.yaml rename {playbooks/legacy/validations-dsvm-functional => roles/validations/tasks}/validations.yaml (81%) diff --git a/.zuul.yaml b/.zuul.yaml index 08b0bf3d..5bf298f6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,13 +1,15 @@ - job: - name: validations-dsvm-functional - parent: legacy-dsvm-base - run: playbooks/legacy/validations-dsvm-functional/run.yaml - post-run: playbooks/legacy/validations-dsvm-functional/post.yaml + name: validations-libs-functional + parent: devstack + run: playbooks/validations-libs-functional.yaml + post-run: playbooks/post.yaml timeout: 7200 required-projects: - - openstack/devstack-gate - openstack/validations-libs - openstack/validations-common + vars: + devstack_localrc: + USE_PYTHON3: true irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -20,8 +22,8 @@ - openstack-python3-victoria-jobs check: jobs: - - validations-dsvm-functional + - validations-libs-functional gate: jobs: - openstack-tox-linters - - validations-dsvm-functional + - validations-libs-functional diff --git a/playbooks/legacy/validations-dsvm-functional/post.yaml b/playbooks/legacy/validations-dsvm-functional/post.yaml deleted file mode 100644 index 26093f55..00000000 --- a/playbooks/legacy/validations-dsvm-functional/post.yaml +++ /dev/null @@ -1,96 +0,0 @@ -- hosts: primary - tasks: - - name: Set fact for validation src directory - set_fact: - src_dir: '/home/zuul/src/opendev.org/openstack' - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ src_dir }} on node - synchronize: - src: '{{ src_dir }}' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*.log - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/validations-dsvm-functional/run.yaml b/playbooks/legacy/validations-dsvm-functional/run.yaml deleted file mode 100644 index 97e0c71b..00000000 --- a/playbooks/legacy/validations-dsvm-functional/run.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- hosts: primary - name: validations-dsvm-functional - tasks: - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - name: Set fact for validation src directory - set_fact: - src_dir: '/home/zuul/src/opendev.org/openstack' - - - name: Set fact for validation executable - set_fact: - val_exec: "python {{ src_dir }}/validations-common/validations_common/validation.py" - - - include: setup.yaml src_dir="{{ src_dir }}" log_dir="/var/log/validations" - - - include: validations.yaml validation_command="{{ item.command }}" val_output="{{ item.output }}" src_dir="{{ src_dir }}" - with_items: - - { output: "{{ src_dir }}/run.log", command: "{{ val_exec }} run --validation check-ftype,512e \ - --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \ - --ansible-base-dir {{ src_dir }}/validations-common/validations_common \ - --output-log {{ src_dir }}/run.log" } - - { output: "{{ src_dir }}/run-group.log", command: "{{ val_exec }} run --group prep \ - --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \ - --ansible-base-dir {{ src_dir }}/validations-common/validations_common \ - --output-log {{ src_dir }}/run-group.log" } - - { output: "{{ src_dir }}/list.log", command: "{{ val_exec }} list \ - --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \ - --ansible-base-dir {{ src_dir }}/validations-common/validations_common \ - --output-log {{ src_dir }}/list.log" } - - { output: "{{ src_dir }}/show.log", command: "{{ val_exec }} show \ - --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \ - --ansible-base-dir {{ src_dir }}/validations-common/validations_common \ - --output-log {{ src_dir }}/show.log" } diff --git a/playbooks/legacy/validations-dsvm-functional/setup.yaml b/playbooks/legacy/validations-dsvm-functional/setup.yaml deleted file mode 100644 index 5c2533ba..00000000 --- a/playbooks/legacy/validations-dsvm-functional/setup.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- name: Instal python pip - ignore_errors: yes - become: yes - package: - name: - - python3-pip - - python-pip - state: present - -- name: Install validations deps - ignore_errors: yes - shell: - cmd: | - sudo pip3 install prettytable - sudo pip3 install setuptools - sudo pip3 install ansible - -- name: Install validations deps - shell: - cmd: | - sudo pip install prettytable - sudo pip install setuptools - sudo pip3 install ansible - -- name: Install validations-libs - ignore_errors: yes - shell: - cmd: | - sudo pip install -r requirements.txt - sudo python setup.py install - executable: /bin/bash - chdir: '{{ src_dir }}/validations-libs' - -- name: Install validations-common - ignore_errors: yes - shell: - cmd: | - sudo pip install -r requirements.txt - sudo python setup.py install - executable: /bin/bash - chdir: '{{ src_dir }}/validations-common' - -- name: Make sure Validations Log dir exists - become: yes - file: - path: '{{ log_dir }}' - state: directory - mode: '0755' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' - recurse: yes - -- name: Make validation script executable - register: output - shell: - cmd: | - chmod +x validations_common/validation.py - executable: /bin/bash - chdir: '{{ src_dir }}/validations-common' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/post.yaml b/playbooks/post.yaml new file mode 100644 index 00000000..debe8842 --- /dev/null +++ b/playbooks/post.yaml @@ -0,0 +1,9 @@ +- hosts: all + vars: + tox_envlist: functional + roles: + - fetch-subunit-output + - fetch-devstack-log-dir + - fetch-output + - fetch-validations + - fetch-python-sdist-output diff --git a/playbooks/validations-libs-functional.yaml b/playbooks/validations-libs-functional.yaml new file mode 100644 index 00000000..08dc26df --- /dev/null +++ b/playbooks/validations-libs-functional.yaml @@ -0,0 +1,14 @@ +--- +- hosts: tempest + name: validations-libs-functional + roles: + - ensure-tox + - ensure-pip + - ensure-virtualenv + - role: ensure-if-python + vars: + zuul_work_dir: "src/opendev.org/openstack/validations-libs" + - role: ensure-if-python + vars: + zuul_work_dir: "src/opendev.org/openstack/validations-common" + - role: validations diff --git a/roles/fetch-validations/defaults/main.yaml b/roles/fetch-validations/defaults/main.yaml new file mode 100644 index 00000000..484f2c31 --- /dev/null +++ b/roles/fetch-validations/defaults/main.yaml @@ -0,0 +1,3 @@ +src_dir: "{{ zuul.project.src_dir }}" +output_dir: "{{ ansible_user_dir }}" +log_path: "{{ zuul.executor.log_root }}" diff --git a/roles/fetch-validations/tasks/main.yaml b/roles/fetch-validations/tasks/main.yaml new file mode 100644 index 00000000..6565517f --- /dev/null +++ b/roles/fetch-validations/tasks/main.yaml @@ -0,0 +1,24 @@ +- name: Ensure local output dirs + delegate_to: localhost + file: + path: "{{ vf_output_dir }}" + state: directory + with_items: + - "{{ log_path }}/validations-logs" + - "{{ zuul.executor.work_root }}/artifacts" + loop_control: + loop_var: vf_output_dir + +- name: Collect logs and artifacts + synchronize: + dest: "{{ vf_output.dest }}/" + mode: pull + src: "{{ output_dir }}/{{ vf_output.src }}/" + verify_host: true + owner: false + group: false + loop: + - dest: "{{ log_path }}/validations-logs" + src: "logs" + loop_control: + loop_var: vf_output diff --git a/roles/validations/defaults/main.yaml b/roles/validations/defaults/main.yaml new file mode 100644 index 00000000..d617ba74 --- /dev/null +++ b/roles/validations/defaults/main.yaml @@ -0,0 +1,23 @@ +log_dir: "{{ ansible_user_dir }}/logs" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +ansible_dir: "{{ zuul_work_virtualenv }}/share/validations-common" +validation_dir: "{{ zuul_work_virtualenv }}/share/validations-common/playbooks" +vf_log_dir: "/var/log/validations" +val_exec: "source {{ zuul_work_virtualenv }}/bin/activate; validation.py" +command: + - { output: "{{ log_dir }}/run.log", command: "{{ val_exec }} run --validation check-ftype,512e \ + --validation-dir {{ validation_dir }} \ + --ansible-base-dir {{ ansible_dir }} \ + --output-log {{ log_dir }}/run.log" } + - { output: "{{ log_dir }}/run-group.log", command: "{{ val_exec }} run --group prep \ + --validation-dir {{ validation_dir }} \ + --ansible-base-dir {{ ansible_dir }} \ + --output-log {{ log_dir }}/run-group.log" } + - { output: "{{ log_dir }}/list.log", command: "{{ val_exec }} list \ + --validation-dir {{ validation_dir }} \ + --ansible-base-dir {{ ansible_dir }} \ + --output-log {{ log_dir }}/list.log" } + - { output: "{{ log_dir }}/show.log", command: "{{ val_exec }} show \ + --validation-dir {{ validation_dir }} \ + --ansible-base-dir {{ ansible_dir }} \ + --output-log {{ log_dir }}/show.log" } diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml new file mode 100644 index 00000000..c8c53edf --- /dev/null +++ b/roles/validations/tasks/main.yaml @@ -0,0 +1,29 @@ +- name: Make sure Validations Log dir exists + become: yes + file: + path: "{{ vf_log_dir }}" + state: directory + mode: "0755" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + recurse: yes + +# @matbu: TODO make the /var/log/validations as a parameter +# We should be able to write validation logs outside of the /var/log path +# This statement will have to be remove then. +- name: Create Validations log symlink + file: + src: "{{ vf_log_dir }}" + dest: "{{ log_dir }}" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + state: link + +# @matbu: TODO make ansible-runner depends on Ansible +- name: Install Ansible + pip: + name: ansible + virtualenv: "{{ zuul_work_virtualenv }}" + +- include: validations.yaml validation_command="{{ item.command }}" val_output="{{ item.output }}" + loop: "{{command}}" diff --git a/playbooks/legacy/validations-dsvm-functional/validations.yaml b/roles/validations/tasks/validations.yaml similarity index 81% rename from playbooks/legacy/validations-dsvm-functional/validations.yaml rename to roles/validations/tasks/validations.yaml index 21760078..b1b682b0 100644 --- a/playbooks/legacy/validations-dsvm-functional/validations.yaml +++ b/roles/validations/tasks/validations.yaml @@ -1,10 +1,12 @@ - name: Execute validation commands - shell: "{{ validation_command }}" + shell: + cmd: "{{ validation_command }}" + executable: /bin/bash - name: Get run results register: result shell: - cmd: "cat {{val_output}}" + cmd: "cat {{ val_output }}" executable: /bin/bash - name: Get json data