diff --git a/bindep.txt b/bindep.txt index bb8f9809d..eaeea6369 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,28 +1,38 @@ # This file contains runtime (non-python) dependencies # More info at: https://docs.openstack.org/infra/bindep/readme.html -# All OSs -gcc [] -python3 [] - # Red Hat +gcc [platform:redhat] +make [platform:redhat] +python3 [platform:redhat] +bzip2-devel [platform:redhat] git [platform:redhat] iproute [platform:redhat] libffi-devel [platform:redhat] openssl-devel [platform:redhat] python3-devel [platform:redhat] +python3-setuptools [platform:redhat] +readline-devel [platform:redhat] +sqlite-devel [platform:redhat] +zlib-devel [platform:redhat] # RHEL/CentOS 7 libselinux-python [platform:rhel-7] libselinux-python [platform:centos-7] -python2-devel [platform:rhel-7] -python2-devel [platform:centos-7] +python-devel [platform:rhel-7] +python-devel [platform:centos-7] +python-virtualenv [platform:centos-7] # RHEL/CentOS 8 python3-libselinux [platform:rhel-8] python3-libselinux [platform:centos-8] +python3-virtualenv [platform:rhel-8] +python3-virtualenv [platform:centos-8] # Ubuntu +gcc [platform:ubuntu] +make [platform:ubuntu] +python3 [platform:ubuntu] git [platform:ubuntu] libffi-dev [platform:ubuntu] libssl-dev [platform:ubuntu] diff --git a/roles/plugin.spec b/roles/plugin.spec index a04c27a9d..6b8ee4a35 100644 --- a/roles/plugin.spec +++ b/roles/plugin.spec @@ -89,8 +89,12 @@ subparsers: help: enable/disable verbose log entries in tests results log file ansible_variable: test_log_debug - - title: Run tox stage + - title: Run stage options: + workflow: + type: Value + help: name of workflow to execute + ansible_variable: test_workflow tox-dir: type: Value help: directory from where run tox (typically test_dir) diff --git a/roles/tests/Vagrantfile b/roles/tests/Vagrantfile index 2544ca0ae..609dd8dc7 100644 --- a/roles/tests/Vagrantfile +++ b/roles/tests/Vagrantfile @@ -8,11 +8,11 @@ VAGRANTFILE_API_VERSION = "2" CPUS = 4 # Customize the amount of memory on the VM -MEMORY = 512 +MEMORY = ENV.fetch("VM_SIZE", "512").to_i # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. -BOX = "generic/centos8" +BOX = ENV.fetch("VM_BOX", "generic/centos8") TOX_INI_DIR = '../..' diff --git a/roles/tobiko-bindep/defaults/main.yaml b/roles/tobiko-bindep/defaults/main.yaml new file mode 100644 index 000000000..00b690696 --- /dev/null +++ b/roles/tobiko-bindep/defaults/main.yaml @@ -0,0 +1,6 @@ +--- + +bindep_command: bindep +bindep_min_version: 2.8 +bindep_file: bindep.txt +bindep_profile: test diff --git a/roles/tobiko-ensure-bindep/meta/main.yaml b/roles/tobiko-bindep/meta/main.yaml similarity index 68% rename from roles/tobiko-ensure-bindep/meta/main.yaml rename to roles/tobiko-bindep/meta/main.yaml index 75b81b773..97b06fff8 100644 --- a/roles/tobiko-ensure-bindep/meta/main.yaml +++ b/roles/tobiko-bindep/meta/main.yaml @@ -1,5 +1,4 @@ --- dependencies: - - role: tobiko-common - role: tobiko-ensure-python3 diff --git a/roles/tobiko-ensure-bindep/tasks/bindep.yaml b/roles/tobiko-bindep/tasks/ensure-bindep.yaml similarity index 60% rename from roles/tobiko-ensure-bindep/tasks/bindep.yaml rename to roles/tobiko-bindep/tasks/ensure-bindep.yaml index 557503263..bd96fb713 100644 --- a/roles/tobiko-ensure-bindep/tasks/bindep.yaml +++ b/roles/tobiko-bindep/tasks/ensure-bindep.yaml @@ -8,29 +8,31 @@ - block: - - name: "get '{{ bindep_executable }}' full path" - command: > - which '{{ bindep_executable }}' + - name: "get '{{ bindep_command }}' full path" + shell: | + export PATH={{ python_exe_path }} + which '{{ bindep_command }}' register: get_bindep_path - - name: "update bindep_executable fact" + - name: "set bindep_executable fact" set_fact: bindep_executable: '{{ get_bindep_path.stdout_lines | first }}' rescue: - - name: "update bindep_executable fact" + - name: "set bindep_executable fact" set_fact: bindep_executable: '{{ ansible_user_dir }}/.local/bin/bindep' -- name: "get installed Bindep version" - command: "'{{ bindep_executable }}' --version" +- name: "get '{{ bindep_executable }}' version" + shell: "'{{ bindep_executable }}' --version 2>&1" register: get_bindep_version + failed_when: '(get_bindep_version.stdout_lines | length) == 0' - name: "update bindep_version fact" set_fact: - bindep_version: '{{ get_bindep_version.stdout }}' + bindep_version: '{{ get_bindep_version.stdout_lines | first }}' - name: "show Bindep facts" diff --git a/roles/tobiko-bindep/tasks/main.yaml b/roles/tobiko-bindep/tasks/main.yaml new file mode 100644 index 000000000..3e057f312 --- /dev/null +++ b/roles/tobiko-bindep/tasks/main.yaml @@ -0,0 +1,36 @@ +--- + +- name: "ensure '{{ bindep_command }}' command is available" + include_tasks: ensure-bindep.yaml + when: + - bindep_executable is not defined + + +- name: "check '{{ bindep_file }}' file exists" + stat: + path: '{{ bindep_file | realpath }}' + register: check_bindep_file + + +- block: + - name: "run '{{ bindep_command }}' with profile '{{ bindep_profile }}'" + command: > + '{{ bindep_executable }}' -b -f '{{ bindep_file }}' {{ bindep_profile }} + when: + - check_bindep_file.stat.readable + register: run_bindep + changed_when: no + + rescue: + - debug: var=run_bindep + when: '(run_bindep.stdout_lines | length) == 0' + failed_when: yes + + - name: 'install missing packages' + become: yes + package: + name: '{{ run_bindep.stdout_lines }}' + register: install_packages + + - name: 'show installed packages' + debug: var=install_packages diff --git a/roles/tobiko-collect/tasks/collect.yaml b/roles/tobiko-collect/tasks/collect.yaml index 9ab482678..d0c8a5b3e 100644 --- a/roles/tobiko-collect/tasks/collect.yaml +++ b/roles/tobiko-collect/tasks/collect.yaml @@ -1,6 +1,6 @@ --- -- name: "ensures local directory '{{ collect_dir }}' exists" +- name: "ensure local directory '{{ collect_dir }}' exists" file: path: '{{ collect_dir }}' state: directory diff --git a/roles/tobiko-deploy/meta/main.yaml b/roles/tobiko-deploy/meta/main.yaml index bfea7e6d1..b76c4ada7 100644 --- a/roles/tobiko-deploy/meta/main.yaml +++ b/roles/tobiko-deploy/meta/main.yaml @@ -4,4 +4,3 @@ dependencies: - role: tobiko-common - role: tobiko-ensure-rsync - role: tobiko-ensure-git - - role: tobiko-ensure-bindep diff --git a/roles/tobiko-deploy/tasks/deploy-dir.yaml b/roles/tobiko-deploy/tasks/deploy-dir.yaml index acc94902f..63a549212 100644 --- a/roles/tobiko-deploy/tasks/deploy-dir.yaml +++ b/roles/tobiko-deploy/tasks/deploy-dir.yaml @@ -63,27 +63,3 @@ - name: "show last change details" debug: var=get_git_log.stdout_lines - - -- block: - - name: "check missing binary dependencies from '{{ bindep_file }}'" - command: - cmd: > - '{{ bindep_executable }}' -b -f '{{ bindep_file }}' {{ bindep_profiles }} - chdir: '{{ dest_dir }}' - register: get_bindeps - changed_when: no - - rescue: - - name: "install missing dependencies" - become: yes - package: - name: '{{ get_bindeps.stdout_lines }}' - - - name: "check missing binary dependencies from '{{ bindep_file }}'" - command: - cmd: > - '{{ bindep_executable }}' -b -f '{{ bindep_file }}' {{ bindep_profiles }} - chdir: '{{ dest_dir }}' - register: get_bindeps - changed_when: no diff --git a/roles/tobiko-ensure-bindep/defaults/main.yaml b/roles/tobiko-ensure-bindep/defaults/main.yaml deleted file mode 100644 index 30b02f515..000000000 --- a/roles/tobiko-ensure-bindep/defaults/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -bindep_executable: bindep -bindep_min_version: 2.8 diff --git a/roles/tobiko-ensure-bindep/tasks/main.yaml b/roles/tobiko-ensure-bindep/tasks/main.yaml deleted file mode 100644 index d01f6312f..000000000 --- a/roles/tobiko-ensure-bindep/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -- name: "ensures '{{ bindep_executable }}' command is available" - include_tasks: bindep.yaml - when: - - bindep_version is not defined diff --git a/roles/tobiko-ensure-git/meta/main.yaml b/roles/tobiko-ensure-git/meta/main.yaml deleted file mode 100644 index 0d4361583..000000000 --- a/roles/tobiko-ensure-git/meta/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -dependencies: - - role: tobiko-common diff --git a/roles/tobiko-ensure-git/tasks/main.yaml b/roles/tobiko-ensure-git/tasks/main.yaml index d06fa0468..e24c1e210 100644 --- a/roles/tobiko-ensure-git/tasks/main.yaml +++ b/roles/tobiko-ensure-git/tasks/main.yaml @@ -1,6 +1,6 @@ --- -- name: "ensures 'git' command is available" +- name: "ensure '{{ git_executable }}' command is available" include_tasks: git.yaml when: - git_version is not defined diff --git a/roles/tobiko-ensure-python3/defaults/main.yaml b/roles/tobiko-ensure-python3/defaults/main.yaml index f188a7dd2..98d1f6044 100644 --- a/roles/tobiko-ensure-python3/defaults/main.yaml +++ b/roles/tobiko-ensure-python3/defaults/main.yaml @@ -1,5 +1,19 @@ --- -python_executable: '/usr/bin/python' +python_exe_dirs: + - ~/.local/bin + - ~/bin + - /usr/local/bin + - /usr/bin + - /usr/local/sbin + - /usr/sbin + +python_exe_path: + "{{ python_exe_dirs | join(':') }}" + python_version: '3' -python3_executable: '/usr/bin/python{{ python_version }}' +python_command: 'python' +python3_command: 'python{{ python_version }}' + +python_packages: {} +python_info: {} diff --git a/roles/tobiko-ensure-python3/tasks/install.yaml b/roles/tobiko-ensure-python3/tasks/install.yaml new file mode 100644 index 000000000..f65057407 --- /dev/null +++ b/roles/tobiko-ensure-python3/tasks/install.yaml @@ -0,0 +1,45 @@ +--- + +- name: "install '{{ python_command }}' packages" + become: true + package: + name: "{{ python_packages[python_command] | flatten }}" + when: 'python_command in python_packages' + register: install_python_packages + + +- name: "show installed packages" + debug: var=install_python_packages.changes + when: install_python_packages is changed + + +- name: "get '{{ python_command }}' full path" + shell: | + export PATH={{ python_exe_path }} + which "{{ python_command }}" + register: get_python_exe + + +- name: "get '{{ get_python_exe.stdout_lines | first }}' info" + script: + cmd: get_python_info.py --base --quiet + executable: '{{ get_python_exe.stdout_lines | first }}' + register: get_python_info + + +- name: "set python_info['{{ python_command }}'] fact" + set_fact: + python_info: > + {{ python_info | combine({python_command: + (get_python_info.stdout | from_json)}) }} + + +- name: "upgrade Pip command to latest version" + command: > + '{{ python_info[python_command].executable }}' -m pip install pip --upgrade --user + + +- name: "show python_info facts" + debug: + msg: + python_info: '{{ python_info }}' diff --git a/roles/tobiko-ensure-python3/tasks/main.yaml b/roles/tobiko-ensure-python3/tasks/main.yaml index 92f9b7f3b..964e17ef7 100644 --- a/roles/tobiko-ensure-python3/tasks/main.yaml +++ b/roles/tobiko-ensure-python3/tasks/main.yaml @@ -1,31 +1,5 @@ --- -- name: "show ansible distro variables" - debug: - msg: - ansible_architecture: '{{ ansible_architecture }}' - ansible_distribution: '{{ ansible_distribution }}' - ansible_distribution_major_version: '{{ ansible_distribution_major_version }}' - ansible_os_family: '{{ ansible_os_family }}' - - -- name: "include OS-specific variables" - include_vars: "{{ item }}" - ignore_errors: yes - with_first_found: - - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" - - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" - - "{{ ansible_distribution }}.yaml" - - "{{ ansible_os_family }}.yaml" - - -- block: - - name: "ensures '{{ python3_executable }}' command is available" - include_tasks: python3.yaml - when: - - python3_info is not defined - - - name: "ensures '{{ python_executable }}' command is available" - include_tasks: python.yaml - when: - - python_info is not defined +- name: "ensure '{{ python3_command }}' command is available" + include_tasks: python3.yaml + when: 'python3_command not in python_info' diff --git a/roles/tobiko-ensure-python3/tasks/platform.yaml b/roles/tobiko-ensure-python3/tasks/platform.yaml new file mode 100644 index 000000000..9b4e20583 --- /dev/null +++ b/roles/tobiko-ensure-python3/tasks/platform.yaml @@ -0,0 +1,31 @@ +--- + +- name: "show ansible distro variables" + debug: + msg: + ansible_architecture: '{{ ansible_architecture }}' + ansible_distribution: '{{ ansible_distribution }}' + ansible_distribution_major_version: '{{ ansible_distribution_major_version }}' + ansible_os_family: '{{ ansible_os_family }}' + + +- name: "include platform variables" + include_vars: "{{ item }}" + ignore_errors: yes + with_first_found: + - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" + - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" + - "{{ ansible_distribution }}.yaml" + - "{{ ansible_os_family }}.yaml" + register: include_platform_vars + + +- name: "set Python platform facts" + set_fact: + python_platform: + '{{ (include_platform_vars.results | first).item | basename | splitext | first }}' + + +- name: 'show Python platform facts' + debug: + msg: '{{ (include_platform_vars.results | first).ansible_facts }}' diff --git a/roles/tobiko-ensure-python3/tasks/python.yaml b/roles/tobiko-ensure-python3/tasks/python.yaml deleted file mode 100644 index c5ab5b66c..000000000 --- a/roles/tobiko-ensure-python3/tasks/python.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -- block: - - name: "get Python info for '{{ python_executable }}'" - script: - cmd: get_python_info.py --base --quiet - executable: '{{ python_executable }}' - register: get_python_info - - rescue: - - block: - - name: "set '{{ python_alternative }}' as default alternative for python" - become: true - command: "alternatives --set python '{{ python_alternative }}'" - - - name: "get Python info for '{{ python_executable }}'" - script: - cmd: get_python_info.py --base --quiet - executable: '{{ python_executable }}' - register: get_python_info - - when: - - python_alternative is defined - - - name: "report '{{ python_executable }}' failure" - fail: - msg: get_python_info - when: get_python_info is failed - - -- name: "set python_info fact" - set_fact: - python_executable: '{{ (get_python_info.stdout | from_json).executables | first }}' - python_info: '{{ get_python_info.stdout | from_json }}' - - -- name: "show '{{ python_executable }}' executables facts" - debug: - msg: - python_executable: '{{ python_executable }}' - python_info: '{{ python_info }}' diff --git a/roles/tobiko-ensure-python3/tasks/python3.yaml b/roles/tobiko-ensure-python3/tasks/python3.yaml index 5e78e0601..13f6e2dcd 100644 --- a/roles/tobiko-ensure-python3/tasks/python3.yaml +++ b/roles/tobiko-ensure-python3/tasks/python3.yaml @@ -1,6 +1,12 @@ --- -- name: "validate python_version value: {{ python_version }}" +- name: "include platform variables" + include_tasks: platform.yaml + when: + - python_platform is not defined + + +- name: "validate Python version: {{ python_version }}" assert: that: - (python_version | string).split(".") | length >= 1 @@ -8,43 +14,33 @@ - (python_version | string).split(".")[0] == '3' +- include_tasks: install.yaml + vars: + python_command: '{{ python3_command }}' + + +- name: "set python3_executable fact" + set_fact: + python3_executable: '{{ python_info[python3_command].executable }}' + + - block: - - name: "get Python info for '{{ python3_executable }}'" - script: - cmd: get_python_info.py --base --quiet - executable: '{{ python3_executable }}' - register: get_python_info + - include_tasks: install.yaml + when: python_command not in python_info + + - name: "set python_executable fact" + set_fact: + python_executable: '{{ python_info[python_command].executable }}' rescue: - - block: - - name: "install Python '{{ python3_executable }}' packages" - become: true - package: - name: "{{ python3_packages }}" + - name: "try setting python alternative to '{{ python_alternative }}'" + become: true + command: "alternatives --set python '{{ python_alternative }}'" + ignore_errors: yes + when: 'python_alternative is defined' - - name: "get Python info for '{{ python3_executable }}'" - script: - cmd: get_python_info.py --base --quiet - executable: '{{ python3_executable }}' - register: get_python_info + - include_tasks: install.yaml - when: - - python3_packages is defined - - - name: "report '{{ python3_executable }}' failure" - fail: - msg: get_python_info - when: get_python_info is failed - - -- name: "set python3_info fact" - set_fact: - python3_executable: '{{ (get_python_info.stdout | from_json).executables | first }}' - python3_info: '{{ get_python_info.stdout | from_json }}' - - -- name: "show '{{ python3_executable }}' facts" - debug: - msg: - python3_executable: '{{ python3_executable }}' - python3_info: '{{ python3_info }}' + - name: "set python_executable fact" + set_fact: + python_executable: '{{ python_info[python_command].executable }}' diff --git a/roles/tobiko-ensure-python3/vars/CentOS-7.yaml b/roles/tobiko-ensure-python3/vars/CentOS-7.yaml new file mode 100644 index 000000000..d7e5cc64f --- /dev/null +++ b/roles/tobiko-ensure-python3/vars/CentOS-7.yaml @@ -0,0 +1,17 @@ +--- + +python_packages: + python: + - bzip2-devel + - gcc + - make + - openssl-devel + - python2-devel + - python2-pip + - readline-devel + - sqlite-devel + - zlib-devel + python3: + - python3 + python3.6: + - python36 diff --git a/roles/tobiko-ensure-python3/vars/MacOSX.yaml b/roles/tobiko-ensure-python3/vars/MacOSX.yaml index bb8cadca5..117dc8761 100644 --- a/roles/tobiko-ensure-python3/vars/MacOSX.yaml +++ b/roles/tobiko-ensure-python3/vars/MacOSX.yaml @@ -1,3 +1,14 @@ --- -python3_executable: python3 +python_exe_dirs: + - ~/Library/Python/3.8/bin + - ~/Library/Python/3.7/bin + - ~/Library/Python/3.6/bin + - /Library/Frameworks/Python.framework/Versions/3.8/bin + - /Library/Frameworks/Python.framework/Versions/3.7/bin + - /Library/Frameworks/Python.framework/Versions/3.6/bin + - /usr/local/bin + - /usr/bin + - /bin + - /usr/sbin + - /sbin diff --git a/roles/tobiko-ensure-python3/vars/RedHat.yaml b/roles/tobiko-ensure-python3/vars/RedHat.yaml index f12692514..4c6476a09 100644 --- a/roles/tobiko-ensure-python3/vars/RedHat.yaml +++ b/roles/tobiko-ensure-python3/vars/RedHat.yaml @@ -2,17 +2,8 @@ python_alternative: '/usr/bin/python3' -python3_package_name: "python{{ python_version | regex_replace('\\.', '') }}" - -python3_packages: - - bzip2-devel - - gcc - - make - - openssl-devel - - readline-devel - - sqlite-devel - - zlib-devel - - '{{ python3_package_name }}' - - '{{ python3_package_name }}-devel' - - '{{ python3_package_name }}-setuptools' - - '{{ python3_package_name }}-virtualenv' +python_packages: + python3: + - python3 + python3.6: + - python36 diff --git a/roles/tobiko-ensure-rsync/defaults/main.yaml b/roles/tobiko-ensure-rsync/defaults/main.yaml new file mode 100644 index 000000000..43019f0dd --- /dev/null +++ b/roles/tobiko-ensure-rsync/defaults/main.yaml @@ -0,0 +1,3 @@ +--- + +rsync_executable: rsync diff --git a/roles/tobiko-ensure-rsync/meta/main.yaml b/roles/tobiko-ensure-rsync/meta/main.yaml deleted file mode 100644 index 0d4361583..000000000 --- a/roles/tobiko-ensure-rsync/meta/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -dependencies: - - role: tobiko-common diff --git a/roles/tobiko-ensure-rsync/tasks/main.yaml b/roles/tobiko-ensure-rsync/tasks/main.yaml index 9804b15ee..6a51ee822 100644 --- a/roles/tobiko-ensure-rsync/tasks/main.yaml +++ b/roles/tobiko-ensure-rsync/tasks/main.yaml @@ -1,6 +1,6 @@ --- -- name: "ensures 'rsync' command is available" +- name: "ensure '{{ rsync_executable }}' command is available" include_tasks: rsync.yaml when: - - test_stage in ['all', 'pre-run', 'rsync'] + - rsync_version is not defined diff --git a/roles/tobiko-ensure-rsync/tasks/rsync.yaml b/roles/tobiko-ensure-rsync/tasks/rsync.yaml index b8760ca06..bf1fb3bcf 100644 --- a/roles/tobiko-ensure-rsync/tasks/rsync.yaml +++ b/roles/tobiko-ensure-rsync/tasks/rsync.yaml @@ -1,23 +1,42 @@ --- - block: - - name: "get installed rsync version" # noqa 303 + - name: "get '{{ rsync_executable }}' full path" shell: > - rsync --version - register: get_rsync_gersion + which '{{ rsync_executable }}' + register: get_rsync_path rescue: - - name: "install rsync packages" + - name: "install Rsync packages" become: true package: name: rsync - - name: "get installed rsync version" # noqa 303 + - name: "get '{{ rsync_executable }}' full path" shell: > - rsync --version - register: get_rsync_gersion + which '{{ rsync_executable }}' + register: get_rsync_path -- name: "show installed rsync version" +- name: "update rsync_executable fact" + set_fact: + rsync_executable: '{{ get_rsync_path.stdout_lines | first }}' + + +- name: "get Rsync version" + command: > + '{{ rsync_executable }}' --version + register: + get_rsync_version + + +- name: update rsync_version fact + set_fact: + rsync_version: '{{ get_rsync_version.stdout_lines | first }}' + + +- name: "show Rsync facts" debug: - var: get_rsync_gersion.stdout_lines + msg: + rsync_executable: '{{ rsync_executable }}' + rsync_version: '{{ rsync_version }}' diff --git a/roles/tobiko-ensure-tox/defaults/main.yaml b/roles/tobiko-ensure-tox/defaults/main.yaml index 5b98a920d..297eee5b1 100644 --- a/roles/tobiko-ensure-tox/defaults/main.yaml +++ b/roles/tobiko-ensure-tox/defaults/main.yaml @@ -1,4 +1,4 @@ --- -tox_executable: tox +tox_command: tox tox_min_version: 3.4 diff --git a/roles/tobiko-ensure-tox/meta/main.yaml b/roles/tobiko-ensure-tox/meta/main.yaml index 75b81b773..97b06fff8 100644 --- a/roles/tobiko-ensure-tox/meta/main.yaml +++ b/roles/tobiko-ensure-tox/meta/main.yaml @@ -1,5 +1,4 @@ --- dependencies: - - role: tobiko-common - role: tobiko-ensure-python3 diff --git a/roles/tobiko-ensure-tox/tasks/main.yaml b/roles/tobiko-ensure-tox/tasks/main.yaml index b1f6c2695..d599ece1d 100644 --- a/roles/tobiko-ensure-tox/tasks/main.yaml +++ b/roles/tobiko-ensure-tox/tasks/main.yaml @@ -1,6 +1,6 @@ --- -- name: "ensures '{{ tox_executable }}' command is available" +- name: "ensure '{{ tox_command }}' command is available" include_tasks: tox.yaml when: - - tox_version is not defined + - tox_executable is not defined diff --git a/roles/tobiko-ensure-tox/tasks/tox.yaml b/roles/tobiko-ensure-tox/tasks/tox.yaml index 53fe78b7c..33a16b809 100644 --- a/roles/tobiko-ensure-tox/tasks/tox.yaml +++ b/roles/tobiko-ensure-tox/tasks/tox.yaml @@ -3,22 +3,23 @@ - name: "ensure Tox is installed" command: | '{{ python_executable }}' -m pip install --user 'tox>={{ tox_min_version }}' - register: result - changed_when: "'Successfully installed' in result.stdout" + register: install_tox + changed_when: "'Successfully installed' in install_tox.stdout" - block: - - name: "get '{{ tox_executable }}' full path" - command: > - which '{{ tox_executable }}' + - name: "get '{{ tox_command }}' full path" + shell: | + export PATH={{ python_exe_path }} + which '{{ tox_command }}' register: get_tox_path - - name: "update tox_executable fact" + - name: "set tox_executable fact" set_fact: tox_executable: '{{ get_tox_path.stdout_lines | first }}' rescue: - - name: "update tox_executable fact" + - name: "set tox_executable fact" set_fact: tox_executable: '{{ ansible_user_dir }}/.local/bin/tox' @@ -30,7 +31,7 @@ - name: "update tox_version fact" set_fact: - tox_version: '{{ get_tox_version.stdout }}' + tox_version: '{{ get_tox_version.stdout_lines | first }}' - name: "show Tox facts" diff --git a/roles/tobiko-run/defaults/main.yaml b/roles/tobiko-run/defaults/main.yaml new file mode 100644 index 000000000..27d2d35b9 --- /dev/null +++ b/roles/tobiko-run/defaults/main.yaml @@ -0,0 +1,5 @@ +--- + +test_workflow: default +tox_step_name: +tox_step_index: 0 diff --git a/roles/tobiko-run/meta/main.yaml b/roles/tobiko-run/meta/main.yaml index d0e9e4f30..0d4361583 100644 --- a/roles/tobiko-run/meta/main.yaml +++ b/roles/tobiko-run/meta/main.yaml @@ -1,4 +1,4 @@ --- dependencies: - - role: tobiko-tox + - role: tobiko-common diff --git a/roles/tobiko-run/tasks/main.yaml b/roles/tobiko-run/tasks/main.yaml new file mode 100644 index 000000000..f616cb954 --- /dev/null +++ b/roles/tobiko-run/tasks/main.yaml @@ -0,0 +1,16 @@ +--- + +- name: "include test workflow steps for name '{{ test_workflow }}'" + include_vars: "{{ test_workflow }}.yaml" + + +- name: "show test workflow steps" + debug: var=test_workflow_steps + + +- name: "run '{{ test_workflow }}' test steps" + include_tasks: run.yaml + loop: '{{ test_workflow_steps }}' + loop_control: + label: '{{ test_step.tox_description }}' + loop_var: test_step diff --git a/roles/tobiko-run/tasks/run.yaml b/roles/tobiko-run/tasks/run.yaml new file mode 100644 index 000000000..a5670ace8 --- /dev/null +++ b/roles/tobiko-run/tasks/run.yaml @@ -0,0 +1,20 @@ +--- + +- name: 'set step variables' + set_fact: + '{{ item.0 }}': '{{ item.1 }}' + loop: '{{ test_step | dictsort }}' + loop_control: + label: '{{ item.0 }} = {{ item.1 }}' + + +- name: 'set test step index' + set_fact: + tox_step_index: "{{ (tox_step_index | int) + 1 }}" + when: + - (test_workflow_steps | length) > 1 + + +- name: '{{ test_step.tox_description }}' + include_role: + name: tobiko-tox diff --git a/roles/tobiko-run/vars/default.yaml b/roles/tobiko-run/vars/default.yaml new file mode 100644 index 000000000..b3e5d3178 --- /dev/null +++ b/roles/tobiko-run/vars/default.yaml @@ -0,0 +1,4 @@ +--- + +test_workflow_steps: + - tox_description: 'run test cases' diff --git a/roles/tobiko-run/vars/faults.yaml b/roles/tobiko-run/vars/faults.yaml new file mode 100644 index 000000000..1618ad9f6 --- /dev/null +++ b/roles/tobiko-run/vars/faults.yaml @@ -0,0 +1,18 @@ +--- + +test_workflow_steps: + - tox_description: 'create workload resources' + tox_envlist: scenario + tox_step_name: create_resources + tox_environment: + TOBIKO_PREVENT_CREATE: no + + - tox_description: 'run disruptive test cases' + tox_envlist: faults + tox_step_name: faults + + - tox_description: 'verify workload resources' + tox_envlist: scenario + tox_step_name: verify_resources' + tox_environment: + TOBIKO_PREVENT_CREATE: yes diff --git a/roles/tobiko-run/vars/functional.yaml b/roles/tobiko-run/vars/functional.yaml new file mode 100644 index 000000000..d70d28606 --- /dev/null +++ b/roles/tobiko-run/vars/functional.yaml @@ -0,0 +1,5 @@ +--- + +test_workflow_steps: + - tox_description: 'run functional test cases' + tox_envlist: functional diff --git a/roles/tobiko-run/vars/unit.yaml b/roles/tobiko-run/vars/unit.yaml new file mode 100644 index 000000000..7b8ebe607 --- /dev/null +++ b/roles/tobiko-run/vars/unit.yaml @@ -0,0 +1,15 @@ +test_workflow_steps: + - tox_description: 'run unit tests using Python 3.6' + tox_envlist: py36 + python_version: '3.6' + bindep_profile: test py36 + + - tox_description: 'run unit tests using Python 3.7' + tox_envlist: py37 + python_version: '3.7' + bindep_profile: test py37 + + - tox_description: 'run unit tests using Python 3.8' + tox_envlist: py38 + python_version: '3.8' + bindep_profile: test py38 diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index 7cbaf6108..b592f37f1 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -1,22 +1,21 @@ --- # Directory where test cases has been deployed to +tox_description: 'run test cases' tox_dir: '{{ test_dir | realpath }}' tox_command: '{{ tox_executable | default(tox) }}' tox_environment: {} tox_envlist: tox_extra_args: '' -tox_command_line: > - {{ tox_command }} - {% if tox_envlist %} -e {{ tox_envlist | quote }} {% endif %} - {{ tox_extra_args }} tox_python: '{{ python3_executable }}' tox_report_dir: '{{ test_report_dir | realpath }}' -tox_report_name: '{{ test_report_name }}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}' +tox_step_name: +tox_step_index: 0 +tox_report_name: + "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}" tox_report_env: TOBIKO_TEST_REPORT_DIR: '{{ tox_report_dir }}' TOBIKO_TEST_REPORT_NAME: '{{ tox_report_name }}' -tox_constrain_env: - PYTHON: '{{ tox_python }}' +tox_constrain_env: {} diff --git a/roles/tobiko-tox/meta/main.yaml b/roles/tobiko-tox/meta/main.yaml index 25dc77cb2..52962d331 100644 --- a/roles/tobiko-tox/meta/main.yaml +++ b/roles/tobiko-tox/meta/main.yaml @@ -2,5 +2,7 @@ dependencies: - role: tobiko-common - - role: tobiko-ensure-python3 + - role: tobiko-bindep + vars: + bindep_file: '{{ tox_dir }}/bindep.txt' - role: tobiko-ensure-tox diff --git a/roles/tobiko-tox/tasks/tox.yaml b/roles/tobiko-tox/tasks/tox.yaml index 81035cab9..4d6fd468c 100644 --- a/roles/tobiko-tox/tasks/tox.yaml +++ b/roles/tobiko-tox/tasks/tox.yaml @@ -1,11 +1,29 @@ --- +- name: "set Tox command line fact" + set_fact: + tox_command_line: > + {{ tox_command }} + {% if tox_envlist %} -e {{ tox_envlist | quote }} {% endif %} + {{ tox_extra_args }} + + - name: "normalize white spaces from Tox command line" set_fact: tox_command_line: '{{ tox_command_line.split() | join(" ") }}' -- name: "run Tox on direcory '{{ tox_dir }}': '{{ tox_command_line }}'" +- name: "show tox variables" + debug: + msg: + tox_command_line: '{{ tox_command_line }}' + tox_description: '{{ tox_description }}' + tox_dir: '{{ tox_dir }}' + tox_environment: '{{ tox_environment | combine(tox_constrain_env) }}' + tox_report_env: '{{ tox_report_env | combine(tox_constrain_env) }}' + + +- name: "{{ tox_description }}" command: chdir: '{{ tox_dir }}' cmd: '{{ tox_command_line }}' diff --git a/tobiko/config.py b/tobiko/config.py index 0f38b6cd1..e3a8099ce 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -339,9 +339,9 @@ def get_bool_env(name): value = get_env(name) if value: value = str(value).lower() - if value in ["true", "1"]: + if value in ['true', '1', 'yes']: return True - elif value in ["false", '0']: + elif value in ['false', '0', 'no']: return False else: LOG.exception("Environment variable %r is not a boolean: %r", diff --git a/tools/setup_infrared.py b/tools/setup_infrared.py index c77bbd403..859d407a8 100644 --- a/tools/setup_infrared.py +++ b/tools/setup_infrared.py @@ -25,8 +25,8 @@ LOG = logging.getLogger(__name__) def main(): setup_logging() add_tobiko_plugin() - import_workspace() - copy_ansible_inventory() + ensure_workspace() + copy_inventory() def setup_logging(level=logging.DEBUG): @@ -36,62 +36,82 @@ def setup_logging(level=logging.DEBUG): format='%(name)-s: %(levelname)-7s %(asctime)-15s | %(message)s') -def add_tobiko_plugin(): - add_plugin('tobiko', os.environ.get('IR_TOBIKO_PLUGIN')) +def add_tobiko_plugin(path=None): + path = path or os.environ.get('IR_TOBIKO_PLUGIN') + if path: + add_plugin('tobiko', path) def add_plugin(name, path): - if path: - path = normalize_path(path) - if os.path.isdir(path): - remove_plugin(name) - execute('ir plugin add "{}"', path) - else: - LOG.debug("Plug-in '%s' directory not found: '%s'", name, path) + path = normalize_path(path) + if not os.path.isdir(path): + message = ("invalid plug-in '{}' directory: '{}'").format(name, path) + raise RuntimeError(message) - else: - LOG.debug("Plug-in '%s' path not specified", name) + remove_plugin(name) + execute('ir plugin add "{}"', path) + LOG.info("plug-in '%s' added from path '%s'", name, path) def remove_plugin(name): try: execute('ir plugin remove "{}"', name) - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as ex: + LOG.debug("plug-in '%s' not removed: %s", name, ex) return False else: + LOG.info("plug-in '%s' removed", name) return True -def import_workspace(filename=None): - filename = filename or os.environ.get('IR_WORKSPACE_FILE') - if filename: - filename = normalize_path(filename) - if os.path.isfile(filename): - try: - execute('ir workspace import "{}"', filename) - except subprocess.CalledProcessError: - # If file was already imported before we checkout to its - # workspace - workspace = name_from_path(filename) - execute('ir workspace checkout "{}"', workspace) +def ensure_workspace(filename=None): + filename = (filename or + os.environ.get('IR_WORKSPACE_FILE') or + 'workspace.tgz') + filename = normalize_path(filename) + workspace = name_from_path(filename) + if os.path.isfile(filename): + try: + execute('ir workspace import "{}"', filename) + except subprocess.CalledProcessError as ex: + LOG.debug("workspace file '%s' not imported: %s", filename, ex) else: - LOG.debug("No such workspace file: '%s'", filename) + LOG.info("workspace imported from file '%s'", filename) + return else: - LOG.debug('Workspace file not specified') + LOG.debug("workspace file not found: '%s'", filename) + + try: + execute('ir workspace checkout "{}"', workspace) + except subprocess.CalledProcessError as ex: + LOG.debug("workspace '%s' not checked out: %s", workspace, ex) + else: + LOG.info("workspace '%s' checked out", workspace) + return + + execute('infrared workspace checkout --create "{}"', workspace) + LOG.info("workspace '%s' created", workspace) -def copy_ansible_inventory(filename=None): - filename = filename or os.environ.get('ANSIBLE_INVENTORY') - if filename: - if os.path.isfile(filename): - destination = execute('ir workspace inventory') - if not os.path.exists(os.path.basename(destination)): - os.makedirs(os.path.basename(destination)) - execute('cp {} {}', filename, destination) - else: - LOG.debug('Inventary file not found: %r', filename) - else: - LOG.debug('Ansible inventory file not specified') +def copy_inventory(filename=None): + filename = (filename or + os.environ.get('ANSIBLE_INVENTORY') or + 'ansible_hosts') + if not os.path.isfile(filename): + LOG.debug('inventary file not found: %r', filename) + return False + + dest_file = execute('ir workspace inventory') + LOG.debug("got workspace inventory file: '%s'", dest_file) + + dest_dir = os.path.basename(dest_file) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + LOG.info("directory created: '%s'", dest_dir) + + execute('cp {} {}', filename, dest_file) + LOG.info("inventary file '%s' copied to '%s'", filename, dest_file) + return True def normalize_path(path): @@ -101,7 +121,7 @@ def normalize_path(path): def execute(command, *args, **kwargs): if args or kwargs: command = command.format(*args, **kwargs) - LOG.info('%s', command) + LOG.debug("execute command: '%s'", command) return subprocess.check_output(command, shell=True, universal_newlines=True) diff --git a/tox.ini b/tox.ini index a14f406e5..d3833e085 100644 --- a/tox.ini +++ b/tox.ini @@ -184,6 +184,8 @@ usedevelop = false skipdist = true skip_install = true sitepackages = true +whitelist_externals = + rm deps = -r infrared-requirements.txt @@ -206,7 +208,11 @@ commands_pre = {envpython} {toxinidir}/tools/setup_infrared.py commands = - ir tobiko --tobiko-src-dir {toxinidir} {posargs} + rm -fR '{toxinidir}/test_results' + ir tobiko \ + --tobiko-src-dir {toxinidir} \ + --collect-dir '{toxinidir}/test_results' \ + {posargs} # --- documentation environments ----------------------------------------------