diff --git a/roles/ensure-tox/README.rst b/roles/ensure-tox/README.rst index 60e0cac34..f3cfa359b 100644 --- a/roles/ensure-tox/README.rst +++ b/roles/ensure-tox/README.rst @@ -2,3 +2,22 @@ Ensure tox is installed If tox is not already installed, it will be installed via pip in the user install directory (i.e., "pip install --user"). + +**Role Variables** + +.. zuul:rolevar:: tox_upgrade + :default: false + + If you want the installation of latest tox, regardless presence on the + system, define `tox_upgrade: true`. + +.. zuul:rolevar:: tox_condition + :default: tox + + Python requirements condition to be used when installing tox. You can define + it to "tox>=3.8.0" to force a minimal version. + +.. zuul:rolevar:: tox_executable + :default: {{ ansible_python.executable }} -m tox + + Location of the tox executable. diff --git a/roles/ensure-tox/defaults/main.yaml b/roles/ensure-tox/defaults/main.yaml new file mode 100644 index 000000000..5d52ca238 --- /dev/null +++ b/roles/ensure-tox/defaults/main.yaml @@ -0,0 +1,7 @@ +# when set, it will always install latest version of tox using --user +# be aware that this may break /usr/bin/tox executable due to changed internals +tox_upgrade: false +tox_condition: tox +tox_environment: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_WARN_SCRIPT_LOCATION: "1" diff --git a/roles/ensure-tox/molecule/default/Dockerfile.j2 b/roles/ensure-tox/molecule/default/Dockerfile.j2 new file mode 100644 index 000000000..84d5ca291 --- /dev/null +++ b/roles/ensure-tox/molecule/default/Dockerfile.j2 @@ -0,0 +1,18 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +USER root + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates python-apt {{ item.pkg_extras | default('') }} && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python3 sudo python3-devel python3-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +CMD ["sh", "-c", "while true; do sleep 10000; done"]N \ No newline at end of file diff --git a/roles/ensure-tox/molecule/default/molecule.yml b/roles/ensure-tox/molecule/default/molecule.yml new file mode 100644 index 000000000..1310dd4dd --- /dev/null +++ b/roles/ensure-tox/molecule/default/molecule.yml @@ -0,0 +1,41 @@ +--- +# The MOLECULE_ vars used here are not officially endorsed by molecule, but we +# hope a future version will add implicit support for them. +driver: + name: docker +platforms: + + - name: centos7 + hostname: centos7 + image: centos:7 + dockerfile: Dockerfile.j2 + easy_install: + - pip + + - name: fedora + hostname: fedora + image: fedora:latest + dockerfile: Dockerfile.j2 + easy_install: + - pip + + - name: ubuntu1804 + hostname: ubuntu1804 + image: ubuntu:18.04 + dockerfile: Dockerfile.j2 + +provisioner: + name: ansible + log: true + inventory: + host_vars: + fedora: + ansible_python_interpreter: /usr/bin/python3 +scenario: + test_sequence: + - destroy + - create + - converge + - destroy +lint: + enabled: false diff --git a/roles/ensure-tox/molecule/default/playbook.yml b/roles/ensure-tox/molecule/default/playbook.yml new file mode 100644 index 000000000..0dd146626 --- /dev/null +++ b/roles/ensure-tox/molecule/default/playbook.yml @@ -0,0 +1,60 @@ +--- + +- name: Converge + hosts: all + gather_facts: true + tasks: + + - name: Include OS specific variables + include_vars: "{{ item }}" + failed_when: false + loop: + - "family-{{ ansible_os_family | lower }}.yml" + - "family-{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[0:2] | join('-') | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[0:3] | join('-') | lower }}.yml" + + - name: installing requirements + package: + name: "{{ pkg_tox_pre|default('curl') }}" + + # current code in ensure-tox makes the assumption that `pip` is installed. + - name: ensure-pip + become: yes + environment: + PIP_NO_WARN_SCRIPT_LOCATION: "1" + shell: | + set -exu + {{ ansible_python.executable }} -m pip --version || { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + {{ ansible_python.executable }} get-pip.py + } + {{ ansible_python.executable }} -m pip --version + + - name: include ensure-tox before installing system tox + include_role: + name: ensure-tox + + - name: include ensure-tox with upgrade + vars: + tox_upgrade: true + include_role: + name: ensure-tox + + - name: installing system tox + package: + name: "{{ pkg_tox | default('python-tox') }}" + update_cache: | + {{ ansible_version.full is version_compare('2.7.0', '>=') or omit }} + + - name: include ensure-tox after installing system tox + include_role: + name: ensure-tox + + - name: include ensure-tox with upgrade after installing system tox + vars: + tox_upgrade: true + include_role: + name: ensure-tox diff --git a/roles/ensure-tox/molecule/default/vars/centos.yml b/roles/ensure-tox/molecule/default/vars/centos.yml new file mode 100644 index 000000000..3b0156265 --- /dev/null +++ b/roles/ensure-tox/molecule/default/vars/centos.yml @@ -0,0 +1,3 @@ +pkg_tox_pre: + - curl + - epel-release diff --git a/roles/ensure-tox/molecule/default/vars/fedora.yml b/roles/ensure-tox/molecule/default/vars/fedora.yml new file mode 100644 index 000000000..afad103a6 --- /dev/null +++ b/roles/ensure-tox/molecule/default/vars/fedora.yml @@ -0,0 +1 @@ +pkg_tox: python3-tox \ No newline at end of file diff --git a/roles/ensure-tox/tasks/main.yaml b/roles/ensure-tox/tasks/main.yaml index b2b7487eb..be08ff5f6 100644 --- a/roles/ensure-tox/tasks/main.yaml +++ b/roles/ensure-tox/tasks/main.yaml @@ -1,2 +1,43 @@ -- name: Ensure tox is installed - shell: type tox || pip install --user tox +- name: Check if tox is accesible # noqa 305 + environment: "{{ tox_environment }}" + shell: | + set -eux + tox --version + {{ ansible_python.executable }} -m tox --version + register: result + failed_when: result.rc != 0 and 'not found' not in result.stderr + changed_when: false + +# installing tox with --user will create tox script inside ~/.local/bin but +# this folder is *not* included PATH in all distros. While most modern ones +# adopted it, there is no guarantee. +- name: Ensure tox is installed # noqa 305 + shell: | + set -eu + # {# for systems that do not have ~/.local/bin in PATH yet #} + # if [[ ":$PATH:" != *":${HOME}/.local/bin:"* ]] && [ -d "${HOME}/.local/bin" ]; then + # export PATH=${HOME}/.local/bin:$PATH + # fi + {% if result is failed or tox_upgrade %} + {{ ansible_python.executable }} -m pip install --user --upgrade '{{ tox_condition }}' + {% else %} + {{ ansible_python.executable }} -m pip install --user '{{ tox_condition }}' + {% endif %} + {{ ansible_python.executable }} -m tox --version + args: + executable: /bin/bash + +# We want to be sure that if someone calls 'tox' without having `~/.local/bin` +# in PATH, they still get a *working* copy of it. +- name: Ensure a system tox is also available + become: true + environment: "{{ tox_environment }}" + shell: | + set -eu + type tox >/dev/null || { + {# -s is key here to prevent it from finding tox from userdir #} + {{ ansible_python.executable }} -s -m pip install '{{ tox_condition }}' + } + tox --version + args: + executable: /bin/bash diff --git a/roles/fetch-tox-output/defaults/main.yaml b/roles/fetch-tox-output/defaults/main.yaml index dddc1c877..5d19ec4be 100644 --- a/roles/fetch-tox-output/defaults/main.yaml +++ b/roles/fetch-tox-output/defaults/main.yaml @@ -2,6 +2,6 @@ # TODO(mordred) This needs to switch back to not being venv - venv is OpenStack # specific. tox_envlist: venv -tox_executable: tox +tox_executable: "{{ ansible_python.executable }} -m tox" zuul_work_dir: "{{ zuul.project.src_dir }}" diff --git a/roles/tox/README.rst b/roles/tox/README.rst index 215c3ff97..5397c9032 100644 --- a/roles/tox/README.rst +++ b/roles/tox/README.rst @@ -12,7 +12,7 @@ Runs tox for a project Which tox environment to run. .. zuul:rolevar:: tox_executable - :default: tox + :default: {{ ansible_python.executable }} -m tox Location of the tox executable. diff --git a/roles/tox/defaults/main.yaml b/roles/tox/defaults/main.yaml index 1178db533..513ee3913 100644 --- a/roles/tox/defaults/main.yaml +++ b/roles/tox/defaults/main.yaml @@ -1,7 +1,9 @@ --- tox_environment: {} tox_envlist: venv -tox_executable: tox +# Calling tox as a module in order to avoid case where tox script gets broken +# because user installed newer version of tox into user packages. +tox_executable: "{{ ansible_python.executable }} -m tox" tox_extra_args: -vv tox_install_siblings: true diff --git a/tox.ini b/tox.ini index 757c0a7f2..db8139bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,26 @@ [tox] minversion = 1.6 skipsdist = True -envlist = linters +envlist = linters,molecule [testenv] +setenv = + ANSIBLE_FORCE_COLOR=1 + ANSIBLE_INVENTORY={toxinidir}/test/hosts.ini + ANSIBLE_NOCOWS=1 + ANSIBLE_RETRY_FILES_ENABLED=0 + ANSIBLE_STDOUT_CALLBACK=debug + PY_COLORS=1 + VIRTUAL_ENV={envdir} + # Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207 + PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command + PIP_DISABLE_PIP_VERSION_CHECK=1 +passenv = + ANSIBLE_* + DOCKER_* + MOLECULE_* + SSH_AUTH_SOCK + TERM basepython = python3 install_command = pip install {opts} {packages} deps = -r{toxinidir}/test-requirements.txt @@ -55,6 +72,20 @@ commands = [testenv:venv] commands = {posargs} +[testenv:molecule] +deps = + ansi2html # GPL (soft-dependency of pytest-html) + molecule[docker]>=2.22rc6 # MIT + ansible>=2.8.0 # GPL (on purpose newer than the one in test-requirements) + pytest # MIT + pytest-cov # MIT + pytest-molecule # MIT + pytest-html # MPL 2.0 + pytest-xdist # MIT + selinux # MIT +commands = + python -m pytest -ra --continue-on-collection-errors --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs} + [flake8] # These are ignored intentionally in openstack-infra projects; # please don't submit patches that solely correct them or enable them. diff --git a/zuul-tests.d/project.yaml b/zuul-tests.d/project.yaml index 7b6dfd12f..3c59c404e 100644 --- a/zuul-tests.d/project.yaml +++ b/zuul-tests.d/project.yaml @@ -8,11 +8,13 @@ - zuul-tox-docs - tox-py27 - tox-py35 + - tox-molecule gate: jobs: - zuul-tox-docs - tox-py27 - tox-py35 + - tox-molecule promote: jobs: - zuul-promote-docs