From a91b1553f259d305350e44d914c71935c47bd2f2 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Thu, 25 Jul 2019 16:38:50 +0100 Subject: [PATCH] WIP: Allow ensure-tox to upgrade tox version Adds new variable `tox_upgrade: false` which can be used to change behavior of ensure-tox role. By default old bahaviour is maintained. This fixes failure to run "Run tox without tests" on some platform which already have an ancient version (like 1.6) on the system. Users encountering this can easily set tox_upgrade to true. Such old versions of tox are not able to upgrade itself and not even able to parse the tox.ini file. The only way to avoid this is to assure that the system has a recent enough version of tox. Tox ability to boostrap itself was added only in 3.8.0 which is not present in most distributions. This change also switches the default tox executable to use python module calling method in order to avoid calling /usr/bin/tox file which could be broken after the upgrade. Change-Id: Ibaeb21281d6db0a802638fd5dd39aa4cb825a09d Needed-By: https://review.rdoproject.org/r/#/c/21594/ --- roles/ensure-tox/README.rst | 19 ++++++ roles/ensure-tox/defaults/main.yaml | 7 +++ .../ensure-tox/molecule/default/Dockerfile.j2 | 18 ++++++ .../ensure-tox/molecule/default/molecule.yml | 41 +++++++++++++ .../ensure-tox/molecule/default/playbook.yml | 60 +++++++++++++++++++ .../molecule/default/vars/centos.yml | 3 + .../molecule/default/vars/fedora.yml | 1 + roles/ensure-tox/tasks/main.yaml | 45 +++++++++++++- roles/fetch-tox-output/defaults/main.yaml | 2 +- roles/tox/README.rst | 2 +- roles/tox/defaults/main.yaml | 4 +- tox.ini | 33 +++++++++- zuul-tests.d/project.yaml | 2 + 13 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 roles/ensure-tox/defaults/main.yaml create mode 100644 roles/ensure-tox/molecule/default/Dockerfile.j2 create mode 100644 roles/ensure-tox/molecule/default/molecule.yml create mode 100644 roles/ensure-tox/molecule/default/playbook.yml create mode 100644 roles/ensure-tox/molecule/default/vars/centos.yml create mode 100644 roles/ensure-tox/molecule/default/vars/fedora.yml diff --git a/roles/ensure-tox/README.rst b/roles/ensure-tox/README.rst index 60e0cac34..f3cfa359b 100644 --- a/roles/ensure-tox/README.rst +++ b/roles/ensure-tox/README.rst @@ -2,3 +2,22 @@ Ensure tox is installed If tox is not already installed, it will be installed via pip in the user install directory (i.e., "pip install --user"). + +**Role Variables** + +.. zuul:rolevar:: tox_upgrade + :default: false + + If you want the installation of latest tox, regardless presence on the + system, define `tox_upgrade: true`. + +.. zuul:rolevar:: tox_condition + :default: tox + + Python requirements condition to be used when installing tox. You can define + it to "tox>=3.8.0" to force a minimal version. + +.. zuul:rolevar:: tox_executable + :default: {{ ansible_python.executable }} -m tox + + Location of the tox executable. diff --git a/roles/ensure-tox/defaults/main.yaml b/roles/ensure-tox/defaults/main.yaml new file mode 100644 index 000000000..5d52ca238 --- /dev/null +++ b/roles/ensure-tox/defaults/main.yaml @@ -0,0 +1,7 @@ +# when set, it will always install latest version of tox using --user +# be aware that this may break /usr/bin/tox executable due to changed internals +tox_upgrade: false +tox_condition: tox +tox_environment: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_WARN_SCRIPT_LOCATION: "1" diff --git a/roles/ensure-tox/molecule/default/Dockerfile.j2 b/roles/ensure-tox/molecule/default/Dockerfile.j2 new file mode 100644 index 000000000..84d5ca291 --- /dev/null +++ b/roles/ensure-tox/molecule/default/Dockerfile.j2 @@ -0,0 +1,18 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +USER root + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates python-apt {{ item.pkg_extras | default('') }} && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python3 sudo python3-devel python3-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +CMD ["sh", "-c", "while true; do sleep 10000; done"]N \ No newline at end of file diff --git a/roles/ensure-tox/molecule/default/molecule.yml b/roles/ensure-tox/molecule/default/molecule.yml new file mode 100644 index 000000000..1310dd4dd --- /dev/null +++ b/roles/ensure-tox/molecule/default/molecule.yml @@ -0,0 +1,41 @@ +--- +# The MOLECULE_ vars used here are not officially endorsed by molecule, but we +# hope a future version will add implicit support for them. +driver: + name: docker +platforms: + + - name: centos7 + hostname: centos7 + image: centos:7 + dockerfile: Dockerfile.j2 + easy_install: + - pip + + - name: fedora + hostname: fedora + image: fedora:latest + dockerfile: Dockerfile.j2 + easy_install: + - pip + + - name: ubuntu1804 + hostname: ubuntu1804 + image: ubuntu:18.04 + dockerfile: Dockerfile.j2 + +provisioner: + name: ansible + log: true + inventory: + host_vars: + fedora: + ansible_python_interpreter: /usr/bin/python3 +scenario: + test_sequence: + - destroy + - create + - converge + - destroy +lint: + enabled: false diff --git a/roles/ensure-tox/molecule/default/playbook.yml b/roles/ensure-tox/molecule/default/playbook.yml new file mode 100644 index 000000000..0dd146626 --- /dev/null +++ b/roles/ensure-tox/molecule/default/playbook.yml @@ -0,0 +1,60 @@ +--- + +- name: Converge + hosts: all + gather_facts: true + tasks: + + - name: Include OS specific variables + include_vars: "{{ item }}" + failed_when: false + loop: + - "family-{{ ansible_os_family | lower }}.yml" + - "family-{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[0:2] | join('-') | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[0:3] | join('-') | lower }}.yml" + + - name: installing requirements + package: + name: "{{ pkg_tox_pre|default('curl') }}" + + # current code in ensure-tox makes the assumption that `pip` is installed. + - name: ensure-pip + become: yes + environment: + PIP_NO_WARN_SCRIPT_LOCATION: "1" + shell: | + set -exu + {{ ansible_python.executable }} -m pip --version || { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + {{ ansible_python.executable }} get-pip.py + } + {{ ansible_python.executable }} -m pip --version + + - name: include ensure-tox before installing system tox + include_role: + name: ensure-tox + + - name: include ensure-tox with upgrade + vars: + tox_upgrade: true + include_role: + name: ensure-tox + + - name: installing system tox + package: + name: "{{ pkg_tox | default('python-tox') }}" + update_cache: | + {{ ansible_version.full is version_compare('2.7.0', '>=') or omit }} + + - name: include ensure-tox after installing system tox + include_role: + name: ensure-tox + + - name: include ensure-tox with upgrade after installing system tox + vars: + tox_upgrade: true + include_role: + name: ensure-tox diff --git a/roles/ensure-tox/molecule/default/vars/centos.yml b/roles/ensure-tox/molecule/default/vars/centos.yml new file mode 100644 index 000000000..3b0156265 --- /dev/null +++ b/roles/ensure-tox/molecule/default/vars/centos.yml @@ -0,0 +1,3 @@ +pkg_tox_pre: + - curl + - epel-release diff --git a/roles/ensure-tox/molecule/default/vars/fedora.yml b/roles/ensure-tox/molecule/default/vars/fedora.yml new file mode 100644 index 000000000..afad103a6 --- /dev/null +++ b/roles/ensure-tox/molecule/default/vars/fedora.yml @@ -0,0 +1 @@ +pkg_tox: python3-tox \ No newline at end of file diff --git a/roles/ensure-tox/tasks/main.yaml b/roles/ensure-tox/tasks/main.yaml index b2b7487eb..be08ff5f6 100644 --- a/roles/ensure-tox/tasks/main.yaml +++ b/roles/ensure-tox/tasks/main.yaml @@ -1,2 +1,43 @@ -- name: Ensure tox is installed - shell: type tox || pip install --user tox +- name: Check if tox is accesible # noqa 305 + environment: "{{ tox_environment }}" + shell: | + set -eux + tox --version + {{ ansible_python.executable }} -m tox --version + register: result + failed_when: result.rc != 0 and 'not found' not in result.stderr + changed_when: false + +# installing tox with --user will create tox script inside ~/.local/bin but +# this folder is *not* included PATH in all distros. While most modern ones +# adopted it, there is no guarantee. +- name: Ensure tox is installed # noqa 305 + shell: | + set -eu + # {# for systems that do not have ~/.local/bin in PATH yet #} + # if [[ ":$PATH:" != *":${HOME}/.local/bin:"* ]] && [ -d "${HOME}/.local/bin" ]; then + # export PATH=${HOME}/.local/bin:$PATH + # fi + {% if result is failed or tox_upgrade %} + {{ ansible_python.executable }} -m pip install --user --upgrade '{{ tox_condition }}' + {% else %} + {{ ansible_python.executable }} -m pip install --user '{{ tox_condition }}' + {% endif %} + {{ ansible_python.executable }} -m tox --version + args: + executable: /bin/bash + +# We want to be sure that if someone calls 'tox' without having `~/.local/bin` +# in PATH, they still get a *working* copy of it. +- name: Ensure a system tox is also available + become: true + environment: "{{ tox_environment }}" + shell: | + set -eu + type tox >/dev/null || { + {# -s is key here to prevent it from finding tox from userdir #} + {{ ansible_python.executable }} -s -m pip install '{{ tox_condition }}' + } + tox --version + args: + executable: /bin/bash diff --git a/roles/fetch-tox-output/defaults/main.yaml b/roles/fetch-tox-output/defaults/main.yaml index dddc1c877..5d19ec4be 100644 --- a/roles/fetch-tox-output/defaults/main.yaml +++ b/roles/fetch-tox-output/defaults/main.yaml @@ -2,6 +2,6 @@ # TODO(mordred) This needs to switch back to not being venv - venv is OpenStack # specific. tox_envlist: venv -tox_executable: tox +tox_executable: "{{ ansible_python.executable }} -m tox" zuul_work_dir: "{{ zuul.project.src_dir }}" diff --git a/roles/tox/README.rst b/roles/tox/README.rst index 215c3ff97..5397c9032 100644 --- a/roles/tox/README.rst +++ b/roles/tox/README.rst @@ -12,7 +12,7 @@ Runs tox for a project Which tox environment to run. .. zuul:rolevar:: tox_executable - :default: tox + :default: {{ ansible_python.executable }} -m tox Location of the tox executable. diff --git a/roles/tox/defaults/main.yaml b/roles/tox/defaults/main.yaml index 1178db533..513ee3913 100644 --- a/roles/tox/defaults/main.yaml +++ b/roles/tox/defaults/main.yaml @@ -1,7 +1,9 @@ --- tox_environment: {} tox_envlist: venv -tox_executable: tox +# Calling tox as a module in order to avoid case where tox script gets broken +# because user installed newer version of tox into user packages. +tox_executable: "{{ ansible_python.executable }} -m tox" tox_extra_args: -vv tox_install_siblings: true diff --git a/tox.ini b/tox.ini index 757c0a7f2..db8139bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,26 @@ [tox] minversion = 1.6 skipsdist = True -envlist = linters +envlist = linters,molecule [testenv] +setenv = + ANSIBLE_FORCE_COLOR=1 + ANSIBLE_INVENTORY={toxinidir}/test/hosts.ini + ANSIBLE_NOCOWS=1 + ANSIBLE_RETRY_FILES_ENABLED=0 + ANSIBLE_STDOUT_CALLBACK=debug + PY_COLORS=1 + VIRTUAL_ENV={envdir} + # Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207 + PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command + PIP_DISABLE_PIP_VERSION_CHECK=1 +passenv = + ANSIBLE_* + DOCKER_* + MOLECULE_* + SSH_AUTH_SOCK + TERM basepython = python3 install_command = pip install {opts} {packages} deps = -r{toxinidir}/test-requirements.txt @@ -55,6 +72,20 @@ commands = [testenv:venv] commands = {posargs} +[testenv:molecule] +deps = + ansi2html # GPL (soft-dependency of pytest-html) + molecule[docker]>=2.22rc6 # MIT + ansible>=2.8.0 # GPL (on purpose newer than the one in test-requirements) + pytest # MIT + pytest-cov # MIT + pytest-molecule # MIT + pytest-html # MPL 2.0 + pytest-xdist # MIT + selinux # MIT +commands = + python -m pytest -ra --continue-on-collection-errors --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs} + [flake8] # These are ignored intentionally in openstack-infra projects; # please don't submit patches that solely correct them or enable them. diff --git a/zuul-tests.d/project.yaml b/zuul-tests.d/project.yaml index 7b6dfd12f..3c59c404e 100644 --- a/zuul-tests.d/project.yaml +++ b/zuul-tests.d/project.yaml @@ -8,11 +8,13 @@ - zuul-tox-docs - tox-py27 - tox-py35 + - tox-molecule gate: jobs: - zuul-tox-docs - tox-py27 - tox-py35 + - tox-molecule promote: jobs: - zuul-promote-docs