From 6c8855ede4516f6d1299844fb1459706b3947261 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 8 Dec 2022 13:12:38 -0800 Subject: [PATCH] Add noxfile and switch to nox Tox v4 behaves significantly differently than v3, and some of the more complex things we do with tox would need an overhaul to continue to use it. Meanwhile, nox is much simpler and more flexible, so let's try using it. This adds a noxfile which should be equivalent to our tox.ini file. We still need to update the docs build (which involves changes to base jobs) before we can completely remove tox. Depends-On: https://review.opendev.org/868134 Change-Id: Ibebb0988d2702d310e46c437e58917db3f091382 --- .gitignore | 1 + .zuul.yaml | 85 +++++++----- noxfile.py | 127 ++++++++++++++++++ .../post-system-logs.yaml | 0 playbooks/{zuul-tox => zuul-nox}/pre.yaml | 0 setup.cfg | 7 + tools/yarn-build.sh | 73 ++++++++++ tox.ini | 7 - 8 files changed, 259 insertions(+), 41 deletions(-) create mode 100644 noxfile.py rename playbooks/{zuul-tox => zuul-nox}/post-system-logs.yaml (100%) rename playbooks/{zuul-tox => zuul-nox}/pre.yaml (100%) create mode 100755 tools/yarn-build.sh diff --git a/.gitignore b/.gitignore index c2f47953b5..ee87c9d503 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .mypy_cache .test .testrepository +.nox .tox .venv .coverage diff --git a/.zuul.yaml b/.zuul.yaml index 635ce5ac3f..c23bc6b862 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -40,44 +40,47 @@ zuul_ansible_version: 6 - job: - name: zuul-tox + name: zuul-nox description: | Zuul unit tests with ZooKeeper running - parent: tox + parent: nox nodeset: ubuntu-jammy - pre-run: playbooks/zuul-tox/pre.yaml - post-run: playbooks/zuul-tox/post-system-logs.yaml + pre-run: playbooks/zuul-nox/pre.yaml + post-run: playbooks/zuul-nox/post-system-logs.yaml vars: - tox_environment: + nox_environment: ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_TEST_ROOT: /tmp/zuul-test YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" + CI: "1" test_setup_environment: ZUUL_TEST_ROOT: /tmp/zuul-test YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" - job: - name: zuul-tox-remote - parent: tox + name: zuul-nox-remote + parent: nox nodeset: ubuntu-jammy timeout: 2700 # 45 minutes - pre-run: playbooks/zuul-tox/pre.yaml - post-run: playbooks/zuul-tox/post-system-logs.yaml + pre-run: playbooks/zuul-nox/pre.yaml + post-run: playbooks/zuul-nox/post-system-logs.yaml vars: - tox_envlist: remote - tox_environment: + nox_session: remote + nox_environment: ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}" ZUUL_REMOTE_KEEP: "true" + CI: "1" - job: + # Zuul cient uses this job so we can't just delete it yet. name: zuul-tox-zuul-client - parent: zuul-tox + parent: zuul-nox description: | Test that Zuul and zuul-client work together. required-projects: @@ -88,28 +91,42 @@ tox_envlist: zuul_client - job: - name: zuul-tox-py311 - parent: zuul-tox + name: zuul-nox-zuul-client + parent: zuul-nox + description: | + Test that Zuul and zuul-client work together. + required-projects: + - zuul/zuul + - zuul/zuul-client + vars: + zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}" + nox_session: zuul_client + +- job: + name: zuul-nox-py311 + parent: zuul-nox timeout: 7200 # 120 minutes vars: - tox_envlist: py311 + nox_keyword: tests + nox_force_python: "3.11" python_version: "3.11" - job: - name: zuul-tox-py38 - parent: zuul-tox + name: zuul-nox-py38 + parent: zuul-nox timeout: 7200 # 120 minutes vars: - tox_envlist: py38 + nox_keyword: tests + nox_force_python: "3.8" python_version: "3.8" nodeset: ubuntu-focal - job: - name: zuul-tox-py311-multi-scheduler - parent: zuul-tox-py311 + name: zuul-nox-py311-multi-scheduler + parent: zuul-nox-py311 voting: false vars: - tox_environment: + nox_environment: ZUUL_SCHEDULER_COUNT: 2 - job: @@ -283,13 +300,13 @@ jobs: - zuul-build-image - zuul-tox-docs - - tox-linters: + - nox-linters: vars: - tox_install_bindep: false + nox_install_bindep: false nodeset: ubuntu-jammy - - zuul-tox-py38 - - zuul-tox-py311 - - zuul-tox-py311-multi-scheduler + - zuul-nox-py38 + - zuul-nox-py311 + - zuul-nox-py311-multi-scheduler - zuul-build-dashboard-openstack-whitelabel - zuul-build-dashboard-software-factory - zuul-build-dashboard-opendev @@ -304,22 +321,22 @@ - web/.* nodeset: ubuntu-jammy - zuul-stream-functional-6 - - zuul-tox-remote + - zuul-nox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-build-image - - zuul-tox-zuul-client + - zuul-nox-zuul-client - zuul-build-python-release gate: jobs: - zuul-upload-image - zuul-tox-docs - - tox-linters: + - nox-linters: vars: - tox_install_bindep: false + nox_install_bindep: false nodeset: ubuntu-jammy - - zuul-tox-py38 - - zuul-tox-py311 + - zuul-nox-py38 + - zuul-nox-py311 - zuul-build-dashboard - nodejs-run-lint: vars: @@ -332,11 +349,11 @@ - web/.* nodeset: ubuntu-jammy - zuul-stream-functional-6 - - zuul-tox-remote + - zuul-nox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-upload-image - - zuul-tox-zuul-client + - zuul-nox-zuul-client - zuul-build-python-release promote: jobs: diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..80bab59220 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,127 @@ +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import multiprocessing +import os + +import nox + + +nox.options.error_on_external_run = True +nox.options.reuse_existing_virtualenvs = True +nox.options.sessions = ["tests-3", "linters"] + + +def set_env(session, var, default): + session.env[var] = os.environ.get(var, default) + + +def set_standard_env_vars(session): + set_env(session, 'OS_LOG_CAPTURE', '1') + set_env(session, 'OS_STDERR_CAPTURE', '1') + set_env(session, 'OS_STDOUT_CAPTURE', '1') + set_env(session, 'OS_TEST_TIMEOUT', '360') + set_env(session, 'SQLALCHEMY_WARN_20', '1') + session.env['PYTHONWARNINGS'] = ','.join([ + 'always::DeprecationWarning:zuul.driver.sql.sqlconnection', + 'always::DeprecationWarning:tests.base', + 'always::DeprecationWarning:tests.unit.test_database', + 'always::DeprecationWarning:zuul.driver.sql.alembic.env', + 'always::DeprecationWarning:zuul.driver.sql.alembic.script', + ]) + + +@nox.session(python='3') +def bindep(session): + set_standard_env_vars(session) + set_env(session, 'SQLALCHEMY_WARN_20', '1') + session.install('bindep') + session.install('.') + session.run('bindep', 'test') + + +@nox.session(python='3') +def cover(session): + set_standard_env_vars(session) + session.env['PYTHON'] = 'coverage run --source zuul --parallel-mode' + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run('stestr', 'run') + session.run('coverage', 'combine') + session.run('coverage', 'html', '-d', 'cover') + session.run('coverage', 'xml', '-o', 'cover/coverage.xml') + + +@nox.session(python='3') +def docs(session): + set_standard_env_vars(session) + session.install('-r', 'doc/requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run('sphinx-build', '-E', '-W', '-d', 'doc/build/doctrees', + '-b', 'html', 'doc/source/', 'doc/build/html') + + +@nox.session(python='3') +def linters(session): + set_standard_env_vars(session) + session.install('flake8', 'openapi-spec-validator') + session.run('flake8') + session.run('openapi-spec-validator', 'web/public/openapi.yaml') + + +@nox.session(python='3') +def tests(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run_always('tools/yarn-build.sh', external=True) + session.run_always('zuul-manage-ansible', '-v') + procs = max(int(multiprocessing.cpu_count() - 1), 1) + session.run('stestr', 'run', '--slowest', f'--concurrency={procs}', + *session.posargs) + + +@nox.session(python='3') +def remote(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run_always('zuul-manage-ansible', '-v') + session.run('stestr', 'run', '--test-path', './tests/remote') + + +@nox.session(python='3') +def venv(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run() + + +@nox.session(python='3') +def zuul_client(session): + set_standard_env_vars(session) + session.install('zuul-client', + '-r', 'test-requirements.txt', + '-r', 'requirements.txt') + session.install('-e', '.') + session.run_always('zuul-manage-ansible', '-v') + session.run( + 'stestr', 'run', '--concurrency=1', + '--test-path', './tests/zuul_client') diff --git a/playbooks/zuul-tox/post-system-logs.yaml b/playbooks/zuul-nox/post-system-logs.yaml similarity index 100% rename from playbooks/zuul-tox/post-system-logs.yaml rename to playbooks/zuul-nox/post-system-logs.yaml diff --git a/playbooks/zuul-tox/pre.yaml b/playbooks/zuul-nox/pre.yaml similarity index 100% rename from playbooks/zuul-tox/pre.yaml rename to playbooks/zuul-nox/pre.yaml diff --git a/setup.cfg b/setup.cfg index 79157c9ed7..e2eec4ba46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,3 +60,10 @@ allow_redefinition = True files = zuul ignore_missing_imports = True python_version = 3.6 + +[flake8] +# These are ignored intentionally in zuul projects; +# please don't submit patches that solely correct them or enable them. +ignore = E124,E125,E129,E252,E402,E741,H,W503,W504 +show-source = True +exclude = .venv,.tox,.nox,dist,doc,build,*.egg,node_modules diff --git a/tools/yarn-build.sh b/tools/yarn-build.sh new file mode 100755 index 0000000000..42eaffc7b8 --- /dev/null +++ b/tools/yarn-build.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2018 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This script checks if yarn is installed in the current path. If it is not, +# it will use nodeenv to install node, npm and yarn. +# Finally, it will install pip things. +if [[ ! $(command -v yarn) ]] +then + pip install nodeenv + # Initialize nodeenv and tell it to re-use the currently active virtualenv + attempts=0 + set +e + until nodeenv --python-virtualenv -n 16.14.0 ; do + ((attempts++)) + if [[ $attempts > 2 ]] + then + echo "Failed creating nodeenv" + exit 1 + fi + done + set -e + # Use -g because inside of the virtualenv '-g' means 'install into the' + # virtualenv - as opposed to installing into the local node_modules. + # Avoid writing a package-lock.json file since we don't use it. + # Avoid writing yarn into package.json. + npm install -g --no-package-lock --no-save yarn +fi +if [[ ! -f zuul/web/static/index.html ]] +then + mkdir -p zuul/web/static + ln -sfn ../zuul/web/static web/build + pushd web/ + if [[ -n "${YARN_REGISTRY}" ]] + then + echo "Using yarn registry: ${YARN_REGISTRY}" + sed -i "s#https://registry.yarnpkg.com#${YARN_REGISTRY}#" yarn.lock + fi + + # Be forgiving of package retrieval errors + attempts=0 + set +e + until yarn install; do + ((attempts++)) + if [[ $attempts > 2 ]] + then + echo "Failed installing npm packages" + exit 1 + fi + done + set -e + + yarn build + if [[ -n "${YARN_REGISTRY}" ]] + then + echo "Resetting yarn registry" + sed -i "s#${YARN_REGISTRY}#https://registry.yarnpkg.com#" yarn.lock + fi + popd +fi diff --git a/tox.ini b/tox.ini index 8d7fb8c22b..9c7fc5d280 100644 --- a/tox.ini +++ b/tox.ini @@ -105,10 +105,3 @@ passenv = ZUUL_ZK_KEY commands = stestr run --test-path ./tests/remote {posargs} - -[flake8] -# These are ignored intentionally in zuul projects; -# please don't submit patches that solely correct them or enable them. -ignore = E124,E125,E129,E252,E402,E741,H,W503,W504 -show-source = True -exclude = .venv,.tox,dist,doc,build,*.egg,node_modules