Add noxfile and switch to nox

Tox v4 behaves significantly differently than v3, and some of the
more complex things we do with tox would need an overhaul to
continue to use it.  Meanwhile, nox is much simpler and more
flexible, so let's try using it.

This adds a noxfile which should be equivalent to our tox.ini file.
We still need to update the docs build (which involves changes to
base jobs) before we can completely remove tox.

Depends-On: https://review.opendev.org/868134
Change-Id: Ibebb0988d2702d310e46c437e58917db3f091382
This commit is contained in:
James E. Blair 2022-12-08 13:12:38 -08:00
parent 07ab843dd3
commit 6c8855ede4
8 changed files with 259 additions and 41 deletions

1
.gitignore vendored
View File

@ -7,6 +7,7 @@
.mypy_cache .mypy_cache
.test .test
.testrepository .testrepository
.nox
.tox .tox
.venv .venv
.coverage .coverage

View File

@ -40,44 +40,47 @@
zuul_ansible_version: 6 zuul_ansible_version: 6
- job: - job:
name: zuul-tox name: zuul-nox
description: | description: |
Zuul unit tests with ZooKeeper running Zuul unit tests with ZooKeeper running
parent: tox parent: nox
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
pre-run: playbooks/zuul-tox/pre.yaml pre-run: playbooks/zuul-nox/pre.yaml
post-run: playbooks/zuul-tox/post-system-logs.yaml post-run: playbooks/zuul-nox/post-system-logs.yaml
vars: vars:
tox_environment: nox_environment:
ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
ZUUL_TEST_ROOT: /tmp/zuul-test ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
CI: "1"
test_setup_environment: test_setup_environment:
ZUUL_TEST_ROOT: /tmp/zuul-test ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
- job: - job:
name: zuul-tox-remote name: zuul-nox-remote
parent: tox parent: nox
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
timeout: 2700 # 45 minutes timeout: 2700 # 45 minutes
pre-run: playbooks/zuul-tox/pre.yaml pre-run: playbooks/zuul-nox/pre.yaml
post-run: playbooks/zuul-tox/post-system-logs.yaml post-run: playbooks/zuul-nox/post-system-logs.yaml
vars: vars:
tox_envlist: remote nox_session: remote
tox_environment: nox_environment:
ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}" ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
ZUUL_REMOTE_KEEP: "true" ZUUL_REMOTE_KEEP: "true"
CI: "1"
- job: - job:
# Zuul cient uses this job so we can't just delete it yet.
name: zuul-tox-zuul-client name: zuul-tox-zuul-client
parent: zuul-tox parent: zuul-nox
description: | description: |
Test that Zuul and zuul-client work together. Test that Zuul and zuul-client work together.
required-projects: required-projects:
@ -88,28 +91,42 @@
tox_envlist: zuul_client tox_envlist: zuul_client
- job: - job:
name: zuul-tox-py311 name: zuul-nox-zuul-client
parent: zuul-tox parent: zuul-nox
description: |
Test that Zuul and zuul-client work together.
required-projects:
- zuul/zuul
- zuul/zuul-client
vars:
zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}"
nox_session: zuul_client
- job:
name: zuul-nox-py311
parent: zuul-nox
timeout: 7200 # 120 minutes timeout: 7200 # 120 minutes
vars: vars:
tox_envlist: py311 nox_keyword: tests
nox_force_python: "3.11"
python_version: "3.11" python_version: "3.11"
- job: - job:
name: zuul-tox-py38 name: zuul-nox-py38
parent: zuul-tox parent: zuul-nox
timeout: 7200 # 120 minutes timeout: 7200 # 120 minutes
vars: vars:
tox_envlist: py38 nox_keyword: tests
nox_force_python: "3.8"
python_version: "3.8" python_version: "3.8"
nodeset: ubuntu-focal nodeset: ubuntu-focal
- job: - job:
name: zuul-tox-py311-multi-scheduler name: zuul-nox-py311-multi-scheduler
parent: zuul-tox-py311 parent: zuul-nox-py311
voting: false voting: false
vars: vars:
tox_environment: nox_environment:
ZUUL_SCHEDULER_COUNT: 2 ZUUL_SCHEDULER_COUNT: 2
- job: - job:
@ -283,13 +300,13 @@
jobs: jobs:
- zuul-build-image - zuul-build-image
- zuul-tox-docs - zuul-tox-docs
- tox-linters: - nox-linters:
vars: vars:
tox_install_bindep: false nox_install_bindep: false
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
- zuul-tox-py38 - zuul-nox-py38
- zuul-tox-py311 - zuul-nox-py311
- zuul-tox-py311-multi-scheduler - zuul-nox-py311-multi-scheduler
- zuul-build-dashboard-openstack-whitelabel - zuul-build-dashboard-openstack-whitelabel
- zuul-build-dashboard-software-factory - zuul-build-dashboard-software-factory
- zuul-build-dashboard-opendev - zuul-build-dashboard-opendev
@ -304,22 +321,22 @@
- web/.* - web/.*
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
- zuul-stream-functional-6 - zuul-stream-functional-6
- zuul-tox-remote - zuul-nox-remote
- zuul-quick-start: - zuul-quick-start:
requires: nodepool-container-image requires: nodepool-container-image
dependencies: zuul-build-image dependencies: zuul-build-image
- zuul-tox-zuul-client - zuul-nox-zuul-client
- zuul-build-python-release - zuul-build-python-release
gate: gate:
jobs: jobs:
- zuul-upload-image - zuul-upload-image
- zuul-tox-docs - zuul-tox-docs
- tox-linters: - nox-linters:
vars: vars:
tox_install_bindep: false nox_install_bindep: false
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
- zuul-tox-py38 - zuul-nox-py38
- zuul-tox-py311 - zuul-nox-py311
- zuul-build-dashboard - zuul-build-dashboard
- nodejs-run-lint: - nodejs-run-lint:
vars: vars:
@ -332,11 +349,11 @@
- web/.* - web/.*
nodeset: ubuntu-jammy nodeset: ubuntu-jammy
- zuul-stream-functional-6 - zuul-stream-functional-6
- zuul-tox-remote - zuul-nox-remote
- zuul-quick-start: - zuul-quick-start:
requires: nodepool-container-image requires: nodepool-container-image
dependencies: zuul-upload-image dependencies: zuul-upload-image
- zuul-tox-zuul-client - zuul-nox-zuul-client
- zuul-build-python-release - zuul-build-python-release
promote: promote:
jobs: jobs:

127
noxfile.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright 2022 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import multiprocessing
import os
import nox
nox.options.error_on_external_run = True
nox.options.reuse_existing_virtualenvs = True
nox.options.sessions = ["tests-3", "linters"]
def set_env(session, var, default):
session.env[var] = os.environ.get(var, default)
def set_standard_env_vars(session):
set_env(session, 'OS_LOG_CAPTURE', '1')
set_env(session, 'OS_STDERR_CAPTURE', '1')
set_env(session, 'OS_STDOUT_CAPTURE', '1')
set_env(session, 'OS_TEST_TIMEOUT', '360')
set_env(session, 'SQLALCHEMY_WARN_20', '1')
session.env['PYTHONWARNINGS'] = ','.join([
'always::DeprecationWarning:zuul.driver.sql.sqlconnection',
'always::DeprecationWarning:tests.base',
'always::DeprecationWarning:tests.unit.test_database',
'always::DeprecationWarning:zuul.driver.sql.alembic.env',
'always::DeprecationWarning:zuul.driver.sql.alembic.script',
])
@nox.session(python='3')
def bindep(session):
set_standard_env_vars(session)
set_env(session, 'SQLALCHEMY_WARN_20', '1')
session.install('bindep')
session.install('.')
session.run('bindep', 'test')
@nox.session(python='3')
def cover(session):
set_standard_env_vars(session)
session.env['PYTHON'] = 'coverage run --source zuul --parallel-mode'
session.install('-r', 'requirements.txt',
'-r', 'test-requirements.txt')
session.install('-e', '.')
session.run('stestr', 'run')
session.run('coverage', 'combine')
session.run('coverage', 'html', '-d', 'cover')
session.run('coverage', 'xml', '-o', 'cover/coverage.xml')
@nox.session(python='3')
def docs(session):
set_standard_env_vars(session)
session.install('-r', 'doc/requirements.txt',
'-r', 'test-requirements.txt')
session.install('-e', '.')
session.run('sphinx-build', '-E', '-W', '-d', 'doc/build/doctrees',
'-b', 'html', 'doc/source/', 'doc/build/html')
@nox.session(python='3')
def linters(session):
set_standard_env_vars(session)
session.install('flake8', 'openapi-spec-validator')
session.run('flake8')
session.run('openapi-spec-validator', 'web/public/openapi.yaml')
@nox.session(python='3')
def tests(session):
set_standard_env_vars(session)
session.install('-r', 'requirements.txt',
'-r', 'test-requirements.txt')
session.install('-e', '.')
session.run_always('tools/yarn-build.sh', external=True)
session.run_always('zuul-manage-ansible', '-v')
procs = max(int(multiprocessing.cpu_count() - 1), 1)
session.run('stestr', 'run', '--slowest', f'--concurrency={procs}',
*session.posargs)
@nox.session(python='3')
def remote(session):
set_standard_env_vars(session)
session.install('-r', 'requirements.txt',
'-r', 'test-requirements.txt')
session.install('-e', '.')
session.run_always('zuul-manage-ansible', '-v')
session.run('stestr', 'run', '--test-path', './tests/remote')
@nox.session(python='3')
def venv(session):
set_standard_env_vars(session)
session.install('-r', 'requirements.txt',
'-r', 'test-requirements.txt')
session.install('-e', '.')
session.run()
@nox.session(python='3')
def zuul_client(session):
set_standard_env_vars(session)
session.install('zuul-client',
'-r', 'test-requirements.txt',
'-r', 'requirements.txt')
session.install('-e', '.')
session.run_always('zuul-manage-ansible', '-v')
session.run(
'stestr', 'run', '--concurrency=1',
'--test-path', './tests/zuul_client')

View File

@ -60,3 +60,10 @@ allow_redefinition = True
files = zuul files = zuul
ignore_missing_imports = True ignore_missing_imports = True
python_version = 3.6 python_version = 3.6
[flake8]
# These are ignored intentionally in zuul projects;
# please don't submit patches that solely correct them or enable them.
ignore = E124,E125,E129,E252,E402,E741,H,W503,W504
show-source = True
exclude = .venv,.tox,.nox,dist,doc,build,*.egg,node_modules

73
tools/yarn-build.sh Executable file
View File

@ -0,0 +1,73 @@
#!/bin/bash
# Copyright 2018 Red Hat, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# This script checks if yarn is installed in the current path. If it is not,
# it will use nodeenv to install node, npm and yarn.
# Finally, it will install pip things.
if [[ ! $(command -v yarn) ]]
then
pip install nodeenv
# Initialize nodeenv and tell it to re-use the currently active virtualenv
attempts=0
set +e
until nodeenv --python-virtualenv -n 16.14.0 ; do
((attempts++))
if [[ $attempts > 2 ]]
then
echo "Failed creating nodeenv"
exit 1
fi
done
set -e
# Use -g because inside of the virtualenv '-g' means 'install into the'
# virtualenv - as opposed to installing into the local node_modules.
# Avoid writing a package-lock.json file since we don't use it.
# Avoid writing yarn into package.json.
npm install -g --no-package-lock --no-save yarn
fi
if [[ ! -f zuul/web/static/index.html ]]
then
mkdir -p zuul/web/static
ln -sfn ../zuul/web/static web/build
pushd web/
if [[ -n "${YARN_REGISTRY}" ]]
then
echo "Using yarn registry: ${YARN_REGISTRY}"
sed -i "s#https://registry.yarnpkg.com#${YARN_REGISTRY}#" yarn.lock
fi
# Be forgiving of package retrieval errors
attempts=0
set +e
until yarn install; do
((attempts++))
if [[ $attempts > 2 ]]
then
echo "Failed installing npm packages"
exit 1
fi
done
set -e
yarn build
if [[ -n "${YARN_REGISTRY}" ]]
then
echo "Resetting yarn registry"
sed -i "s#${YARN_REGISTRY}#https://registry.yarnpkg.com#" yarn.lock
fi
popd
fi

View File

@ -105,10 +105,3 @@ passenv =
ZUUL_ZK_KEY ZUUL_ZK_KEY
commands = commands =
stestr run --test-path ./tests/remote {posargs} stestr run --test-path ./tests/remote {posargs}
[flake8]
# These are ignored intentionally in zuul projects;
# please don't submit patches that solely correct them or enable them.
ignore = E124,E125,E129,E252,E402,E741,H,W503,W504
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg,node_modules