diff --git a/.gitignore b/.gitignore index c2f47953b5..ee87c9d503 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .mypy_cache .test .testrepository +.nox .tox .venv .coverage diff --git a/.zuul.yaml b/.zuul.yaml index 635ce5ac3f..c23bc6b862 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -40,44 +40,47 @@ zuul_ansible_version: 6 - job: - name: zuul-tox + name: zuul-nox description: | Zuul unit tests with ZooKeeper running - parent: tox + parent: nox nodeset: ubuntu-jammy - pre-run: playbooks/zuul-tox/pre.yaml - post-run: playbooks/zuul-tox/post-system-logs.yaml + pre-run: playbooks/zuul-nox/pre.yaml + post-run: playbooks/zuul-nox/post-system-logs.yaml vars: - tox_environment: + nox_environment: ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_TEST_ROOT: /tmp/zuul-test YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" + CI: "1" test_setup_environment: ZUUL_TEST_ROOT: /tmp/zuul-test YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" - job: - name: zuul-tox-remote - parent: tox + name: zuul-nox-remote + parent: nox nodeset: ubuntu-jammy timeout: 2700 # 45 minutes - pre-run: playbooks/zuul-tox/pre.yaml - post-run: playbooks/zuul-tox/post-system-logs.yaml + pre-run: playbooks/zuul-nox/pre.yaml + post-run: playbooks/zuul-nox/post-system-logs.yaml vars: - tox_envlist: remote - tox_environment: + nox_session: remote + nox_environment: ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}" ZUUL_REMOTE_KEEP: "true" + CI: "1" - job: + # Zuul cient uses this job so we can't just delete it yet. name: zuul-tox-zuul-client - parent: zuul-tox + parent: zuul-nox description: | Test that Zuul and zuul-client work together. required-projects: @@ -88,28 +91,42 @@ tox_envlist: zuul_client - job: - name: zuul-tox-py311 - parent: zuul-tox + name: zuul-nox-zuul-client + parent: zuul-nox + description: | + Test that Zuul and zuul-client work together. + required-projects: + - zuul/zuul + - zuul/zuul-client + vars: + zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}" + nox_session: zuul_client + +- job: + name: zuul-nox-py311 + parent: zuul-nox timeout: 7200 # 120 minutes vars: - tox_envlist: py311 + nox_keyword: tests + nox_force_python: "3.11" python_version: "3.11" - job: - name: zuul-tox-py38 - parent: zuul-tox + name: zuul-nox-py38 + parent: zuul-nox timeout: 7200 # 120 minutes vars: - tox_envlist: py38 + nox_keyword: tests + nox_force_python: "3.8" python_version: "3.8" nodeset: ubuntu-focal - job: - name: zuul-tox-py311-multi-scheduler - parent: zuul-tox-py311 + name: zuul-nox-py311-multi-scheduler + parent: zuul-nox-py311 voting: false vars: - tox_environment: + nox_environment: ZUUL_SCHEDULER_COUNT: 2 - job: @@ -283,13 +300,13 @@ jobs: - zuul-build-image - zuul-tox-docs - - tox-linters: + - nox-linters: vars: - tox_install_bindep: false + nox_install_bindep: false nodeset: ubuntu-jammy - - zuul-tox-py38 - - zuul-tox-py311 - - zuul-tox-py311-multi-scheduler + - zuul-nox-py38 + - zuul-nox-py311 + - zuul-nox-py311-multi-scheduler - zuul-build-dashboard-openstack-whitelabel - zuul-build-dashboard-software-factory - zuul-build-dashboard-opendev @@ -304,22 +321,22 @@ - web/.* nodeset: ubuntu-jammy - zuul-stream-functional-6 - - zuul-tox-remote + - zuul-nox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-build-image - - zuul-tox-zuul-client + - zuul-nox-zuul-client - zuul-build-python-release gate: jobs: - zuul-upload-image - zuul-tox-docs - - tox-linters: + - nox-linters: vars: - tox_install_bindep: false + nox_install_bindep: false nodeset: ubuntu-jammy - - zuul-tox-py38 - - zuul-tox-py311 + - zuul-nox-py38 + - zuul-nox-py311 - zuul-build-dashboard - nodejs-run-lint: vars: @@ -332,11 +349,11 @@ - web/.* nodeset: ubuntu-jammy - zuul-stream-functional-6 - - zuul-tox-remote + - zuul-nox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-upload-image - - zuul-tox-zuul-client + - zuul-nox-zuul-client - zuul-build-python-release promote: jobs: diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..80bab59220 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,127 @@ +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import multiprocessing +import os + +import nox + + +nox.options.error_on_external_run = True +nox.options.reuse_existing_virtualenvs = True +nox.options.sessions = ["tests-3", "linters"] + + +def set_env(session, var, default): + session.env[var] = os.environ.get(var, default) + + +def set_standard_env_vars(session): + set_env(session, 'OS_LOG_CAPTURE', '1') + set_env(session, 'OS_STDERR_CAPTURE', '1') + set_env(session, 'OS_STDOUT_CAPTURE', '1') + set_env(session, 'OS_TEST_TIMEOUT', '360') + set_env(session, 'SQLALCHEMY_WARN_20', '1') + session.env['PYTHONWARNINGS'] = ','.join([ + 'always::DeprecationWarning:zuul.driver.sql.sqlconnection', + 'always::DeprecationWarning:tests.base', + 'always::DeprecationWarning:tests.unit.test_database', + 'always::DeprecationWarning:zuul.driver.sql.alembic.env', + 'always::DeprecationWarning:zuul.driver.sql.alembic.script', + ]) + + +@nox.session(python='3') +def bindep(session): + set_standard_env_vars(session) + set_env(session, 'SQLALCHEMY_WARN_20', '1') + session.install('bindep') + session.install('.') + session.run('bindep', 'test') + + +@nox.session(python='3') +def cover(session): + set_standard_env_vars(session) + session.env['PYTHON'] = 'coverage run --source zuul --parallel-mode' + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run('stestr', 'run') + session.run('coverage', 'combine') + session.run('coverage', 'html', '-d', 'cover') + session.run('coverage', 'xml', '-o', 'cover/coverage.xml') + + +@nox.session(python='3') +def docs(session): + set_standard_env_vars(session) + session.install('-r', 'doc/requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run('sphinx-build', '-E', '-W', '-d', 'doc/build/doctrees', + '-b', 'html', 'doc/source/', 'doc/build/html') + + +@nox.session(python='3') +def linters(session): + set_standard_env_vars(session) + session.install('flake8', 'openapi-spec-validator') + session.run('flake8') + session.run('openapi-spec-validator', 'web/public/openapi.yaml') + + +@nox.session(python='3') +def tests(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run_always('tools/yarn-build.sh', external=True) + session.run_always('zuul-manage-ansible', '-v') + procs = max(int(multiprocessing.cpu_count() - 1), 1) + session.run('stestr', 'run', '--slowest', f'--concurrency={procs}', + *session.posargs) + + +@nox.session(python='3') +def remote(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run_always('zuul-manage-ansible', '-v') + session.run('stestr', 'run', '--test-path', './tests/remote') + + +@nox.session(python='3') +def venv(session): + set_standard_env_vars(session) + session.install('-r', 'requirements.txt', + '-r', 'test-requirements.txt') + session.install('-e', '.') + session.run() + + +@nox.session(python='3') +def zuul_client(session): + set_standard_env_vars(session) + session.install('zuul-client', + '-r', 'test-requirements.txt', + '-r', 'requirements.txt') + session.install('-e', '.') + session.run_always('zuul-manage-ansible', '-v') + session.run( + 'stestr', 'run', '--concurrency=1', + '--test-path', './tests/zuul_client') diff --git a/playbooks/zuul-tox/post-system-logs.yaml b/playbooks/zuul-nox/post-system-logs.yaml similarity index 100% rename from playbooks/zuul-tox/post-system-logs.yaml rename to playbooks/zuul-nox/post-system-logs.yaml diff --git a/playbooks/zuul-tox/pre.yaml b/playbooks/zuul-nox/pre.yaml similarity index 100% rename from playbooks/zuul-tox/pre.yaml rename to playbooks/zuul-nox/pre.yaml diff --git a/setup.cfg b/setup.cfg index 79157c9ed7..e2eec4ba46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,3 +60,10 @@ allow_redefinition = True files = zuul ignore_missing_imports = True python_version = 3.6 + +[flake8] +# These are ignored intentionally in zuul projects; +# please don't submit patches that solely correct them or enable them. +ignore = E124,E125,E129,E252,E402,E741,H,W503,W504 +show-source = True +exclude = .venv,.tox,.nox,dist,doc,build,*.egg,node_modules diff --git a/tools/yarn-build.sh b/tools/yarn-build.sh new file mode 100755 index 0000000000..42eaffc7b8 --- /dev/null +++ b/tools/yarn-build.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2018 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This script checks if yarn is installed in the current path. If it is not, +# it will use nodeenv to install node, npm and yarn. +# Finally, it will install pip things. +if [[ ! $(command -v yarn) ]] +then + pip install nodeenv + # Initialize nodeenv and tell it to re-use the currently active virtualenv + attempts=0 + set +e + until nodeenv --python-virtualenv -n 16.14.0 ; do + ((attempts++)) + if [[ $attempts > 2 ]] + then + echo "Failed creating nodeenv" + exit 1 + fi + done + set -e + # Use -g because inside of the virtualenv '-g' means 'install into the' + # virtualenv - as opposed to installing into the local node_modules. + # Avoid writing a package-lock.json file since we don't use it. + # Avoid writing yarn into package.json. + npm install -g --no-package-lock --no-save yarn +fi +if [[ ! -f zuul/web/static/index.html ]] +then + mkdir -p zuul/web/static + ln -sfn ../zuul/web/static web/build + pushd web/ + if [[ -n "${YARN_REGISTRY}" ]] + then + echo "Using yarn registry: ${YARN_REGISTRY}" + sed -i "s#https://registry.yarnpkg.com#${YARN_REGISTRY}#" yarn.lock + fi + + # Be forgiving of package retrieval errors + attempts=0 + set +e + until yarn install; do + ((attempts++)) + if [[ $attempts > 2 ]] + then + echo "Failed installing npm packages" + exit 1 + fi + done + set -e + + yarn build + if [[ -n "${YARN_REGISTRY}" ]] + then + echo "Resetting yarn registry" + sed -i "s#${YARN_REGISTRY}#https://registry.yarnpkg.com#" yarn.lock + fi + popd +fi diff --git a/tox.ini b/tox.ini index 8d7fb8c22b..9c7fc5d280 100644 --- a/tox.ini +++ b/tox.ini @@ -105,10 +105,3 @@ passenv = ZUUL_ZK_KEY commands = stestr run --test-path ./tests/remote {posargs} - -[flake8] -# These are ignored intentionally in zuul projects; -# please don't submit patches that solely correct them or enable them. -ignore = E124,E125,E129,E252,E402,E741,H,W503,W504 -show-source = True -exclude = .venv,.tox,dist,doc,build,*.egg,node_modules