From 58d622019d8e92b301699787180aafe16cde53ee Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Tue, 12 Nov 2019 10:56:02 -0600 Subject: [PATCH] Remove requirements-integration job This job was a legacy job that attempted to verify each project could be independently installed after all global-requirements had been installed. This job is no longer necessary since we have much better coverage with requirements-check and other jobs that make sure the requirements are compatible. It is also a legacy zuulv2 job, uses the long deprecated devstack-gate, and runs with Python 2.7. For historical reference, this was added originally to PBR back in 2013 [0] and moved to requirements in 2014 [1]. [0] https://opendev.org/openstack/pbr/commit/378261a8cba818a29b7e3d33b917de5ed0751579 [1] https://opendev.org/openstack/requirements/commit/8e073738db06086f949d2be377999356aee2d8d0 Change-Id: I29abcba929995f9307c392110160d889a29b47c3 Signed-off-by: Sean McGinnis --- .zuul.d/jobs.yaml | 93 ------------ .zuul.d/project.yaml | 2 - bindep.txt | 2 +- global-requirements.txt | 2 - .../requirements-integration-dsvm/post.yaml | 15 -- .../requirements-integration-dsvm/run.yaml | 135 ------------------ tools/README.txt | 5 - tools/integration.sh | 118 --------------- 8 files changed, 1 insertion(+), 371 deletions(-) delete mode 100644 playbooks/requirements-integration-dsvm/post.yaml delete mode 100644 playbooks/requirements-integration-dsvm/run.yaml delete mode 100644 tools/integration.sh diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index 236db0099c..680617ab9e 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -56,96 +56,3 @@ vars: tox_envlist: bindep -- job: - name: requirements-integration - parent: legacy-dsvm-base - run: playbooks/requirements-integration-dsvm/run.yaml - post-run: playbooks/requirements-integration-dsvm/post.yaml - timeout: 7800 - required-projects: - - openstack/devstack - - openstack/grenade - - openstack/pbr - - openstack/devstack-gate - - openstack/tripleo-ci - - openstack/aodh - - openstack/automaton - - openstack/ceilometer - - openstack/ceilometermiddleware - - openstack/cinder - - openstack/cliff - - openstack/debtcollector - - openstack/dib-utils - - openstack/diskimage-builder - - openstack/django_openstack_auth - - openstack/futurist - - openstack/glance - - openstack/glance_store - - openstack/heat - - openstack/heat-cfntools - - openstack/heat-templates - - openstack/horizon - - openstack/ironic - - openstack/ironic-lib - - openstack/ironic-python-agent - - openstack/keystone - - openstack/keystoneauth - - openstack/keystonemiddleware - - openstack/manila - - openstack/manila-ui - - openstack/neutron - - openstack/neutron-fwaas - - openstack/neutron-vpnaas - - openstack/nova - - openstack/octavia - - openstack/os-apply-config - - openstack/os-brick - - openstack/os-client-config - - openstack/os-collect-config - - openstack/os-net-config - - openstack/os-refresh-config - - openstack/osc-lib - - openstack/oslo.cache - - openstack/oslo.concurrency - - openstack/oslo.config - - openstack/oslo.context - - openstack/oslo.db - - openstack/oslo.i18n - - openstack/oslo.log - - openstack/oslo.messaging - - openstack/oslo.middleware - - openstack/oslo.policy - - openstack/oslo.reports - - openstack/oslo.rootwrap - - openstack/oslo.serialization - - openstack/oslo.service - - openstack/oslo.utils - - openstack/oslo.versionedobjects - - openstack/oslo.vmware - - openstack/pycadf - - openstack/python-cinderclient - - openstack/python-glanceclient - - openstack/python-heatclient - - openstack/python-ironicclient - - openstack/python-keystoneclient - - openstack/python-manilaclient - - openstack/python-neutronclient - - openstack/python-novaclient - - openstack/python-openstackclient - - openstack/python-saharaclient - - openstack/python-swiftclient - - openstack/python-troveclient - - openstack/python-zaqarclient - - openstack/requirements - - openstack/sahara - - openstack/sahara-dashboard - - openstack/stevedore - - openstack/swift - - openstack/taskflow - - openstack/tempest - - openstack/tooz - - openstack/tripleo-heat-templates - - openstack/tripleo-image-elements - - openstack/trove - - openstack/trove-dashboard - - openstack/zaqar diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index d93b6239a4..38ac196582 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -46,7 +46,6 @@ - cross-swift-py27 - cross-swift-py36 - openstack-tox-py36 - - requirements-integration - neutron-grenade-multinode: irrelevant-files: - ^(test-|)requirements.txt$ @@ -100,7 +99,6 @@ - cross-swift-py27 - cross-swift-py36 - openstack-tox-py36 - - requirements-integration - neutron-grenade-multinode: irrelevant-files: - ^(test-|)requirements.txt$ diff --git a/bindep.txt b/bindep.txt index 82ec960920..d248bb34e2 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,5 +1,5 @@ # These are needed to build all the things in global-requirements, which we do -# for integration testing. (see tools/integration.sh) +# for integration testing. app-crypt/mit-krb5 [platform:gentoo] krb5-devel [platform:redhat] diff --git a/global-requirements.txt b/global-requirements.txt index 3adf74757c..ee3b6b7de9 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -245,8 +245,6 @@ zVMCloudConnector;sys_platform!='win32' # Apache 2.0 License opentracing # Apache-2.0 jaeger-client # Apache-2.0 -# NOTE(flaper87): This added 2 packages to `integration.sh` -# uuid-dev and swig pyngus # Apache-2.0 pyOpenSSL # Apache-2.0 diff --git a/playbooks/requirements-integration-dsvm/post.yaml b/playbooks/requirements-integration-dsvm/post.yaml deleted file mode 100644 index e07f5510ae..0000000000 --- a/playbooks/requirements-integration-dsvm/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/requirements-integration-dsvm/run.yaml b/playbooks/requirements-integration-dsvm/run.yaml deleted file mode 100644 index b2faf0c281..0000000000 --- a/playbooks/requirements-integration-dsvm/run.yaml +++ /dev/null @@ -1,135 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-requirements-integration-dsvm from old job gate-requirements-integration-dsvm-ubuntu-xenial - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - https://opendev.org \ - openstack/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - # Define the entire projects list here so that what we - # test requirements against is independent of what d-g - # thinks is relevant. - export PROJECTS="openstack/devstack-gate $PROJECTS" - export PROJECTS="openstack/devstack $PROJECTS" - export PROJECTS="openstack/grenade $PROJECTS" - export PROJECTS="openstack/pbr $PROJECTS" - export PROJECTS="openstack/tripleo-ci $PROJECTS" - export PROJECTS="openstack/aodh $PROJECTS" - export PROJECTS="openstack/automaton $PROJECTS" - export PROJECTS="openstack/ceilometer $PROJECTS" - export PROJECTS="openstack/ceilometermiddleware $PROJECTS" - export PROJECTS="openstack/cinder $PROJECTS" - export PROJECTS="openstack/cliff $PROJECTS" - export PROJECTS="openstack/debtcollector $PROJECTS" - export PROJECTS="openstack/dib-utils $PROJECTS" - export PROJECTS="openstack/diskimage-builder $PROJECTS" - export PROJECTS="openstack/django_openstack_auth $PROJECTS" - export PROJECTS="openstack/futurist $PROJECTS" - export PROJECTS="openstack/glance $PROJECTS" - export PROJECTS="openstack/glance_store $PROJECTS" - export PROJECTS="openstack/heat $PROJECTS" - export PROJECTS="openstack/heat-cfntools $PROJECTS" - export PROJECTS="openstack/heat-templates $PROJECTS" - export PROJECTS="openstack/horizon $PROJECTS" - export PROJECTS="openstack/ironic $PROJECTS" - export PROJECTS="openstack/ironic-lib $PROJECTS" - export PROJECTS="openstack/ironic-python-agent $PROJECTS" - export PROJECTS="openstack/keystone $PROJECTS" - export PROJECTS="openstack/keystoneauth $PROJECTS" - export PROJECTS="openstack/keystonemiddleware $PROJECTS" - export PROJECTS="openstack/manila $PROJECTS" - export PROJECTS="openstack/manila-ui $PROJECTS" - export PROJECTS="openstack/zaqar $PROJECTS" - export PROJECTS="openstack/neutron $PROJECTS" - export PROJECTS="openstack/neutron-fwaas $PROJECTS" - export PROJECTS="openstack/octavia $PROJECTS" - export PROJECTS="openstack/neutron-vpnaas $PROJECTS" - export PROJECTS="openstack/nova $PROJECTS" - export PROJECTS="openstack/os-apply-config $PROJECTS" - export PROJECTS="openstack/os-brick $PROJECTS" - export PROJECTS="openstack/os-client-config $PROJECTS" - export PROJECTS="openstack/os-collect-config $PROJECTS" - export PROJECTS="openstack/os-net-config $PROJECTS" - export PROJECTS="openstack/os-refresh-config $PROJECTS" - export PROJECTS="openstack/osc-lib $PROJECTS" - export PROJECTS="openstack/oslo.cache $PROJECTS" - export PROJECTS="openstack/oslo.concurrency $PROJECTS" - export PROJECTS="openstack/oslo.config $PROJECTS" - export PROJECTS="openstack/oslo.context $PROJECTS" - export PROJECTS="openstack/oslo.db $PROJECTS" - export PROJECTS="openstack/oslo.i18n $PROJECTS" - export PROJECTS="openstack/oslo.log $PROJECTS" - export PROJECTS="openstack/oslo.messaging $PROJECTS" - export PROJECTS="openstack/oslo.middleware $PROJECTS" - export PROJECTS="openstack/oslo.policy $PROJECTS" - export PROJECTS="openstack/oslo.reports $PROJECTS" - export PROJECTS="openstack/oslo.rootwrap $PROJECTS" - export PROJECTS="openstack/oslo.utils $PROJECTS" - export PROJECTS="openstack/oslo.serialization $PROJECTS" - export PROJECTS="openstack/oslo.service $PROJECTS" - export PROJECTS="openstack/oslo.versionedobjects $PROJECTS" - export PROJECTS="openstack/oslo.vmware $PROJECTS" - export PROJECTS="openstack/pycadf $PROJECTS" - export PROJECTS="openstack/python-cinderclient $PROJECTS" - export PROJECTS="openstack/python-glanceclient $PROJECTS" - export PROJECTS="openstack/python-heatclient $PROJECTS" - export PROJECTS="openstack/python-ironicclient $PROJECTS" - export PROJECTS="openstack/python-keystoneclient $PROJECTS" - export PROJECTS="openstack/python-manilaclient $PROJECTS" - export PROJECTS="openstack/python-zaqarclient $PROJECTS" - export PROJECTS="openstack/python-neutronclient $PROJECTS" - export PROJECTS="openstack/python-novaclient $PROJECTS" - export PROJECTS="openstack/python-openstackclient $PROJECTS" - export PROJECTS="openstack/python-saharaclient $PROJECTS" - export PROJECTS="openstack/python-swiftclient $PROJECTS" - export PROJECTS="openstack/python-troveclient $PROJECTS" - export PROJECTS="openstack/requirements $PROJECTS" - export PROJECTS="openstack/sahara $PROJECTS" - export PROJECTS="openstack/sahara-dashboard $PROJECTS" - export PROJECTS="openstack/stevedore $PROJECTS" - export PROJECTS="openstack/swift $PROJECTS" - export PROJECTS="openstack/taskflow $PROJECTS" - export PROJECTS="openstack/tempest $PROJECTS" - export PROJECTS="openstack/tooz $PROJECTS" - export PROJECTS="openstack/tripleo-heat-templates $PROJECTS" - export PROJECTS="openstack/tripleo-image-elements $PROJECTS" - export PROJECTS="openstack/trove $PROJECTS" - export PROJECTS="openstack/trove-dashboard $PROJECTS" - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_REQS_INTEGRATION=1 - export PIP_PROCESS_DEPENDENCY_LINKS=1 - export USE_PYTHON3=true - function gate_hook { - set -x - bash $BASE/new/devstack-gate/devstack-vm-gate.sh && \ - bash -xe $BASE/new/requirements/tools/integration.sh $(cat $BASE/new/requirements/projects.txt) - } - export -f gate_hook - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/tools/README.txt b/tools/README.txt index cfa8b87fc8..72f4914459 100644 --- a/tools/README.txt +++ b/tools/README.txt @@ -50,11 +50,6 @@ grep-all.sh List a requirements specification and constratint for a given libarary -integration.sh --------------- - -Used in the gate! Install all the "$PROJECTS" to verify that g-r and the each project are compatible - noop-change.sh -------------- diff --git a/tools/integration.sh b/tools/integration.sh deleted file mode 100644 index 9c05967f98..0000000000 --- a/tools/integration.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -xe -# Parameters -# PBR_PIP_VERSION :- if not set, run pip's latest release, if set must be a -# valid reference file entry describing what pip to use. -# Bootstrappping the mkenv needs to install *a* pip -export PIPVERSION=pip -PIPFLAGS=${PIPFLAGS:-} - -function mkvenv { - venv=$1 - - rm -rf $venv - virtualenv $venv - $venv/bin/pip install -U $PIPVERSION wheel pbr -} - -function install_all_of_gr { - mkvenv $tmpdir/all_requirements - $tmpdir/all_requirements/bin/pip install -r $REPODIR/requirements/global-requirements.txt -} - -# BASE should be a directory with a subdir called "new" and in that -# dir, there should be a git repository for every entry in PROJECTS -BASE=${BASE:-/opt/stack} - -REPODIR=${REPODIR:-$BASE/new} - -root=$(dirname $0)/.. -sudo -H pip install bindep -sudo apt-get install -y --force-yes $(bindep -b python -f $root/bindep.txt) - -# And use ccache explitly -export PATH=/usr/lib/ccache:$PATH - -tmpdir=$(mktemp -d) - -# Set up a wheelhouse -mkvenv $tmpdir/wheelhouse -# Specific PIP version - must succeed to be useful. -# - build/download a local wheel so we don't hit the network on each venv. -if [ -n "${PBR_PIP_VERSION:-}" ]; then - td=$(mktemp -d) - $tmpdir/wheelhouse/bin/pip wheel -w $td $PBR_PIP_VERSION - # This version will now be installed in every new venv. - export PIPVERSION="$td/$(ls $td)" - $tmpdir/wheelhouse/bin/pip install -U $PIPVERSION - # We have pip in global-requirements as open-ended requirements, - # but since we don't use -U in any other invocations, our version - # of pip should be sticky. -fi - -#BRANCH -BRANCH=${OVERRIDE_ZUUL_BRANCH=:-master} -# PROJECTS is a list of projects that we're testing -PROJECTS=$* - -projectdir=$tmpdir/projects -mkdir -p $projectdir - -# Attempt to install all of global requirements -install_all_of_gr - -# Install requirementsrrepo itself. -$tmpdir/all_requirements/bin/pip install $REPODIR/requirements -UPDATE="$tmpdir/all_requirements/bin/update-requirements" - -# Check that we can generate an upper-requirements.txt file with the change -# that is being proposed. -$tmpdir/all_requirements/bin/generate-constraints -p /usr/bin/python2.7 \ - -p /usr/bin/python3 -b $REPODIR/requirements/blacklist.txt \ - -r $REPODIR/requirements/global-requirements.txt - -for PROJECT in $PROJECTS ; do - SHORT_PROJECT=$(basename $PROJECT) - if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/setup.py >/dev/null 2>&1 - then - # project doesn't use pbr - continue - fi - if [ $SHORT_PROJECT = 'tempest' ]; then - # Tempest doesn't really install - continue - fi - if [ $SHORT_PROJECT = 'requirements' ]; then - # requirements doesn't really install - continue - fi - if [[ "$BRANCH" =~ "stable" ]]; then - # When testing stable, only attempt to sync to projects that also - # have a corresponding stable branch. This prevents us from trying and - # failing to sync stable requirements to a library's master branch, - # when that same library may be listed and capped in global-requirements.txt. - proj_branch="$(cd $REPODIR/$SHORT_PROJECT && git rev-parse --symbolic-full-name --abbrev-ref HEAD)" - if [ "$proj_branch" != "$BRANCH" ]; then - continue - fi - fi - - # Clone from synced repo - shortprojectdir=$projectdir/$SHORT_PROJECT - git clone $REPODIR/$SHORT_PROJECT $shortprojectdir - $UPDATE --source $REPODIR/requirements $shortprojectdir - - # Test python setup.py install - installvenv=$tmpdir/install - mkvenv $installvenv - - installprojectdir=$projectdir/install$SHORT_PROJECT - git clone $shortprojectdir $installprojectdir - $UPDATE --source $REPODIR/requirements $installprojectdir - cd $installprojectdir - $installvenv/bin/python setup.py install - - # Ensure the install_package_data is doing the thing it should do - if [ $SHORT_PROJECT = 'nova' ]; then - find $installvenv | grep migrate.cfg - fi -done