Select a pypi mirror before running tox.

Change-Id: I62a86b6a0ad30ff8cb387ddcc05668128248c6d0 Reviewed-on: https://review.openstack.org/23513 Reviewed-by: Jeremy Stanley <fungi@yuggoth.org> Approved: James E. Blair <corvus@inaugust.com> Tested-by: Jenkins
2013-03-04 21:26:37 -08:00 · 2013-03-04 21:26:37 -08:00 · 8c6df9f19c
commit 8c6df9f19c
parent 07c1647531
21 changed files with 326 additions and 51 deletions
--- a/modules/jenkins/files/slave_scripts/run-cover.sh
+++ b/modules/jenkins/files/slave_scripts/run-cover.sh
@ -4,6 +4,20 @@
 # resulting environment at the end so that we have a record of exactly
 # what packages we ended up testing.

+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 export NOSE_COVER_HTML=1

 venv=cover
--- a/modules/jenkins/files/slave_scripts/run-docs.sh
+++ b/modules/jenkins/files/slave_scripts/run-docs.sh
@ -7,6 +7,20 @@
 # what packages we ended up testing.
 #

+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 venv=venv

 mkdir -p doc/build
--- a/modules/jenkins/files/slave_scripts/run-pep8.sh
+++ b/modules/jenkins/files/slave_scripts/run-pep8.sh
@ -0,0 +1,34 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+set -o pipefail
+tox -v -epep8 | tee pep8.txt
+set +o pipefail
+
--- a/modules/jenkins/files/slave_scripts/run-pyflakes.sh
+++ b/modules/jenkins/files/slave_scripts/run-pyflakes.sh
@ -0,0 +1,31 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+tox -v -epyflakes
--- a/modules/jenkins/files/slave_scripts/run-pylint.sh
+++ b/modules/jenkins/files/slave_scripts/run-pylint.sh
@ -0,0 +1,34 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+set -o pipefail
+tox -v -epylint | tee pylint.txt
+set +o pipefail
+
--- a/modules/jenkins/files/slave_scripts/run-selenium.sh
+++ b/modules/jenkins/files/slave_scripts/run-selenium.sh
@ -7,6 +7,20 @@
 # what packages we ended up testing.
 #

+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 venv=venv

 VDISPLAY=99
--- a/modules/jenkins/files/slave_scripts/run-tarball.sh
+++ b/modules/jenkins/files/slave_scripts/run-tarball.sh
@ -0,0 +1,37 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+rm -f dist/*.tar.gz
+tox -evenv python setup.py sdist
+
+echo "SHA1sum: "
+sha1sum dist/*
+echo "MD5sum: "
+md5sum dist/*
--- a/modules/jenkins/files/slave_scripts/run-tox.sh
+++ b/modules/jenkins/files/slave_scripts/run-tox.sh
@ -13,10 +13,16 @@
 # "py27"/"jenkins27" respectively.

 version=$1
+org=$2
+project=$3

-if [ -z "$version" ]
+if [[ -z "$version" || -z "$org" || -z "$project" ]]
 then
-  echo "The tox environment python version (eg '27') must be the first argument."
+  echo "Usage: $? VERSION ORG PROJECT"
+  echo
+  echo "VERSION: The tox environment python version (eg '27')"
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
  exit 1
 fi

@ -32,6 +38,8 @@ trap "rm -rf $TMPDIR" EXIT

 sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh pre

+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 tox -e$venv
 result=$?

--- a/modules/jenkins/files/slave_scripts/select-mirror.sh
+++ b/modules/jenkins/files/slave_scripts/select-mirror.sh
@ -0,0 +1,51 @@
+#!/bin/bash -x
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+rm -f ~/.pydistutils.cfg
+mkdir -p ~/.pip
+rm -f ~/.pip/pip.conf
+
+# For OpenStack projects, use the pypi.openstack.org mirror exclusively
+if [ "$org" == "openstack" ]
+then
+echo $org
+    cat <<EOF > ~/.pydistutils.cfg
+[easy_install]
+index_url = http://pypi.openstack.org
+EOF
+    cat <<EOF > ~/.pip/pip.conf
+[global]
+index-url = http://pypi.openstack.org
+EOF
+else
+    cat <<EOF > ~/.pip/pip.conf
+[global]
+index-url = http://pypi.openstack.org
+extra-index-url = http://pypi.python.org/simple
+EOF
+fi
--- a/modules/jenkins/manifests/jenkinsuser.pp
+++ b/modules/jenkins/manifests/jenkinsuser.pp
@ -42,20 +42,6 @@ class jenkins::jenkinsuser(
    require => File['/home/jenkins'],
  }

-  file { '/home/jenkins/.pip/pip.conf':
-    ensure  => present,
-    owner   => 'jenkins',
-    group   => 'jenkins',
-    mode    => '0640',
-    source  => 'puppet:///modules/jenkins/pip.conf',
-    require => File['/home/jenkins/.pip'],
-  }
-
-  file { '/home/jenkins/.pydistutils.cfg':
-    ensure  => absent,
-    require => File['/home/jenkins'],
-  }
-
  file { '/home/jenkins/.gitconfig':
    ensure  => present,
    owner   => 'jenkins',
--- a/modules/openstack_project/files/jenkins_job_builder/config/heat.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/heat.yaml
@ -7,11 +7,14 @@

    builders:
      - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack
+          project: heat

    publishers:
      - coverage-log
      - console-log
+
 - job:
    name: 'dev-python-heatclient-coverage'
    node: 'oneiric'
@ -21,7 +24,9 @@

    builders:
      - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack
+          project: python-heatclient

    publishers:
      - coverage-log
--- a/modules/openstack_project/files/jenkins_job_builder/config/horizon.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/horizon.yaml
@ -7,7 +7,9 @@

    builders:
      - gerrit-git-prep
-      - selenium
+      - selenium:
+          github-org: openstack
+          project: horizon

    publishers:
      - console-log
--- a/modules/openstack_project/files/jenkins_job_builder/config/job-builder.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/job-builder.yaml
@ -8,7 +8,9 @@

    builders:
      - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: jenkins-job-builder

    publishers:
      - scp:
--- a/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml
@ -6,12 +6,12 @@
 - builder:
    name: coverage
    builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh {github-org} {project}"

 - builder:
    name: docs
    builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh {github-org} {project}"

 - builder:
    name: maven-test
@ -50,17 +50,17 @@
 - builder:
    name: pep8
    builders:
-      - shell: "set -o pipefail ; tox -v -epep8 | tee pep8.txt ; set +o pipefail"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pep8.sh {github-org} {project}"

 - builder:
    name: pylint
    builders:
-      - shell: "set -o pipefail ; tox -v -epylint | tee pylint.txt ; set +o pipefail"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pylint.sh {github-org} {project}"

 - builder:
    name: pyflakes
    builders:
-      - shell: "tox -v -epyflakes"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pyflakes.sh {github-org} {project}"

 - builder:
    name: puppet-syntax
@ -74,7 +74,7 @@
 - builder:
    name: selenium
    builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh {github-org} {project}"

 - builder:
    name: python26
@ -114,16 +114,7 @@
 - builder:
    name: tarball
    builders:
-      - shell: |
-          #!/bin/bash -xe
-
-          rm -f dist/*.tar.gz
-          tox -evenv python setup.py sdist
-
-          echo "SHA1sum: "
-          sha1sum dist/*
-          echo "MD5sum: "
-          md5sum dist/*
+      - shell: "/usr/local/jenkins/slave_scripts/run-tarball.sh {github-org} {project}"

 - builder:
    name: devstack-checkout
--- a/modules/openstack_project/files/jenkins_job_builder/config/openstack-ci-puppet.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/openstack-ci-puppet.yaml
@ -8,7 +8,9 @@
    builders:
      - gerrit-git-prep
      - puppet-syntax
-      - pyflakes
+      - pyflakes:
+          github-org: openstack-infra
+          project: config

    publishers:
      - console-log
@ -38,7 +40,7 @@

    builders:
      - gerrit-git-prep
-      - shell: "./tools/run-compare-xml.sh"
+      - shell: "./tools/run-compare-xml.sh openstack-infra config"

    publishers:
      - console-log
@ -53,7 +55,7 @@

    builders:
      - gerrit-git-prep
-      - shell: "./tools/run-layout.sh"
+      - shell: "./tools/run-layout.sh openstack-infra config"

    publishers:
      - console-log
@ -69,7 +71,9 @@

    builders:
      - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: config

    publishers:
      - scp:
--- a/modules/openstack_project/files/jenkins_job_builder/config/openstack-publish-jobs.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/openstack-publish-jobs.yaml
@ -7,7 +7,9 @@

    builders:
      - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - ftp:
--- a/modules/openstack_project/files/jenkins_job_builder/config/python-bitrot-jobs.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/python-bitrot-jobs.yaml
@ -70,7 +70,9 @@
      - timed: '0 6 * * *'

    builders:
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - console-log-periodic
--- a/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml
@ -12,7 +12,9 @@

    builders:
      - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - coverage-log-post
@ -29,7 +31,9 @@

    builders:
      - gerrit-git-prep
-      - pep8
+      - pep8:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - pep8
@ -46,7 +50,9 @@

    builders:
      - gerrit-git-prep
-      - pylint
+      - pylint:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - pylint
@ -215,7 +221,9 @@

    builders:
      - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - console-log
@ -232,7 +240,9 @@

    builders:
      - gerrit-git-prep
-      - pyflakes
+      - pyflakes:
+          github-org: '{github-org}'
+          project: '{name}'

    publishers:
      - console-log
@ -248,6 +258,7 @@
    builders:
      - gerrit-git-prep
      - tarball:
+          github-org: '{github-org}'
          project: '{name}'

    publishers:
@ -267,12 +278,13 @@

    builders:
      - gerrit-git-prep
+      - tarball:
+          github-org: '{github-org}'
+          project: '{name}'
      - shell: |
          #!/bin/bash -xe
          BRANCH=$ZUUL_REFNAME
          BRANCH_PATH=`echo $BRANCH | tr / -`
-
-          tox -v -evenv python setup.py sdist
          mv dist/* dist/{name}-$BRANCH_PATH.tar.gz

    publishers:
--- a/modules/openstack_project/files/jenkins_job_builder/config/zuul.yaml
+++ b/modules/openstack_project/files/jenkins_job_builder/config/zuul.yaml
@ -8,7 +8,9 @@

    builders:
      - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: zuul

    publishers:
      - scp:
@ -28,7 +30,9 @@

    builders:
      - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack-infra
+          project: zuul

    publishers:
      - coverage-log
--- a/tools/run-compare-xml.sh
+++ b/tools/run-compare-xml.sh
@ -16,6 +16,20 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.

+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  #TODO: make fatal in subsequent change: exit 1
+else
+  /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+fi
+
 rm -fr .test
 mkdir .test
 cd .test
--- a/tools/run-layout.sh
+++ b/tools/run-layout.sh
@ -14,6 +14,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.

+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  #TODO: make fatal in subsequent change: exit 1
+else
+  /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+fi
+
 rm -fr .test
 mkdir .test
 cd .test