Select a pypi mirror before running tox.

Change-Id: I62a86b6a0ad30ff8cb387ddcc05668128248c6d0
Reviewed-on: https://review.openstack.org/23513
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins

modules/jenkins/files/slave_scripts/run-cover.sh

@@ -4,6 +4,20 @@
 # resulting environment at the end so that we have a record of exactly
 # what packages we ended up testing.
 
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 export NOSE_COVER_HTML=1
 
 venv=cover

modules/jenkins/files/slave_scripts/run-docs.sh

@@ -7,6 +7,20 @@
 # what packages we ended up testing.
 #
 
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 venv=venv
 
 mkdir -p doc/build

modules/jenkins/files/slave_scripts/run-pep8.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+set -o pipefail
+tox -v -epep8 | tee pep8.txt
+set +o pipefail
+

modules/jenkins/files/slave_scripts/run-pyflakes.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+tox -v -epyflakes

modules/jenkins/files/slave_scripts/run-pylint.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+set -o pipefail
+tox -v -epylint | tee pylint.txt
+set +o pipefail
+

modules/jenkins/files/slave_scripts/run-selenium.sh

@@ -7,6 +7,20 @@
 # what packages we ended up testing.
 #
 
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 venv=venv
 
 VDISPLAY=99

modules/jenkins/files/slave_scripts/run-tarball.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -xe
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
+rm -f dist/*.tar.gz
+tox -evenv python setup.py sdist
+
+echo "SHA1sum: "
+sha1sum dist/*
+echo "MD5sum: "
+md5sum dist/*

modules/jenkins/files/slave_scripts/run-tox.sh

@@ -13,10 +13,16 @@
 # "py27"/"jenkins27" respectively.
 
 version=$1
+org=$2
+project=$3
 
-if [ -z "$version" ]
+if [[ -z "$version" || -z "$org" || -z "$project" ]]
 then
-  echo "The tox environment python version (eg '27') must be the first argument."
+  echo "Usage: $? VERSION ORG PROJECT"
+  echo
+  echo "VERSION: The tox environment python version (eg '27')"
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
   exit 1
 fi
 
@@ -32,6 +38,8 @@ trap "rm -rf $TMPDIR" EXIT
 
 sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh pre
 
+/usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+
 tox -e$venv
 result=$?
 

modules/jenkins/files/slave_scripts/select-mirror.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -x
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  exit 1
+fi
+
+rm -f ~/.pydistutils.cfg
+mkdir -p ~/.pip
+rm -f ~/.pip/pip.conf
+
+# For OpenStack projects, use the pypi.openstack.org mirror exclusively
+if [ "$org" == "openstack" ]
+then
+echo $org
+    cat <<EOF > ~/.pydistutils.cfg
+[easy_install]
+index_url = http://pypi.openstack.org
+EOF
+    cat <<EOF > ~/.pip/pip.conf
+[global]
+index-url = http://pypi.openstack.org
+EOF
+else
+    cat <<EOF > ~/.pip/pip.conf
+[global]
+index-url = http://pypi.openstack.org
+extra-index-url = http://pypi.python.org/simple
+EOF
+fi

modules/jenkins/manifests/jenkinsuser.pp

@@ -42,20 +42,6 @@ class jenkins::jenkinsuser(
     require => File['/home/jenkins'],
   }
 
-  file { '/home/jenkins/.pip/pip.conf':
-    ensure  => present,
-    owner   => 'jenkins',
-    group   => 'jenkins',
-    mode    => '0640',
-    source  => 'puppet:///modules/jenkins/pip.conf',
-    require => File['/home/jenkins/.pip'],
-  }
-
-  file { '/home/jenkins/.pydistutils.cfg':
-    ensure  => absent,
-    require => File['/home/jenkins'],
-  }
-
   file { '/home/jenkins/.gitconfig':
     ensure  => present,
     owner   => 'jenkins',

modules/openstack_project/files/jenkins_job_builder/config/heat.yaml

@@ -7,11 +7,14 @@
 
     builders:
       - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack
+          project: heat
 
     publishers:
       - coverage-log
       - console-log
+
 - job:
     name: 'dev-python-heatclient-coverage'
     node: 'oneiric'
@@ -21,7 +24,9 @@
 
     builders:
       - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack
+          project: python-heatclient
 
     publishers:
       - coverage-log

modules/openstack_project/files/jenkins_job_builder/config/horizon.yaml

@@ -7,7 +7,9 @@
 
     builders:
       - gerrit-git-prep
-      - selenium
+      - selenium:
+          github-org: openstack
+          project: horizon
 
     publishers:
       - console-log

modules/openstack_project/files/jenkins_job_builder/config/job-builder.yaml

@@ -8,7 +8,9 @@
 
     builders:
       - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: jenkins-job-builder
 
     publishers:
       - scp:

modules/openstack_project/files/jenkins_job_builder/config/macros.yaml

@@ -6,12 +6,12 @@
 - builder:
     name: coverage
     builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh {github-org} {project}"
 
 - builder:
     name: docs
     builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh {github-org} {project}"
 
 - builder:
     name: maven-test
@@ -50,17 +50,17 @@
 - builder:
     name: pep8
     builders:
-      - shell: "set -o pipefail ; tox -v -epep8 | tee pep8.txt ; set +o pipefail"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pep8.sh {github-org} {project}"
 
 - builder:
     name: pylint
     builders:
-      - shell: "set -o pipefail ; tox -v -epylint | tee pylint.txt ; set +o pipefail"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pylint.sh {github-org} {project}"
 
 - builder:
     name: pyflakes
     builders:
-      - shell: "tox -v -epyflakes"
+      - shell: "/usr/local/jenkins/slave_scripts/run-pyflakes.sh {github-org} {project}"
 
 - builder:
     name: puppet-syntax
@@ -74,7 +74,7 @@
 - builder:
     name: selenium
     builders:
-      - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh"
+      - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh {github-org} {project}"
 
 - builder:
     name: python26
@@ -114,16 +114,7 @@
 - builder:
     name: tarball
     builders:
-      - shell: |
+      - shell: "/usr/local/jenkins/slave_scripts/run-tarball.sh {github-org} {project}"
-          #!/bin/bash -xe
-
-          rm -f dist/*.tar.gz
-          tox -evenv python setup.py sdist
-
-          echo "SHA1sum: "
-          sha1sum dist/*
-          echo "MD5sum: "
-          md5sum dist/*
 
 - builder:
     name: devstack-checkout

modules/openstack_project/files/jenkins_job_builder/config/openstack-ci-puppet.yaml

@@ -8,7 +8,9 @@
     builders:
       - gerrit-git-prep
       - puppet-syntax
-      - pyflakes
+      - pyflakes:
+          github-org: openstack-infra
+          project: config
 
     publishers:
       - console-log
@@ -38,7 +40,7 @@
 
     builders:
       - gerrit-git-prep
-      - shell: "./tools/run-compare-xml.sh"
+      - shell: "./tools/run-compare-xml.sh openstack-infra config"
 
     publishers:
       - console-log
@@ -53,7 +55,7 @@
 
     builders:
       - gerrit-git-prep
-      - shell: "./tools/run-layout.sh"
+      - shell: "./tools/run-layout.sh openstack-infra config"
 
     publishers:
       - console-log
@@ -69,7 +71,9 @@
 
     builders:
       - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: config
 
     publishers:
       - scp:

modules/openstack_project/files/jenkins_job_builder/config/openstack-publish-jobs.yaml

@@ -7,7 +7,9 @@
 
     builders:
       - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - ftp:

modules/openstack_project/files/jenkins_job_builder/config/python-bitrot-jobs.yaml

@@ -70,7 +70,9 @@
       - timed: '0 6 * * *'
 
     builders:
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - console-log-periodic

modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml

@@ -12,7 +12,9 @@
 
     builders:
       - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - coverage-log-post
@@ -29,7 +31,9 @@
 
     builders:
       - gerrit-git-prep
-      - pep8
+      - pep8:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - pep8
@@ -46,7 +50,9 @@
 
     builders:
       - gerrit-git-prep
-      - pylint
+      - pylint:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - pylint
@@ -215,7 +221,9 @@
 
     builders:
       - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - console-log
@@ -232,7 +240,9 @@
 
     builders:
       - gerrit-git-prep
-      - pyflakes
+      - pyflakes:
+          github-org: '{github-org}'
+          project: '{name}'
 
     publishers:
       - console-log
@@ -248,6 +258,7 @@
     builders:
       - gerrit-git-prep
       - tarball:
+          github-org: '{github-org}'
           project: '{name}'
 
     publishers:
@@ -267,12 +278,13 @@
 
     builders:
       - gerrit-git-prep
+      - tarball:
+          github-org: '{github-org}'
+          project: '{name}'
       - shell: |
           #!/bin/bash -xe
           BRANCH=$ZUUL_REFNAME
           BRANCH_PATH=`echo $BRANCH | tr / -`
-
-          tox -v -evenv python setup.py sdist
           mv dist/* dist/{name}-$BRANCH_PATH.tar.gz
 
     publishers:

modules/openstack_project/files/jenkins_job_builder/config/zuul.yaml

@@ -8,7 +8,9 @@
 
     builders:
       - gerrit-git-prep
-      - docs
+      - docs:
+          github-org: openstack-infra
+          project: zuul
 
     publishers:
       - scp:
@@ -28,7 +30,9 @@
 
     builders:
       - gerrit-git-prep
-      - coverage
+      - coverage:
+          github-org: openstack-infra
+          project: zuul
 
     publishers:
       - coverage-log

tools/run-compare-xml.sh

@@ -16,6 +16,20 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  #TODO: make fatal in subsequent change: exit 1
+else
+  /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+fi
+
 rm -fr .test
 mkdir .test
 cd .test

tools/run-layout.sh

@@ -14,6 +14,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+org=$1
+project=$2
+
+if [[ -z "$org" || -z "$project" ]]
+then
+  echo "Usage: $0 ORG PROJECT"
+  echo
+  echo "ORG: The project organization (eg 'openstack')"
+  echo "PROJECT: The project name (eg 'nova')"
+  #TODO: make fatal in subsequent change: exit 1
+else
+  /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project
+fi
+
 rm -fr .test
 mkdir .test
 cd .test