Run pypi uploads more securely.

To run pypi uploads more securely perform the `python setup.py sdist` on
a normal build slave, then copy the sdist to the pypi slave and perform
only the upload from that host.

Change-Id: Ie68d484ef6d129749186c792d0ced812ac25818b
Reviewed-on: https://review.openstack.org/16335
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins

manifests/site.pp

@@ -196,6 +196,13 @@ node 'tx.slave.openstack.org' {
   }
 }
 
+node 'pypi.slave.openstack.org' {
+  class { 'openstack_project::pypi_slave':
+    pypi_username => 'openstackci',
+    pypi_password => hiera('pypi_password')
+  }
+}
+
 # Rollout cgroups to precise slaves.
 node /^precise.*\.slave\.openstack\.org$/ {
   include jenkins::cgroups

modules/jenkins/files/slave_scripts/pypi-upload.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+PROJECT=$1
+
+FILENAME=`ls ${PROJECT}*.tar.gz`
+# Strip project name and extension leaving only the version.
+VERSION=`echo ${FILENAME} | sed -n "s/${PROJECT}-\(.*\).tar.gz/\1/p"`
+MD5_DIGEST=`md5sum ${FILENAME} | cut -d' ' -f1`
+
+curl --config /home/jenkins/.pypicurl \
+     -F "content=@${FILENAME};filename=${FILENAME}" \
+     -F ":action=file_upload" \
+     -F "protocol_version=1" \
+     -F "name=${PROJECT}" \
+     -F "version=${VERSION}" \
+     -F "file_type=sdist" \
+     -F "md5_digest=${MD5_DIGEST}" \
+     http://pypi.python.org/pypi > /dev/null 2>&1
+
+exit $?

modules/openstack_project/files/jenkins_job_builder/config/pypi-jobs.yaml

@@ -21,7 +21,6 @@
           site: '{tarball-publisher-site}'
       - console-log-post
 
-
 - job-template:
     name: '{name}-pypi'
     node: pypi
@@ -37,6 +36,48 @@
       - tarball:
           project: '{name}'
           site: '{tarball-publisher-site}'
+
+
+- job-template:
+    name: '{name}-pypi-sdist'
+    concurrent: false
+    block-downstream: true
+    node: precise
+
+    triggers:
+      - zuul-post
+
+    builders:
+      - gerrit-git-prep
+      - shell: tox -v -evenv python setup.py sdist
+
+    publishers:
+      - archive:
+          artifacts: 'dist/{name}*.tar.gz'
+          latest_only: true
+      - tarball:
+          project: '{name}'
+          site: '{tarball-publisher-site}'
+      - trigger:
+          project: '{name}-pypi-upload'
+      - console-log-post
+
+
+- job-template:
+    name: '{name}-pypi-upload'
+    concurrent: false
+    node: pypi
+
+    builders:
+      - copyartifact:
+          project: '{name}-pypi'
+          filter: '{name}*.tar.gz'
+      - shell: |
+          /usr/local/jenkins/slave_scripts/pypi-upload.sh {name}
+          # Remove copied artifact
+          rm -rf {name}*.tar.gz
+
+    publishers:
       - console-log-post
 
 
@@ -46,3 +87,5 @@
       - '{name}-docs'
       - '{name}-sdist-tarball'
       - '{name}-pypi'
+      - '{name}-pypi-sdist'
+      - '{name}-pypi-upload'

modules/openstack_project/files/zuul/layout.yaml

@@ -146,7 +146,7 @@ projects:
     post:
       - gerritbot-sdist-tarball
     publish:
-      - gerritbot-pypi
+      - gerritbot-pypi-sdist
 
   - name: openstack-ci/gerritlib
     check:
@@ -158,7 +158,7 @@ projects:
         - gate-gerritlib-pep8
         - gate-gerritlib-pyflakes
     publish:
-      - gerritlib-pypi
+      - gerritlib-pypi-sdist
 
   - name: openstack-ci/gerrit-verification-status-plugin
     check:
@@ -182,7 +182,7 @@ projects:
       - jenkins-job-builder-sdist-tarball
       - jenkins-job-builder-docs
     publish:
-      - jenkins-job-builder-pypi
+      - jenkins-job-builder-pypi-sdist
 
   - name: openstack-ci/meetbot
     check:

modules/openstack_project/manifests/pypi_slave.pp

@@ -0,0 +1,15 @@
+class openstack_project::pypi_slave (
+  pypi_password,
+  pypi_username = 'openstackci'
+) {
+  include openstack_project::slave
+
+  file { '/home/jenkins/.pypicurl':
+    ensure  => present,
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    mode    => '0600',
+    content => template('openstack_project/pypicurl.erb'),
+    require => File['/home/jenkins'],
+  }
+}

modules/openstack_project/templates/pypicurl.erb

@@ -0,0 +1 @@
+user = "<%= pypi_username %>:<%= pypi_password %>"