make maven versioning and deployment workflow similiar to python

This commit fixes bug 1182154

The maven workflow for versioning and deployment is at odds with how
we like to do it.  For versioning, our convention is to get info
from SCM to set the project build versions.  For deployments
to remote repositories we prefer to use curl instead of mvn deploy
due to maven security vulnerabilities.  Our python builds
have already been setup to set package versions from SCM and
deploy to pypi using curl so this commit is to make
maven versioning and deployments similiar our python
versioning and deployment workflow.

This commit does the following:

Setup a project version string as an environement variable
so we can pass it to maven builds. The version string is
retrieved from information in git.  This makes the build versioning
workflow similar to how we build python packages.

  This setup expects a variable called '$project-version' in
  the root 'version' element (i.e. <version>${project-version}</version>)
  of the maven project's pom.xml file.

  For general (throw away CI) builds we do the following:
    1. generate a package 'myplugin.hpi' with version '1.3.0.4.a0bc21f'
       in the MANIFEST.MF file.  The '4' is the number of commits since
       last tag.
    2. publish 'myplugin-1.3.0.4.a0bc21f.hpi' to tarballs.o.o

  For release builds we do the following:
    1. generate a package 'myplugin.hpi' with version '1.3.1'
     in the MANIFEST.MF file.
    2. publish 'myplugin-1.3.1.hpi' to tarballs.o.o
    3. publish 'myplugin-1.3.1.hpi' to repo.jenkins-ci.org

Passes the jenkins credentials from hiera to the pypi slave so
we can use it to deploy released plugins to repo.jenkins-ci.org

Creates a generic jenkinsci-upload job that will deploy
released jenkins plugin artifacts to a remote repository with
user credentials from hiera. It will use the same curl deployment
method as the pypi-upload job.

Change-Id: If1306523a28da94ee970d96b7a788ca116348de7
Reviewed-on: https://review.openstack.org/31875
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins

.gitignore

@@ -9,3 +9,4 @@ manifests/secrets.pp
 .test/
 openstack_infra_config.egg-info/
 /.project
+/.pydevproject

manifests/site.pp

@@ -344,6 +344,8 @@ node 'pypi.slave.openstack.org' {
     pypi_username          => 'openstackci',
     pypi_password          => hiera('pypi_password'),
     jenkins_ssh_public_key => $openstack_project::jenkins_ssh_key,
+    jenkinsci_username     => hiera('jenkins_ci_org_user'),
+    jenkinsci_password     => hiera('jenkins_ci_org_password'),
   }
 }
 

modules/jenkins/files/slave_scripts/maven-properties.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -ex
+
+# This file is a helper for versioning and deployment of
+# maven projects.  It sets up environment variables to
+# pass to maven build commands so that we can generate
+# versioned builds within the gerrit workflow.
+
+# get version info from scm
+SCM_TAG=$(git describe --abbrev=0 --tags) || true
+SCM_SHA=$(git rev-parse --short HEAD)
+
+# assumes format is like this  '0.0.4-2-g135721c'
+COMMITS_SINCE_TAG=`git describe | awk '{split($0,a,"-"); print a[2]}'`
+
+# just use git sha if there is no tag yet.
+if [[ "${SCM_TAG}" == "" ]]; then
+    SCM_TAG=$SCM_SHA
+fi
+
+# General build version should be something like '0.0.4.3.d4ee90c'
+# Release build version should be something like '0.0.5'
+if [[ "${COMMITS_SINCE_TAG}" == "" ]]; then
+    PROJECT_VER=$SCM_TAG
+else
+    PROJECT_VER="$SCM_TAG.$COMMITS_SINCE_TAG.$SCM_SHA";
+fi
+
+echo "SCM_SHA=$SCM_SHA" >maven.properties
+echo "COMMITS_SINCE_TAG=$COMMITS_SINCE_TAG" >>maven.properties
+echo "PROJECT_VER=$PROJECT_VER" >>maven.properties

modules/jenkins/files/slave_scripts/maven-upload.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright 2013  Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Upload java binaries to maven repositories
+
+PROJECT=$1
+REPO_URL=$2
+REPO_CRED_FILE=$3
+
+FILENAME=`ls ${PROJECT}*.hpi`
+# Strip project name and extension leaving only the version.
+VERSION=`echo ${FILENAME} | sed -n "s/${PROJECT}-\(.*\).hpi/\1/p"`
+
+curl -X PUT \
+     -u --config ${REPO_CRED_FILE} \
+     --data-binary @${FILENAME} \
+     -i "${REPO_URL}/${PROJECT}/${VERSION}/${FILENAME}" > /dev/null 2>&1
+
+exit $?

modules/openstack_project/files/jenkins_job_builder/config/jenkins-plugin-jobs.yaml

@@ -3,6 +3,11 @@
     project-type: maven
     node: precise
 
+    properties:
+      - inject:
+          script-file: /usr/local/jenkins/slave_scripts/maven-properties.sh
+          properties-file: maven.properties
+
     prebuilders:
       - gerrit-git-prep
 
@@ -12,7 +17,7 @@
         group-id: '{maven-group-id}'
         artifact-id: '{name}'
       root-pom: pom.xml
-      goals: 'clean package'
+      goals: 'clean package -Dproject-version=$PROJECT_VER'
 
     publishers:
       - console-log
@@ -22,6 +27,11 @@
     project-type: maven
     node: precise
 
+    properties:
+      - inject:
+          script-file: /usr/local/jenkins/slave_scripts/maven-properties.sh
+          properties-file: maven.properties
+
     prebuilders:
       - gerrit-git-prep
 
@@ -31,13 +41,35 @@
         group-id: '{maven-group-id}'
         artifact-id: '{name}'
       root-pom: pom.xml
-      goals: 'clean package -Dmaven.test.skip=true'
+      goals: 'clean package -Dmaven.test.skip=true -Dproject-version=$PROJECT_VER'
 
     publishers:
       - war:
           site: 'tarballs.openstack.org'
           warfile: 'target/{name}.hpi'
-          target: 'tarballs/ci/'
+          target: 'tarballs/ci/{name}/{name}-$PROJECT_VER.hpi'
+      - console-log-post
+
+- job-template:
+    name: '{name}-jenkinsci-upload'
+    node: pypi
+
+    builders:
+      - shell: |
+          #!/bin/bash -xe
+          TAG=`echo $ZUUL_REF | sed 's/^refs.tags.//'`
+          FILENAME="{name}-$TAG.hpi"
+
+          # copy plugin plugin artifacts from tarballs to local workspace
+          rm -rf *.hpi
+          curl -o $FILENAME http://{tarball-site}/ci/{name}/$FILENAME
+
+          # deploy plugin artifacts from workspace to repo.jenkins-ci.org
+          JENKINSCI_REPO="http://repo.jenkins-ci.org/list/releases/org/jenkins-ci/plugins"
+          JENKINSCI_REPO_CREDS="/home/jenkins/.jenkinsci-curl"
+          /usr/local/jenkins/slave_scripts/maven-upload.sh {name} $JENKINSCI_REPO $JENKINSCI_REPO_CREDS
+
+    publishers:
       - console-log-post
 
 - job-group:
@@ -45,3 +77,4 @@
     jobs:
       - 'gate-{name}-build'
       - '{name}-hpi-artifact'
+      - '{name}-jenkinsci-upload'

modules/openstack_project/files/zuul/layout.yaml

@@ -1538,6 +1538,9 @@ projects:
       - gate-gearman-plugin-build
     post:
       - gearman-plugin-hpi-artifact
+    release:
+      - gearman-plugin-hpi-artifact:
+        - gearman-plugin-jenkinsci-upload
 
   - name: openstack-infra/zmq-event-publisher
     check:

modules/openstack_project/manifests/pypi_slave.pp

@@ -12,12 +12,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-# Class to install dependencies for uploading python packages to pypi.
+# Class to install dependencies for uploading python packages to pypi and
+# maven repositories
 #
 class openstack_project::pypi_slave (
   $pypi_password,
   $jenkins_ssh_public_key,
-  $pypi_username = 'openstackci'
+  $pypi_username = 'openstackci',
+  $jenkinsci_username,
+  $jenkinsci_password
 ) {
   class { 'openstack_project::slave':
     ssh_key => $jenkins_ssh_public_key,
@@ -39,4 +42,14 @@ class openstack_project::pypi_slave (
     content => template('openstack_project/pypicurl.erb'),
     require => File['/home/jenkins'],
   }
+
+  file { '/home/jenkins/.jenkinsci-curl':
+    ensure  => present,
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    mode    => '0600',
+    content => template('openstack_project/jenkinsci-curl.erb'),
+    require => File['/home/jenkins'],
+  }
+
 }

modules/openstack_project/templates/jenkinsci-curl.erb

@@ -0,0 +1 @@
+user = "<%= jenkinsci_username %>:<%= jenkinsci_password %>"