From 782501b22806ef0c9510d71ad53dc51c73f8e426 Mon Sep 17 00:00:00 2001 From: zaro0508 Date: Wed, 12 Jun 2013 14:26:41 -0700 Subject: [PATCH] make maven versioning and deployment workflow similiar to python This commit fixes bug 1182154 The maven workflow for versioning and deployment is at odds with how we like to do it. For versioning, our convention is to get info from SCM to set the project build versions. For deployments to remote repositories we prefer to use curl instead of mvn deploy due to maven security vulnerabilities. Our python builds have already been setup to set package versions from SCM and deploy to pypi using curl so this commit is to make maven versioning and deployments similiar our python versioning and deployment workflow. This commit does the following: Setup a project version string as an environement variable so we can pass it to maven builds. The version string is retrieved from information in git. This makes the build versioning workflow similar to how we build python packages. This setup expects a variable called '$project-version' in the root 'version' element (i.e. ${project-version}) of the maven project's pom.xml file. For general (throw away CI) builds we do the following: 1. generate a package 'myplugin.hpi' with version '1.3.0.4.a0bc21f' in the MANIFEST.MF file. The '4' is the number of commits since last tag. 2. publish 'myplugin-1.3.0.4.a0bc21f.hpi' to tarballs.o.o For release builds we do the following: 1. generate a package 'myplugin.hpi' with version '1.3.1' in the MANIFEST.MF file. 2. publish 'myplugin-1.3.1.hpi' to tarballs.o.o 3. publish 'myplugin-1.3.1.hpi' to repo.jenkins-ci.org Passes the jenkins credentials from hiera to the pypi slave so we can use it to deploy released plugins to repo.jenkins-ci.org Creates a generic jenkinsci-upload job that will deploy released jenkins plugin artifacts to a remote repository with user credentials from hiera. It will use the same curl deployment method as the pypi-upload job. Change-Id: If1306523a28da94ee970d96b7a788ca116348de7 Reviewed-on: https://review.openstack.org/31875 Reviewed-by: Monty Taylor Reviewed-by: Clark Boylan Reviewed-by: Jeremy Stanley Approved: Clark Boylan Tested-by: Jenkins --- .gitignore | 1 + manifests/site.pp | 2 + .../files/slave_scripts/maven-properties.sh | 30 ++++++++++++++ .../files/slave_scripts/maven-upload.sh | 32 +++++++++++++++ .../config/jenkins-plugin-jobs.yaml | 39 +++++++++++++++++-- .../openstack_project/files/zuul/layout.yaml | 3 ++ .../openstack_project/manifests/pypi_slave.pp | 17 +++++++- .../templates/jenkinsci-curl.erb | 1 + 8 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 modules/jenkins/files/slave_scripts/maven-properties.sh create mode 100644 modules/jenkins/files/slave_scripts/maven-upload.sh create mode 100644 modules/openstack_project/templates/jenkinsci-curl.erb diff --git a/.gitignore b/.gitignore index 14aa128097..eb9aba1f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ manifests/secrets.pp .test/ openstack_infra_config.egg-info/ /.project +/.pydevproject diff --git a/manifests/site.pp b/manifests/site.pp index bb944a7b79..c5bd570b49 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -344,6 +344,8 @@ node 'pypi.slave.openstack.org' { pypi_username => 'openstackci', pypi_password => hiera('pypi_password'), jenkins_ssh_public_key => $openstack_project::jenkins_ssh_key, + jenkinsci_username => hiera('jenkins_ci_org_user'), + jenkinsci_password => hiera('jenkins_ci_org_password'), } } diff --git a/modules/jenkins/files/slave_scripts/maven-properties.sh b/modules/jenkins/files/slave_scripts/maven-properties.sh new file mode 100644 index 0000000000..ca5328fd05 --- /dev/null +++ b/modules/jenkins/files/slave_scripts/maven-properties.sh @@ -0,0 +1,30 @@ +#!/bin/bash -ex + +# This file is a helper for versioning and deployment of +# maven projects. It sets up environment variables to +# pass to maven build commands so that we can generate +# versioned builds within the gerrit workflow. + +# get version info from scm +SCM_TAG=$(git describe --abbrev=0 --tags) || true +SCM_SHA=$(git rev-parse --short HEAD) + +# assumes format is like this '0.0.4-2-g135721c' +COMMITS_SINCE_TAG=`git describe | awk '{split($0,a,"-"); print a[2]}'` + +# just use git sha if there is no tag yet. +if [[ "${SCM_TAG}" == "" ]]; then + SCM_TAG=$SCM_SHA +fi + +# General build version should be something like '0.0.4.3.d4ee90c' +# Release build version should be something like '0.0.5' +if [[ "${COMMITS_SINCE_TAG}" == "" ]]; then + PROJECT_VER=$SCM_TAG +else + PROJECT_VER="$SCM_TAG.$COMMITS_SINCE_TAG.$SCM_SHA"; +fi + +echo "SCM_SHA=$SCM_SHA" >maven.properties +echo "COMMITS_SINCE_TAG=$COMMITS_SINCE_TAG" >>maven.properties +echo "PROJECT_VER=$PROJECT_VER" >>maven.properties diff --git a/modules/jenkins/files/slave_scripts/maven-upload.sh b/modules/jenkins/files/slave_scripts/maven-upload.sh new file mode 100644 index 0000000000..1610e0d1dc --- /dev/null +++ b/modules/jenkins/files/slave_scripts/maven-upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Upload java binaries to maven repositories + +PROJECT=$1 +REPO_URL=$2 +REPO_CRED_FILE=$3 + +FILENAME=`ls ${PROJECT}*.hpi` +# Strip project name and extension leaving only the version. +VERSION=`echo ${FILENAME} | sed -n "s/${PROJECT}-\(.*\).hpi/\1/p"` + +curl -X PUT \ + -u --config ${REPO_CRED_FILE} \ + --data-binary @${FILENAME} \ + -i "${REPO_URL}/${PROJECT}/${VERSION}/${FILENAME}" > /dev/null 2>&1 + +exit $? diff --git a/modules/openstack_project/files/jenkins_job_builder/config/jenkins-plugin-jobs.yaml b/modules/openstack_project/files/jenkins_job_builder/config/jenkins-plugin-jobs.yaml index 0a3492ecc9..85423c9e92 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/jenkins-plugin-jobs.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/jenkins-plugin-jobs.yaml @@ -3,6 +3,11 @@ project-type: maven node: precise + properties: + - inject: + script-file: /usr/local/jenkins/slave_scripts/maven-properties.sh + properties-file: maven.properties + prebuilders: - gerrit-git-prep @@ -12,7 +17,7 @@ group-id: '{maven-group-id}' artifact-id: '{name}' root-pom: pom.xml - goals: 'clean package' + goals: 'clean package -Dproject-version=$PROJECT_VER' publishers: - console-log @@ -22,6 +27,11 @@ project-type: maven node: precise + properties: + - inject: + script-file: /usr/local/jenkins/slave_scripts/maven-properties.sh + properties-file: maven.properties + prebuilders: - gerrit-git-prep @@ -31,13 +41,35 @@ group-id: '{maven-group-id}' artifact-id: '{name}' root-pom: pom.xml - goals: 'clean package -Dmaven.test.skip=true' + goals: 'clean package -Dmaven.test.skip=true -Dproject-version=$PROJECT_VER' publishers: - war: site: 'tarballs.openstack.org' warfile: 'target/{name}.hpi' - target: 'tarballs/ci/' + target: 'tarballs/ci/{name}/{name}-$PROJECT_VER.hpi' + - console-log-post + +- job-template: + name: '{name}-jenkinsci-upload' + node: pypi + + builders: + - shell: | + #!/bin/bash -xe + TAG=`echo $ZUUL_REF | sed 's/^refs.tags.//'` + FILENAME="{name}-$TAG.hpi" + + # copy plugin plugin artifacts from tarballs to local workspace + rm -rf *.hpi + curl -o $FILENAME http://{tarball-site}/ci/{name}/$FILENAME + + # deploy plugin artifacts from workspace to repo.jenkins-ci.org + JENKINSCI_REPO="http://repo.jenkins-ci.org/list/releases/org/jenkins-ci/plugins" + JENKINSCI_REPO_CREDS="/home/jenkins/.jenkinsci-curl" + /usr/local/jenkins/slave_scripts/maven-upload.sh {name} $JENKINSCI_REPO $JENKINSCI_REPO_CREDS + + publishers: - console-log-post - job-group: @@ -45,3 +77,4 @@ jobs: - 'gate-{name}-build' - '{name}-hpi-artifact' + - '{name}-jenkinsci-upload' diff --git a/modules/openstack_project/files/zuul/layout.yaml b/modules/openstack_project/files/zuul/layout.yaml index a4e6ae1de0..da8374ae78 100644 --- a/modules/openstack_project/files/zuul/layout.yaml +++ b/modules/openstack_project/files/zuul/layout.yaml @@ -1538,6 +1538,9 @@ projects: - gate-gearman-plugin-build post: - gearman-plugin-hpi-artifact + release: + - gearman-plugin-hpi-artifact: + - gearman-plugin-jenkinsci-upload - name: openstack-infra/zmq-event-publisher check: diff --git a/modules/openstack_project/manifests/pypi_slave.pp b/modules/openstack_project/manifests/pypi_slave.pp index 3787d00887..b6f44c4334 100644 --- a/modules/openstack_project/manifests/pypi_slave.pp +++ b/modules/openstack_project/manifests/pypi_slave.pp @@ -12,12 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. # -# Class to install dependencies for uploading python packages to pypi. +# Class to install dependencies for uploading python packages to pypi and +# maven repositories # class openstack_project::pypi_slave ( $pypi_password, $jenkins_ssh_public_key, - $pypi_username = 'openstackci' + $pypi_username = 'openstackci', + $jenkinsci_username, + $jenkinsci_password ) { class { 'openstack_project::slave': ssh_key => $jenkins_ssh_public_key, @@ -39,4 +42,14 @@ class openstack_project::pypi_slave ( content => template('openstack_project/pypicurl.erb'), require => File['/home/jenkins'], } + + file { '/home/jenkins/.jenkinsci-curl': + ensure => present, + owner => 'jenkins', + group => 'jenkins', + mode => '0600', + content => template('openstack_project/jenkinsci-curl.erb'), + require => File['/home/jenkins'], + } + } diff --git a/modules/openstack_project/templates/jenkinsci-curl.erb b/modules/openstack_project/templates/jenkinsci-curl.erb new file mode 100644 index 0000000000..d475bf8f8b --- /dev/null +++ b/modules/openstack_project/templates/jenkinsci-curl.erb @@ -0,0 +1 @@ +user = "<%= jenkinsci_username %>:<%= jenkinsci_password %>"