From 697a5df0a12693a7901d3ef259fa44e5f2e96f6b Mon Sep 17 00:00:00 2001
From: Emilien Macchi <emilien@redhat.com>
Date: Mon, 4 Apr 2016 18:09:23 -0400
Subject: [PATCH] Implement periodic job to update Puppet OpenStack constraints

In Puppet OpenStack CI, we use r10k to clone Puppet OpenStack modules
and all their dependencies from different places.
The Puppetfile is useful for both unit and functional tests, so we have
a consistency in what we test.

Until now, we were managing this Puppetfile without automation.
From time to time, we were updating the file but it does not scale,
regarding the work that needs to be done and the amount of dependencies
that never stop to grow up.

We recently implemented a script that is called
generate_puppetfile.sh (see dependency). This script aims to generate a
Puppetfile by cloning OpenStack & External modules.
The script will fetch tags and take the most recent one.
The script works as it is now, please look:
https://review.openstack.org/300696
And the Puppetfile that is generated.

This patch is an attempt to automate the run of this script, by using
the periodic job that is already in place for many other projects.

More features will come in next iterations:
* support stable branches
* add more logic in the script to detect whether or not tags exist (all
  our current modules have tags, but we never know...)

Change-Id: Id23ba86487d2223861b21941fa04acac0d49e3f3
---
 jenkins/jobs/puppet-module-jobs.yaml   | 15 +++++++
 jenkins/scripts/generate_puppetfile.sh | 57 ++++++++++++++++++++++++++
 jenkins/scripts/propose_update.sh      |  7 ++++
 zuul/layout.yaml                       |  2 +
 4 files changed, 81 insertions(+)
 create mode 100755 jenkins/scripts/generate_puppetfile.sh

diff --git a/jenkins/jobs/puppet-module-jobs.yaml b/jenkins/jobs/puppet-module-jobs.yaml
index 9255feea5a..0864b892a5 100644
--- a/jenkins/jobs/puppet-module-jobs.yaml
+++ b/jenkins/jobs/puppet-module-jobs.yaml
@@ -339,6 +339,21 @@
     publishers:
       - console-log
 
+- job:
+    name: propose-puppet-openstack-constraints
+    node: proposal
+
+    builders:
+      - revoke-sudo
+      - link-logs
+      - branch-git-prep:
+          branch: master
+      - net-info
+      - shell: |
+          export ZUUL_REFNAME=master
+          /usr/local/jenkins/slave_scripts/propose_update.sh \
+              puppet-openstack-constraints
+
 - job-group:
     name: 'puppet-check-jobs'
     puppet_version:
diff --git a/jenkins/scripts/generate_puppetfile.sh b/jenkins/scripts/generate_puppetfile.sh
new file mode 100755
index 0000000000..8a1fa489a5
--- /dev/null
+++ b/jenkins/scripts/generate_puppetfile.sh
@@ -0,0 +1,57 @@
+#!/bin/bash -xe
+#
+# Build a Puppetfile with latest dependencies
+#
+
+# cleanup
+rm -rf modules Puppetfile
+mkdir modules
+
+# header
+echo -e "# Auto-generated Puppetfile for Puppet OpenStack project\n" > Puppetfile
+
+# OpenStack Modules
+echo "## OpenStack modules" >> Puppetfile
+for p in $(cat openstack_modules.txt); do
+    # hack for puppet-openstack-integration
+    # where namespace is openstack_integration
+    title=$(echo $p | sed 's/-/_/g')
+    # TODO(emilien) we need to add support for stable branches
+    cat >> Puppetfile <<EOF
+mod '$title',
+  :git => 'https://git.openstack.org/openstack/puppet-$p',
+  :ref => 'master'
+
+EOF
+done
+
+# External Modules
+echo -e "## External modules" >> Puppetfile
+for e in $(cat external_modules.txt); do
+    namespace=$(echo $e | awk -F'/' '{print $1}' | cut -d "," -f 1)
+    module=$(echo $e | awk -F'/' '{print $2}' | cut -d "," -f 1)
+    title=$(echo $module | awk -F'/' '{print $1}' | cut -d "-" -f 2)
+    pin=$(echo $e | grep "," | cut -d "," -f 2)
+    if [ ! -z "$pin" ]; then
+        git ls-remote --exit-code https://github.com/$namespace/$module $pin
+        if (($? == 2)); then
+            echo "Wrong pin: $pin does not exist in $module module."
+            exit 1
+        else
+            tag=$pin
+        fi
+    else
+        git clone https://github.com/$namespace/$module modules/$module
+        tag=$(cd modules/$module; git describe --tags $(git rev-list --tags --max-count=1))
+        rm -rf modules/$module
+    fi
+    cat >> Puppetfile <<EOF
+mod '$title',
+  :git => 'https://github.com/$namespace/$module',
+  :ref => '$tag'
+
+EOF
+done
+
+# for debug
+cat Puppetfile
diff --git a/jenkins/scripts/propose_update.sh b/jenkins/scripts/propose_update.sh
index 58771f7335..ef0628c4c5 100755
--- a/jenkins/scripts/propose_update.sh
+++ b/jenkins/scripts/propose_update.sh
@@ -57,6 +57,13 @@ elif [ "$OWN_PROJECT" == "devstack-plugins-list" ] ; then
     function update {
         bash -ex tools/generate-devstack-plugins-list.sh $1
     }
+elif [ "$OWN_PROJECT" == "puppet-openstack-constraints" ] ; then
+    INITIAL_COMMIT_MSG="Updated from Puppet OpenStack modules constraints"
+    TOPIC="openstack/puppet/constraints"
+    PROJECTS=openstack/puppet-openstack-integration
+    function update {
+        bash /usr/local/jenkins/slave_scripts/generate_puppetfile.sh
+    }
 else
     echo "Unknown project $1" >2
     exit 1
diff --git a/zuul/layout.yaml b/zuul/layout.yaml
index 741808bec1..cf781bd4d8 100755
--- a/zuul/layout.yaml
+++ b/zuul/layout.yaml
@@ -9492,6 +9492,8 @@ projects:
       - name: puppet-check-jobs
       - name: puppet-openstack-integration-jobs-all
       - name: periodic-jobs-with-puppet-4-openstack-integration-jobs-all
+    periodic:
+      - propose-puppet-openstack-constraints
     experimental:
       - 'gate-puppet-openstack-integration-dsvm-multinode-nv'