Initial librarian-puppet-simple configuration

This change adds support to leverage a Puppetfile located in the
deployment folder to add additional upstream modules.  A new script
called update_modules.sh is now available in the deployment directory
that will run librarian-puppet-simple to update the modules.  The noop
tests have been updated to leverage this script prior to running the
tests. Additionally the fuel-library spec has been updated to execute
the update_modules.sh script as part of the rpm build process.

It should be noted that this is going to use librarian-puppet-simple
which only supports git references and tarballs and provides no
dependancy lookup. This is a much simpler way of pulling in the modules.

Partial blueprint: fuel-puppet-librarian

Change-Id: I5e628f159d2d11121af741bcc1218f292cd2b96e

deployment/.gitignore

@@ -0,0 +1,4 @@
+Puppetfile.lock
+.tmp
+
+# add modules being managed via librarian-puppet under here

deployment/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+# this is used to bundle install librarian puppet for the deployment folder
+
+group :development, :test do
+  gem 'librarian-puppet-simple', :require => false
+end
+
+if puppetversion = ENV['PUPPET_GEM_VERSION']
+  gem 'puppet', puppetversion, :require => false
+else
+  gem 'puppet', :require => false
+end
+
+# vim:ft=ruby

deployment/Puppetfile

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+#^syntax detection
+# See https://github.com/bodepd/librarian-puppet-simple for additional docs
+#
+# Important information for fuel-library:
+# With librarian-puppet-simple you *must* remove the existing folder from the
+# repo prior to trying to run librarian-puppet as it will not remove the folder
+# for you and you may run into some errors.
+#
+############
+# Examples #
+############
+# From git repo
+#  mod 'stdlib',
+#    :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git',
+#    :ref => '4.6.x'
+#
+# From tarbal
+#  mod 'stdlib',
+#    :tarbal => 'https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.6.0.tar.gz'
+#
+#

deployment/update_modules.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+###############################################################################
+#
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+###############################################################################
+#
+# update_modules.sh
+#
+#  This script uses librarian-puppet-simple to populate the puppet folder with
+#  upstream puppet modules.  By default, it assumes librarian-puppet-simple is
+#  already available to the environment or it will fail. You can provide command
+#  line options to have the script use bundler to install librarian-puppet-simple
+#  if neccessary.
+#
+# Parameters:
+#  -b - Use bundler to install librarian-puppet (optional)
+#  -p <puppet_version> - Puppet version to use with bundler (optional)
+#  -h <bundle_dir> - Folder to be used as the home directory for bundler (optional)
+#  -g <gem_home> - Folder to be used as the gem directory (optional)
+#  -v - Verbose printing, turns on set -x (optional)
+#  -? - This usage information
+#
+# Variables:
+#  PUPPET_GEM_VERSION - the version of puppet to be pulled down by bundler
+#                       Defaults to '3.4.3'
+#  BUNDLE_DIR - The folder to store the bundle gems in.
+#               Defaults to '/var/tmp/.bundle_home'
+#  GEM_HOME - The folder to store the gems in to not require root.
+#               Defaults to '/var/tmp/.gem_home'
+#
+#  NOTE: These variables can be overriden via bash environment variable with the
+#        same name or via the command line paramters.
+#
+# Author: Alex Schultz <aschultz@mirantis.com>
+#
+###############################################################################
+set -e
+
+usage() {
+  cat <<EOF
+  Usage: $(basename $0) [-b] [-p <puppet_version>] [-h <bundle_dir>] [-g <gem_home>] [-?]
+
+Options:
+  -b - Use bundler instead of assuming librarian-puppet is available
+  -p <puppet_version> - Puppet version to use with bundler
+  -h <bundle_dir> - Folder to be used as the home directory for bundler
+  -g <gem_home> - Folder to be used as the gem directory
+  -v - Verbose printing of commands
+  -? - This usage information
+
+EOF
+  exit 1
+}
+
+while getopts ":bp:l:h:v" opt; do
+  case $opt in
+    b)
+      USE_BUNDLER=true
+      BUNDLER_EXEC="bundle exec"
+      ;;
+    p)
+      PUPPET_GEM_VERSION=$OPTARG
+      ;;
+    h)
+      BUNDLE_DIR=$OPTARG
+      ;;
+    g)
+      GEM_HOME=$OPTARG
+      ;;
+    v)
+      set -x
+      ;;
+    \?)
+      usage
+      ;;
+    :)
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+shift "$((OPTIND-1))"
+
+DEPLOYMENT_DIR=$(cd `dirname $0` && pwd -P)
+export PUPPET_GEM_VERSION=${PUPPET_GEM_VERSION:-'3.4.3'}
+export BUNDLE_DIR=${BUNDLE_DIR:-'/var/tmp/.bundle_home'}
+export GEM_HOME=${GEM_HOME:-'/var/tmp/.gem_home'}
+
+# We need to be in the deployment directory to run librarian-puppet-simple
+cd $DEPLOYMENT_DIR
+
+if [ "$USE_BUNDLER" = true ]; then
+  # ensure bundler is installed
+  bundle --version
+
+  # update bundler modules
+  bundle update
+fi
+
+# run librarian-puppet install to populate the modules
+$BUNDLER_EXEC librarian-puppet install --path=puppet

specs/fuel-library7.0.spec

@@ -13,6 +13,7 @@ Source0: %{name}-%{version}.tar.gz
 Provides: fuel-library
 BuildArch: noarch
 BuildRoot: %{_tmppath}/fuel-library-%{version}-%{release}
+BuildRequires: ruby21-rubygem-librarian-puppet-simple
 Requires: fuel-misc
 
 %define files_source %{_builddir}/%{name}-%{version}/files
@@ -29,6 +30,11 @@ This package contains deployment manifests and code to execute provisioning of m
 %prep
 %setup -cq
 
+%build
+if test -x %{_builddir}/%{name}-%{version}/deployment/update_modules.sh; then
+  %{_builddir}/%{name}-%{version}/deployment/update_modules.sh
+fi
+
 %install
 mkdir -p %{buildroot}/etc/puppet/%{openstack_version}/modules/
 mkdir -p %{buildroot}/etc/puppet/%{openstack_version}/manifests/

utils/jenkins/fuel_noop_tests.rb

@@ -110,6 +110,9 @@ module NoopTests
         ENV['SPEC_PUPPET_LOGS_DIR'] = dir
         @options[:puppet_logs_dir] = dir
       end
+      opts.on('-u', '--update-librarian-puppet', 'Run librarian-puppet update in the deployment directory prior to testing') do
+        @options[:update_librarian_puppet] = true
+      end
     end
     optparse.parse!
     @options
@@ -159,6 +162,12 @@ module NoopTests
     File.expand_path File.join File.dirname(__FILE__), '..', '..', 'deployment', 'puppet', 'osnailyfacter', 'modular'
   end
 
+  # the base directory that houses all the puppet module directory
+  # @return [String]
+  def self.deployment_directory
+    File.expand_path File.join File.dirname(__FILE__), '..', '..', 'deployment'
+  end
+
   # LISTERS #
 
   # find all modular task files
@@ -211,6 +220,16 @@ module NoopTests
     result
   end
 
+  # run the clode block inside the deployment driectory
+  # and then return back
+  def self.inside_deployment_directory
+    current_directory = Dir.pwd
+    Dir.chdir deployment_directory
+    result = yield
+    Dir.chdir current_directory if current_directory
+    result
+  end
+
   # find all modular tasks that have no corresponding specs
   # @return [Array<String>]
   def self.puppet_tasks_without_specs
@@ -369,6 +388,22 @@ module NoopTests
     end
   end
 
+  # run librarian-puppet to fetch modules as necessary
+  def self.prepare_library
+    # these are needed to ensure we have the correctly bundle
+    ENV['PUPPET_GEM_VERSION'] = PUPPET_GEM_VERSION unless ENV['PUPPET_GEM_VERSION']
+    ENV['BUNDLE_DIR'] = File.join workspace, BUNDLE_DIR
+    command = './update_modules.sh -v'
+    # pass the bundle parameter to update_modules if specified for this script
+    command = command + ' -b' if options[:bundle]
+    inside_deployment_directory do
+      puts "-> Starting update_modules script"
+      system command
+      raise 'Unable to update upstream puppet modules using librarian-puppet' if $?.exitstatus != 0
+      puts "-> Finished update_modules script"
+    end
+  end
+
   # add color codes to a line
   # @param [Integer] code Color code
   # @param [String] string Text string
@@ -458,6 +493,11 @@ module NoopTests
       exit 0
     end
 
+    # run librarian-puppet update to prepare the library
+    if options[:update_librarian_puppet]
+        prepare_library
+    end
+
     success, result = for_every_astute_yaml do
       if options[:run_individually]
         run_all_specs_individually

utils/jenkins/fuel_noop_tests.sh

@@ -5,5 +5,5 @@ DIR=`dirname $0`
 echo 'Ruby version:'
 ruby --version
 
-"${DIR}/fuel_noop_tests.rb" -b -d -m ${@}
+"${DIR}/fuel_noop_tests.rb" -b -d -u -m ${@}