create external lib for all actions

move all code to an external library
This commit is contained in:
Dan Bode
2012-12-03 12:18:19 -08:00
parent 3081627a36
commit d5531da2fb
2 changed files with 306 additions and 239 deletions

257
Rakefile
View File

@@ -1,55 +1,14 @@
require 'yaml'
require 'rubygems'
def cmd_system (cmd)
result = system cmd
raise(RuntimeError, $?) unless $?.success?
result
end
def git_cmd(cmd)
command = 'git ' + cmd
Open3.popen3(*command) do |i, o, e, t|
raise StandardError, e.read unless (t ? t.value : $?).success?
o.read.split("\n")
end
end
def on_box (box, cmd)
cmd_system("vagrant ssh #{box} -c '#{cmd}'")
end
def deploy_two_node
require 'vagrant'
env = Vagrant::Environment.new(:cwd => base_dir, :ui_class => Vagrant::UI::Colored)
build(:openstack_controller, env)
build(:compute1, env)
end
def base_dir
File.expand_path(File.dirname(__FILE__))
end
# bring vagrant vm with image name up
def build(instance, env)
unless vm = env.vms[instance]
puts "invalid VM: #{instance}"
else
if vm.created?
puts "VM: #{instance} was already created"
else
# be very fault tolerant :)
begin
# this will always fail
vm.up(:provision => true)
rescue Exception => e
puts e.class
puts e
end
end
end
end
require File.join(base_dir, 'lib', 'puppetlabs', 'os_tester')
include Puppetlabs::OsTester
namespace :openstack do
@@ -58,10 +17,9 @@ namespace :openstack do
cmd_system('librarian-puppet install')
end
desc 'destroy all vms'
task 'destroy' do
puts "About to destroy all vms..."
cmd_system('vagrant destroy -f')
puts "Destroyed all vms"
destroy_all_vms
end
desc 'deploys the entire environment'
@@ -72,54 +30,23 @@ namespace :openstack do
end
remote_name = 'bodepd'
namespace :git do
cwd = base_dir
desc 'for all repos in the module directory, add a read/write remote'
desc 'for all repos in the module directory, add a read/write remote (hardcoded to bodepd)'
task :dev_setup do
each_repo do |module_name|
# need to handle more failure cases
remotes = git_cmd('remote')
if remotes.include?(remote_name)
puts "Did not have to add remote #{remote_name} to #{module_name}"
elsif ! remotes.include?('origin')
raise(Exception, "Repo #{module_name} has no remote called origin, failing")
else
remote_url = git_cmd('remote show origin').detect {|x| x =~ /\s+Push\s+URL: / }
if remote_url =~ /(git|https?):\/\/(.+)\/(.+)?\/(.+)/
url = "git@#{$2}:#{remote_name}/#{$4}"
else
puts "remote_url #{remote_url} did not have the expected format. weird..."
end
puts "Adding remote #{remote_name} as #{url}"
git_cmd("remote add #{remote_name} #{url}")
end
end
dev_setup('bodped')
end
desc 'pull the latest version of all code'
task :pull_all do
each_repo do |module_name|
puts "Pulling repo: #{module_name}"
puts ' ' + git_cmd('pull').join("\n ")
end
pull_all
end
desc 'shows the current state of code that has not been commited'
task :status_all do
each_repo do |module_name|
status = git_cmd('status')
if status.include?('nothing to commit (working directory clean)')
puts "Module #{module_name} has not changed" if verbose
else
puts "Uncommitted changes for: #{module_name}"
puts " #{status.join("\n ")}"
end
end
status_all
end
desc 'make sure that the current version from the module file matches the last tagged version'
@@ -127,28 +54,12 @@ namespace :git do
# I need to be able to return this as a data structure
# when I start to do more complicated things like
# automated releases, I will need this data
each_repo do |module_name|
require 'puppet'
if ! args.project_name || args.project_name == module_name
modulefile = File.join(Dir.getwd, 'Modulefile')
if File.exists?(modulefile)
print module_name
metadata = ::Puppet::ModuleTool::Metadata.new
::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
print ':' + metadata.version
branch_output = git_cmd('branch')
if branch_output.first =~ /\* (.+)/
puts ":#{$1}"
puts ' ' + git_cmd("log #{metadata.version}..HEAD --oneline").join("\n ")
puts ''
else
puts ' ' + branch_output.join("\n ")
end
else
puts "#{module_name} does not have a Modulefile"
end
end
check_tags(args.project_name)
end
desc 'make sure that the current version from the module file matches the last tagged version'
task :check_all_tags do
check_tags
end
task :check_sha_all do
@@ -171,100 +82,22 @@ namespace :git do
end
end
# list of users that can approve PRs that should run through the integration
# tests
admin_users = ['bodepd']
test_with_this_body = 'test_it'
namespace :github do
desc 'pick a single pull request to test. Accepts the project name and number of PR to test'
# you can also specify the OPERATINGSYSTEM to test as an ENV variable
task :test_pull_request, [:project_name, :number] do |t, args|
# TODO - this is way too much overhead, I am reusing each_repo,
# but I should write some kind of repo select
each_repo do |repo_name|
#require 'ruby-debug';debugger
if repo_name == args.project_name
require 'curb'
require 'json'
project_url = "https://api.github.com/repos/puppetlabs/puppetlabs-#{args.project_name}"
pull_request_url = "#{project_url}/pulls/#{args.number}"
resp = Curl.get(pull_request_url)
pr = JSON.parse(resp.body_str)
if ! pr['merged']
if pr['mergeable']
if pr['comments'] > 0
resp = Curl.get("#{project_url}/issues/#{args.number}/comments")
comments = JSON.parse(resp.body_str)
puts 'going through comments'
comments.each do |comment|
if admin_users.include?(comment['user']['login'])
test_file = File.join(base_dir, '.current_tests')
if File.exists?(test_file)
loaded_pr = YAML.load_file(test_file)
puts "Branch already checkout out for testing #{loaded_pr[:project]}/#{loaded_pr[:number]}"
end
if comment['body'] == 'test_it'
clone_url = pr['head']['repo']['clone_url']
remote_name = pr['head']['user']['login']
sha = pr['head']['sha']
File.open(test_file, 'w') do |fh|
fh.write({
:project => args.project_name,
:number => args.number
}.to_yaml)
end
puts 'found one that we should test'
# TODO I am not sure how reliable all of this is going
# to be
remotes = git_cmd('remote')
unless remotes.include?(remote_name)
git_cmd("remote add #{remote_name} #{clone_url}")
end
git_cmd("fetch #{remote_name}")
# TODO does that work if master has been updated?
git_cmd("checkout #{sha}")
end
end
end
else
puts "PR: #{args.number} from #{args.project_name} has no commits.\
I will not test it. We only test things approved.
"
end
else
puts "PR: #{args.number} from #{args.project_name} cannot be merged, will not test"
end
else
puts "PR: #{args.number} from #{args.project_name} was already merged, will not test"
end
end
end
#GET /repos/:owner/:repo/pulls/:number/comments
checkout_pr(args.project_name, args.number, ['bodepd'], 'test_it')
end
end
def each_testable_pull_request(&block)
end
namespace :test do
desc 'test openstack with basic test script on redhat and ubuntu'
task 'two_node' do
require 'yaml'
#Rake::Task['openstack:setup'.to_sym].invoke
['redhat', 'ubuntu'].each do |os|
cfg = File.join(base_dir, 'config.yaml')
yml = YAML.load_file(cfg).merge({'operatingsystem' => os})
File.open(cfg, 'w') {|f| f.write(yml.to_yaml) }
cmd_system('vagrant destroy -f')
deploy_two_node
# I should check this to see if the last line is cirros
on_box('openstack_controller', 'sudo bash /tmp/test_nova.sh;exit $?')
end
test_two_node(['redhat', 'ubunut'])
end
desc 'test all in one deployment on redhat/ubuntu (not yet implemented)'
@@ -272,60 +105,8 @@ namespace :test do
end
task :test do
desc 'test that openstack can boot an image from the vagrant bog'
task :controller_test do
on_box('openstack_controller', 'sudo bash /tmp/foo.sh')
end
end
def contributor_hash
repos_i_care_about = ['nova', 'glance', 'openstack', 'keystone', 'swift', 'horizon', 'cinder']
contributors = {}
each_repo do |module_name|
if repos_i_care_about.include?(module_name)
logs = git_cmd('log --format=short')
user_lines = logs.select {|x| x =~ /^Author:\s+(.*)$/ }
user_lines.collect do |x|
if x =~ /^Author:\s+(.*)?\s+<((\S+)@(\S+))>$/
unless ['root', 'vagrant', 'Dan'].include?($1)
if contributors[$1]
contributors[$1][:repos] = contributors[$1][:repos] | [module_name]
else
contributors[$1] = {:email => $2, :repos => [module_name] }
end
else
# trimming out extra users
end
else
puts "Skipping unexpected line #{x}"
end
end
end
end
contributors
end
def each_repo(&block)
require 'librarian/puppet'
require 'librarian/puppet/source/git'
# create a manifest
# TODO replace this to use librarian puppet
env = Librarian::Puppet::Environment.new()
# this is the lock file, so it assumes that install has been run
env.lock.manifests.each do |manifest|
# I only care about git sources
if manifest.source.is_a? Librarian::Puppet::Source::Git
module_name = manifest.name.split('/', 2)[1]
module_path = File.join(env.install_path,module_name)
if File.directory?(module_path)
Dir.chdir(module_path) do
yield module_name
end
else
puts "Module directory #{module_path} does not exist... How strange."
end
else
puts "Found a non-git manifest: #{manifest.class}, ignoring"
end
end
end

286
lib/puppetlabs/os_tester.rb Normal file
View File

@@ -0,0 +1,286 @@
#
# class that hold utilities that I use to test openstack
#
module Puppetlabs
module OsTester
def cmd_system (cmd)
result = system cmd
raise(RuntimeError, $?) unless $?.success?
result
end
def git_cmd(cmd)
command = 'git ' + cmd
Open3.popen3(*command) do |i, o, e, t|
raise StandardError, e.read unless (t ? t.value : $?).success?
o.read.split("\n")
end
end
def on_box (box, cmd)
cmd_system("vagrant ssh #{box} -c '#{cmd}'")
end
def deploy_two_node
require 'vagrant'
env = Vagrant::Environment.new(:cwd => base_dir, :ui_class => Vagrant::UI::Colored)
build(:openstack_controller, env)
build(:compute1, env)
end
# bring vagrant vm with image name up
def build(instance, env)
unless vm = env.vms[instance]
puts "invalid VM: #{instance}"
else
if vm.created?
puts "VM: #{instance} was already created"
else
# be very fault tolerant :)
begin
# this will always fail
vm.up(:provision => true)
rescue Exception => e
puts e.class
puts e
end
end
end
end
def each_repo(&block)
require 'librarian/puppet'
require 'librarian/puppet/source/git'
# create a manifest
# TODO replace this to use librarian puppet
env = Librarian::Puppet::Environment.new()
# this is the lock file, so it assumes that install has been run
env.lock.manifests.each do |manifest|
# I only care about git sources
if manifest.source.is_a? Librarian::Puppet::Source::Git
module_name = manifest.name.split('/', 2)[1]
module_path = File.join(env.install_path,module_name)
if File.directory?(module_path)
Dir.chdir(module_path) do
yield module_name
end
else
puts "Module directory #{module_path} does not exist... How strange."
end
else
puts "Found a non-git manifest: #{manifest.class}, ignoring"
end
end
end
def contributor_hash(
repos_i_care_about = ['nova', 'glance', 'openstack', 'keystone', 'swift', 'horizon', 'cinder']
)
contributors = {}
each_repo do |module_name|
if repos_i_care_about.include?(module_name)
logs = git_cmd('log --format=short')
user_lines = logs.select {|x| x =~ /^Author:\s+(.*)$/ }
user_lines.collect do |x|
if x =~ /^Author:\s+(.*)?\s+<((\S+)@(\S+))>$/
unless ['root', 'vagrant', 'Dan'].include?($1)
if contributors[$1]
contributors[$1][:repos] = contributors[$1][:repos] | [module_name]
else
contributors[$1] = {:email => $2, :repos => [module_name] }
end
else
# trimming out extra users
end
else
puts "Skipping unexpected line #{x}"
end
end
end
end
contributors
end
# destroy all vagrant instances
def destroy_all_vms
puts "About to destroy all vms..."
cmd_system('vagrant destroy -f')
puts "Destroyed all vms"
end
# adds the specified remote name as a read/write remote
def dev_setup(remote_name)
each_repo do |module_name|
# need to handle more failure cases
remotes = git_cmd('remote')
if remotes.include?(remote_name)
puts "Did not have to add remote #{remote_name} to #{module_name}"
elsif ! remotes.include?('origin')
raise(Exception, "Repo #{module_name} has no remote called origin, failing")
else
remote_url = git_cmd('remote show origin').detect {|x| x =~ /\s+Push\s+URL: / }
if remote_url =~ /(git|https?):\/\/(.+)\/(.+)?\/(.+)/
url = "git@#{$2}:#{remote_name}/#{$4}"
else
puts "remote_url #{remote_url} did not have the expected format. weird..."
end
puts "Adding remote #{remote_name} as #{url}"
git_cmd("remote add #{remote_name} #{url}")
end
end
end
def pull_all
each_repo do |module_name|
puts "Pulling repo: #{module_name}"
puts ' ' + git_cmd('pull').join("\n ")
end
end
def status_all
each_repo do |module_name|
status = git_cmd('status')
if status.include?('nothing to commit (working directory clean)')
puts "Module #{module_name} has not changed" if verbose
else
puts "Uncommitted changes for: #{module_name}"
puts " #{status.join("\n ")}"
end
end
end
def check_tags(project_name=nil)
each_repo do |module_name|
require 'puppet'
if ! project_name || project_name == module_name
modulefile = File.join(Dir.getwd, 'Modulefile')
if File.exists?(modulefile)
print module_name
metadata = ::Puppet::ModuleTool::Metadata.new
::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
print ':' + metadata.version
branch_output = git_cmd('branch')
if branch_output.first =~ /\* (.+)/
puts ":#{$1}"
puts ' ' + git_cmd("log #{metadata.version}..HEAD --oneline").join("\n ")
puts ''
else
puts ' ' + branch_output.join("\n ")
end
else
puts "#{module_name} does not have a Modulefile"
end
end
end
end
# given a pull request, return true if we should test it.
# this means that is can be merged, and has a comment where one of the admin users
# has specified the expected body.
def testable_pull_request?(
pr,
admin_users,
project_base_url = 'https://api.github.com/repos/puppetlabs/',
expected_body = 'test_it'
)
project_url = project_base_url + pr['base']['repo']['name']
if ! pr['merged']
if pr['mergeable']
if pr['comments'] > 0
resp = Curl.get("#{project_url}/issues/#{pr['number']}/comments")
comments = JSON.parse(resp.body_str)
puts 'going through comments'
comments.each do |comment|
if admin_users.include?(comment['user']['login'])
if comment['body'] == expected_body
return true
end
else
end
end
else
puts "PR: #{pr['number']} from #{project_name} has no commits.\
I will not test it. We only test things approved.
"
end
else
puts "PR: #{pr['number']} from #{project_name} cannot be merged, will not test"
end
else
puts "PR: #{pr['number']} from #{project_name} was already merged, will not test"
end
return false
end
def checkout_pr(project_name, number, admin_users, expected_body)
# but I should write some kind of repo select
# depends on https://github.com/peter-murach/github
require 'github_api'
require 'curb'
require 'json'
each_repo do |repo_name|
if repo_name == project_name
project_url = "https://api.github.com/repos/puppetlabs/puppetlabs-#{project_name}"
pull_request_url = "#{project_url}/pulls/#{number}"
resp = Curl.get(pull_request_url)
pr = JSON.parse(resp.body_str)
# need to be able to override this?
test_file = File.join(base_dir, '.current_testing')
if File.exists?(test_file)
loaded_pr = YAML.load_file(test_file)
puts "Branch already checked out for testing #{loaded_pr[:project]}/#{loaded_pr[:number]}"
exit 1
end
if testable_pull_request?(pr, admin_users)
clone_url = pr['head']['repo']['clone_url']
remote_name = pr['head']['user']['login']
sha = pr['head']['sha']
File.open(test_file, 'w') do |fh|
fh.write({
:project => project_name,
:number => number
}.to_yaml)
end
puts 'found one that we should test'
# TODO I am not sure how reliable all of this is going
# to be
remotes = git_cmd('remote')
unless remotes.include?(remote_name)
git_cmd("remote add #{remote_name} #{clone_url}")
end
git_cmd("fetch #{remote_name}")
# TODO does that work if master has been updated?
git_cmd("checkout #{sha}")
end
end
end
end
def test_two_node(oses = [])
require 'yaml'
#Rake::Task['openstack:setup'.to_sym].invoke
oses.each do |os|
cfg = File.join(base_dir, 'config.yaml')
yml = YAML.load_file(cfg).merge({'operatingsystem' => os})
File.open(cfg, 'w') {|f| f.write(yml.to_yaml) }
cmd_system('vagrant destroy -f')
deploy_two_node
# I should check this to see if the last line is cirros
on_box('openstack_controller', 'sudo bash /tmp/test_nova.sh;exit $?')
end
end
# iterate through each testable pull request
def each_testable_pull_request(&block)
end
end
end