create external lib for all actions

move all code to an external library

Rakefile

@@ -1,55 +1,14 @@
 require 'yaml'
 require 'rubygems'
 
-
-def cmd_system (cmd)
-  result = system cmd
-  raise(RuntimeError, $?) unless $?.success?
-  result
-end
-
-def git_cmd(cmd)
-  command = 'git ' + cmd
-  Open3.popen3(*command) do |i, o, e, t|
-    raise StandardError, e.read unless (t ? t.value : $?).success?
-    o.read.split("\n")
-  end
-end
-
-def on_box (box, cmd)
-  cmd_system("vagrant ssh #{box} -c '#{cmd}'")
-end
-
-def deploy_two_node
-  require 'vagrant'
-  env = Vagrant::Environment.new(:cwd => base_dir, :ui_class => Vagrant::UI::Colored)
-  build(:openstack_controller, env)
-  build(:compute1, env)
-end
-
 def base_dir
   File.expand_path(File.dirname(__FILE__))
 end
 
-# bring vagrant vm with image name up
+require File.join(base_dir, 'lib', 'puppetlabs', 'os_tester')
-def build(instance, env)
+
-  unless vm = env.vms[instance]
+include Puppetlabs::OsTester
-    puts "invalid VM: #{instance}"
+
-  else
-    if vm.created?
-      puts "VM: #{instance} was already created"
-    else
-      # be very fault tolerant :)
-      begin
-        # this will always fail
-        vm.up(:provision => true)
-      rescue Exception => e
-        puts e.class
-        puts e
-      end
-    end
-  end
-end
 
 namespace :openstack do
 
@@ -58,10 +17,9 @@ namespace :openstack do
     cmd_system('librarian-puppet install')
   end
 
+  desc 'destroy all vms'
   task 'destroy' do
-    puts "About to destroy all vms..."
+    destroy_all_vms
-    cmd_system('vagrant destroy -f')
-    puts "Destroyed all vms"
   end
 
   desc 'deploys the entire environment'
@@ -72,54 +30,23 @@ namespace :openstack do
 end
 
 
-remote_name = 'bodepd'
-
 namespace :git do
 
   cwd = base_dir
 
-  desc 'for all repos in the module directory, add a read/write remote'
+  desc 'for all repos in the module directory, add a read/write remote (hardcoded to bodepd)'
   task :dev_setup do
-
+    dev_setup('bodped')
-    each_repo do |module_name|
-      # need to handle more failure cases
-      remotes = git_cmd('remote')
-      if remotes.include?(remote_name)
-        puts "Did not have to add remote #{remote_name} to #{module_name}"
-      elsif ! remotes.include?('origin')
-        raise(Exception, "Repo #{module_name} has no remote called origin, failing")
-      else
-        remote_url = git_cmd('remote show origin').detect {|x| x =~ /\s+Push\s+URL: / }
-        if remote_url =~ /(git|https?):\/\/(.+)\/(.+)?\/(.+)/
-          url = "git@#{$2}:#{remote_name}/#{$4}"
-        else
-          puts "remote_url #{remote_url} did not have the expected format. weird..."
-        end
-        puts "Adding remote #{remote_name} as #{url}"
-        git_cmd("remote add #{remote_name} #{url}")
-      end
-    end
   end
 
   desc 'pull the latest version of all code'
   task :pull_all do
-    each_repo do |module_name|
+    pull_all
-      puts "Pulling repo: #{module_name}"
-      puts '  ' + git_cmd('pull').join("\n  ")
-    end
   end
 
   desc 'shows the current state of code that has not been commited'
   task :status_all do
-    each_repo do |module_name|
+    status_all
-      status = git_cmd('status')
-      if status.include?('nothing to commit (working directory clean)')
-        puts "Module #{module_name} has not changed" if verbose
-      else
-        puts "Uncommitted changes for: #{module_name}"
-        puts "  #{status.join("\n  ")}"
-      end
-    end
   end
 
   desc 'make sure that the current version from the module file matches the last tagged version'
@@ -127,28 +54,12 @@ namespace :git do
     # I need to be able to return this as a data structure
     # when I start to do more complicated things like
     # automated releases, I will need this data
-    each_repo do |module_name|
+    check_tags(args.project_name)
-      require 'puppet'
-      if ! args.project_name || args.project_name == module_name
-        modulefile = File.join(Dir.getwd, 'Modulefile')
-        if File.exists?(modulefile)
-          print module_name
-          metadata  = ::Puppet::ModuleTool::Metadata.new
-          ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
-          print ':' + metadata.version
-          branch_output = git_cmd('branch')
-          if branch_output.first =~ /\* (.+)/
-            puts ":#{$1}"
-            puts '  ' + git_cmd("log #{metadata.version}..HEAD --oneline").join("\n  ")
-            puts ''
-          else
-            puts '  ' + branch_output.join("\n  ")
-          end
-        else
-          puts "#{module_name} does not have a Modulefile"
-        end
-      end
   end
+
+  desc 'make sure that the current version from the module file matches the last tagged version'
+  task :check_all_tags do
+    check_tags
   end
 
   task :check_sha_all do
@@ -171,100 +82,22 @@ namespace :git do
   end
 end
 
-# list of users that can approve PRs that should run through the integration
-# tests
-admin_users         = ['bodepd']
-test_with_this_body = 'test_it'
 
 namespace :github do
 
   desc 'pick a single pull request to test. Accepts the project name and number of PR to test'
     # you can also specify the OPERATINGSYSTEM to test as an ENV variable
   task :test_pull_request, [:project_name, :number] do |t, args|
-    # TODO - this is way too much overhead, I am reusing each_repo,
+    checkout_pr(args.project_name, args.number, ['bodepd'], 'test_it')
-    # but I should write some kind of repo select
-    each_repo do |repo_name|
-      #require 'ruby-debug';debugger
-      if repo_name == args.project_name
-        require 'curb'
-        require 'json'
-        project_url = "https://api.github.com/repos/puppetlabs/puppetlabs-#{args.project_name}"
-        pull_request_url = "#{project_url}/pulls/#{args.number}"
-        resp = Curl.get(pull_request_url)
-        pr   = JSON.parse(resp.body_str)
-
-        if ! pr['merged']
-          if pr['mergeable']
-            if pr['comments'] > 0
-              resp = Curl.get("#{project_url}/issues/#{args.number}/comments")
-              comments = JSON.parse(resp.body_str)
-              puts 'going through comments'
-              comments.each do |comment|
-                if admin_users.include?(comment['user']['login'])
-                  test_file = File.join(base_dir, '.current_tests')
-                  if File.exists?(test_file)
-                    loaded_pr = YAML.load_file(test_file)
-                    puts "Branch already checkout out for testing #{loaded_pr[:project]}/#{loaded_pr[:number]}"
-                  end
-                  if comment['body'] == 'test_it'
-                    clone_url   = pr['head']['repo']['clone_url']
-                    remote_name = pr['head']['user']['login']
-                    sha         = pr['head']['sha']
-                    File.open(test_file, 'w') do |fh|
-                      fh.write({
-                        :project => args.project_name,
-                        :number  => args.number
-                      }.to_yaml)
-                    end
-                    puts 'found one that we should test'
-                    # TODO I am not sure how reliable all of this is going
-                    # to be
-                    remotes = git_cmd('remote')
-                    unless remotes.include?(remote_name)
-                      git_cmd("remote add #{remote_name} #{clone_url}")
-                    end
-                    git_cmd("fetch #{remote_name}")
-                    # TODO does that work if master has been updated?
-                    git_cmd("checkout #{sha}")
-                  end
-                end
-              end
-            else
-              puts "PR: #{args.number} from #{args.project_name} has no commits.\
-              I will not test it. We only test things approved.
-              "
-            end
-          else
-            puts "PR: #{args.number} from #{args.project_name} cannot be merged, will not test"
-          end
-        else
-          puts "PR: #{args.number} from #{args.project_name} was already merged, will not test"
-        end
-      end
-    end
-    #GET /repos/:owner/:repo/pulls/:number/comments
   end
 
 end
 
-def each_testable_pull_request(&block)
-end
-
 namespace :test do
 
   desc 'test openstack with basic test script on redhat and ubuntu'
   task 'two_node' do
-    require 'yaml'
+    test_two_node(['redhat', 'ubunut'])
-    #Rake::Task['openstack:setup'.to_sym].invoke
-    ['redhat', 'ubuntu'].each do |os|
-      cfg = File.join(base_dir, 'config.yaml')
-      yml = YAML.load_file(cfg).merge({'operatingsystem' => os})
-      File.open(cfg, 'w') {|f| f.write(yml.to_yaml) }
-      cmd_system('vagrant destroy -f')
-      deploy_two_node
-      # I should check this to see if the last line is cirros
-      on_box('openstack_controller', 'sudo bash /tmp/test_nova.sh;exit $?')
-    end
   end
 
   desc 'test all in one deployment on redhat/ubuntu (not yet implemented)'
@@ -272,60 +105,8 @@ namespace :test do
 
   end
 
-  task :test do
+  desc 'test that openstack can boot an image from the vagrant bog'
+  task :controller_test do
     on_box('openstack_controller', 'sudo bash /tmp/foo.sh')
   end
 end
-
-def contributor_hash
-  repos_i_care_about = ['nova', 'glance', 'openstack', 'keystone', 'swift', 'horizon', 'cinder']
-  contributors = {}
-  each_repo do |module_name|
-    if repos_i_care_about.include?(module_name)
-      logs = git_cmd('log --format=short')
-      user_lines = logs.select {|x| x =~ /^Author:\s+(.*)$/ }
-      user_lines.collect do |x|
-        if x =~ /^Author:\s+(.*)?\s+<((\S+)@(\S+))>$/
-          unless ['root', 'vagrant', 'Dan'].include?($1)
-            if contributors[$1]
-              contributors[$1][:repos] = contributors[$1][:repos] | [module_name]
-            else
-              contributors[$1] = {:email => $2, :repos => [module_name] }
-            end
-          else
-            # trimming out extra users
-          end
-        else
-          puts "Skipping unexpected line #{x}"
-        end
-      end
-    end
-  end
-  contributors
-end
-
-
-def each_repo(&block)
-  require 'librarian/puppet'
-  require 'librarian/puppet/source/git'
-  # create a manifest
-  # TODO replace this to use librarian puppet
-  env = Librarian::Puppet::Environment.new()
-  # this is the lock file, so it assumes that install has been run
-  env.lock.manifests.each do |manifest|
-    # I only care about git sources
-    if manifest.source.is_a? Librarian::Puppet::Source::Git
-      module_name = manifest.name.split('/', 2)[1]
-      module_path = File.join(env.install_path,module_name)
-      if File.directory?(module_path)
-        Dir.chdir(module_path) do
-          yield module_name
-        end
-      else
-        puts "Module directory #{module_path} does not exist... How strange."
-      end
-    else
-      puts "Found a non-git manifest: #{manifest.class}, ignoring"
-    end
-  end
-end

lib/puppetlabs/os_tester.rb

@@ -0,0 +1,286 @@
+#
+# class that hold utilities that I use to test openstack
+#
+module Puppetlabs
+  module OsTester
+
+    def cmd_system (cmd)
+      result = system cmd
+      raise(RuntimeError, $?) unless $?.success?
+      result
+    end
+
+    def git_cmd(cmd)
+      command = 'git ' + cmd
+      Open3.popen3(*command) do |i, o, e, t|
+        raise StandardError, e.read unless (t ? t.value : $?).success?
+        o.read.split("\n")
+      end
+    end
+
+    def on_box (box, cmd)
+      cmd_system("vagrant ssh #{box} -c '#{cmd}'")
+    end
+
+    def deploy_two_node
+      require 'vagrant'
+      env = Vagrant::Environment.new(:cwd => base_dir, :ui_class => Vagrant::UI::Colored)
+      build(:openstack_controller, env)
+      build(:compute1, env)
+    end
+
+    # bring vagrant vm with image name up
+    def build(instance, env)
+      unless vm = env.vms[instance]
+        puts "invalid VM: #{instance}"
+      else
+        if vm.created?
+          puts "VM: #{instance} was already created"
+        else
+          # be very fault tolerant :)
+          begin
+            # this will always fail
+            vm.up(:provision => true)
+          rescue Exception => e
+            puts e.class
+            puts e
+          end
+        end
+      end
+    end
+
+    def each_repo(&block)
+      require 'librarian/puppet'
+      require 'librarian/puppet/source/git'
+      # create a manifest
+      # TODO replace this to use librarian puppet
+      env = Librarian::Puppet::Environment.new()
+      # this is the lock file, so it assumes that install has been run
+      env.lock.manifests.each do |manifest|
+        # I only care about git sources
+        if manifest.source.is_a? Librarian::Puppet::Source::Git
+          module_name = manifest.name.split('/', 2)[1]
+          module_path = File.join(env.install_path,module_name)
+          if File.directory?(module_path)
+            Dir.chdir(module_path) do
+              yield module_name
+            end
+          else
+            puts "Module directory #{module_path} does not exist... How strange."
+          end
+        else
+          puts "Found a non-git manifest: #{manifest.class}, ignoring"
+        end
+      end
+    end
+
+
+    def contributor_hash(
+      repos_i_care_about = ['nova', 'glance', 'openstack', 'keystone', 'swift', 'horizon', 'cinder']
+    )
+      contributors = {}
+      each_repo do |module_name|
+        if repos_i_care_about.include?(module_name)
+          logs = git_cmd('log --format=short')
+          user_lines = logs.select {|x| x =~ /^Author:\s+(.*)$/ }
+          user_lines.collect do |x|
+            if x =~ /^Author:\s+(.*)?\s+<((\S+)@(\S+))>$/
+              unless ['root', 'vagrant', 'Dan'].include?($1)
+                if contributors[$1]
+                  contributors[$1][:repos] = contributors[$1][:repos] | [module_name]
+                else
+                  contributors[$1] = {:email => $2, :repos => [module_name] }
+                end
+              else
+                # trimming out extra users
+              end
+            else
+              puts "Skipping unexpected line #{x}"
+            end
+          end
+        end
+      end
+      contributors
+    end
+
+
+    # destroy all vagrant instances
+    def destroy_all_vms
+      puts "About to destroy all vms..."
+      cmd_system('vagrant destroy -f')
+      puts "Destroyed all vms"
+    end
+
+
+    # adds the specified remote name as a read/write remote
+    def dev_setup(remote_name)
+      each_repo do |module_name|
+        # need to handle more failure cases
+        remotes = git_cmd('remote')
+        if remotes.include?(remote_name)
+          puts "Did not have to add remote #{remote_name} to #{module_name}"
+        elsif ! remotes.include?('origin')
+          raise(Exception, "Repo #{module_name} has no remote called origin, failing")
+        else
+          remote_url = git_cmd('remote show origin').detect {|x| x =~ /\s+Push\s+URL: / }
+          if remote_url =~ /(git|https?):\/\/(.+)\/(.+)?\/(.+)/
+            url = "git@#{$2}:#{remote_name}/#{$4}"
+          else
+            puts "remote_url #{remote_url} did not have the expected format. weird..."
+          end
+          puts "Adding remote #{remote_name} as #{url}"
+          git_cmd("remote add #{remote_name} #{url}")
+        end
+      end
+    end
+
+
+    def pull_all
+      each_repo do |module_name|
+        puts "Pulling repo: #{module_name}"
+        puts '  ' + git_cmd('pull').join("\n  ")
+      end
+    end
+
+
+    def status_all
+      each_repo do |module_name|
+        status = git_cmd('status')
+        if status.include?('nothing to commit (working directory clean)')
+          puts "Module #{module_name} has not changed" if verbose
+        else
+          puts "Uncommitted changes for: #{module_name}"
+          puts "  #{status.join("\n  ")}"
+        end
+      end
+    end
+
+    def check_tags(project_name=nil)
+      each_repo do |module_name|
+        require 'puppet'
+        if ! project_name || project_name == module_name
+          modulefile = File.join(Dir.getwd, 'Modulefile')
+          if File.exists?(modulefile)
+            print module_name
+            metadata  = ::Puppet::ModuleTool::Metadata.new
+            ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
+            print ':' + metadata.version
+            branch_output = git_cmd('branch')
+            if branch_output.first =~ /\* (.+)/
+              puts ":#{$1}"
+              puts '  ' + git_cmd("log #{metadata.version}..HEAD --oneline").join("\n  ")
+              puts ''
+            else
+              puts '  ' + branch_output.join("\n  ")
+            end
+          else
+            puts "#{module_name} does not have a Modulefile"
+          end
+        end
+      end
+    end
+
+    # given a pull request, return true if we should test it.
+    # this means that is can be merged, and has a comment where one of the admin users
+    # has specified the expected body.
+    def testable_pull_request?(
+      pr,
+      admin_users,
+      project_base_url = 'https://api.github.com/repos/puppetlabs/',
+      expected_body = 'test_it'
+    )
+      project_url = project_base_url + pr['base']['repo']['name']
+      if ! pr['merged']
+        if pr['mergeable']
+          if pr['comments'] > 0
+            resp = Curl.get("#{project_url}/issues/#{pr['number']}/comments")
+            comments = JSON.parse(resp.body_str)
+            puts 'going through comments'
+            comments.each do |comment|
+              if admin_users.include?(comment['user']['login'])
+                if comment['body'] == expected_body
+                  return true
+                end
+              else
+              end
+            end
+          else
+            puts "PR: #{pr['number']} from #{project_name} has no commits.\
+            I will not test it. We only test things approved.
+            "
+          end
+        else
+          puts "PR: #{pr['number']} from #{project_name} cannot be merged, will not test"
+        end
+      else
+        puts "PR: #{pr['number']} from #{project_name} was already merged, will not test"
+      end
+      return false
+    end
+
+    def checkout_pr(project_name, number, admin_users, expected_body)
+      # but I should write some kind of repo select
+      # depends on https://github.com/peter-murach/github
+      require 'github_api'
+      require 'curb'
+      require 'json'
+
+      each_repo do |repo_name|
+        if repo_name == project_name
+          project_url = "https://api.github.com/repos/puppetlabs/puppetlabs-#{project_name}"
+          pull_request_url = "#{project_url}/pulls/#{number}"
+          resp = Curl.get(pull_request_url)
+          pr   = JSON.parse(resp.body_str)
+          # need to be able to override this?
+          test_file = File.join(base_dir, '.current_testing')
+          if File.exists?(test_file)
+            loaded_pr = YAML.load_file(test_file)
+            puts "Branch already checked out for testing #{loaded_pr[:project]}/#{loaded_pr[:number]}"
+            exit 1
+          end
+
+          if testable_pull_request?(pr, admin_users)
+            clone_url   = pr['head']['repo']['clone_url']
+            remote_name = pr['head']['user']['login']
+            sha         = pr['head']['sha']
+            File.open(test_file, 'w') do |fh|
+              fh.write({
+                :project => project_name,
+                :number  => number
+              }.to_yaml)
+            end
+            puts 'found one that we should test'
+            # TODO I am not sure how reliable all of this is going
+            # to be
+            remotes = git_cmd('remote')
+            unless remotes.include?(remote_name)
+              git_cmd("remote add #{remote_name} #{clone_url}")
+            end
+            git_cmd("fetch #{remote_name}")
+            # TODO does that work if master has been updated?
+            git_cmd("checkout #{sha}")
+          end
+        end
+      end
+    end
+
+    def test_two_node(oses = [])
+      require 'yaml'
+      #Rake::Task['openstack:setup'.to_sym].invoke
+      oses.each do |os|
+        cfg = File.join(base_dir, 'config.yaml')
+        yml = YAML.load_file(cfg).merge({'operatingsystem' => os})
+        File.open(cfg, 'w') {|f| f.write(yml.to_yaml) }
+        cmd_system('vagrant destroy -f')
+        deploy_two_node
+        # I should check this to see if the last line is cirros
+        on_box('openstack_controller', 'sudo bash /tmp/test_nova.sh;exit $?')
+      end
+    end
+
+    # iterate through each testable pull request
+    def each_testable_pull_request(&block)
+    end
+
+  end
+end