From 8697c43752d48cb6feb197e0e575e792fc03c0fd Mon Sep 17 00:00:00 2001
From: Kyrylo Galanov <kgalanov@mirantis.com>
Date: Fri, 18 Sep 2015 13:25:38 +0300
Subject: [PATCH] Add multithreading to url_available check

* Checks are performed in parallel.
* Puppet exception is raised after the first failure.
* Add unit test.

Change-Id: Ie09930d63198ff2bcd3f1b0ed4a94ec4b8a59fd6
Closes-Bug: #1457190
---
 .../puppet/parser/functions/url_available.rb  | 16 +++++--
 .../spec/functions/url_available_spec.rb      | 44 +++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100644 deployment/puppet/osnailyfacter/spec/functions/url_available_spec.rb

diff --git a/deployment/puppet/osnailyfacter/lib/puppet/parser/functions/url_available.rb b/deployment/puppet/osnailyfacter/lib/puppet/parser/functions/url_available.rb
index 4185dd7260..6baf9591a0 100644
--- a/deployment/puppet/osnailyfacter/lib/puppet/parser/functions/url_available.rb
+++ b/deployment/puppet/osnailyfacter/lib/puppet/parser/functions/url_available.rb
@@ -1,6 +1,7 @@
 require 'pp'
 require 'timeout'
 require 'net/http'
+require 'open-uri'
 require 'uri'
 
 Puppet::Parser::Functions::newfunction(:url_available, :doc => <<-EOS
@@ -40,6 +41,8 @@ EOS
 ) do |argv|
   url = argv[0]
   http_proxy = argv[1]
+  threads_count = 16
+  Thread.abort_on_exception=true
 
   def fetch(url, http_proxy = nil)
     # proxy variables, set later if http_proxy is provided or there is a proxy
@@ -76,7 +79,7 @@ function. Must be of type String or Hash."
       end
     end
 
-    puts "Checking #{uri}"
+    puts "Checking #{uri}\n"
     begin
       out = Timeout::timeout(180) do
         u = URI.parse(uri)
@@ -94,9 +97,16 @@ function. Must be of type String or Hash."
   end
 
   # if passed an array, iterate through the array an check each element
+  # within a thread pool equal to threads_count
   if url.instance_of? Array
-    url.each do |u|
-      fetch(u, http_proxy)
+    url.each_slice(threads_count) do |group|
+      threads = []
+      group.each do |u|
+        threads << Thread.new do
+          fetch(u, http_proxy)
+        end
+      end
+      threads.each(&:join)
     end
   else
     fetch(url, http_proxy)
diff --git a/deployment/puppet/osnailyfacter/spec/functions/url_available_spec.rb b/deployment/puppet/osnailyfacter/spec/functions/url_available_spec.rb
new file mode 100644
index 0000000000..d58c96579f
--- /dev/null
+++ b/deployment/puppet/osnailyfacter/spec/functions/url_available_spec.rb
@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe 'the structure function' do
+  let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
+
+  let(:valid_urls) do
+    [
+      "http://archive.ubuntu.com/ubuntu/",
+      "http://mirror.fuel-infra.org/mos/ubuntu/",
+      "http://apt.postgresql.org/pub/repos/apt/"
+    ]
+  end
+
+  let(:invalid_urls) do
+    [
+      "http://invalid-url.ubuntu.com/ubuntu/",
+      "http://mirror.fuel-infra.org/invalid-url"
+    ]
+  end
+
+  it 'should exist' do
+    expect(Puppet::Parser::Functions.function('url_available')).to eq 'function_url_available'
+  end
+
+  context 'with single values' do
+    it 'should be able to process a single value' do
+      expect(scope.function_url_available([valid_urls[0]])).to be true
+    end
+
+    it 'should throw exception on invalid url' do
+      expect{ scope.function_url_available([invalid_urls[0]]) }.to raise_error(Puppet::Error)
+    end
+  end
+
+  context 'with multiple values' do
+    it 'should be able to process an array of values' do
+      expect(scope.function_url_available([valid_urls])).to be true
+    end
+
+    it 'should throw exception on invalid urls' do
+      expect{ scope.function_url_available([invalid_urls]) }.to raise_error(Puppet::Error)
+    end
+  end
+end