Connectivity tests for external services

In order to provide better error handling around configured external
services, we are adding additional checks to some deployment tasks
where we know that there are either external services or services
that can be user configurable.

*) Adding a repository connectivity test task to be run after
netconfig to ensure that software repository access is OK before
proceeding with the rest of the deployment.
*) Adding in post connectivity tests before NTP server to ensure that
those services will be able to reach the configured settings.

With this change, we are adding two custom puppet functions
(url_available and ntp_available) to the osnailyfacter module. These
functions will throw a Puppet::Error if unable to properly
communicate with the services.

Change-Id: I6b0302ce403871384d377aceb7e94b09126b885e
Closes-Bug: 1261940

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/ntp_available.rb

@@ -0,0 +1,75 @@
+require 'pp'
+require 'socket'
+require 'timeout'
+
+Puppet::Parser::Functions::newfunction(:ntp_available, :doc => <<-EOS
+The ntp_available function attempts to make an NTP request to a server or
+servers and throws a puppet error if unable to make at least one successful
+request. The ntp_available function takes a single parameter that can be one
+of the following:
+1) String - a single hostname
+2) Array - an array of hostname strings
+
+Examples:
+ntp_available('pool.ntp.org')
+ntp_available(['0.pool.ntp.org', '1.pool.ntp.org', '2.pool.ntp.org'])
+EOS
+) do |argv|
+  host = argv[0]
+
+  def ntp_query(host)
+    # time since unix epoch, RFC 868
+    time_offset = 2208988800
+    # timeout to wait for a response
+    timeout = 10
+    # an ntp packet
+    # http://blog.mattcrampton.com/post/88291892461/query-an-ntp-server-from-python
+    ntp_msg = "\x1b" + ("\0" * 47)
+    # our UDP socket
+    sock = UDPSocket.new
+    begin
+      # open up a socket to connect to the ntp host
+      sock.connect(host, 'ntp')
+      # send our ntp message to the ntp server
+      sock.print ntp_msg
+      sock.flush
+      # read the response
+      read, write, error = IO.select [sock], nil, nil, timeout
+      if read.nil?
+        raise Timeout::Error
+      else
+        data, _ = sock.recvfrom(960)
+        # un pack the response
+        # https://github.com/zencoder/net-ntp/blob/master/lib/net/ntp/ntp.rb#L194
+        parsed_data = data.unpack("a C3   n B16 n B16 H8   N B32 N B32   N B32 N B32")
+        # attempt to parse the time we got back
+        t = Time.at(parsed_data[13] - time_offset)
+        puts "Time from #{host} is #{t}"
+      end
+      sock.close if sock
+    rescue
+      sock.close if sock
+      return false
+    end
+    true
+  end
+
+  # our check boolean used to indicate we had at least one successful request
+  ok = false
+  # if passed an array, iterate throught he array and check each element
+  if host.instance_of? Array
+    host.each do |h|
+      if (ntp_query(h))
+        ok = true
+      end
+    end
+  else
+    ok = ntp_query(host)
+  end
+  # we need at least one successful request
+  if !ok
+    raise Puppet::Error, "ERROR: Unable to communicate with at least one of NTP server, checked the following host(s): #{host}"
+  end
+  return ok
+end
+# vim: set ts=2 sw=2 et :

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/url_available.rb

@@ -0,0 +1,64 @@
+require 'pp'
+require 'timeout'
+require 'net/http'
+require 'uri'
+
+Puppet::Parser::Functions::newfunction(:url_available, :doc => <<-EOS
+The url_available function attempts to make a http request to a url and throws
+a puppet error if the URL is unavailable.  The url_available function takes
+a single parameter that can be one of the following:
+1) String - a single url string
+3) Hash - a hash with the url set to the key of 'uri'
+2) Array - an array of url strings or an array of hashes that match the
+previous format.
+
+Examples:
+url_available('http://www.google.com')
+url_available(['http://www.google.com', 'http://www.mirantis.com'])
+url_available({ 'uri' => 'http://www.google.com' })
+url_available([{ 'uri' => 'http://www.google.com' },
+               { 'uri' => 'http://www.mirantis.com' }]
+EOS
+) do |argv|
+  url = argv[0]
+
+  def fetch(url)
+    # check the type of url being passed, if hash look for the uri key
+    if url.instance_of? Hash
+      if url.has_key?('uri')
+        uri = url['uri']
+      end
+    elsif url.instance_of? String
+      uri = url
+    else
+      raise Puppet::ParseError, "Invalid url type passed to the url_available
+function. Must be of type String or Hash."
+    end
+    puts "Checking #{uri}"
+    begin
+      out = Timeout::timeout(15) do
+        u = URI.parse(uri)
+        http = Net::HTTP.new(u.host, u.port)
+        http.open_timeout = 5
+        http.read_timeout = 5
+        request = Net::HTTP::Get.new(u.request_uri)
+        response = http.request(request)
+      end
+    rescue OpenURI::HTTPError => error
+      raise Puppet::Error, "ERROR: Unable to fetch url '#{uri}', error '#{error.io}'. Please verify node connectivity to this URL, or remove it from the settings page if it is invalid."
+    rescue Exception => e
+      raise Puppet::Error, "ERROR: Unable to fetch url '#{uri}', error '#{e}'. Please verify node connectivity to this URL, or remove it from the settings page if it is invalid."
+    end
+  end
+
+  # if passed an array, iterate through the array an check each element
+  if url.instance_of? Array
+    url.each do |u|
+      fetch(u)
+    end
+  else
+    fetch(url)
+  end
+  return true
+end
+# vim: set ts=2 sw=2 et :

deployment/puppet/osnailyfacter/modular/netconfig/connectivity_tests.pp

@@ -0,0 +1,5 @@
+notice('MODULAR: connectivity_tests.pp')
+# Pull the list of repos from hiera
+$repo_setup = hiera('repo_setup')
+# test that the repos are accessible
+url_available($repo_setup['repos'])

deployment/puppet/osnailyfacter/modular/netconfig/tasks.yaml

@@ -11,3 +11,14 @@
     cmd: ruby /etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig_pre.rb
   test_post:
     cmd: ruby /etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig_post.rb
+
+- id: connectivity_tests
+  type: puppet
+  groups: [primary-controller, controller, cinder, cinder-vmware, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
+  required_for: [firewall, hosts]
+  requires: [netconfig]
+  parameters:
+    puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/netconfig/connectivity_tests.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 600
+    cwd: /

deployment/puppet/osnailyfacter/modular/ntp/ntp-check.pp

@@ -0,0 +1,6 @@
+notice('MODULAR: ntp-check.pp')
+# get the ntp configuration from hiera
+$ntp_servers = hiera('external_ntp')
+# take the comma seperated list and turn it into an array of servers and then
+# pass it to the ntp_available function to check that at least 1 server works
+ntp_available(strip(split($ntp_servers['ntp_list'], ',')))

deployment/puppet/osnailyfacter/modular/ntp/tasks.yaml

@@ -19,3 +19,14 @@
     puppet_modules: /etc/puppet/modules
     timeout: 3600
     cwd: /
+
+- id: ntp-check
+  type: puppet
+  role: [primary-controller, controller]
+  required_for: [ntp-server]
+  requires: [dns-client]
+  parameters:
+    puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/ntp/ntp-check.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 600
+    cwd: /

tests/noop/spec/hosts/netconfig/connectivity_tests_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'shared-examples'
+manifest = 'netconfig/connectivity_tests.pp'
+
+describe manifest do
+  test_ubuntu_and_centos manifest
+end

tests/noop/spec/hosts/ntp/ntp-check_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'shared-examples'
+manifest = 'ntp/ntp-check.pp'
+
+describe manifest do
+  test_ubuntu_and_centos manifest
+end