Change the way monitor IP are discovered

This changes allows for a couple things:
 - support for IPv6 addresses (provided that the configuration
   is stateful or use_tempaddr is set to 0 in a stateless conf)
 - support multiple NICs (public_network must be defined)
 - if public_network is not defined, we fall back to the previous
   behavior.

Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>

libraries/default.rb

@@ -1,3 +1,5 @@
+require 'ipaddr'
+
 def is_crowbar?()
   return defined?(Chef::Recipe::Barclamp) != nil
 end
@@ -20,30 +22,87 @@ def get_mon_nodes(extra_search=nil)
   return mons
 end
 
+# If public_network is specified
+# we need to search for the monitor IP
+# in the node environment.
+# 1. We look if the network is IPv6 or IPv4
+# 2. We look for a route matching the network
+# 3. We grab the IP and return it with the port
+def find_node_ip_in_network(network, nodeish=nil)
+  nodeish = node unless nodeish
+  net = IPAddr.new(network)
+  node["network"]["interfaces"].each do |iface|
+    if iface[1]["routes"].nil?
+      next
+    end
+    if net.ipv4?
+      iface[1]["routes"].each_with_index do |route, index|
+        if iface[1]["routes"][index]["destination"] == network
+          return "#{iface[1]["routes"][index]["src"]}:6789"
+        end
+      end
+    else
+      # Here we are getting an IPv6. We assume that
+      # the configuration is stateful.
+      # For this configuration to not fail in a stateless
+      # configuration, you should run:
+      #  echo "0" > /proc/sys/net/ipv6/conf/*/use_tempaddr
+      # on each server, this will disabe temporary addresses
+      # See: http://en.wikipedia.org/wiki/IPv6_address#Temporary_addresses
+      iface[1]["routes"].each_with_index do |route, index|
+        if iface[1]["routes"][index]["destination"] == network
+          iface[1]["addresses"].each do |k,v|
+            if v["scope"] == "Global" and v["family"] == "inet6"
+              return "[#{k}]:6789"
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
 def get_mon_addresses()
-  mons = []
+  mon_ips = []
 
-  # make sure if this node runs ceph-mon, it's always included even if
+  if File.exists?("/var/run/ceph/ceph-mon.#{node['hostname']}.asok")
-  # search is laggy; put it first in the hopes that clients will talk
+    mon_ips = get_quorum_members_ips()
-  # primarily to local node
-  if node['roles'].include? 'ceph-mon'
-    mons << node
-  end
-
-  mons += get_mon_nodes()
-
-  if is_crowbar?
-    mon_addresses = mons.map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address }
   else
-    mon_addresses = mons.map { |node| node["ipaddress"] }
+    mons = []
-  end
+    # make sure if this node runs ceph-mon, it's always included even if
+    # search is laggy; put it first in the hopes that clients will talk
+    # primarily to local node
+    if node['roles'].include? 'ceph-mon'
+      mons << node
+    end
 
-  mon_addresses = mon_addresses.map { |ip| ip + ":6789" }
+    mons += get_mon_nodes()
-  return mon_addresses.uniq
+    if is_crowbar?
+      mon_ips = mons.map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address }
+    else
+      if node['ceph']['config']['global'] && node['ceph']['config']['global']['public network']
+        mon_ips = mons.map { |node| find_node_ip_in_network(node['ceph']['config']['global']['public network'], node) }
+      else
+        mon_ips = mons.map { |node| node['ipaddress'] + ":6789" }
+      end
+    end
+  end
+  return mon_ips.uniq
+end
+
+def get_quorum_members_ips()
+  mon_ips = []
+  mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
+  raise 'getting quorum members failed' unless $?.exitstatus == 0
+
+  mons = JSON.parse(mon_status)['monmap']['mons']
+  mons.each do |k|
+    mon_ips.push(k['addr'][0..-3])
+  end
+  return mon_ips
 end
 
 QUORUM_STATES = ['leader', 'peon']
-
 def have_quorum?()
     # "ceph auth get-or-create-key" would hang if the monitor wasn't
     # in quorum yet, which is highly likely on the first run. This