Isolate the Ceph storage and storage_mgmt nets

This patch updates the Ceph configuration for the puppet
implementation so that it isolates the Ceph traffic
for the public and cluster interfaces. By default public traffic
runs on the "storage" network and the cluster traffic runs on the
"storage mgmt" network.

If network isolation is not enabled then the default
ctlplane address's will be used for both the public and
cluster interfaces.

Change-Id: I791244d72c8f42142d9de99e0cf0acdca19e62b0

ceph-storage.yaml

@@ -28,6 +28,11 @@ parameters:
     default: ''
     description: The list of ip/names to use as Ceph monitors
     type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
 
 resources:
   CephStorage:

overcloud-without-mergepy.yaml

@@ -516,6 +516,8 @@ parameters:
       RabbitMqNetwork: internal_api
       RedisNetwork: internal_api
       MysqlNetwork: internal_api
+      CephClusterNetwork: storage_mgmt
+      CephPublicNetwork: storage
     description: Mapping of service_name -> network name. Typically set
                  via parameter_defaults in the resource registry.
     type: json
@@ -782,6 +784,7 @@ resources:
           KeyName: {get_param: KeyName}
           Flavor: {get_param: OvercloudCephStorageFlavor}
           NtpServer: {get_param: NtpServer}
+          ServiceNetMap: {get_param: ServiceNetMap}
 
   ControllerIpListMap:
     type: OS::TripleO::Network::Ports::NetIpListMap
@@ -980,7 +983,7 @@ resources:
       ceph_mon_key: {get_param: CephMonKey}
       ceph_admin_key: {get_param: CephAdminKey}
       ceph_mon_names: {get_attr: [Controller, hostname]}
-      ceph_mon_ips: {get_attr: [Controller, ip_address]}
+      ceph_mon_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 
   ControllerClusterConfig:
     type: OS::Heat::StructuredConfig

puppet/ceph-storage-puppet.yaml

@@ -28,6 +28,11 @@ parameters:
     default: 'false'
     description: Set to true to enable package installation via Puppet
     type: boolean
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
 
 resources:
   CephStorage:
@@ -61,6 +66,12 @@ resources:
       StorageIpSubnet: {get_attr: [StoragePort, ip_subnet]}
       StorageMgmtIpSubnet: {get_attr: [StorageMgmtPort, ip_subnet]}
 
+  NetIpSubnetMap:
+    type: OS::TripleO::Network::Ports::NetIpMap
+    properties:
+      StorageIp: {get_attr: [StoragePort, ip_subnet]}
+      StorageMgmtIp: {get_attr: [StorageMgmtPort, ip_subnet]}
+
   NetworkDeployment:
     type: OS::TripleO::SoftwareDeployment
     properties:
@@ -79,6 +90,8 @@ resources:
             params:
               server: {get_param: NtpServer}
         enable_package_install: {get_param: EnablePackageInstall}
+        ceph_cluster_network: {get_attr: [NetIpSubnetMap, net_ip_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]}
+        ceph_public_network: {get_attr: [NetIpSubnetMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 
   CephStorageConfig:
     type: OS::Heat::StructuredConfig
@@ -88,7 +101,6 @@ resources:
         hiera:
           hierarchy:
             - heat_config_%{::deploy_config_name}
-            - cephstorage
             - ceph_cluster # provided by CephClusterConfig
             - ceph
             - '"%{::osfamily}"'
@@ -98,10 +110,11 @@ resources:
               raw_data: {get_file: hieradata/common.yaml}
             ceph:
               raw_data: {get_file: hieradata/ceph.yaml}
-            cephstorage:
               mapped_data:
                 ntp::servers: {get_input: ntp_servers}
                 enable_package_install: {get_input: enable_package_install}
+                ceph::profile::params::cluster_network: {get_input: ceph_cluster_network}
+                ceph::profile::params::public_network: {get_input: ceph_public_network}
 
 outputs:
   hosts_entry:

puppet/controller-puppet.yaml

@@ -498,6 +498,15 @@ resources:
       StorageMgmtIp: {get_attr: [StorageMgmtPort, ip_address]}
       TenantIp: {get_attr: [TenantPort, ip_address]}
 
+  NetIpSubnetMap:
+    type: OS::TripleO::Network::Ports::NetIpMap
+    properties:
+      ExternalIp: {get_attr: [ExternalPort, ip_subnet]}
+      InternalApiIp: {get_attr: [InternalApiPort, ip_subnet]}
+      StorageIp: {get_attr: [StoragePort, ip_subnet]}
+      StorageMgmtIp: {get_attr: [StorageMgmtPort, ip_subnet]}
+      TenantIp: {get_attr: [TenantPort, ip_subnet]}
+
   NetworkConfig:
     type: OS::TripleO::Controller::Net::SoftwareConfig
     properties:
@@ -727,6 +736,9 @@ resources:
         redis_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RedisNetwork]}]}
         memcached_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}
         mysql_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
+        ceph_cluster_network: {get_attr: [NetIpSubnetMap, net_ip_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]}
+        ceph_public_network: {get_attr: [NetIpSubnetMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
+        ceph_public_ip: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 
   # Map heat metadata into hiera datafiles
   ControllerConfig:
@@ -752,6 +764,10 @@ resources:
               raw_data: {get_file: hieradata/common.yaml}
             ceph:
               raw_data: {get_file: hieradata/ceph.yaml}
+              mapped_data:
+                ceph::profile::params::cluster_network: {get_input: ceph_cluster_network}
+                ceph::profile::params::public_network: {get_input: ceph_public_network}
+                ceph::mon::public_addr: {get_input: ceph_public_ip}
             object:
               raw_data: {get_file: hieradata/object.yaml}
             controller: