Support for using external Ceph clusters

This patch adds support for using an externally managed Ceph
cluster with the TripleO Heat templates.

For an externally managed Ceph cluster we initially
only deploy the Ceph client tools, install the 'openstack' user
keyring, and generate the ceph.conf. This matches what we do
for managed Ceph installations and is a good first start.
No other Ceph related services are installed or managed.

To enable use of a Ceph external cluster simply add
the custom Heat environment file environments/puppet-ceph-external.yaml
to your heat stack create/update command and make sure to
set the required CephClientKey, CephExternalMonHost, and CephClusterFSID
variables.

Change-Id: I0a8b213ce9dfa2fc4e62ae1e7631466e5179fc2b

ceph-cluster-config.yaml

@@ -2,6 +2,18 @@ heat_template_version: 2015-04-30
 description: 'Ceph Cluster config data'
 
 parameters:
+  ceph_storage_count:
+    default: 0
+    type: number
+    description: Number of Ceph storage nodes. Used to enable/disable managed Ceph installation.
+  ceph_external_mon_ips:
+    default: ''
+    type: string
+    description: List of external Ceph Mon host IPs.
+  ceph_client_key:
+    default: ''
+    type: string
+    description: Ceph key used to create the 'openstack' user keyring.
   ceph_fsid:
     default: ''
     type: string

environments/puppet-ceph-external.yaml

@@ -0,0 +1,18 @@
+# A Heat environment file which can be used to enable the
+# use of an externally managed Ceph cluster.
+resource_registry:
+  OS::TripleO::CephClusterConfig::SoftwareConfig: ../puppet/extraconfig/ceph/ceph-external-config.yaml
+
+parameters:
+  # NOTE: These example parameters are required when using Ceph External
+  #CephClusterFSID: '4b5c8c0a-ff60-454b-a1b4-9747aa737d19'
+  #CephClientKey: 'AQDLOh1VgEp6FRAAFzT7Zw+Y9V6JJExQAsRnRQ=='
+  #CephExternalMonHost: '172.16.1.7, 172.16.1.8'
+
+  # the following parameters enable Ceph backends for Cinder, Glance, and Nova
+  NovaEnableRbdBackend: true
+  CinderEnableRbdBackend: true
+  GlanceBackend: rbd
+
+  # finally we disable the Cinder LVM backend
+  CinderEnableIscsiBackend: false

overcloud-without-mergepy.yaml

@@ -46,6 +46,14 @@ parameters:
     default: false
     description: Whether to enable or not the NFS backend for Cinder
     type: boolean
+  CephClientKey:
+    default: ''
+    description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
+    type: string
+  CephExternalMonHost:
+    default: ''
+    type: string
+    description: List of externally managed Ceph Mon Host IPs. Only used for external Ceph deployments.
   CinderEnableIscsiBackend:
     default: true
     description: Whether to enable or not the Iscsi backend for Cinder
@@ -1168,9 +1176,12 @@ resources:
   CephClusterConfig:
     type: OS::TripleO::CephClusterConfig::SoftwareConfig
     properties:
+      ceph_storage_count: {get_param: CephStorageCount}
       ceph_fsid: {get_param: CephClusterFSID}
       ceph_mon_key: {get_param: CephMonKey}
       ceph_admin_key: {get_param: CephAdminKey}
+      ceph_client_key: {get_param: CephClientKey}
+      ceph_external_mon_ips: {get_param: CephExternalMonHost}
       ceph_mon_names: {get_attr: [Controller, hostname]}
       ceph_mon_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 

puppet/ceph-cluster-config.yaml

@@ -2,6 +2,18 @@ heat_template_version: 2015-04-30
 description: 'Ceph Cluster config data for Puppet'
 
 parameters:
+  ceph_storage_count:
+    default: 0
+    type: number
+    description: Number of Ceph storage nodes. Used to enable/disable managed Ceph installation.
+  ceph_external_mon_ips:
+    default: ''
+    type: string
+    description: List of external Ceph Mon host IPs.
+  ceph_client_key:
+    default: ''
+    type: string
+    description: Ceph key used to create the 'openstack' user keyring.
   ceph_fsid:
     default: ''
     type: string
@@ -26,6 +38,7 @@ resources:
           datafiles:
             ceph_cluster:
               mapped_data:
+                ceph_storage_count: {get_param: ceph_storage_count}
                 ceph_mon_initial_members:
                   list_join:
                   - ','

puppet/extraconfig/ceph/ceph-external-config.yaml

@@ -0,0 +1,65 @@
+heat_template_version: 2015-04-30
+description: 'Configure parameters for an external Ceph cluster via Puppet.'
+
+parameters:
+  ceph_storage_count:
+    default: 0
+    type: number
+    description: Number of Ceph storage nodes. Used to enable/disable managed Ceph installation.
+  ceph_external_mon_ips:
+    default: ''
+    type: string
+    description: List of external Ceph Mon host IPs.
+  ceph_client_key:
+    default: ''
+    type: string
+    description: Ceph key used to create the 'openstack' user keyring.
+  ceph_fsid:
+    default: ''
+    type: string
+  # The following parameters are unused for external Ceph clusters and
+  # are here and exist for compatibility
+  ceph_admin_key:
+    default: ''
+    type: string
+  ceph_mon_key:
+    default: ''
+    type: string
+  ceph_mon_names:
+    type: comma_delimited_list
+  ceph_mon_ips:
+    type: comma_delimited_list
+
+resources:
+  CephClusterConfigImpl:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        hiera:
+          datafiles:
+            ceph_cluster:
+              mapped_data:
+                ceph_storage_count: {get_param: ceph_storage_count}
+                enable_external_ceph: true
+                ceph::profile::params::mon_host: {get_param: ceph_external_mon_ips}
+                ceph::profile::params::fsid: {get_param: ceph_fsid}
+                ceph::profile::params::client_keys:
+                  str_replace:
+                    template: "{
+                      client.openstack: {
+                        secret: 'CLIENT_KEY',
+                        mode: '0644',
+                        cap_mon: 'allow r',
+                        cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rwx pool=images'
+                      }
+                    }"
+                    params:
+                      CLIENT_KEY: {get_param: ceph_client_key}
+
+
+outputs:
+  config_id:
+    description: The ID of the CephClusterConfigImpl resource.
+    value:
+      {get_resource: CephClusterConfigImpl}

puppet/manifests/overcloud_controller.pp

@@ -122,8 +122,7 @@ if hiera('step') >= 2 {
   # pre-install swift here so we can build rings
   include ::swift
 
-  $cinder_enable_rbd_backend = hiera('cinder_enable_rbd_backend', false)
-  $enable_ceph = $cinder_enable_rbd_backend
+  $enable_ceph = hiera('ceph_storage_count', 0) > 0
 
   if $enable_ceph {
     class { 'ceph::profile::params':
@@ -147,10 +146,13 @@ if hiera('step') >= 2 {
       } -> Class['ceph::profile::osd']
     }
 
-    include ::ceph::profile::client
     include ::ceph::profile::osd
   }
 
+  if str2bool(hiera('enable_external_ceph', 'false')) {
+    include ::ceph::profile::client
+  }
+
 } #END STEP 2
 
 if hiera('step') >= 3 {
@@ -276,9 +278,14 @@ if hiera('step') >= 3 {
 
     $ceph_pools = hiera('ceph_pools')
     ceph::pool { $ceph_pools : }
+
+    $cinder_pool_requires = [Ceph::Pool['volumes']]
+
+  } else {
+    $cinder_pool_requires = []
   }
 
-  if $cinder_enable_rbd_backend {
+  if hiera('cinder_enable_rbd_backend', false) {
     $cinder_rbd_backend = 'tripleo_ceph'
 
     cinder_config {
@@ -289,7 +296,7 @@ if hiera('step') >= 3 {
       rbd_pool        => 'volumes',
       rbd_user        => 'openstack',
       rbd_secret_uuid => hiera('ceph::profile::params::fsid'),
-      require         => Ceph::Pool['volumes'],
+      require         => $cinder_pool_requires,
     }
   }
 

puppet/manifests/overcloud_controller_pacemaker.pp

@@ -438,8 +438,7 @@ MYSQL_HOST=localhost\n",
   include ::swift
 
   # Ceph
-  $cinder_enable_rbd_backend = hiera('cinder_enable_rbd_backend', false)
-  $enable_ceph = $cinder_enable_rbd_backend
+  $enable_ceph = hiera('ceph_storage_count', 0) > 0
 
   if $enable_ceph {
     class { 'ceph::profile::params':
@@ -463,10 +462,13 @@ MYSQL_HOST=localhost\n",
       } -> Class['ceph::profile::osd']
     }
 
-    include ::ceph::profile::client
     include ::ceph::profile::osd
   }
 
+  if str2bool(hiera('enable_external_ceph', 'false')) {
+    include ::ceph::profile::client
+  }
+
 
 } #END STEP 2
 
@@ -643,9 +645,14 @@ if hiera('step') >= 3 {
 
     $ceph_pools = hiera('ceph_pools')
     ceph::pool { $ceph_pools : }
+
+    $cinder_pool_requires = [Ceph::Pool['volumes']]
+
+  } else {
+    $cinder_pool_requires = []
   }
 
-  if $cinder_enable_rbd_backend {
+  if hiera('cinder_enable_rbd_backend', false) {
     $cinder_rbd_backend = 'tripleo_ceph'
 
     cinder_config {
@@ -656,7 +663,7 @@ if hiera('step') >= 3 {
       rbd_pool        => 'volumes',
       rbd_user        => 'openstack',
       rbd_secret_uuid => hiera('ceph::profile::params::fsid'),
-      require         => Ceph::Pool['volumes'],
+      require         => $cinder_pool_requires,
     }
   }