Puppet: First support Ceph

This is a first implementation of Ceph support in TripleO with Puppet:

* Install ceph-mon on controller node
* Install ceph-osd on cephstorage node

Co-Authored-By: Giulio Fidente <gfidente@redhat.com>

Change-Id: I48488cbe950047fae5e746e458106d6edb9a6183

ceph-cluster-config.yaml

@@ -0,0 +1,45 @@
+heat_template_version: 2014-10-16
+description: 'Ceph Cluster config data'
+
+parameters:
+  ceph_fsid:
+    default: ''
+    type: string
+  ceph_admin_key:
+    default: ''
+    type: string
+  ceph_mon_key:
+    default: ''
+    type: string
+  ceph_mon_names:
+    type: comma_delimited_list
+  ceph_mon_ips:
+    type: comma_delimited_list
+
+resources:
+  CephClusterConfigImpl:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        ceph_cluster:
+          mon_initial_members:
+            list_join:
+            - ','
+            - {get_param: ceph_mon_names}
+          mon_host:
+            list_join:
+            - ','
+            - {get_param: ceph_mon_ips}
+          fsid: {get_param: ceph_fsid}
+          admin_key: {get_param: ceph_admin_key}
+          mon_key: {get_param: ceph_mon_key}
+          # We would need a dedicated key for OSD bootstrap
+          bootstrap_osd_key: {get_param: ceph_mon_key}
+          osds: '{"/srv/data": {}}'
+
+outputs:
+  config_id:
+    description: The ID of the CephClusterConfigImpl resource.
+    value:
+      {get_resource: CephClusterConfigImpl}

ceph-storage-post.yaml

@@ -0,0 +1,8 @@
+heat_template_version: 2014-10-16
+description: 'Ceph Storage Post Deployment'
+# NOTE: this is a noop for os-apply-config style deployments because
+# post deployment ordering is controlled by tripleo-image-elements
+
+parameters:
+  servers:
+    type: json

ceph-storage.yaml

@@ -27,7 +27,7 @@ parameters:
   CephMonitors:
     default: ''
     description: The list of ip/names to use as Ceph monitors
-    type: string
+    type: json
 resources:
   CephStorage:
     type: OS::Nova::Server

overcloud-resource-registry-puppet.yaml

@@ -10,7 +10,9 @@ resource_registry:
   OS::TripleO::ComputePostDeployment: puppet/compute-post-puppet.yaml
   OS::TripleO::ObjectStoragePostDeployment: puppet/swift-storage-post.yaml
   OS::TripleO::BlockStoragePostDeployment: puppet/cinder-storage-post.yaml
+  OS::TripleO::CephStoragePostDeployment: puppet/ceph-storage-post-puppet.yaml
   OS::TripleO::SwiftDevicesAndProxy::SoftwareConfig: puppet/swift-devices-and-proxy-config.yaml
+  OS::TripleO::CephClusterConfig::SoftwareConfig: puppet/ceph-cluster-config.yaml
   OS::TripleO::AllNodes::SoftwareConfig: puppet/all-nodes-config.yaml
   OS::TripleO::BootstrapNode::SoftwareConfig: puppet/bootstrap-config.yaml
 

overcloud-resource-registry.yaml

@@ -10,6 +10,8 @@ resource_registry:
   OS::TripleO::ComputePostDeployment: compute-post.yaml
   OS::TripleO::ObjectStoragePostDeployment: swift-storage-post.yaml
   OS::TripleO::BlockStoragePostDeployment: cinder-storage-post.yaml
+  OS::TripleO::CephStoragePostDeployment: ceph-storage-post.yaml
   OS::TripleO::SwiftDevicesAndProxy::SoftwareConfig: swift-devices-and-proxy-config.yaml
+  OS::TripleO::CephClusterConfig::SoftwareConfig: ceph-cluster-config.yaml
   OS::TripleO::AllNodes::SoftwareConfig: all-nodes-config.yaml
   OS::TripleO::BootstrapNode::SoftwareConfig: bootstrap-config.yaml

overcloud-without-mergepy.yaml

@@ -25,6 +25,19 @@ parameters:
     description: The password for the ceilometer service account.
     type: string
     hidden: true
+  # This has to be an UUID so for now we generate it outside the template
+  CephClusterFSID:
+    default: ''
+    type: string
+    description: The Ceph cluster FSID.
+  CephMonKey:
+    default: ''
+    description: The Ceph monitors key.
+    type: string
+  CephAdminKey:
+    default: ''
+    description: The Ceph admin client key.
+    type: string
   CloudName:
     default: ''
     description: The DNS name of this cloud. E.g. ci-overcloud.tripleo.org
@@ -194,18 +207,6 @@ parameters:
     description: The user password for SNMPd with readonly rights running on all Overcloud nodes
     type: string
     hidden: true
-  CephClusterFSID:
-    default: ''
-    type: string
-    description: The Ceph cluster FSID.
-  CephMonKey:
-    default: ''
-    description: The Ceph monitors key.
-    type: string
-  CephAdminKey:
-    default: ''
-    description: The Ceph admin client key.
-    type: string
 
   # Controller-specific params
   AdminToken:
@@ -490,6 +491,8 @@ parameters:
     default: baremetal
     description: Flavor for Ceph storage nodes to request when deploying.
     type: string
+    constraints:
+      - custom_constraint: nova.flavor
 
 resources:
 
@@ -663,10 +666,7 @@ resources:
           Image: {get_param: CephStorageImage}
           KeyName: {get_param: KeyName}
           Flavor: {get_param: OvercloudCephStorageFlavor}
-          CephClusterFSID: {get_param: CephClusterFSID}
-          CephMonKey: {get_param: CephMonKey}
-          CephAdminKey: {get_param: CephAdminKey}
-          CephMonitors: {get_attr: [Controller, corosync_node]}
+          NtpServer: {get_param: NtpServer}
 
   allNodesConfig:
     type: OS::TripleO::AllNodes::SoftwareConfig
@@ -743,6 +743,27 @@ resources:
       object_store_swift_devices: {get_attr: [ObjectStorage, swift_device]}
       controller_swift_proxy_memcaches: {get_attr: [Controller, swift_proxy_memcache]}
 
+  ControllerCephDeployment:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      config: {get_attr: [CephClusterConfig, config_id]}
+      servers: {get_attr: [Controller, attributes, nova_server_resource]}
+
+  CephStorageCephDeployment:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      config: {get_attr: [CephClusterConfig, config_id]}
+      servers: {get_attr: [CephStorage, attributes, nova_server_resource]}
+
+  CephClusterConfig:
+    type: OS::TripleO::CephClusterConfig::SoftwareConfig
+    properties:
+      ceph_fsid: {get_param: CephClusterFSID}
+      ceph_mon_key: {get_param: CephMonKey}
+      ceph_admin_key: {get_param: CephAdminKey}
+      ceph_mon_names: {get_attr: [Controller, hostname]}
+      ceph_mon_ips: {get_attr: [Controller, ip_address]}
+
   ControllerClusterConfig:
     type: OS::Heat::StructuredConfig
     properties:
@@ -789,10 +810,16 @@ resources:
       config: {get_attr: [allNodesConfig, config_id]}
       servers: {get_attr: [ObjectStorage, attributes, nova_server_resource]}
 
+  CephStorageAllNodesDeployment:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      config: {get_attr: [allNodesConfig, config_id]}
+      servers: {get_attr: [CephStorage, attributes, nova_server_resource]}
+
   # Nested stack deployment runs after all other controller deployments
   ControllerNodesPostDeployment:
     type: OS::TripleO::ControllerPostDeployment
-    depends_on: [ControllerBootstrapNodeDeployment, ControllerAllNodesDeployment, ControllerSwiftDeployment]
+    depends_on: [ControllerBootstrapNodeDeployment, ControllerAllNodesDeployment, ControllerSwiftDeployment, ControllerCephDeployment]
     properties:
       servers: {get_attr: [Controller, attributes, nova_server_resource]}
 
@@ -814,6 +841,12 @@ resources:
     properties:
       servers: {get_attr: [BlockStorage, attributes, nova_server_resource]}
 
+  CephStorageNodesPostDeployment:
+    type: OS::TripleO::CephStoragePostDeployment
+    depends_on: [ControllerNodesPostDeployment, CephStorageCephDeployment, CephStorageAllNodesDeployment]
+    properties:
+      servers: {get_attr: [CephStorage, attributes, nova_server_resource]}
+
 outputs:
   KeystoneURL:
     description: URL for the Overcloud Keystone service

puppet/ceph-cluster-config.yaml

@@ -0,0 +1,48 @@
+heat_template_version: 2014-10-16
+description: 'Ceph Cluster config data for Puppet'
+
+parameters:
+  ceph_fsid:
+    default: ''
+    type: string
+  ceph_admin_key:
+    default: ''
+    type: string
+  ceph_mon_key:
+    default: ''
+    type: string
+  ceph_mon_names:
+    type: comma_delimited_list
+  ceph_mon_ips:
+    type: comma_delimited_list
+
+resources:
+  CephClusterConfigImpl:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        hiera:
+          datafiles:
+            ceph_cluster:
+              mapped_data:
+                ceph_mon_initial_members:
+                  list_join:
+                  - ','
+                  - {get_param: ceph_mon_names}
+                ceph::profile::params::mon_host:
+                  list_join:
+                  - ','
+                  - {get_param: ceph_mon_ips}
+                ceph::profile::params::fsid: {get_param: ceph_fsid}
+                ceph::profile::params::admin_key: {get_param: ceph_admin_key}
+                ceph::profile::params::mon_key: {get_param: ceph_mon_key}
+                # We would need a dedicated key for OSD
+                ceph::profile::params::bootstrap_osd_key: {get_param: ceph_mon_key}
+                ceph::profile::params::osds: '{"/srv/data": {}}'
+
+outputs:
+  config_id:
+    description: The ID of the CephClusterConfigImpl resource.
+    value:
+      {get_resource: CephClusterConfigImpl}

puppet/ceph-storage-post-puppet.yaml

@@ -0,0 +1,24 @@
+heat_template_version: 2014-10-16
+
+description: >
+  OpenStack Ceph Storage node post deployment for Puppet.
+
+parameters:
+  servers:
+    type: json
+
+resources:
+  CephStoragePuppetConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: puppet
+      outputs:
+      - name: result
+      config:
+        get_file: manifests/overcloud_cephstorage.pp
+
+  CephStoragePuppetDeployment:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      servers:  {get_param: servers}
+      config: {get_resource: CephStoragePuppetConfig}

puppet/ceph-storage-puppet.yaml

@@ -1,2 +1,88 @@
 heat_template_version: 2014-10-16
-description: 'Common Ceph Storage Configuration by Puppet'
+description: 'Common Ceph Storage Configuration for Puppet'
+parameters:
+  Flavor:
+    description: Flavor for the Ceph Storage node.
+    type: string
+    constraints:
+      - custom_constraint: nova.flavor
+  Image:
+    type: string
+    default: overcloud-ceph-storage
+    constraints:
+      - custom_constraint: glance.image
+  ImageUpdatePolicy:
+    default: 'REBUILD_PRESERVE_EPHEMERAL'
+    description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt.
+    type: string
+  KeyName:
+    description: Name of an existing EC2 KeyPair to enable SSH access to the instances
+    type: string
+    default: default
+    constraints:
+      - custom_constraint: nova.keypair
+  NtpServer:
+    type: string
+    default: ''
+  EnablePackageInstall:
+    default: 'false'
+    description: Set to true to enable package installation via Puppet
+    type: boolean
+
+resources:
+  CephStorage:
+    type: OS::Nova::Server
+    properties:
+      image: {get_param: Image}
+      image_update_policy: {get_param: ImageUpdatePolicy}
+      flavor: {get_param: Flavor}
+      key_name: {get_param: KeyName}
+      networks:
+        - network: ctlplane
+      user_data_format: SOFTWARE_CONFIG
+  CephStorageDeployment:
+    type: OS::Heat::StructuredDeployment
+    properties:
+      config: {get_resource: CephStorageConfig}
+      server: {get_resource: CephStorage}
+      input_values:
+        ntp_servers:
+          str_replace:
+            template: '["server"]'
+            params:
+              server: {get_param: NtpServer}
+        enable_package_install: {get_param: EnablePackageInstall}
+  CephStorageConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        hiera:
+          hierarchy:
+            - heat_config_%{::deploy_config_name}
+            - cephstorage
+            - ceph_cluster # provided by CephClusterConfig
+            - ceph
+            - common
+          datafiles:
+            common:
+              raw_data: {get_file: hieradata/common.yaml}
+            ceph:
+              raw_data: {get_file: hieradata/ceph.yaml}
+            cephstorage:
+              mapped_data:
+                ntp::servers: {get_input: ntp_servers}
+                enable_package_install: {get_input: enable_package_install}
+
+outputs:
+  hosts_entry:
+    value:
+      str_replace:
+        template: "IP HOST HOST.novalocal"
+        params:
+          IP: {get_attr: [CephStorage, networks, ctlplane, 0]}
+          HOST: {get_attr: [CephStorage, name]}
+  nova_server_resource:
+    description: Heat resource handle for the ceph storage server
+    value:
+      {get_resource: CephStorage}

puppet/controller-puppet.yaml

@@ -556,11 +556,15 @@ resources:
             - object
             - swift_devices_and_proxy # provided by SwiftDevicesAndProxyConfig
             - rabbit # provided by allNodesConfig
+            - ceph_cluster # provided by CephClusterConfig
+            - ceph
             - bootstrap_node # provided by BootstrapNodeConfig
             - common
           datafiles:
             common:
               raw_data: {get_file: hieradata/common.yaml}
+            ceph:
+              raw_data: {get_file: hieradata/ceph.yaml}
             object:
               raw_data: {get_file: hieradata/object.yaml}
             controller:

puppet/hieradata/ceph.yaml

@@ -0,0 +1,7 @@
+ceph::profile::params::osd_journal_size: 1024
+ceph::profile::params::osd_pool_default_pg_num: 128
+ceph::profile::params::osd_pool_default_pgp_num: 128
+ceph::profile::params::osd_pool_default_size: 3
+ceph::profile::params::osd_pool_default_min_size: 1
+ceph::profile::params::manage_repo: false
+ceph::profile::params::authentication_type: cephx

puppet/manifests/overcloud_cephstorage.pp

@@ -0,0 +1,35 @@
+# Copyright 2015 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+if !str2bool(hiera('enable_package_install', 'false')) {
+  case $::osfamily {
+    'RedHat': {
+      Package { provider => 'norpm' } # provided by tripleo-puppet
+    }
+    default: {
+      warning('enable_package_install option not supported.')
+    }
+  }
+}
+
+if count(hiera('ntp::servers')) > 0 {
+  include ::ntp
+}
+
+class { 'ceph::profile::params':
+  mon_initial_members => downcase(hiera('ceph_mon_initial_members'))
+}
+include ::ceph::profile::client
+include ::ceph::profile::osd

puppet/manifests/overcloud_controller.pp

@@ -150,6 +150,14 @@ if hiera('step') >= 1 {
   # pre-install swift here so we can build rings
   include ::swift
 
+  # don't install Ceph if FSID is not provided
+  if hiera('ceph::profile::params::fsid', false) {
+    class { 'ceph::profile::params':
+      mon_initial_members => downcase(hiera('ceph_mon_initial_members'))
+    }
+    include ::ceph::profile::mon
+  }
+
 } #END STEP 1
 
 if hiera('step') >= 2 {