Deploy Ceph in containers using ceph-ansible via external workflow

Add docker profiles to deploy Ceph in containers via ceph-ansible. This is
implemented by triggering a Mistral workflow during one of the overcloud
deployment steps, as provided by [1].

Some new service-specific parameters are available to determine the workflow to
execute and the ansible playbook to use. A new `CephAnsibleExtraConfig`
parameter can be used to provide arbitrary config variables consumed by `ceph-ansible`.

The pre-existing template params consumed up until the Pike release to
drive `puppet-ceph` continue to work and are translated, when possible, into
the equivalent `ceph-ansible` variable.

A new environment file is added to enable use of ceph-ansible;
the pre-existing puppet-ceph implementation remains unchanged and usable
for non-containerized deployments.

1. https://review.openstack.org/#/c/463324/

Change-Id: I81d44a1e198c83a4ef8b109b4eb6c611555dcdc5
This commit is contained in:
Giulio Fidente 2017-05-26 01:35:53 +02:00 committed by John Fulton
parent ed0b77ff93
commit d11e256eed
7 changed files with 453 additions and 0 deletions

View File

@ -0,0 +1,205 @@
heat_template_version: pike
description: >
Ceph base service. Shared by all Ceph services.
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
CephAnsibleWorkflowName:
type: string
description: Name of the Mistral workflow to execute
default: tripleo.storage.v1.ceph-install
CephAnsiblePlaybook:
type: string
description: Path to the ceph-ansible playbook to execute
default: /usr/share/ceph-ansible/site-docker.yml.sample
CephAnsibleExtraConfig:
type: json
description: Extra vars for the ceph-ansible playbook
default: {}
CephClusterFSID:
type: string
description: The Ceph cluster FSID. Must be a UUID.
CephPoolDefaultPgNum:
description: default pg_num to use for the RBD pools
type: number
default: 32
CephPools:
description: >
It can be used to override settings for one of the predefined pools, or to create
additional ones. Example:
{
"volumes": {
"size": 5,
"pg_num": 128,
"pgp_num": 128
}
}
default: {}
type: json
CinderRbdPoolName:
default: volumes
type: string
CinderBackupRbdPoolName:
default: backups
type: string
GlanceRbdPoolName:
default: images
type: string
GnocchiRbdPoolName:
default: metrics
type: string
NovaRbdPoolName:
default: vms
type: string
CephClientKey:
description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
type: string
hidden: true
CephClientUserName:
default: openstack
type: string
CephPoolDefaultSize:
description: default minimum replication for RBD copies
type: number
default: 3
CephIPv6:
default: False
type: boolean
DockerCephDaemonImage:
description: image
type: string
default: 'ceph/daemon:tag-build-master-jewel-centos-7'
conditions:
custom_registry_host:
yaql:
data: {get_param: DockerCephDaemonImage}
expression: $.data.split('/')[0].matches('(\.|:)')
outputs:
role_data:
description: Role data for the Ceph base service.
value:
service_name: ceph_base
upgrade_tasks: []
step_config: ''
puppet_config:
config_image: ''
config_volume: ''
step_config: ''
docker_config: {}
service_workflow_tasks:
step2:
- name: ceph_base_ansible_workflow
workflow: { get_param: CephAnsibleWorkflowName }
input:
ceph_ansible_extra_vars: {get_param: CephAnsibleExtraConfig}
ceph_ansible_playbook: {get_param: CephAnsiblePlaybook}
config_settings:
ceph_common_ansible_vars:
fsid: { get_param: CephClusterFSID }
docker: true
ceph_docker_registry:
if:
- custom_registry_host
- yaql:
expression: regex('(?:https?://)?(.*)/').split($.data)[1]
data: {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
- docker.io
ceph_docker_image:
if:
- custom_registry_host
- yaql:
expression: regex('(?:https?://)?(.*)/').split($.data)[2]
data: {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
- {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
ceph_docker_image_tag: {str_split: [':', {get_param: DockerCephDaemonImage}, 1]}
containerized_deployment: true
public_network: {get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephMonNetwork]}]}
cluster_network: {get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]}
user_config: true
ceph_stable: true
ceph_origin: distro
openstack_config: true
openstack_pools:
list_concat:
- repeat:
template:
name: <%pool%>
pg_num: {get_param: CephPoolDefaultPgNum}
rule_name: ""
for_each:
<%pool%>:
- {get_param: CinderRbdPoolName}
- {get_param: CinderBackupRbdPoolName}
- {get_param: NovaRbdPoolName}
- {get_param: GlanceRbdPoolName}
- {get_param: GnocchiRbdPoolName}
- repeat:
template:
name: <%pool%>
pg_num: {get_param: CephPoolDefaultPgNum}
rule_name: ""
for_each:
<%pool%>: {get_param: CephPools}
openstack_keys: &openstack_keys
- name:
list_join:
- '.'
- - client
- {get_param: CephClientUserName}
key: {get_param: CephClientKey}
mon_cap: "allow r"
osd_cap:
str_replace:
template: "allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL"
params:
NOVA_POOL: {get_param: NovaRbdPoolName}
CINDER_POOL: {get_param: CinderRbdPoolName}
CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
GLANCE_POOL: {get_param: GlanceRbdPoolName}
GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
acls:
- "u:glance:r--"
- "u:nova:r--"
- "u:cinder:r--"
- "u:gnocchi:r--"
keys: *openstack_keys
pools: []
ceph_conf_overrides:
global:
osd_pool_default_size: {get_param: CephPoolDefaultSize}
osd_pool_default_pg_num: {get_param: CephPoolDefaultPgNum}
ntp_service_enabled: false
generate_fsid: false
ip_version:
if:
- {get_param: CephIPv6}
- ipv6
- ipv4

View File

@ -0,0 +1,58 @@
heat_template_version: pike
description: >
Ceph Client service.
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
resources:
CephBase:
type: ./ceph-base.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for the Ceph Client service.
value:
service_name: ceph_client
upgrade_tasks: []
step_config: ''
puppet_config:
config_image: ''
config_volume: ''
step_config: ''
docker_config: {}
service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
config_settings: {}

View File

@ -0,0 +1,86 @@
heat_template_version: pike
description: >
Ceph Monitor service.
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
CephMonKey:
description: The Ceph monitors key. Can be created with ceph-authtool --gen-print-key.
type: string
hidden: true
CephAdminKey:
default: ''
description: The Ceph admin client key. Can be created with ceph-authtool --gen-print-key.
type: string
hidden: true
CephValidationRetries:
type: number
default: 40
description: Number of retry attempts for Ceph validation
CephValidationDelay:
type: number
default: 30
description: Interval (in seconds) in between validation checks
resources:
CephBase:
type: ./ceph-base.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for the Ceph Monitor service.
value:
service_name: ceph_mon
upgrade_tasks: []
step_config: ''
puppet_config:
config_image: ''
config_volume: ''
step_config: ''
docker_config: {}
service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
config_settings:
map_merge:
- tripleo.ceph_mon.firewall_rules:
'110 ceph_mon':
dport:
- 6789
- ceph_mon_ansible_vars:
map_merge:
- {get_attr: [CephBase, role_data, config_settings, ceph_common_ansible_vars]}
- monitor_secret: {get_param: CephMonKey}
admin_secret: {get_param: CephAdminKey}
monitor_interface: br_ex

View File

@ -0,0 +1,75 @@
heat_template_version: pike
description: >
Ceph OSD service.
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
CephAnsibleDisksConfig:
type: json
description: Disks config settings for ceph-ansible
default:
devices:
- /dev/vdb
journal_size: 512
journal_collocation: true
resources:
CephBase:
type: ./ceph-base.yaml
properties:
ServiceData: {get_param: ServiceData}
ServiceNetMap: {get_param: ServiceNetMap}
DefaultPasswords: {get_param: DefaultPasswords}
EndpointMap: {get_param: EndpointMap}
RoleName: {get_param: RoleName}
RoleParameters: {get_param: RoleParameters}
outputs:
role_data:
description: Role data for the Ceph OSD service.
value:
service_name: ceph_osd
upgrade_tasks: []
step_config: ''
puppet_config:
config_image: ''
config_volume: ''
step_config: ''
docker_config: {}
service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
config_settings:
map_merge:
- tripleo.ceph_osd.firewall_rules:
'111 ceph_osd':
dport:
- '6800-7300'
- ceph_osd_ansible_vars:
map_merge:
- {get_attr: [CephBase, role_data, config_settings, ceph_common_ansible_vars]}
- {get_param: CephAnsibleDisksConfig}

View File

@ -0,0 +1,12 @@
resource_registry:
OS::TripleO::Services::CephMon: ../../docker/services/ceph-ansible/ceph-mon.yaml
OS::TripleO::Services::CephOSD: ../../docker/services/ceph-ansible/ceph-osd.yaml
OS::TripleO::Services::CephClient: ../../docker/services/ceph-ansible/ceph-client.yaml
parameter_defaults:
CinderEnableIscsiBackend: false
CinderEnableRbdBackend: true
CinderBackupBackend: ceph
NovaEnableRbdBackend: true
GlanceBackend: rbd
GnocchiBackend: rbd

View File

@ -0,0 +1,14 @@
---
prelude: >
Deployment of Ceph in containers is implemented using a Mistral workflow.
other:
- |
It is possible to deploy Ceph in docker containers in the overcloud. This
is implemented by triggering `ceph-ansible` via a Mistral workflow. A new
`CephAnsibleExtraConfig` parameter has been added to the templates and can
be used to provide arbitrary config variables consumed by `ceph-ansible`.
The pre-existing template params consumed by the TripleO Pike release to
drive `puppet-ceph` continue to work and are translated, when possible, into
their equivalent `ceph-ansible` variable. To enable the deployment of Ceph
in containers use `environments/ceph-ansible/ceph-ansible.yaml` when
deploying the overcloud.

View File

@ -31,6 +31,7 @@ envs_containing_endpoint_map = ['tls-endpoints-public-dns.yaml',
'tls-endpoints-public-ip.yaml',
'tls-everywhere-endpoints-dns.yaml']
ENDPOINT_MAP_FILE = 'endpoint_map.yaml'
OPTIONAL_SECTIONS = ['service_workflow_tasks']
REQUIRED_DOCKER_SECTIONS = ['service_name', 'docker_config', 'puppet_config',
'config_settings', 'step_config']
OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks',
@ -271,6 +272,8 @@ def validate_docker_service(filename, tpl):
else:
if section_name in OPTIONAL_DOCKER_SECTIONS:
continue
elif section_name in OPTIONAL_SECTIONS:
continue
else:
print('ERROR: %s is extra in role_data for %s.'
% (section_name, filename))