tripleo-heat-templates/docker/services/ceph-ansible/ceph-base.yaml

636 lines
27 KiB
YAML

heat_template_version: rocky
description: >
Ceph base service. Shared by all Ceph services.
parameters:
ServiceData:
default: {}
description: Dictionary packing service data
type: json
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
via parameter_defaults in the resource registry. This
mapping overrides those in ServiceNetMapDefaults.
type: json
DefaultPasswords:
default: {}
type: json
RoleName:
default: ''
description: Role name on which the service is applied
type: string
RoleParameters:
default: {}
description: Parameters specific to the role
type: json
EndpointMap:
default: {}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
type: json
StackUpdateType:
type: string
description: >
Type of update, to differentiate between UPGRADE and UPDATE cases
when StackAction is UPDATE (both are the same stack action).
constraints:
- allowed_values: ['', 'UPGRADE', 'FASTFORWARDUPGRADE']
default: ''
NodeDataLookup:
type: json
default: {}
description: json containing per-node configuration map
DeploymentServerBlacklist:
default: []
type: comma_delimited_list
description: >
List of server hostnames to blacklist from any triggered deployments.
CephAnsiblePlaybook:
type: comma_delimited_list
description: >
List of paths to the ceph-ansible playbooks to execute. If not
specified, the playbook will be determined automatically
depending on type of operation being performed
(deploy/update/upgrade).
default: ['default']
CephAnsibleExtraConfig:
type: json
description: Extra vars for the ceph-ansible playbook
default: {}
CephAnsibleSkipTags:
type: string
description: List of ceph-ansible tags to skip
default: 'package-install,with_pkg'
CephConfigOverrides:
type: json
description: Extra config settings to dump into ceph.conf
default: {}
CephClusterFSID:
type: string
description: The Ceph cluster FSID. Must be a UUID.
CephClusterName:
type: string
default: ceph
description: The Ceph cluster name.
constraints:
- allowed_pattern: "[a-zA-Z0-9]+"
description: >
The Ceph cluster name must be at least 1 character and contain only
letters and numbers.
CephPoolDefaultPgNum:
description: default pg_num to use for the RBD pools
type: number
default: 128
CephPools:
description: >
It can be used to override settings for one of the predefined pools, or to create
additional ones. Example:
[{"name": "volumes", "pg_num": 64, "rule_name": "replicated_rule"}]
default: []
type: json
CinderRbdPoolName:
default: volumes
type: string
CinderRbdExtraPools:
default: []
description: >
List of extra Ceph pools for use with RBD backends for Cinder. An
extra Cinder RBD backend driver is created for each pool in the
list. This is in addition to the standard RBD backend driver
associated with the CinderRbdPoolName.
type: comma_delimited_list
CinderBackupRbdPoolName:
default: backups
type: string
GlanceRbdPoolName:
default: images
type: string
GnocchiRbdPoolName:
default: metrics
type: string
NovaRbdPoolName:
default: vms
type: string
CephClientKey:
description: The Ceph client key. Can be created with ceph-authtool --gen-print-key.
type: string
hidden: true
CephClientUserName:
default: openstack
type: string
CephRgwClientName:
default: radosgw
type: string
CephRgwKey:
description: The cephx key for the radosgw client. Can be created
with ceph-authtool --gen-print-key.
type: string
hidden: true
CephPoolDefaultSize:
description: default minimum replication for RBD copies
type: number
default: 3
ManilaCephFSCephFSAuthId:
default: manila
type: string
CephManilaClientKey:
default: ''
description: The Ceph client key. Can be created with ceph-authtool --gen-print-key.
type: string
hidden: true
CephIPv6:
default: False
type: boolean
SwiftPassword:
description: The password for the swift service account
type: string
hidden: true
DockerCephDaemonImage:
description: image
type: string
CephAnsiblePlaybookVerbosity:
default: 1
description: The number of '-v', '-vv', etc. passed to ansible-playbook command
type: number
constraints:
- range: { min: 1, max: 5 }
CephAnsibleEnvironmentVariables:
default: {}
description: Mapping of Ansible environment variables to override defaults.
type: json
SwiftFetchDirGetTempurl:
default: ''
description: A temporary Swift URL to download the fetch_directory from.
type: string
SwiftFetchDirPutTempurl:
default: ''
description: A temporary Swift URL to upload the fetch_directory to.
type: string
LocalCephAnsibleFetchDirectoryBackup:
default: ''
description: Filesystem path on undercloud to persist a copy of the data
from the ceph-ansible fetch directory. Used as an alternative
to backing up the fetch_directory in Swift. Path must be
writable and readable by the user running ansible from
config-download, e.g. the mistral user in the mistral-executor
container is able to read/write to /var/lib/mistral/ceph_fetch
type: string
conditions:
custom_registry_host:
yaql:
data: {get_param: DockerCephDaemonImage}
expression: $.data.split('/')[0].matches('(\.|:)')
perform_upgrade:
equals: [{get_param: StackUpdateType}, 'UPGRADE']
ceph_ansible_skip_tags_set:
not:
equals:
- {get_param: CephAnsibleSkipTags}
- ''
resources:
DockerImageUrlParts:
type: OS::Heat::Value
properties:
type: json
value:
host:
if:
- custom_registry_host
- yaql:
expression: let(location => $.data.rightSplit(':', 1)[0]) -> regex('(?:https?://)?(.*?)/(.*)').split($location)[1]
data: {get_param: DockerCephDaemonImage}
- docker.io
image:
if:
- custom_registry_host
- yaql:
expression: let(location => $.data.rightSplit(':', 1)[0]) -> regex('(?:https?://)?(.*?)/(.*)').split($location)[2]
data: {get_param: DockerCephDaemonImage}
- yaql:
expression: $.data.rightSplit(':', 1)[0]
data: {get_param: DockerCephDaemonImage}
image_tag:
yaql:
expression: $.data.rightSplit(':', 1)[1]
data: {get_param: DockerCephDaemonImage}
CephBaseAnsibleVars:
type: OS::Heat::Value
properties:
type: json
value:
vars:
fsid: { get_param: CephClusterFSID }
cluster: { get_param: CephClusterName }
docker: true
ceph_docker_registry: {get_attr: [DockerImageUrlParts, value, host]}
ceph_docker_image: {get_attr: [DockerImageUrlParts, value, image]}
ceph_docker_image_tag: {get_attr: [DockerImageUrlParts, value, image_tag]}
containerized_deployment: true
public_network:
list_join:
- ','
- get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephMonNetwork]}]
monitor_address_block:
list_join:
- ','
- get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephMonNetwork]}]
cluster_network:
list_join:
- ','
- get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]
user_config: true
ceph_stable: true
ceph_origin: distro
openstack_config: true
openstack_pools:
yaql:
expression: $.data.toDict($.get('name')).values()
data:
list_concat_unique:
- repeat:
template:
name: <%pool%>
pg_num: {get_param: CephPoolDefaultPgNum}
rule_name: replicated_rule
application: rbd
for_each:
<%pool%>:
list_concat_unique:
- - {get_param: CinderRbdPoolName}
- {get_param: CinderBackupRbdPoolName}
- {get_param: NovaRbdPoolName}
- {get_param: GlanceRbdPoolName}
# CinderRbdExtraPools is a list (do not indent further)
- {get_param: CinderRbdExtraPools}
- if:
- equals: [{get_param: GnocchiRbdPoolName}, '']
- []
- - name: {get_param: GnocchiRbdPoolName}
pg_num: {get_param: CephPoolDefaultPgNum}
rule_name: replicated_rule
application: openstack_gnocchi
- {get_param: CephPools}
openstack_keys: &openstack_keys
- name:
list_join:
- '.'
- - client
- {get_param: CephClientUserName}
key: {get_param: CephClientKey}
caps:
mgr: "allow *"
mon: "profile rbd"
osd:
list_join:
- ', '
- repeat:
template: 'profile rbd pool=<%pool%>'
for_each:
<%pool%>:
list_concat_unique:
- - {get_param: CinderRbdPoolName}
- {get_param: CinderBackupRbdPoolName}
- {get_param: NovaRbdPoolName}
- {get_param: GlanceRbdPoolName}
- if:
- equals: [{get_param: GnocchiRbdPoolName}, '']
- []
- [{get_param: GnocchiRbdPoolName}]
# CinderRbdExtraPools is a list (do not indent further)
- {get_param: CinderRbdExtraPools}
- yaql:
data: {get_param: CephPools}
expression: $.data.select($.name)
mode: "0600"
- name:
list_join:
- '.'
- - client
- {get_param: ManilaCephFSCephFSAuthId}
key: {get_param: CephManilaClientKey}
caps:
mgr: "allow *"
mon: "allow r, allow command 'auth del', allow command 'auth caps', allow command 'auth get', allow command 'auth get-or-create'"
mds: "allow *"
osd: "allow rw"
mode: "0600"
- name:
list_join:
- '.'
- - client
- {get_param: CephRgwClientName}
key: {get_param: CephRgwKey}
caps:
mgr: "allow *"
mon: "allow rw"
osd: "allow rwx"
mode: "0600"
keys: *openstack_keys
pools: []
ceph_conf_overrides:
global:
map_merge:
- osd_pool_default_size: {get_param: CephPoolDefaultSize}
osd_pool_default_pg_num: {get_param: CephPoolDefaultPgNum}
osd_pool_default_pgp_num: {get_param: CephPoolDefaultPgNum}
rgw_keystone_api_version: 3
rgw_keystone_url: {get_param: [EndpointMap, KeystoneInternal, uri_no_suffix]}
rgw_keystone_accepted_roles: 'Member, admin'
rgw_keystone_admin_domain: default
rgw_keystone_admin_project: service
rgw_keystone_admin_user: swift
rgw_keystone_admin_password: {get_param: SwiftPassword}
rgw_keystone_implicit_tenants: 'true'
rgw_keystone_revocation_interval: '0'
rgw_s3_auth_use_keystone: 'true'
- {get_param: CephConfigOverrides}
ntp_service_enabled: false
generate_fsid: false
ip_version:
if:
- {get_param: CephIPv6}
- ipv6
- ipv4
outputs:
role_data:
description: Role data for the Ceph base service.
value:
service_name: ceph_base
upgrade_tasks: []
puppet_config:
config_image: ''
config_volume: ''
step_config: ''
docker_config: {}
config_settings: {}
external_deploy_tasks:
- name: ceph_base_external_deploy_init
when: step == '1'
tags: ceph
block:
- name: set blacklisted_hostnames
set_fact:
blacklisted_hostnames: {get_param: DeploymentServerBlacklist}
- name: create ceph-ansible temp dirs
file:
path: "{{item}}"
state: directory
with_items:
- "{{playbook_dir}}/ceph-ansible/group_vars"
- "{{playbook_dir}}/ceph-ansible/host_vars"
- "{{playbook_dir}}/ceph-ansible/fetch_dir"
- name: generate inventory
copy:
dest: "{{playbook_dir}}/ceph-ansible/inventory.yml"
content: |
{%- set ceph_groups = ['mgr', 'mon', 'osd', 'mds', 'rgw', 'nfs', 'rbdmirror', 'client'] -%}
{%- for ceph_group in ceph_groups -%}
{%- if 'ceph_' ~ ceph_group in groups -%}
{%- set ceph_group_hosts = groups['ceph_' ~ ceph_group] | difference(blacklisted_hostnames) -%}
{%- if ceph_group_hosts|length > 0 %}
{{ ceph_group ~ 's:' }}
hosts:
{% for host in ceph_group_hosts -%}
{{ host }}:
ansible_user: {{ hostvars.raw_get(host)['ansible_ssh_user'] | default('root') }}
ansible_host: {{ hostvars.raw_get(host)['ansible_host'] | default(host) }}
{% if hostvars.raw_get(host)['ansible_connection'] | default('') == 'local' -%}
ansible_connection: local
{% endif -%}
ansible_become: true
{% endfor -%}
{%- endif -%}
{%- endif -%}
{%- endfor %}
- name: set ceph-ansible group vars all
set_fact:
ceph_ansible_group_vars_all: {get_attr: [CephBaseAnsibleVars, value, vars]}
- name: generate ceph-ansible group vars all
copy:
dest: "{{playbook_dir}}/ceph-ansible/group_vars/all.yml"
content: "{{ceph_ansible_group_vars_all|to_nice_yaml}}"
- name: set ceph-ansible extra vars
set_fact:
ceph_ansible_extra_vars:
map_merge:
- {get_param: CephAnsibleExtraConfig}
- ireallymeanit: "yes"
fetch_directory: "{{playbook_dir}}/ceph-ansible/fetch_dir"
- name: generate ceph-ansible extra vars
copy:
dest: "{{playbook_dir}}/ceph-ansible/extra_vars.yml"
content: "{{ceph_ansible_extra_vars|to_nice_yaml}}"
- name: generate nodes-uuid data file
copy:
dest: "{{playbook_dir}}/ceph-ansible/nodes_uuid_data.json"
content:
yaql:
expression: dict($.data.keys().select($.toLower()).zip($.data.values()))
data: {get_param: NodeDataLookup}
- name: generate nodes-uuid playbook
copy:
dest: "{{playbook_dir}}/ceph-ansible/nodes_uuid_playbook.yml"
content: |
- hosts: all
gather_facts: no
tasks:
- name: set nodes data
set_fact:
nodes_data: "{% raw %}{{ lookup('file','./nodes_uuid_data.json') | from_json }}{% endraw %}"
- name: register machine id
command: dmidecode -s system-uuid | tr A-F a-f
register: machine_uuid
# NOTE(tonyb): 0 == no error, 1 == -EPERM or bad data and 2 == Command not found
# 1 and 2 aren't great but shouldn't cause the deploy to fail. If we're using
# the node specific data we'll fail then. If we aren't then lets keep moving
failed_when: machine_uuid.rc not in [0, 1, 2]
- name: generate host vars from nodes data
local_action:
module: copy
content: "{% raw %}{{nodes_data[machine_uuid.stdout|default('')]|default({})|to_nice_yaml}}{% endraw %}"
dest: "{{playbook_dir}}/ceph-ansible/host_vars/{% raw %}{{inventory_hostname}}{% endraw %}.yml"
- name: ceph_base_external_deploy_task
when: step == '2'
tags: ceph
block:
- name: run nodes-uuid
shell:
list_join:
- ' '
- - ANSIBLE_LOG_PATH="{{playbook_dir}}/ceph-ansible/nodes_uuid_command.log"
- ANSIBLE_CONFIG="{{playbook_dir}}/ansible.cfg"
- ANSIBLE_REMOTE_TEMP=/tmp/nodes_uuid_tmp
- ansible-playbook
- '{% if ansible_ssh_private_key_file is defined %}--private-key {{ansible_ssh_private_key_file}}{% endif %}'
- '-i'
- '{{playbook_dir}}/ceph-ansible/inventory.yml'
- '{% if ansible_python_interpreter is defined %}-e ansible_python_interpreter={{ansible_python_interpreter}}{% endif %}'
- '{{playbook_dir}}/ceph-ansible/nodes_uuid_playbook.yml'
- name: set ceph-ansible params from Heat
set_fact:
ceph_ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
ceph_ansible_playbooks_param: {get_param: CephAnsiblePlaybook}
- name: set ceph-ansible playbooks
set_fact:
ceph_ansible_playbooks: >
{%- if ceph_ansible_playbooks_param != ['default'] -%}
{{ ceph_ansible_playbooks_param }}
{%- else -%}
{{ ceph_ansible_playbooks_default|default(['/usr/share/ceph-ansible/site-docker.yml.sample']) }}
{%- endif -%}
- name: was path for local ceph-ansible fetch directory backups set?
set_fact:
local_ceph_ansible_fetch_directory_backup: {get_param: LocalCephAnsibleFetchDirectoryBackup}
ceph_ansible_tarball_name: "temporary_dir.tar.gz"
- block: # local backup
- name: look for requested ceph-ansible fetch directory for local backup
stat: path="{{local_ceph_ansible_fetch_directory_backup}}"
register: local_backup_directory
ignore_errors: True
- name: autocreate new directory for ceph-ansible fetch directory backup
file:
path: "{{local_ceph_ansible_fetch_directory_backup}}"
state: directory
mode: 0700
when: local_backup_directory.stat.exists == False
- name: look for tarball of ceph-ansible fetch directory in local backup
stat: path="{{local_ceph_ansible_fetch_directory_backup}}/{{ceph_ansible_tarball_name}}"
register: local_backup_file
ignore_errors: True
- name: untar local backup of ceph-ansible fetch directory
# unarchive module hit https://github.com/ansible/ansible/issues/35645
shell: "/usr/bin/gtar --gzip --extract --file {{local_ceph_ansible_fetch_directory_backup}}/{{ceph_ansible_tarball_name}} -C {{playbook_dir}}/ceph-ansible/fetch_dir"
when: local_backup_file.stat.exists == True
when: local_ceph_ansible_fetch_directory_backup != ""
- block: # swift backup
- name: set facts for swift back up of ceph-ansible fetch directory
set_fact:
swift_get_url: {get_param: SwiftFetchDirGetTempurl}
swift_put_url: {get_param: SwiftFetchDirPutTempurl}
old_ceph_ansible_tarball_name: "temporary_dir_old.tar.gz"
new_ceph_ansible_tarball_name: "temporary_dir_new.tar.gz"
- name: attempt download of fetch directory tarball from swift backup
shell: "curl -s -o /tmp/{{old_ceph_ansible_tarball_name}} -w '%{http_code}' -X GET \"{{ swift_get_url }}\""
register: curl_get_http_status
ignore_errors: True
- name: ensure we create a new fetch_directory or use the old fetch_directory
fail:
msg: "Received HTTP: {{curl_get_http_status.stdout}} when attempting to GET from {{swift_get_url}}"
when:
- curl_get_http_status.changed
- curl_get_http_status.stdout != "200" # deployment update
- curl_get_http_status.stdout != "404" # new deployment
- name: unpack downloaded ceph-ansible fetch tarball to fetch directory
# unarchive module hit https://github.com/ansible/ansible/issues/35645
shell: "/usr/bin/gtar --gzip --extract --file /tmp/{{old_ceph_ansible_tarball_name}} -C {{playbook_dir}}/ceph-ansible/fetch_dir"
when:
- curl_get_http_status.changed
- curl_get_http_status.stdout == "200"
- name: remove downloaded ceph-ansible fetch directory tarball from filesystem
file:
path: "/tmp/{{old_ceph_ansible_tarball_name}}"
state: absent
when:
- curl_get_http_status.changed
- curl_get_http_status.stdout == "200"
when: local_ceph_ansible_fetch_directory_backup == ""
- name: set ceph-ansible command
set_fact:
ceph_ansible_command:
list_join:
- ' '
- - ANSIBLE_ACTION_PLUGINS=/usr/share/ceph-ansible/plugins/actions/
- ANSIBLE_CALLBACK_PLUGINS=/usr/share/ceph-ansible/plugins/callback/
- ANSIBLE_ROLES_PATH=/usr/share/ceph-ansible/roles/
- ANSIBLE_LOG_PATH="{{playbook_dir}}/ceph-ansible/ceph_ansible_command.log"
- ANSIBLE_LIBRARY=/usr/share/ceph-ansible/library/
- ANSIBLE_CONFIG=/usr/share/ceph-ansible/ansible.cfg
- ANSIBLE_REMOTE_TEMP=/tmp/ceph_ansible_tmp
- ANSIBLE_FORKS=25
- yaql:
data: {get_param: CephAnsibleEnvironmentVariables}
expression: $.data.items().select($[0] + '=' + $[1]).join(' ')
- ansible-playbook
- '{% if ansible_ssh_private_key_file is defined %}--private-key {{ansible_ssh_private_key_file}}{% endif %}'
- '{% if ansible_python_interpreter is defined %}-e ansible_python_interpreter={{ansible_python_interpreter}}{% endif %}'
- '-{%- for number in range(0, ceph_ansible_playbook_verbosity) -%}v{% endfor %}'
- if:
- ceph_ansible_skip_tags_set
- list_join:
- ' '
- - '--skip-tags'
- {get_param: CephAnsibleSkipTags}
- ''
- '-i'
- '{{playbook_dir}}/ceph-ansible/inventory.yml'
- '--extra-vars'
- '@{{playbook_dir}}/ceph-ansible/extra_vars.yml'
- name: run ceph-ansible (immediate log at {{playbook_dir}}/ceph-ansible/ceph_ansible_command.log)
shell: |
set -e
{% for playbook in ceph_ansible_playbooks %}
echo "Running ceph-ansible playbook {{playbook}}"
{{ceph_ansible_command}} {{playbook}} 2>&1
{% endfor %}
# We want the output chunked into bits to prevent
# overflowing Zaqar message size
no_log: true
failed_when: false
register: outputs
- name: print ceph-ansible output in case of failure
debug:
var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
failed_when: outputs.rc != 0
when:
- outputs.changed
- outputs.rc != 0
- name: create ceph-ansible fetch directory tarball in local backup
archive:
path: "{{playbook_dir}}/ceph-ansible/fetch_dir"
dest: "{{local_ceph_ansible_fetch_directory_backup}}/{{ceph_ansible_tarball_name}}"
when: local_ceph_ansible_fetch_directory_backup != ""
- block: # swift backup
- name: create temporary ceph-ansible fetch directory tarball for swift backup
archive:
path: "{{playbook_dir}}/ceph-ansible/fetch_dir"
dest: "/tmp/{{new_ceph_ansible_tarball_name}}"
- name: backup temporary ceph-ansible fetch directory tarball in swift
shell: "curl -s -o /dev/null -w '%{http_code}' -X PUT -T /tmp/{{new_ceph_ansible_tarball_name}} \"{{ swift_put_url }}\""
register: curl_put_http_status
- fail:
msg: 'Received HTTP: {{curl_put_http_status.stdout}} when attempting to PUT to {{swift_put_url}}'
name: ensure we were able to backup temporary fetch directory to swift
when:
- curl_put_http_status.changed
- curl_put_http_status.stdout != "200"
- curl_put_http_status.stdout != "201"
- name: clean temporary fetch directory after swift backup
file:
path: "/tmp/{{new_ceph_ansible_tarball_name}}"
state: absent
when: (curl_put_http_status.changed and (curl_put_http_status.stdout == "200" or
curl_put_http_status.stdout == "201"))
when: local_ceph_ansible_fetch_directory_backup == ""
external_update_tasks:
- when: step|int == 0
tags: ceph
block:
- name: set ceph_ansible_playbooks_default
set_fact:
ceph_ansible_playbooks_default: ["/usr/share/ceph-ansible/infrastructure-playbooks/rolling_update.yml"]
external_upgrade_tasks:
- when: step|int == 0
tags: ceph
block:
- name: set ceph_ansible_playbooks_default
set_fact:
ceph_ansible_playbooks_default: ["/usr/share/ceph-ansible/infrastructure-playbooks/rolling_update.yml"]