openstack
/
kolla
Code Issues Proposed changes

External Ceph - Implementation Cinder 

This patch adds support for external Ceph clusters for Cinder.

For clean integration the backend configuration mechanism had to be
slightly adjusted.

We now have the option to enable multiple backends for Cinder
independently.

Currently, the flags cinder_backend_iscsi and cinder_backend_ceph are
used to toggle backends.

Documentation on how to use external ceph was added.

Change-Id: I7e0267b90d62d6d881f24f063cdb894422ec8618
Partially-Implements: Blueprint: external-ceph
This commit is contained in:
Mathias Ewald 2016-07-18 10:39:09 +00:00
parent 85300a4f8f
commit 3894883871
11 changed files with 234 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ansible/group_vars/all.yml
View File

@ -244,13 +244,15 @@ kibana_user: "kibana"
#################################
# Cinder - Block Storage options
# Cinder options
#################################
cinder_volume_driver: "{{ 'ceph' if enable_ceph | bool else 'lvm' }}"
cinder_volume_group: "{{ 'cinder-volumes' if enable_iscsi | bool else '' }}"
cinder_volume_backend_name: "{{ 'cinder-volumes' if enable_iscsi | bool else '' }}"
cinder_iscsi_helper: "{{ 'tgtadm' if enable_iscsi | bool else '' }}"
cinder_iscsi_protocol: "{{ 'iscsi' if enable_iscsi | bool else '' }}"
cinder_backend_iscsi: "{{ enable_iscsi }}"
cinder_backend_ceph: "{{ enable_ceph }}"
cinder_volume_driver: "{{ 'lvm' if cinder_backend_iscsi }}"
cinder_volume_group: "{{ 'cinder-volumes' if cinder_backend_iscsi | bool else '' }}"
cinder_volume_backend_name: "{{ 'cinder-volumes' if cinder_backend_iscsi | bool else '' }}"
cinder_iscsi_helper: "{{ 'tgtadm' if cinder_backend_iscsi | bool else '' }}"
cinder_iscsi_protocol: "{{ 'iscsi' if cinder_backend_iscsi | bool else '' }}"
###################
# Ceph options

4
ansible/roles/cinder/tasks/bootstrap.yml
View File

@ -45,11 +45,11 @@
action: "create_volume"
common_options: "{{ docker_common_options }}"
name: "cinder"
when: "{{ enable_iscsi | bool }}"
when: "{{ cinder_backend_iscsi | bool }}"
- name: Creating iscsi info volume
kolla_docker:
action: "create_volume"
common_options: "{{ docker_common_options }}"
name: "iscsi_info"
when: "{{ enable_iscsi | bool }}"
when: "{{ cinder_backend_iscsi | bool }}"

8
ansible/roles/cinder/tasks/deploy.yml
View File

@ -1,13 +1,19 @@
---
- include: ceph.yml
when:
- enable_ceph | bool
- (enable_ceph | bool) and (cinder_backend_ceph | bool)
- inventory_hostname in groups['ceph-mon'] or
inventory_hostname in groups['cinder-api'] or
inventory_hostname in groups['cinder-volume'] or
inventory_hostname in groups['cinder-scheduler'] or
inventory_hostname in groups['cinder-backup']
- include: external_ceph.yml
when:
- (enable_ceph | bool == False) and (cinder_backend_ceph | bool)
- inventory_hostname in groups['cinder-volume'] or
inventory_hostname in groups['cinder-backup']
- include: register.yml
when: inventory_hostname in groups['cinder-api']

36
ansible/roles/cinder/tasks/external_ceph.yml Normal file
View File

@ -0,0 +1,36 @@
---
- name: Ensuring config directory exists
file:
path: "{{ node_config_directory }}/cinder-{{ item }}"
state: "directory"
when: inventory_hostname in groups['cinder-volume'] or
inventory_hostname in groups['cinder-backup']
with_items:
- volume
- backup
- name: Copying over ceph.conf for Cinder
merge_configs:
vars:
service_name: "{{ item }}"
sources:
- "{{ node_custom_config }}/cinder/ceph.conf"
- "{{ node_custom_config }}/cinder/{{ item }}/ceph.conf"
dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
with_items:
- "cinder-backup"
- "cinder-volume"
- name: Copy over Ceph keyring files for cinder-volume
copy:
src: "{{ item }}"
dest: "{{ node_config_directory }}/cinder-volume/"
with_fileglob:
- "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"
- name: Copy over Ceph keyring files for cinder-backup
copy:
src: "{{ item }}"
dest: "{{ node_config_directory }}/cinder-backup/"
with_fileglob:
- "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"

4
ansible/roles/cinder/tasks/start.yml
View File

@ -30,8 +30,8 @@
- "/etc/localtime:/etc/localtime:ro"
- "/dev/:/dev/"
- "/run/:/run/"
- "{% if enable_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
- "{% if enable_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
- "{% if cinder_backend_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
- "{% if cinder_backend_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
- "kolla_logs:/var/log/kolla/"
- name: Starting cinder-volume container

26
ansible/roles/cinder/templates/cinder-backup.json.j2
View File

@ -6,27 +6,13 @@
"dest": "/etc/cinder/cinder.conf",
"owner": "cinder",
"perm": "0600"
},
}{% if cinder_backend_ceph | bool %},
{
"source": "{{ container_config_directory }}/ceph.client.cinder-backup.keyring",
"dest": "/etc/ceph/ceph.client.cinder-backup.keyring",
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not enable_ceph | bool) | string | lower }}
},
{
"source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
"dest": "/etc/ceph/ceph.client.cinder.keyring",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not enable_ceph | bool) | string | lower }}
},
{
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not enable_ceph | bool) | string | lower }}
}
"perm": "0700",
"optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
}{% endif %}
]
}

14
ansible/roles/cinder/templates/cinder-volume.json.j2
View File

@ -6,20 +6,18 @@
"dest": "/etc/cinder/cinder.conf",
"owner": "cinder",
"perm": "0600"
},
}{% if cinder_backend_ceph | bool %},
{
"source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
"dest": "/etc/ceph/ceph.client.cinder.keyring",
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not enable_ceph | bool) | string | lower }}
"perm": "0700"
},
{
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not enable_ceph | bool) | string | lower }}
}
"perm": "0600"
}{% endif %}
]
}

13
ansible/roles/cinder/templates/cinder.conf.j2
View File

@ -16,14 +16,14 @@ glance_api_version = 2
os_region_name = {{ openstack_region_name }}
{% if cinder_volume_driver == "lvm" %}
enabled_backends = {{ cinder_volume_backend_name }}
{% elif cinder_volume_driver == "ceph" %}
{% if enable_ceph | bool and cinder_backend_ceph | bool %}
default_volume_type = rbd-1
enabled_backends = rbd-1
{% elif cinder_backend_iscsi | bool %}
enabled_backends = {{ cinder_volume_backend_name }}
{% endif %}
{% if service_name == "cinder-backup" and cinder_volume_driver == "ceph" %}
{% if service_name == "cinder-backup" and enable_ceph | bool and cinder_backend_ceph | bool %}
backup_driver = cinder.backup.drivers.ceph
backup_ceph_conf = /etc/ceph/ceph.conf
backup_ceph_user = cinder-backup
@ -76,7 +76,7 @@ rabbit_ha_queues = true
rabbit_hosts = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
{% if cinder_volume_driver == "lvm" %}
{% if cinder_backend_iscsi | bool %}
[{{ cinder_volume_backend_name }}]
volume_group = {{ cinder_volume_group }}
volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
@ -84,7 +84,8 @@ volume_backend_name = {{ cinder_volume_backend_name }}
iscsi_ip_address = {{ hostvars[groups['tgtd'][0]]['ansible_' + hostvars[groups['tgtd'][0]]['storage_interface']]['ipv4']['address'] }}
iscsi_helper = {{ cinder_iscsi_helper }}
iscsi_protocol = {{ cinder_iscsi_protocol }}
{% elif cinder_volume_driver == "ceph" %}
{% elif enable_ceph | bool and cinder_backend_ceph | bool %}
[rbd-1]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
rbd_pool = {{ ceph_cinder_pool_name }}

153
doc/external-ceph-guide.rst Normal file
View File

@ -0,0 +1,153 @@
.. _external-ceph-guide:
=============
External Ceph
=============
Sometimes it is necessary to connect OpenStack services to an existing Ceph
cluster instead of deploying it with Kolla. This can be achieved with only a
few configuration steps in Kolla.
Requirements
============
* An existing installation of Ceph
* Existing Ceph storage pools
* Existing credentials in Ceph for OpenStack services to connect to Ceph
(Glance, Cinder)
Enabling External Ceph
======================
Using external Ceph with Kolla means not to deploy Ceph via Kolla. Therefore,
disable Ceph deployment in ``/etc/kolla/global.yml``
::
enable_ceph: "no"
There are flags indicating individual services to use ceph or not which default
to the value of ``enable_ceph``. Those flags now need to be activated in order
to activate external Ceph integration. This can be done individually per
service in ``/etc/kolla/global.yml``:
::
glance_enable_ceph: "yes"
cinder_backend_ceph: "yes"
The combination of ``enable_ceph: "no"`` and ``<service>_enable_ceph: "yes"``
triggers the activation of external ceph mechanism in Kolla.
Configuring External Ceph
=========================
Glance
------
Configuring Glance for Ceph includes three steps:
1) Configure RBD backend in glance-api.conf
2) Create Ceph configuration file in /etc/ceph/ceph.conf
3) Create Ceph keyring file in /etc/ceph/ceph.client.<username>.keyring
Step 1 is done by using Kolla's INI merge mechanism: Create a file in
``/etc/kolla/config/glance/glance-api.conf`` with the following contents:
::
[DEFAULT]
show_image_direct_url = True
[glance_store]
stores = rbd
default_store = rbd
rbd_store_chunk_size = 8
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rados_connect_timeout = 0
[image_format]
container_formats = bare
disk_formats = raw
Now put ceph.conf and the keyring file (name depends on the username created in
Ceph) into the same directory, for example:
/etc/kolla/config/glance/ceph.conf
::
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_initial_members = ceph-0
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
/etc/kolla/config/glance/ceph.client.glance.keyring
::
[client.glance]
key = AQAg5YRXS0qxLRAAXe6a4R1a15AoRx7ft80DhA==
Kolla will pick up all files named ceph.* in this directory an copy them to the
/etc/ceph/ directory of the container.
Cinder
------
Configuring external Ceph for Cinder works very similar to
Glance. The required Cinder configuration goes into
/etc/kolla/config/cinder/cinder-volume.conf:
::
[DEFAULT]
enabled_backends=rbd-1
[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
Next, place the ceph.conf file into
/etc/kolla/config/cinder/ceph.conf:
::
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_initial_members = ceph-0
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
Separate configuration options can be configured for
cinder-volume and cinder-backup by adding ceph.conf files to
/etc/kolla/config/cinder/cinder-volume and
/etc/kolla/config/cinder/cinder-backup respectively. They
will be merged with /etc/kolla/config/cinder/ceph.conf.
Ceph keyrings are deployed per service and placed into
cinder-volume and cinder-backup directories:
::
root@deploy:/etc/kolla/config# cat
cinder/cinder-backup/ceph.client.cinder.keyring
[client.cinder]
key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
root@deploy:/etc/kolla/config# cat
cinder/cinder-volume/ceph.client.cinder.keyring
[client.cinder]
key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
It is important that the files are named ceph.client*.

1
doc/index.rst
View File

@ -51,6 +51,7 @@ Kolla Services
:maxdepth: 1
ceph-guide
external-ceph-guide
cinder-guide
ironic-guide
manila-guide

7
etc/kolla/globals.yml
View File

@ -129,6 +129,7 @@ neutron_external_interface: "eth1"
# when ceph is not deployed by Kolla.
#glance_enable_ceph: "{{ enable_ceph }}"
###################
# Ceph options
###################
@ -144,8 +145,12 @@ neutron_external_interface: "eth1"
#######################
# Cinder iSCSI options
# Cinder options
#######################
# Enable / disable Cinder backends
cinder_backend_iscsi: "{{ enable_iscsi }}"
cinder_backend_ceph: "{{ enable_ceph }}"
# Cinder's iSCSI backend !!!REQUIRES!!! two parameters:
# 1 - IP address of the server hosting LVM Volume group
# 2 - The name of Volume group which Cinder will use.