openstack/kolla-ansible
Code Issues Proposed changes

Merge "External Ceph: keys as variables"

Browse Source
This commit is contained in:
Zuul 2020-01-23 12:43:43 +00:00 committed by Gerrit Code Review
commit 13dea3f931
14 changed files with 206 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ansible/group_vars/all.yml
View File

@ -1032,6 +1032,14 @@ ceph_mgr_host_type: "INVENTORY"
ceph_osd_host_type: "IP"
ceph_mds_host_type: "INVENTORY"
# External Ceph keyrings
ceph_cinder_keyring: "ceph.client.cinder.keyring"
ceph_cinder_backup_keyring: "ceph.client.cinder-backup.keyring"
ceph_glance_keyring: "ceph.client.glance.keyring"
ceph_gnocchi_keyring: "ceph.client.gnocchi.keyring"
ceph_manila_keyring: "ceph.client.manila.keyring"
ceph_nova_keyring: "{% if enable_ceph | bool %}ceph.client.nova.keyring{% else %}{{ ceph_cinder_keyring }}{% endif %}"
#####################
# VMware support
######################

11
ansible/roles/cinder/tasks/external_ceph.yml
View File

@ -21,12 +21,10 @@
- name: Copy over Ceph keyring files for cinder-volume
copy:
src: "{{ item }}"
src: "{{ node_custom_config }}/cinder/cinder-volume/{{ ceph_cinder_keyring }}"
dest: "{{ node_config_directory }}/cinder-volume/"
mode: "0660"
become: true
with_fileglob:
- "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"
when:
- external_ceph_cephx_enabled | bool
- inventory_hostname in groups['cinder-volume']
@ -36,13 +34,14 @@
- name: Copy over Ceph keyring files for cinder-backup
copy:
src: "{{ item }}"
src: "{{ node_custom_config }}/cinder/{{ item }}"
dest: "{{ node_config_directory }}/cinder-backup/"
mode: "0660"
become: true
register: cinder_backup_ceph_keyring
with_fileglob:
- "{{ node_custom_config }}/cinder/cinder-backup/ceph.client*"
with_items:
- "cinder-backup/{{ ceph_cinder_keyring }}"
- "cinder-backup/{{ ceph_cinder_backup_keyring }}"
when:
- external_ceph_cephx_enabled | bool
- inventory_hostname in groups['cinder-backup']

20
ansible/roles/cinder/templates/cinder-backup.json.j2
View File

@ -14,10 +14,24 @@
"perm": "0600"
}{% endif %}{% if cinder_backend_ceph | bool %},
{
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "cinder",
"perm": "0700",
"perm": "0600",
"optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
},
{
"source": "{{ container_config_directory }}/{{ ceph_cinder_keyring }}",
"dest": "/etc/ceph/{{ ceph_cinder_keyring }}",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
},
{
"source": "{{ container_config_directory }}/{{ ceph_cinder_backup_keyring }}",
"dest": "/etc/ceph/{{ ceph_cinder_backup_keyring }}",
"owner": "cinder",
"perm": "0600",
"optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
}{% endif %}
],

6
ansible/roles/cinder/templates/cinder-volume.json.j2
View File

@ -8,10 +8,10 @@
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"source": "{{ container_config_directory }}/{{ ceph_cinder_keyring }}",
"dest": "/etc/ceph/{{ ceph_cinder_keyring }}",
"owner": "cinder",
"perm": "0700",
"perm": "0600",
"optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
},
{

32
ansible/roles/glance/tasks/external_ceph.yml
View File

@ -1,27 +1,29 @@
---
- name: Copy over ceph files
copy:
src: "{{ item }}"
dest: "{{ node_config_directory }}/glance-api/"
- name: Copy over ceph.conf for Glance
template:
src: "{{ node_custom_config }}/glance/ceph.conf"
dest: "{{ node_config_directory }}/glance-api/ceph.conf"
mode: "0660"
become: true
when:
- glance_services['glance-api'].host_in_groups | bool
- glance_services['glance-api'].enabled | bool
with_fileglob:
- "{{ node_custom_config }}/glance/ceph*"
when: inventory_hostname in groups['glance-api']
notify:
- Restart glance-api container
- name: Copy over ceph Glance keyring
copy:
src: "{{ node_custom_config }}/glance/{{ ceph_glance_keyring }}"
dest: "{{ node_config_directory }}/glance-api/{{ ceph_glance_keyring }}"
mode: "0660"
become: true
when: inventory_hostname in groups['glance-api']
notify:
- Restart glance-api container
- name: Ensuring config directory has correct owner and permission
file:
path: "{{ node_config_directory }}/{{ item }}"
path: "{{ node_config_directory }}/glance-api"
recurse: yes
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
become: true
when:
- glance_services[item].host_in_groups | bool
- glance_services[item].enabled | bool
with_items:
- "glance-api"
when: inventory_hostname in groups['glance-api']

12
ansible/roles/glance/templates/glance-api.json.j2
View File

@ -14,10 +14,16 @@
"perm": "0600"
}{% endif %}{% if glance_backend_ceph | bool %},
{
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"source": "{{ container_config_directory }}/{{ ceph_glance_keyring }}",
"dest": "/etc/ceph/{{ ceph_glance_keyring }}",
"owner": "glance",
"perm": "0700"
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "glance",
"perm": "0600"
}{% endif %}{% if glance_backend_swift | bool %},
{
"source": "{{ container_config_directory }}/glance-swift.conf",

4
ansible/roles/gnocchi/tasks/external_ceph.yml
View File

@ -15,8 +15,8 @@
- name: Copy over ceph gnocchi keyring
copy:
src: "{{ node_custom_config }}/gnocchi/ceph.client.gnocchi.keyring"
dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keyring"
src: "{{ node_custom_config }}/gnocchi/{{ ceph_gnocchi_keyring }}"
dest: "{{ node_config_directory }}/{{ item }}/{{ ceph_gnocchi_keyring }}"
mode: "0660"
become: true
when: inventory_hostname in groups[item]

4
ansible/roles/gnocchi/templates/gnocchi-api.json.j2
View File

@ -28,8 +28,8 @@
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring",
"dest": "/etc/ceph/ceph.client.gnocchi.keyring",
"source": "{{ container_config_directory }}/{{ ceph_gnocchi_keyring }}",
"dest": "/etc/ceph/{{ ceph_gnocchi_keyring }}",
"owner": "gnocchi",
"perm": "0600"
}{% endif %}

6
ansible/roles/manila/tasks/external_ceph.yml
View File

@ -12,12 +12,10 @@
- name: Copy over Ceph keyring files for manila
copy:
src: "{{ item }}"
dest: "{{ node_config_directory }}/manila-share/"
src: "{{ node_custom_config }}/manila/{{ ceph_manila_keyring }}"
dest: "{{ node_config_directory }}/manila-share/{{ ceph_manila_keyring }}"
mode: "0600"
become: true
with_fileglob:
- "{{ node_custom_config }}/manila/ceph.client*"
when:
- inventory_hostname in groups['manila-share']
notify:

10
ansible/roles/manila/templates/manila-share.json.j2
View File

@ -8,8 +8,14 @@
"perm": "0600"
}{% if enable_manila_backend_cephfs_native | bool or enable_manila_backend_cephfs_nfs | bool %},
{
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "manila",
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/{{ ceph_manila_keyring }}",
"dest": "/etc/ceph/{{ ceph_manila_keyring }}",
"owner": "manila",
"perm": "0600"
}{% endif %}{% if manila_policy_file is defined %},

5
ansible/roles/nova-cell/tasks/external_ceph.yml
View File

@ -11,7 +11,7 @@
- name: Check nova keyring file
stat:
path: "{{ node_custom_config }}/nova/ceph.client.nova.keyring"
path: "{{ node_custom_config }}/nova/{{ ceph_nova_keyring }}"
delegate_to: localhost
run_once: True
register: nova_cephx_keyring_file
@ -22,7 +22,7 @@
- name: Check cinder keyring file
stat:
path: "{{ node_custom_config }}/nova/ceph.client.cinder.keyring"
path: "{{ node_custom_config }}/nova/{{ ceph_cinder_keyring }}"
delegate_to: localhost
run_once: True
register: cinder_cephx_keyring_file
@ -39,7 +39,6 @@
become: true
with_items:
- nova-compute
- nova-libvirt
when:
- inventory_hostname in groups[nova_cell_compute_group]
- nova_backend == "rbd"

12
ansible/roles/nova-cell/templates/nova-compute.json.j2
View File

@ -14,10 +14,16 @@
"perm": "0600"
}{% endif %}{% if nova_backend == "rbd" %},
{
"source": "{{ container_config_directory }}/ceph.*",
"dest": "/etc/ceph/",
"source": "{{ container_config_directory }}/{{ ceph_nova_keyring }}",
"dest": "/etc/ceph/{{ ceph_nova_keyring }}",
"owner": "nova",
"perm": "0700"
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/ceph.conf",
"dest": "/etc/ceph/ceph.conf",
"owner": "nova",
"perm": "0600"
}{% endif %}{% if nova_compute_virt_type == "vmware" and not vmware_vcenter_insecure | bool %},
{
"source": "{{ container_config_directory }}/vmware_ca",

271
doc/source/reference/storage/external-ceph-guide.rst
View File

@ -64,229 +64,176 @@ Configuring External Ceph
Glance
------
Configuring Glance for Ceph includes three steps:
Configuring Glance for Ceph includes the following steps:
#. Configure RBD back end in ``glance-api.conf``
#. Create Ceph configuration file in ``/etc/ceph/ceph.conf``
#. Create Ceph keyring file in ``/etc/ceph/ceph.client.<username>.keyring``
Step 1 is done by using Kolla's INI merge mechanism: Create a file in
``/etc/kolla/config/glance/glance-api.conf`` with the following contents:
.. path /etc/kolla/config/glance/glance-api.conf
.. code-block:: ini
.. code-block:: ini
[glance_store]
stores = rbd
default_store = rbd
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
[glance_store]
stores = rbd
default_store = rbd
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
#. Copy Ceph configuration file to ``/etc/kolla/config/glance/ceph.conf``
Now put ceph.conf and the keyring file (name depends on the username created in
Ceph) into the same directory, for example:
.. path /etc/kolla/config/glance/ceph.conf
.. code-block:: ini
.. path /etc/kolla/config/glance/ceph.conf
.. code-block:: ini
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_initial_members = ceph-0
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_initial_members = ceph-0
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
#. Configure Ceph authentication details in ``/etc/kolla/globals.yml``:
.. code-block:: console
* ``ceph_glance_keyring`` (default: ``ceph.client.glance.keyring``)
$ cat /etc/kolla/config/glance/ceph.client.glance.keyring
[client.glance]
key = AQAg5YRXS0qxLRAAXe6a4R1a15AoRx7ft80DhA==
Kolla will pick up all files named ``ceph.*`` in this directory and copy them
to the ``/etc/ceph/`` directory of the container.
#. Copy Ceph keyring to ``/etc/kolla/config/glance/<ceph_glance_keyring>``
Cinder
------
Configuring external Ceph for Cinder works very similar to
Glance.
Configuring Cinder for Ceph includes following steps:
Modify ``/etc/kolla/config/cinder/cinder-volume.conf`` file according to
the following configuration:
#. Configure RBD backend in ``cinder-volume.conf`` and ``cinder-backup.conf``
.. code-block:: ini
.. path /etc/kolla/config/cinder/cinder-volume.conf
.. code-block:: ini
[DEFAULT]
enabled_backends=rbd-1
[DEFAULT]
enabled_backends=rbd-1
[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_secret_uuid = {{ cinder_rbd_secret_uuid }}
[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_secret_uuid = {{ cinder_rbd_secret_uuid }}
.. note::
.. note::
``cinder_rbd_secret_uuid`` can be found in ``/etc/kolla/passwords.yml`` file.
``cinder_rbd_secret_uuid`` can be found in ``/etc/kolla/passwords.yml``.
Modify ``/etc/kolla/config/cinder/cinder-backup.conf`` file according to
the following configuration:
.. path /etc/kolla/config/cinder/cinder-backup.conf
.. code-block:: ini
.. code-block:: ini
[DEFAULT]
backup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user=cinder-backup
backup_ceph_chunk_size = 134217728
backup_ceph_pool=backups
backup_driver = cinder.backup.drivers.ceph.CephBackupDriver
backup_ceph_stripe_unit = 0
backup_ceph_stripe_count = 0
restore_discard_excess_bytes = true
[DEFAULT]
backup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user=cinder-backup
backup_ceph_chunk_size = 134217728
backup_ceph_pool=backups
backup_driver = cinder.backup.drivers.ceph.CephBackupDriver
backup_ceph_stripe_unit = 0
backup_ceph_stripe_count = 0
restore_discard_excess_bytes = true
For more information about the Cinder backup configuration, see
:cinder-doc:`Ceph backup driver
<configuration/block-storage/backup/ceph-backup-driver.html>`.
For more information about the Cinder backup configuration, see
:cinder-doc:`Ceph backup driver <configuration/block-storage/backup/ceph-backup-driver.html>`.
#. Copy Ceph configuration file to ``/etc/kolla/config/cinder/ceph.conf``
Next, copy the ``ceph.conf`` file into ``/etc/kolla/config/cinder/``:
Separate configuration options can be configured for
cinder-volume and cinder-backup by adding ceph.conf files to
``/etc/kolla/config/cinder/cinder-volume`` and
``/etc/kolla/config/cinder/cinder-backup`` respectively. They
will be merged with ``/etc/kolla/config/cinder/ceph.conf``.
.. code-block:: ini
#. Configure Ceph authentication details in ``/etc/kolla/globals.yml``:
* ``ceph_cinder_keyring`` (default: ``ceph.client.cinder.keyring``)
* ``ceph_cinder_backup_keyring``
(default: ``ceph.client.cinder-backup.keyring``)
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_initial_members = ceph-0
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
Separate configuration options can be configured for
cinder-volume and cinder-backup by adding ceph.conf files to
``/etc/kolla/config/cinder/cinder-volume`` and
``/etc/kolla/config/cinder/cinder-backup`` respectively. They
will be merged with ``/etc/kolla/config/cinder/ceph.conf``.
Ceph keyrings are deployed per service and placed into
``cinder-volume`` and ``cinder-backup`` directories, put the keyring files
to these directories, for example:
#. Copy Ceph keyring files to:
* ``/etc/kolla/config/cinder/cinder-volume/<ceph_cinder_keyring>``
* ``/etc/kolla/config/cinder/cinder-backup/<ceph_cinder_keyring>``
* ``/etc/kolla/config/cinder/cinder-backup/<ceph_cinder_backup_keyring>``
.. note::
``cinder-backup`` requires two keyrings for accessing volumes
and backup pool.
.. code-block:: console
$ cat /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring
[client.cinder]
key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
.. code-block:: console
$ cat /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring
[client.cinder-backup]
key = AQC9wNBYrD8MOBAAwUlCdPKxWZlhkrWIDE1J/w==
.. code-block:: console
$ cat /etc/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring
[client.cinder]
key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
It is important that the files are named ``ceph.client*``.
Nova
----
Put ceph.conf, nova client keyring file and cinder client keyring file into
``/etc/kolla/config/nova``:
Configuring Nova for Ceph includes following steps:
.. warning::
#. Copy Ceph configuration file to ``/etc/kolla/config/nova/ceph.conf``
#. Configure Ceph authentication details in ``/etc/kolla/globals.yml``:
If you are using ceph-ansible - please copy ceph.client.cinder.keyring
as /etc/kolla/config/nova/ceph.client.nova.keyring
* ``ceph_cinder_keyring`` (default: ``ceph.client.cinder.keyring``)
* ``ceph_nova_keyring`` (by default it's the same as ceph_cinder_keyring)
.. code-block:: console
#. Copy Ceph keyring file(s) to:
$ ls /etc/kolla/config/nova
ceph.client.cinder.keyring ceph.client.nova.keyring ceph.conf
* ``/etc/kolla/config/nova/<ceph_cinder_keyring>``
* ``/etc/kolla/config/nova/<ceph_nova_keyring>`` (if your Ceph deployment
created one)
Configure nova-compute to use Ceph as the ephemeral back end by creating
``/etc/kolla/config/nova/nova-compute.conf`` and adding the following
configurations:
.. warning::
.. code-block:: ini
If you are using ceph-ansible or another deployment tool that doesn't
create separate key for Nova just copy the Cinder key.
[libvirt]
images_rbd_pool=vms
images_type=rbd
images_rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=nova
#. Configure nova-compute to use Ceph as the ephemeral back end by creating
``/etc/kolla/config/nova/nova-compute.conf`` and adding the following
configurations:
.. note::
.. code-block:: ini
``rbd_user`` might vary depending on your environment.
[libvirt]
images_rbd_pool=vms
images_type=rbd
images_rbd_ceph_conf=/etc/ceph/ceph.conf
Gnocchi
-------
Modify ``/etc/kolla/config/gnocchi.conf`` file according to
the following configuration:
Configuring Gnocchi for Ceph includes following steps:
.. code-block:: ini
#. Copy Ceph configuration file to ``/etc/kolla/config/gnocchi/ceph.conf``
#. Configure Ceph authentication details in ``/etc/kolla/globals.yml``:
[storage]
driver = ceph
ceph_username = gnocchi
ceph_keyring = /etc/ceph/ceph.client.gnocchi.keyring
ceph_conffile = /etc/ceph/ceph.conf
* ``ceph_gnocchi_keyring``
(default: ``ceph.client.gnocchi.keyring``)
Put ceph.conf and gnocchi client keyring file in
``/etc/kolla/config/gnocchi``:
#. Copy Ceph keyring to ``/etc/kolla/config/gnocchi/<ceph_gnocchi_keyring>``
#. Modify ``/etc/kolla/config/gnocchi.conf`` file according to the following
configuration:
.. code-block:: console
.. code-block:: ini
$ ls /etc/kolla/config/gnocchi
ceph.client.gnocchi.keyring ceph.conf gnocchi.conf
[storage]
driver = ceph
ceph_username = gnocchi
ceph_keyring = /etc/ceph/ceph.client.gnocchi.keyring
ceph_conffile = /etc/ceph/ceph.conf
Manila
------
Configuring Manila for Ceph includes four steps:
Configuring Manila for Ceph includes following steps:
#. Configure CephFS backend, setting ``enable_manila_backend_cephfs_native``
#. Create Ceph configuration file in ``/etc/ceph/ceph.conf``
#. Create Ceph keyring file in ``/etc/ceph/ceph.client.<username>.keyring``
#. Configure CephFS backend by setting ``enable_manila_backend_cephfs_native``
to ``true``
#. Configure Ceph authentication details in ``/etc/kolla/globals.yml``:
* ``ceph_manila_keyring`` (default: ``ceph.client.manila.keyring``)
#. Copy Ceph configuration file to ``/etc/kolla/config/manila/ceph.conf``
#. Copy Ceph keyring to ``/etc/kolla/config/manila/<ceph_manila_keyring>``
#. Setup Manila in the usual way
Step 1 is done by using setting ``enable_manila_backend_cephfs_native=true``
Now put ceph.conf and the keyring file (name depends on the username created
in Ceph) into the same directory, for example:
.. path /etc/kolla/config/manila/ceph.conf
.. code-block:: ini
[global]
fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3
mon_host = 192.168.0.56
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
.. code-block:: console
$ cat /etc/kolla/config/manila/ceph.client.manila.keyring
[client.manila]
key = AQAg5YRXS0qxLRAAXe6a4R1a15AoRx7ft80DhA==
For more details on the rest of the Manila setup, such as creating the share
type ``default_share_type``, please see :doc:`Manila in Kolla <manila-guide>`.

13
releasenotes/notes/ceph-keys-vars-6857d19d291c401d.yaml Normal file
View File

@ -0,0 +1,13 @@
---
features:
- |
Introduce user modifiable variables instead of fixed names for Ceph
keyring files used by external Ceph functionality.
upgrade:
- |
For cinder (cinder-volume and cinder-backup), glance-api and manila
keyrings behavior has changed and kolla-ansible deployment will not copy
those keys using wildcards (ceph.*), instead will use newly introduced
variables. Your environment may render unusable after an upgrade if your
keys in /etc/kolla/config do not match default values for introduced
variables.