ceph-ansible integration

Adds ceph-ansible integration to OSA using the upstream
roles ceph-common, ceph-mon, and ceph-osd to support deployment
of a Ceph cluster for use with OSA.

The integration should be considered experimental at this point,
however an integrated functional testing scenario named "ceph"
is added which fully exercises the ceph-ansible deployment and
configures OSA to use the deployed Ceph cluster. Once this is added
to the OSA gate tests for the integrated build, that should help
quickly stabilize any issues that arise with this integration.

Change-Id: I275708096a71f66d73be1400c8a592c598382c1d

ansible-role-requirements.yml

@@ -170,3 +170,19 @@
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-os_designate
   version: master
+- name: ceph.ceph-common
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-common
+  version: master
+- name: ceph.ceph-docker-common
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-docker-common
+  version: master
+- name: ceph-mon
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-mon
+  version: master
+- name: ceph-osd
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-osd
+  version: master

deploy-guide/source/app-ceph.rst

@@ -0,0 +1,18 @@
+====================================
+Appendix G: Ceph-Ansible integration
+====================================
+
+OpenStack-Ansible allows `Ceph storage <https://ceph.com>`_ cluster integration
+using the roles maintained by the `Ceph-Ansible`_ project>. Deployers can
+enable the ``ceph-install`` playbook by adding hosts to the
+``ceph-mon_hosts`` and ``ceph-osd_hosts`` groups in
+``openstack_user_config.yml``, and then configuring `Ceph-Ansible specific vars
+<https://github.com/ceph/ceph-ansible/blob/master/group_vars/all.yml.sample>`_
+in the OpenStack-Ansible ``user_variables.yml`` file.
+
+.. warning::
+
+   Ceph-Ansible integration in OpenStack-Ansible should be considered
+   experimental and for testing purposes only.
+
+.. _Ceph-Ansible: https://github.com/ceph/ceph-ansible/

deploy-guide/source/app-resources.rst

@@ -1,5 +1,5 @@
 =================================
-Appendix G: Additional resources
+Appendix H: Additional resources
 =================================
 
 Ansible resources:

deploy-guide/source/app.rst

@@ -11,4 +11,5 @@ Appendices
    app-security.rst
    app-networking.rst
    app-advanced-config-options.rst
+   app-ceph.rst
    app-resources.rst

etc/openstack_deploy/conf.d/ceph.yml.aio

@@ -0,0 +1,9 @@
+# The infra nodes where the Ceph mon services will run
+ceph-mon_hosts:
+  aio1:
+    ip: 172.29.236.100
+
+# The nodes that the Ceph OSD disks will be running on
+ceph-osd_hosts:
+  aio1:
+    ip: 172.29.236.100

playbooks/ceph-install.yml

@@ -0,0 +1,90 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install ceph mons
+  hosts: ceph-mon
+  max_fail_percentage: 20
+  user: root
+  pre_tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-ceph"
+            dest: "/var/log/ceph"
+    - include: common-tasks/os-lxc-container-setup.yml
+      vars:
+        list_of_bind_mounts:
+          - bind_dir_path: "/var/lib/ceph"
+            mount_path: "/openstack/{{ inventory_hostname }}"
+    - include: common-tasks/package-cache-proxy.yml
+  roles:
+    - role: "ceph-mon"
+      tags:
+        - skip_ansible_lint
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: ceph_log_rotate
+      rsyslog_client_log_dir: "/var/log/ceph"
+      rsyslog_client_config_name: "99-ceph-rsyslog-client.conf"
+      tags:
+        - rsyslog
+    - role: "system_crontab_coordination"
+      tags:
+        - crontab
+  vars:
+    is_metal: "{{ properties.is_metal|default(false) }}"
+  tags:
+    - ceph
+    - ceph-mon
+
+- name: Gather ceph-mon facts
+  hosts: ceph-mon
+  user: root
+  gather_facts: no
+  tasks:
+    - name: Gather ceph-mon facts
+      action: setup
+      tags:
+        - ceph-mon-facts
+  tags:
+    - ceph
+    - ceph-osd
+
+- name: Install ceph osds
+  hosts: ceph-osd
+  max_fail_percentage: 20
+  user: root
+  pre_tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-ceph"
+            dest: "/var/log/ceph"
+    - include: common-tasks/package-cache-proxy.yml
+  roles:
+    - role: "ceph-osd"
+      tags:
+        - skip_ansible_lint
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: ceph_log_rotate
+      rsyslog_client_log_dir: "/var/log/ceph"
+      rsyslog_client_config_name: "99-ceph-rsyslog-client.conf"
+      tags:
+        - rsyslog
+    - role: "system_crontab_coordination"
+      tags:
+        - crontab
+  tags:
+    - ceph
+    - ceph-osd

playbooks/inventory/env.d/ceph.yml

@@ -0,0 +1,53 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+component_skel:
+  ceph-mon:
+    belongs_to:
+      - ceph_all
+  ceph-osd:
+    belongs_to:
+      - ceph_all
+
+container_skel:
+  ceph-mon_container:
+    belongs_to:
+      - ceph-mon_containers
+    contains:
+      - ceph-mon
+    properties:
+      service_name: ceph
+  ceph-osd_container:
+    belongs_to:
+      - ceph-osd_containers
+    contains:
+      - ceph-osd
+    properties:
+      is_metal: true
+      service_name: ceph
+
+physical_skel:
+  ceph-mon_containers:
+    belongs_to:
+      - all_containers
+  ceph-mon_hosts:
+    belongs_to:
+      - hosts
+  ceph-osd_containers:
+    belongs_to:
+      - all_containers
+  ceph-osd_hosts:
+    belongs_to:
+      - hosts

playbooks/inventory/group_vars/all.yml

@@ -452,3 +452,17 @@ ansible_ssh_extra_args: >
   -o ForwardX11=no
   -o ForwardAgent=yes
   -T
+
+## ceph-ansible configuration
+mon_group_name: ceph-mon
+osd_group_name: ceph-osd
+ceph_stable: true
+# The _stable_release var is used by both the OSA ceph_client role and the
+# ceph-ansible roles. It is defaulted in ceph_client but set here to keep the
+# OSA/ceph-ansible integrations in sync.
+ceph_stable_release: jewel
+fetch_directory: /etc/openstack_deploy/ceph-fetch/
+# tries to create /var/log/ceph as a directory and fails if the log link already
+# exists. we handle the log dir creation so this is not something we need
+# ceph-common to prepare for us.
+rbd_client_directories: false

playbooks/setup-infrastructure.yml

@@ -20,5 +20,6 @@
 - include: galera-install.yml
 - include: rabbitmq-install.yml
 - include: etcd-install.yml
+- include: ceph-install.yml
 - include: utility-install.yml
 - include: rsyslog-install.yml

releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml

@@ -0,0 +1,15 @@
+---
+features:
+  - Playbooks for `ceph-ansible <https://github.com/ceph/ceph-ansible/>`_ have
+    been added to facilitate gate testing of the OpenStack-Ansible integration
+    with Ceph clusters, and can be used to integrate the two projects so that
+    OpenStack-Ansible can deploy and consume its own Ceph installation using
+    ceph-ansible. This should be considered an experimental integration until
+    further testing is been completed by deployers and the OpenStack-Ansible
+    gate to fine tune its stability and completeness. The ceph-install playbook
+    can be activated by adding hosts to the ``ceph-mon_hosts`` and
+    ``ceph-osd_hosts`` in the OSA inventory. A variety of ceph-ansible specific
+    variables will likely need to be configured in ``user_variables.yml`` to
+    configure ceph-ansible for your environment. Please reference the
+    `ceph-ansible repo <https://github.com/ceph/ceph-ansible/blob/master/group_vars/all.yml.sample>`_
+    for a list of variables the project supports.

tests/bootstrap-aio.yml

@@ -22,7 +22,7 @@
     - role: "pip_install"
     - role: "bootstrap-host"
       openstack_confd_entries: "{{ confd_overrides[scenario] }}"
-      scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
+      bootstrap_host_scenario: "{{ scenario }}"
       confd_overrides:
         aio:
           - name: aodh.yml.aio
@@ -37,7 +37,17 @@
           - name: neutron.yml.aio
           - name: nova.yml.aio
           - name: swift.yml.aio
+        ceph:
+          - name: ceph.yml.aio
+          - name: cinder.yml.aio
+          - name: glance.yml.aio
+          - name: heat.yml.aio
+          - name: horizon.yml.aio
+          - name: keystone.yml.aio
+          - name: neutron.yml.aio
+          - name: nova.yml.aio
   vars:
+    scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
     sshd:
       ListenAddress:
         - 0.0.0.0

tests/roles/bootstrap-host/defaults/main.yml

@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Scenario used to bootstrap the host
+bootstrap_host_scenario: aio
+
 # Boolean option to implement OpenStack-Ansible configuration for an AIO
 # Switch to no for a multi-node configuration
 bootstrap_host_aio_config: yes
@@ -44,6 +47,16 @@ bootstrap_host_loopback_nova: yes
 # Size of the Nova loopback disk in gigabytes (GB).
 bootstrap_host_loopback_nova_size: 1024
 
+# Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph
+bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}"
+# Size of the Ceph OSD loopbacks
+bootstrap_host_loopback_ceph_size: 1024
+# Ceph OSDs to create on the AIO host
+ceph_osd_images:
+  - 'ceph1'
+  - 'ceph2'
+  - 'ceph3'
+
 ## Network configuration
 # The AIO bootstrap configures bridges for use with the AIO deployment.
 # By default, these bridges are configured to be independent of any physical

tests/roles/bootstrap-host/tasks/main.yml

@@ -83,6 +83,13 @@
   tags:
     - prepare-loopback-swift
 
+# Prepare the Ceph cluster UUID and loopback disks
+- include: prepare_ceph.yml
+  when:
+    - bootstrap_host_ceph | bool
+  tags:
+    - prepare-ceph
+
 # Prepare the network interfaces
 - include: prepare_networking.yml
   tags:

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -106,7 +106,7 @@
 
 - name: Set the user_variables
   config_template:
-    src: user_variables.aio.yml.j2
+    src: "user_variables.{{ bootstrap_host_scenario }}.yml.j2"
     dest: /etc/openstack_deploy/user_variables.yml
     config_overrides: "{{ user_variables_overrides | default({}) }}"
     config_type: yaml

tests/roles/bootstrap-host/tasks/prepare_ceph.yml

@@ -0,0 +1,74 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create sparse ceph OSD files
+  command: truncate -s {{ bootstrap_host_loopback_ceph_size }}G /openstack/{{ item }}.img
+  args:
+    creates: "/openstack/{{ item }}.img"
+  with_items: "{{ ceph_osd_images }}"
+  register: ceph_create
+  changed_when: false
+  tags:
+    - ceph-file-create
+
+- name: Create the ceph loopback device
+  command: losetup -f /openstack/{{ item.item }}.img --show
+  with_items: "{{ ceph_create.results }}"
+  register: ceph_create_loopback
+  when: not item|skipped
+  changed_when: false
+  tags:
+    - skip_ansible_lint
+
+- name: Ensure that rc.local exists
+  file:
+    path: /etc/rc.local
+    state: touch
+    mode: "u+x"
+  tags:
+    - ceph-rc-file
+
+- name: Create ceph loopback at boot time
+  lineinfile:
+    dest: /etc/rc.local
+    line: "losetup -f /openstack/{{ item }}.img"
+  with_items: "{{ ceph_osd_images }}"
+
+# TODO(logan): Move these vars to user_variables.ceph.yml.j2 once LP #1649381
+# is fixed and eliminate this task.
+- name: Write ceph cluster config
+  copy:
+    content: |
+      ---
+      devices: {{ ceph_create_loopback.results | map(attribute='stdout') | list | to_yaml | trim }}
+      # The OSA ceph_client role does not support loading IPs from an inventory group,
+      # so we have to feed it a list of IPs
+      ceph_mons: "{% raw %}[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]{% endraw %}"
+      cinder_backends:
+        "RBD":
+          volume_driver: cinder.volume.drivers.rbd.RBDDriver
+          rbd_pool: volumes
+          rbd_ceph_conf: /etc/ceph/ceph.conf
+          rbd_store_chunk_size: 8
+          volume_backend_name: rbddriver
+          rbd_user: cinder
+          rbd_secret_uuid: "{% raw %}{{ cinder_ceph_client_uuid }}{% endraw %}"
+          report_discard_supported: true
+    dest: /etc/openstack_deploy/user_ceph_aio.yml
+    force: no
+  become: false
+  when: not ceph_create_loopback|skipped
+  tags:
+    - skip_ansible_lint

tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2

@@ -0,0 +1,16 @@
+{% include 'user_variables.aio.yml.j2' %}
+
+## ceph-ansible AIO settings
+common_single_host_mode: true
+monitor_interface: eth1 # Management network in the AIO
+public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}"
+journal_size: 100
+journal_collocation: true
+pool_default_pg_num: 32
+openstack_config: true # Ceph ansible automatically creates pools & keys
+cinder_ceph_client: cinder
+cinder_default_volume_type: RBD
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms

tests/test_inventory.py

@@ -152,6 +152,17 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
         'ceilometer_api_container',
         'ceilometer_collector',
         'ceilometer_collector_container',
+        'ceph_all',
+        'ceph-mon_all',
+        'ceph-mon_containers',
+        'ceph-mon_container',
+        'ceph-mon_hosts',
+        'ceph-mon',
+        'ceph-osd_all',
+        'ceph-osd_containers',
+        'ceph-osd_container',
+        'ceph-osd_hosts',
+        'ceph-osd',
         'cinder_all',
         'cinder_api',
         'cinder_api_container',