Merge "ceph-ansible integration"

ansible-role-requirements.yml

@@ -170,3 +170,19 @@
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-os_designate
   version: master
+- name: ceph.ceph-common
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-common
+  version: master
+- name: ceph.ceph-docker-common
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-docker-common
+  version: master
+- name: ceph-mon
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-mon
+  version: master
+- name: ceph-osd
+  scm: git
+  src: https://github.com/ceph/ansible-ceph-osd
+  version: master

deploy-guide/source/app-ceph.rst

@@ -0,0 +1,18 @@
+====================================
+Appendix G: Ceph-Ansible integration
+====================================
+
+OpenStack-Ansible allows `Ceph storage <https://ceph.com>`_ cluster integration
+using the roles maintained by the `Ceph-Ansible`_ project>. Deployers can
+enable the ``ceph-install`` playbook by adding hosts to the
+``ceph-mon_hosts`` and ``ceph-osd_hosts`` groups in
+``openstack_user_config.yml``, and then configuring `Ceph-Ansible specific vars
+<https://github.com/ceph/ceph-ansible/blob/master/group_vars/all.yml.sample>`_
+in the OpenStack-Ansible ``user_variables.yml`` file.
+
+.. warning::
+
+   Ceph-Ansible integration in OpenStack-Ansible should be considered
+   experimental and for testing purposes only.
+
+.. _Ceph-Ansible: https://github.com/ceph/ceph-ansible/

deploy-guide/source/app-resources.rst

@@ -1,5 +1,5 @@
 =================================
-Appendix G: Additional resources
+Appendix H: Additional resources
 =================================
 
 Ansible resources:

deploy-guide/source/app.rst

@@ -11,4 +11,5 @@ Appendices
    app-security.rst
    app-networking.rst
    app-advanced-config-options.rst
+   app-ceph.rst
    app-resources.rst

etc/openstack_deploy/conf.d/ceph.yml.aio

@@ -0,0 +1,9 @@
+# The infra nodes where the Ceph mon services will run
+ceph-mon_hosts:
+  aio1:
+    ip: 172.29.236.100
+
+# The nodes that the Ceph OSD disks will be running on
+ceph-osd_hosts:
+  aio1:
+    ip: 172.29.236.100

playbooks/ceph-install.yml

@@ -0,0 +1,90 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install ceph mons
+  hosts: ceph-mon
+  max_fail_percentage: 20
+  user: root
+  pre_tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-ceph"
+            dest: "/var/log/ceph"
+    - include: common-tasks/os-lxc-container-setup.yml
+      vars:
+        list_of_bind_mounts:
+          - bind_dir_path: "/var/lib/ceph"
+            mount_path: "/openstack/{{ inventory_hostname }}"
+    - include: common-tasks/package-cache-proxy.yml
+  roles:
+    - role: "ceph-mon"
+      tags:
+        - skip_ansible_lint
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: ceph_log_rotate
+      rsyslog_client_log_dir: "/var/log/ceph"
+      rsyslog_client_config_name: "99-ceph-rsyslog-client.conf"
+      tags:
+        - rsyslog
+    - role: "system_crontab_coordination"
+      tags:
+        - crontab
+  vars:
+    is_metal: "{{ properties.is_metal|default(false) }}"
+  tags:
+    - ceph
+    - ceph-mon
+
+- name: Gather ceph-mon facts
+  hosts: ceph-mon
+  user: root
+  gather_facts: no
+  tasks:
+    - name: Gather ceph-mon facts
+      action: setup
+      tags:
+        - ceph-mon-facts
+  tags:
+    - ceph
+    - ceph-osd
+
+- name: Install ceph osds
+  hosts: ceph-osd
+  max_fail_percentage: 20
+  user: root
+  pre_tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-ceph"
+            dest: "/var/log/ceph"
+    - include: common-tasks/package-cache-proxy.yml
+  roles:
+    - role: "ceph-osd"
+      tags:
+        - skip_ansible_lint
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: ceph_log_rotate
+      rsyslog_client_log_dir: "/var/log/ceph"
+      rsyslog_client_config_name: "99-ceph-rsyslog-client.conf"
+      tags:
+        - rsyslog
+    - role: "system_crontab_coordination"
+      tags:
+        - crontab
+  tags:
+    - ceph
+    - ceph-osd

playbooks/inventory/env.d/ceph.yml

@@ -0,0 +1,53 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+component_skel:
+  ceph-mon:
+    belongs_to:
+      - ceph_all
+  ceph-osd:
+    belongs_to:
+      - ceph_all
+
+container_skel:
+  ceph-mon_container:
+    belongs_to:
+      - ceph-mon_containers
+    contains:
+      - ceph-mon
+    properties:
+      service_name: ceph
+  ceph-osd_container:
+    belongs_to:
+      - ceph-osd_containers
+    contains:
+      - ceph-osd
+    properties:
+      is_metal: true
+      service_name: ceph
+
+physical_skel:
+  ceph-mon_containers:
+    belongs_to:
+      - all_containers
+  ceph-mon_hosts:
+    belongs_to:
+      - hosts
+  ceph-osd_containers:
+    belongs_to:
+      - all_containers
+  ceph-osd_hosts:
+    belongs_to:
+      - hosts

playbooks/inventory/group_vars/all.yml

@@ -452,3 +452,17 @@ ansible_ssh_extra_args: >
   -o ForwardX11=no
   -o ForwardAgent=yes
   -T
+
+## ceph-ansible configuration
+mon_group_name: ceph-mon
+osd_group_name: ceph-osd
+ceph_stable: true
+# The _stable_release var is used by both the OSA ceph_client role and the
+# ceph-ansible roles. It is defaulted in ceph_client but set here to keep the
+# OSA/ceph-ansible integrations in sync.
+ceph_stable_release: jewel
+fetch_directory: /etc/openstack_deploy/ceph-fetch/
+# tries to create /var/log/ceph as a directory and fails if the log link already
+# exists. we handle the log dir creation so this is not something we need
+# ceph-common to prepare for us.
+rbd_client_directories: false

playbooks/setup-infrastructure.yml

@@ -20,5 +20,6 @@
 - include: galera-install.yml
 - include: rabbitmq-install.yml
 - include: etcd-install.yml
+- include: ceph-install.yml
 - include: utility-install.yml
 - include: rsyslog-install.yml

releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml

@@ -0,0 +1,15 @@
+---
+features:
+  - Playbooks for `ceph-ansible <https://github.com/ceph/ceph-ansible/>`_ have
+    been added to facilitate gate testing of the OpenStack-Ansible integration
+    with Ceph clusters, and can be used to integrate the two projects so that
+    OpenStack-Ansible can deploy and consume its own Ceph installation using
+    ceph-ansible. This should be considered an experimental integration until
+    further testing is been completed by deployers and the OpenStack-Ansible
+    gate to fine tune its stability and completeness. The ceph-install playbook
+    can be activated by adding hosts to the ``ceph-mon_hosts`` and
+    ``ceph-osd_hosts`` in the OSA inventory. A variety of ceph-ansible specific
+    variables will likely need to be configured in ``user_variables.yml`` to
+    configure ceph-ansible for your environment. Please reference the
+    `ceph-ansible repo <https://github.com/ceph/ceph-ansible/blob/master/group_vars/all.yml.sample>`_
+    for a list of variables the project supports.

tests/bootstrap-aio.yml

@@ -22,7 +22,7 @@
     - role: "pip_install"
     - role: "bootstrap-host"
       openstack_confd_entries: "{{ confd_overrides[scenario] }}"
-      scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
+      bootstrap_host_scenario: "{{ scenario }}"
       confd_overrides:
         aio:
           - name: aodh.yml.aio
@@ -37,7 +37,17 @@
           - name: neutron.yml.aio
           - name: nova.yml.aio
           - name: swift.yml.aio
+        ceph:
+          - name: ceph.yml.aio
+          - name: cinder.yml.aio
+          - name: glance.yml.aio
+          - name: heat.yml.aio
+          - name: horizon.yml.aio
+          - name: keystone.yml.aio
+          - name: neutron.yml.aio
+          - name: nova.yml.aio
   vars:
+    scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
     sshd:
       ListenAddress:
         - 0.0.0.0

tests/roles/bootstrap-host/defaults/main.yml

@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Scenario used to bootstrap the host
+bootstrap_host_scenario: aio
+
 # Boolean option to implement OpenStack-Ansible configuration for an AIO
 # Switch to no for a multi-node configuration
 bootstrap_host_aio_config: yes
@@ -44,6 +47,16 @@ bootstrap_host_loopback_nova: yes
 # Size of the Nova loopback disk in gigabytes (GB).
 bootstrap_host_loopback_nova_size: 1024
 
+# Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph
+bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}"
+# Size of the Ceph OSD loopbacks
+bootstrap_host_loopback_ceph_size: 1024
+# Ceph OSDs to create on the AIO host
+ceph_osd_images:
+  - 'ceph1'
+  - 'ceph2'
+  - 'ceph3'
+
 ## Network configuration
 # The AIO bootstrap configures bridges for use with the AIO deployment.
 # By default, these bridges are configured to be independent of any physical

tests/roles/bootstrap-host/tasks/main.yml

@@ -83,6 +83,13 @@
   tags:
     - prepare-loopback-swift
 
+# Prepare the Ceph cluster UUID and loopback disks
+- include: prepare_ceph.yml
+  when:
+    - bootstrap_host_ceph | bool
+  tags:
+    - prepare-ceph
+
 # Prepare the network interfaces
 - include: prepare_networking.yml
   tags:

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -106,7 +106,7 @@
 
 - name: Set the user_variables
   config_template:
-    src: user_variables.aio.yml.j2
+    src: "user_variables.{{ bootstrap_host_scenario }}.yml.j2"
     dest: /etc/openstack_deploy/user_variables.yml
     config_overrides: "{{ user_variables_overrides | default({}) }}"
     config_type: yaml

tests/roles/bootstrap-host/tasks/prepare_ceph.yml

@@ -0,0 +1,74 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create sparse ceph OSD files
+  command: truncate -s {{ bootstrap_host_loopback_ceph_size }}G /openstack/{{ item }}.img
+  args:
+    creates: "/openstack/{{ item }}.img"
+  with_items: "{{ ceph_osd_images }}"
+  register: ceph_create
+  changed_when: false
+  tags:
+    - ceph-file-create
+
+- name: Create the ceph loopback device
+  command: losetup -f /openstack/{{ item.item }}.img --show
+  with_items: "{{ ceph_create.results }}"
+  register: ceph_create_loopback
+  when: not item|skipped
+  changed_when: false
+  tags:
+    - skip_ansible_lint
+
+- name: Ensure that rc.local exists
+  file:
+    path: /etc/rc.local
+    state: touch
+    mode: "u+x"
+  tags:
+    - ceph-rc-file
+
+- name: Create ceph loopback at boot time
+  lineinfile:
+    dest: /etc/rc.local
+    line: "losetup -f /openstack/{{ item }}.img"
+  with_items: "{{ ceph_osd_images }}"
+
+# TODO(logan): Move these vars to user_variables.ceph.yml.j2 once LP #1649381
+# is fixed and eliminate this task.
+- name: Write ceph cluster config
+  copy:
+    content: |
+      ---
+      devices: {{ ceph_create_loopback.results | map(attribute='stdout') | list | to_yaml | trim }}
+      # The OSA ceph_client role does not support loading IPs from an inventory group,
+      # so we have to feed it a list of IPs
+      ceph_mons: "{% raw %}[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]{% endraw %}"
+      cinder_backends:
+        "RBD":
+          volume_driver: cinder.volume.drivers.rbd.RBDDriver
+          rbd_pool: volumes
+          rbd_ceph_conf: /etc/ceph/ceph.conf
+          rbd_store_chunk_size: 8
+          volume_backend_name: rbddriver
+          rbd_user: cinder
+          rbd_secret_uuid: "{% raw %}{{ cinder_ceph_client_uuid }}{% endraw %}"
+          report_discard_supported: true
+    dest: /etc/openstack_deploy/user_ceph_aio.yml
+    force: no
+  become: false
+  when: not ceph_create_loopback|skipped
+  tags:
+    - skip_ansible_lint

tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2

@@ -0,0 +1,16 @@
+{% include 'user_variables.aio.yml.j2' %}
+
+## ceph-ansible AIO settings
+common_single_host_mode: true
+monitor_interface: eth1 # Management network in the AIO
+public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}"
+journal_size: 100
+journal_collocation: true
+pool_default_pg_num: 32
+openstack_config: true # Ceph ansible automatically creates pools & keys
+cinder_ceph_client: cinder
+cinder_default_volume_type: RBD
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms

tests/test_inventory.py

@@ -152,6 +152,17 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
         'ceilometer_api_container',
         'ceilometer_collector',
         'ceilometer_collector_container',
+        'ceph_all',
+        'ceph-mon_all',
+        'ceph-mon_containers',
+        'ceph-mon_container',
+        'ceph-mon_hosts',
+        'ceph-mon',
+        'ceph-osd_all',
+        'ceph-osd_containers',
+        'ceph-osd_container',
+        'ceph-osd_hosts',
+        'ceph-osd',
         'cinder_all',
         'cinder_api',
         'cinder_api_container',