diff --git a/ansible-role-requirements.yml b/ansible-role-requirements.yml index c7abacd2f1..49dc8d0dfd 100644 --- a/ansible-role-requirements.yml +++ b/ansible-role-requirements.yml @@ -170,3 +170,19 @@ scm: git src: https://git.openstack.org/openstack/openstack-ansible-os_designate version: master +- name: ceph.ceph-common + scm: git + src: https://github.com/ceph/ansible-ceph-common + version: master +- name: ceph.ceph-docker-common + scm: git + src: https://github.com/ceph/ansible-ceph-docker-common + version: master +- name: ceph-mon + scm: git + src: https://github.com/ceph/ansible-ceph-mon + version: master +- name: ceph-osd + scm: git + src: https://github.com/ceph/ansible-ceph-osd + version: master diff --git a/deploy-guide/source/app-ceph.rst b/deploy-guide/source/app-ceph.rst new file mode 100644 index 0000000000..260dba6567 --- /dev/null +++ b/deploy-guide/source/app-ceph.rst @@ -0,0 +1,18 @@ +==================================== +Appendix G: Ceph-Ansible integration +==================================== + +OpenStack-Ansible allows `Ceph storage `_ cluster integration +using the roles maintained by the `Ceph-Ansible`_ project>. Deployers can +enable the ``ceph-install`` playbook by adding hosts to the +``ceph-mon_hosts`` and ``ceph-osd_hosts`` groups in +``openstack_user_config.yml``, and then configuring `Ceph-Ansible specific vars +`_ +in the OpenStack-Ansible ``user_variables.yml`` file. + +.. warning:: + + Ceph-Ansible integration in OpenStack-Ansible should be considered + experimental and for testing purposes only. + +.. _Ceph-Ansible: https://github.com/ceph/ceph-ansible/ diff --git a/deploy-guide/source/app-resources.rst b/deploy-guide/source/app-resources.rst index 9117f00897..b5285ea0f3 100644 --- a/deploy-guide/source/app-resources.rst +++ b/deploy-guide/source/app-resources.rst @@ -1,5 +1,5 @@ ================================= -Appendix G: Additional resources +Appendix H: Additional resources ================================= Ansible resources: diff --git a/deploy-guide/source/app.rst b/deploy-guide/source/app.rst index 2eb069078a..4272663259 100644 --- a/deploy-guide/source/app.rst +++ b/deploy-guide/source/app.rst @@ -11,4 +11,5 @@ Appendices app-security.rst app-networking.rst app-advanced-config-options.rst + app-ceph.rst app-resources.rst diff --git a/etc/openstack_deploy/conf.d/ceph.yml.aio b/etc/openstack_deploy/conf.d/ceph.yml.aio new file mode 100644 index 0000000000..b772765713 --- /dev/null +++ b/etc/openstack_deploy/conf.d/ceph.yml.aio @@ -0,0 +1,9 @@ +# The infra nodes where the Ceph mon services will run +ceph-mon_hosts: + aio1: + ip: 172.29.236.100 + +# The nodes that the Ceph OSD disks will be running on +ceph-osd_hosts: + aio1: + ip: 172.29.236.100 diff --git a/playbooks/ceph-install.yml b/playbooks/ceph-install.yml new file mode 100644 index 0000000000..d7d0e61d40 --- /dev/null +++ b/playbooks/ceph-install.yml @@ -0,0 +1,90 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install ceph mons + hosts: ceph-mon + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-ceph" + dest: "/var/log/ceph" + - include: common-tasks/os-lxc-container-setup.yml + vars: + list_of_bind_mounts: + - bind_dir_path: "/var/lib/ceph" + mount_path: "/openstack/{{ inventory_hostname }}" + - include: common-tasks/package-cache-proxy.yml + roles: + - role: "ceph-mon" + tags: + - skip_ansible_lint + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: ceph_log_rotate + rsyslog_client_log_dir: "/var/log/ceph" + rsyslog_client_config_name: "99-ceph-rsyslog-client.conf" + tags: + - rsyslog + - role: "system_crontab_coordination" + tags: + - crontab + vars: + is_metal: "{{ properties.is_metal|default(false) }}" + tags: + - ceph + - ceph-mon + +- name: Gather ceph-mon facts + hosts: ceph-mon + user: root + gather_facts: no + tasks: + - name: Gather ceph-mon facts + action: setup + tags: + - ceph-mon-facts + tags: + - ceph + - ceph-osd + +- name: Install ceph osds + hosts: ceph-osd + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-ceph" + dest: "/var/log/ceph" + - include: common-tasks/package-cache-proxy.yml + roles: + - role: "ceph-osd" + tags: + - skip_ansible_lint + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: ceph_log_rotate + rsyslog_client_log_dir: "/var/log/ceph" + rsyslog_client_config_name: "99-ceph-rsyslog-client.conf" + tags: + - rsyslog + - role: "system_crontab_coordination" + tags: + - crontab + tags: + - ceph + - ceph-osd diff --git a/playbooks/inventory/env.d/ceph.yml b/playbooks/inventory/env.d/ceph.yml new file mode 100644 index 0000000000..707e6e9841 --- /dev/null +++ b/playbooks/inventory/env.d/ceph.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +component_skel: + ceph-mon: + belongs_to: + - ceph_all + ceph-osd: + belongs_to: + - ceph_all + +container_skel: + ceph-mon_container: + belongs_to: + - ceph-mon_containers + contains: + - ceph-mon + properties: + service_name: ceph + ceph-osd_container: + belongs_to: + - ceph-osd_containers + contains: + - ceph-osd + properties: + is_metal: true + service_name: ceph + +physical_skel: + ceph-mon_containers: + belongs_to: + - all_containers + ceph-mon_hosts: + belongs_to: + - hosts + ceph-osd_containers: + belongs_to: + - all_containers + ceph-osd_hosts: + belongs_to: + - hosts diff --git a/playbooks/inventory/group_vars/all.yml b/playbooks/inventory/group_vars/all.yml index 9c7bc507ff..cbe4b5522f 100644 --- a/playbooks/inventory/group_vars/all.yml +++ b/playbooks/inventory/group_vars/all.yml @@ -452,3 +452,17 @@ ansible_ssh_extra_args: > -o ForwardX11=no -o ForwardAgent=yes -T + +## ceph-ansible configuration +mon_group_name: ceph-mon +osd_group_name: ceph-osd +ceph_stable: true +# The _stable_release var is used by both the OSA ceph_client role and the +# ceph-ansible roles. It is defaulted in ceph_client but set here to keep the +# OSA/ceph-ansible integrations in sync. +ceph_stable_release: jewel +fetch_directory: /etc/openstack_deploy/ceph-fetch/ +# tries to create /var/log/ceph as a directory and fails if the log link already +# exists. we handle the log dir creation so this is not something we need +# ceph-common to prepare for us. +rbd_client_directories: false diff --git a/playbooks/setup-infrastructure.yml b/playbooks/setup-infrastructure.yml index 820f80d3b2..96e0d5a0e8 100644 --- a/playbooks/setup-infrastructure.yml +++ b/playbooks/setup-infrastructure.yml @@ -20,5 +20,6 @@ - include: galera-install.yml - include: rabbitmq-install.yml - include: etcd-install.yml +- include: ceph-install.yml - include: utility-install.yml - include: rsyslog-install.yml diff --git a/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml b/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml new file mode 100644 index 0000000000..e947f236f2 --- /dev/null +++ b/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml @@ -0,0 +1,15 @@ +--- +features: + - Playbooks for `ceph-ansible `_ have + been added to facilitate gate testing of the OpenStack-Ansible integration + with Ceph clusters, and can be used to integrate the two projects so that + OpenStack-Ansible can deploy and consume its own Ceph installation using + ceph-ansible. This should be considered an experimental integration until + further testing is been completed by deployers and the OpenStack-Ansible + gate to fine tune its stability and completeness. The ceph-install playbook + can be activated by adding hosts to the ``ceph-mon_hosts`` and + ``ceph-osd_hosts`` in the OSA inventory. A variety of ceph-ansible specific + variables will likely need to be configured in ``user_variables.yml`` to + configure ceph-ansible for your environment. Please reference the + `ceph-ansible repo `_ + for a list of variables the project supports. diff --git a/tests/bootstrap-aio.yml b/tests/bootstrap-aio.yml index f934409a79..f15baac324 100644 --- a/tests/bootstrap-aio.yml +++ b/tests/bootstrap-aio.yml @@ -22,7 +22,7 @@ - role: "pip_install" - role: "bootstrap-host" openstack_confd_entries: "{{ confd_overrides[scenario] }}" - scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}" + bootstrap_host_scenario: "{{ scenario }}" confd_overrides: aio: - name: aodh.yml.aio @@ -37,7 +37,17 @@ - name: neutron.yml.aio - name: nova.yml.aio - name: swift.yml.aio + ceph: + - name: ceph.yml.aio + - name: cinder.yml.aio + - name: glance.yml.aio + - name: heat.yml.aio + - name: horizon.yml.aio + - name: keystone.yml.aio + - name: neutron.yml.aio + - name: nova.yml.aio vars: + scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}" sshd: ListenAddress: - 0.0.0.0 diff --git a/tests/roles/bootstrap-host/defaults/main.yml b/tests/roles/bootstrap-host/defaults/main.yml index ea239bd6fd..39f0628412 100644 --- a/tests/roles/bootstrap-host/defaults/main.yml +++ b/tests/roles/bootstrap-host/defaults/main.yml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Scenario used to bootstrap the host +bootstrap_host_scenario: aio + # Boolean option to implement OpenStack-Ansible configuration for an AIO # Switch to no for a multi-node configuration bootstrap_host_aio_config: yes @@ -44,6 +47,16 @@ bootstrap_host_loopback_nova: yes # Size of the Nova loopback disk in gigabytes (GB). bootstrap_host_loopback_nova_size: 1024 +# Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph +bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}" +# Size of the Ceph OSD loopbacks +bootstrap_host_loopback_ceph_size: 1024 +# Ceph OSDs to create on the AIO host +ceph_osd_images: + - 'ceph1' + - 'ceph2' + - 'ceph3' + ## Network configuration # The AIO bootstrap configures bridges for use with the AIO deployment. # By default, these bridges are configured to be independent of any physical diff --git a/tests/roles/bootstrap-host/tasks/main.yml b/tests/roles/bootstrap-host/tasks/main.yml index fa74d169df..de23ccc760 100644 --- a/tests/roles/bootstrap-host/tasks/main.yml +++ b/tests/roles/bootstrap-host/tasks/main.yml @@ -83,6 +83,13 @@ tags: - prepare-loopback-swift +# Prepare the Ceph cluster UUID and loopback disks +- include: prepare_ceph.yml + when: + - bootstrap_host_ceph | bool + tags: + - prepare-ceph + # Prepare the network interfaces - include: prepare_networking.yml tags: diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index 0eefdd3801..735ddb05e8 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -106,7 +106,7 @@ - name: Set the user_variables config_template: - src: user_variables.aio.yml.j2 + src: "user_variables.{{ bootstrap_host_scenario }}.yml.j2" dest: /etc/openstack_deploy/user_variables.yml config_overrides: "{{ user_variables_overrides | default({}) }}" config_type: yaml diff --git a/tests/roles/bootstrap-host/tasks/prepare_ceph.yml b/tests/roles/bootstrap-host/tasks/prepare_ceph.yml new file mode 100644 index 0000000000..e1794f3660 --- /dev/null +++ b/tests/roles/bootstrap-host/tasks/prepare_ceph.yml @@ -0,0 +1,74 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create sparse ceph OSD files + command: truncate -s {{ bootstrap_host_loopback_ceph_size }}G /openstack/{{ item }}.img + args: + creates: "/openstack/{{ item }}.img" + with_items: "{{ ceph_osd_images }}" + register: ceph_create + changed_when: false + tags: + - ceph-file-create + +- name: Create the ceph loopback device + command: losetup -f /openstack/{{ item.item }}.img --show + with_items: "{{ ceph_create.results }}" + register: ceph_create_loopback + when: not item|skipped + changed_when: false + tags: + - skip_ansible_lint + +- name: Ensure that rc.local exists + file: + path: /etc/rc.local + state: touch + mode: "u+x" + tags: + - ceph-rc-file + +- name: Create ceph loopback at boot time + lineinfile: + dest: /etc/rc.local + line: "losetup -f /openstack/{{ item }}.img" + with_items: "{{ ceph_osd_images }}" + +# TODO(logan): Move these vars to user_variables.ceph.yml.j2 once LP #1649381 +# is fixed and eliminate this task. +- name: Write ceph cluster config + copy: + content: | + --- + devices: {{ ceph_create_loopback.results | map(attribute='stdout') | list | to_yaml | trim }} + # The OSA ceph_client role does not support loading IPs from an inventory group, + # so we have to feed it a list of IPs + ceph_mons: "{% raw %}[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]{% endraw %}" + cinder_backends: + "RBD": + volume_driver: cinder.volume.drivers.rbd.RBDDriver + rbd_pool: volumes + rbd_ceph_conf: /etc/ceph/ceph.conf + rbd_store_chunk_size: 8 + volume_backend_name: rbddriver + rbd_user: cinder + rbd_secret_uuid: "{% raw %}{{ cinder_ceph_client_uuid }}{% endraw %}" + report_discard_supported: true + dest: /etc/openstack_deploy/user_ceph_aio.yml + force: no + become: false + when: not ceph_create_loopback|skipped + tags: + - skip_ansible_lint diff --git a/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 new file mode 100644 index 0000000000..212f2ad7c8 --- /dev/null +++ b/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 @@ -0,0 +1,16 @@ +{% include 'user_variables.aio.yml.j2' %} + +## ceph-ansible AIO settings +common_single_host_mode: true +monitor_interface: eth1 # Management network in the AIO +public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}" +journal_size: 100 +journal_collocation: true +pool_default_pg_num: 32 +openstack_config: true # Ceph ansible automatically creates pools & keys +cinder_ceph_client: cinder +cinder_default_volume_type: RBD +glance_ceph_client: glance +glance_default_store: rbd +glance_rbd_store_pool: images +nova_libvirt_images_rbd_pool: vms diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 97bdebae10..8f96f7dad6 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -152,6 +152,17 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase): 'ceilometer_api_container', 'ceilometer_collector', 'ceilometer_collector_container', + 'ceph_all', + 'ceph-mon_all', + 'ceph-mon_containers', + 'ceph-mon_container', + 'ceph-mon_hosts', + 'ceph-mon', + 'ceph-osd_all', + 'ceph-osd_containers', + 'ceph-osd_container', + 'ceph-osd_hosts', + 'ceph-osd', 'cinder_all', 'cinder_api', 'cinder_api_container',