From c700fdba0da5563f07935f0d7cba7d75ad586a23 Mon Sep 17 00:00:00 2001 From: Logan V Date: Sun, 5 Nov 2017 22:24:33 -0600 Subject: [PATCH] Ceph RadosGW integration This adds a new scenario for Ceph Rados GW integration: - It adds the RGW into haproxy to the default swift port if swift port isn't deployed already - It adds tempest swift API testing on the rados gw in the check scenario - It adds ceph rgw in default inventories. Change-Id: I5f6ff3fa05a4a8019bf5b695b02184d9f065bc2e Co-Authored-By: Jean-Philippe Evrard Co-Authored-By: Maxime Guyot --- doc/source/user/ceph/full-deploy.rst | 6 +- etc/openstack_deploy/conf.d/ceph.yml.aio | 5 + etc/openstack_deploy/user_secrets.yml | 3 + inventory/env.d/ceph.yml | 14 ++ inventory/group_vars/all/ceph.yml | 1 + inventory/group_vars/ceph_all.yml | 42 +++++ inventory/group_vars/haproxy_all/haproxy.yml | 12 ++ inventory/group_vars/utility_all.yml | 2 +- playbooks/ceph-rgw-install.yml | 175 ++++++++++++++++++ playbooks/setup-openstack.yml | 5 + .../templates/user_variables_ceph.yml.j2 | 2 +- tests/test_inventory.py | 5 + 12 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 playbooks/ceph-rgw-install.yml diff --git a/doc/source/user/ceph/full-deploy.rst b/doc/source/user/ceph/full-deploy.rst index ca963bc6a2..b7f75d9d0f 100644 --- a/doc/source/user/ceph/full-deploy.rst +++ b/doc/source/user/ceph/full-deploy.rst @@ -35,9 +35,9 @@ integration in two ways: in ``user_variables.yml`` * deploying a ceph cluster by using the roles maintained by the `Ceph-Ansible`_ project. Deployers can enable the ``ceph-install`` - playbook by adding hosts to the ``ceph-mon_hosts`` and ``ceph-osd_hosts`` - groups in ``openstack_user_config.yml``, and then configuring - `Ceph-Ansible specific vars + playbook by adding hosts to the ``ceph-mon_hosts``, ``ceph-osd_hosts`` and + ``ceph-rgw_hosts`` groups in ``openstack_user_config.yml``, and then + configuring `Ceph-Ansible specific vars `_ in the OpenStack-Ansible ``user_variables.yml`` file. diff --git a/etc/openstack_deploy/conf.d/ceph.yml.aio b/etc/openstack_deploy/conf.d/ceph.yml.aio index b772765713..1e7ad3ad56 100644 --- a/etc/openstack_deploy/conf.d/ceph.yml.aio +++ b/etc/openstack_deploy/conf.d/ceph.yml.aio @@ -7,3 +7,8 @@ ceph-mon_hosts: ceph-osd_hosts: aio1: ip: 172.29.236.100 + +# The nodes that the Ceph RadosGW object gateways will be running on +ceph-rgw_hosts: + aio1: + ip: 172.29.236.100 diff --git a/etc/openstack_deploy/user_secrets.yml b/etc/openstack_deploy/user_secrets.yml index b6af625cb9..d7d61cd3b3 100644 --- a/etc/openstack_deploy/user_secrets.yml +++ b/etc/openstack_deploy/user_secrets.yml @@ -160,3 +160,6 @@ molteniron_container_mysql_password: tacker_rabbitmq_password: tacker_service_password: tacker_container_mysql_password: + +## Ceph RadosGW Keystone password +radosgw_admin_password: diff --git a/inventory/env.d/ceph.yml b/inventory/env.d/ceph.yml index cfe6f26ae2..4ec1b7c541 100644 --- a/inventory/env.d/ceph.yml +++ b/inventory/env.d/ceph.yml @@ -20,6 +20,9 @@ component_skel: ceph-osd: belongs_to: - ceph_all + ceph-rgw: + belongs_to: + - ceph_all container_skel: ceph-mon_container: @@ -34,6 +37,11 @@ container_skel: - ceph-osd properties: is_metal: true + ceph-rgw_container: + belongs_to: + - ceph-rgw_containers + contains: + - ceph-rgw physical_skel: ceph-mon_containers: @@ -48,3 +56,9 @@ physical_skel: ceph-osd_hosts: belongs_to: - hosts + ceph-rgw_containers: + belongs_to: + - all_containers + ceph-rgw_hosts: + belongs_to: + - hosts diff --git a/inventory/group_vars/all/ceph.yml b/inventory/group_vars/all/ceph.yml index a089e52cc1..9350ddb116 100644 --- a/inventory/group_vars/all/ceph.yml +++ b/inventory/group_vars/all/ceph.yml @@ -19,6 +19,7 @@ ceph_client_package_state: "{{ package_state }}" mon_group_name: ceph-mon mgr_group_name: "{{ mon_group_name }}" osd_group_name: ceph-osd +rgw_group_name: ceph-rgw ceph_origin: "{{ (ansible_pkg_mgr == 'zypper') | ternary('distro', 'repository') }}" ceph_repository: community # The _stable_release var is used by both the OSA ceph_client role and the diff --git a/inventory/group_vars/ceph_all.yml b/inventory/group_vars/ceph_all.yml index 951bcdb992..f706257727 100644 --- a/inventory/group_vars/ceph_all.yml +++ b/inventory/group_vars/ceph_all.yml @@ -22,3 +22,45 @@ ceph_container_bind_mounts: # needed. nfs_file_gw: False nfs_obj_gw: False + +# radosgw + +radosgw_service_name: "radosgw" +radosgw_service_type: "object-store" +radosgw_service_description: "Object Storage Service" +radosgw_service_region: "{{ service_region }}" +radosgw_admin_user: radosgw +radosgw_admin_tenant: service + +radosgw_service_port: "{{ (groups['swift_proxy'] is defined and groups['swift_proxy'] | length > 0) | ternary(7980,8080) }}" +radosgw_address: "{{ container_address }}" +radosgw_service_proto: http +radosgw_service_publicuri_proto: "{{ openstack_service_publicuri_proto | default(radosgw_service_proto) }}" +radosgw_service_adminuri_proto: "{{ openstack_service_adminuri_proto | default(radosgw_service_proto) }}" +radosgw_service_internaluri_proto: "{{ openstack_service_internaluri_proto | default(radosgw_service_proto) }}" +radosgw_service_publicuri: "{{ radosgw_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ radosgw_service_port }}" +radosgw_service_publicurl: "{{ radosgw_service_publicuri }}/swift/v1" +radosgw_service_adminuri: "{{ radosgw_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ radosgw_service_port }}" +radosgw_service_adminurl: "{{ radosgw_service_adminuri }}/swift/v1" +radosgw_service_internaluri: "{{ radosgw_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ radosgw_service_port }}" +radosgw_service_internalurl: "{{ radosgw_service_internaluri }}/swift/v1" + +# This is the endpoint hostname that will be configured in the Keystone catalog for object storage +radosgw_dns_name: "{{ ansible_host }}" +# To extend ceph_conf_overrides while keeping this configuration, create a +# dict of config_template style overrides for ceph.conf in a var, ie. +# ceph_conf_overrides_custom, then use: +# ceph_conf_overrides: "{{ ceph_conf_overrides_rgw | combine(ceph_conf_overrides_custom, recursive=True) }}" +ceph_conf_overrides: "{{ ceph_conf_overrides_rgw }}" +ceph_conf_overrides_rgw: + "client.rgw.{{ hostvars[inventory_hostname]['ansible_hostname'] }}": + # OpenStack integration with Keystone + rgw_keystone_url: "{{ keystone_service_adminuri }}" + rgw_keystone_api_version: 3 + rgw_keystone_admin_user: "{{ radosgw_admin_user }}" + rgw_keystone_admin_password: "{{ radosgw_admin_password }}" + rgw_keystone_admin_tenant: "{{ radosgw_admin_tenant }}" + rgw_keystone_admin_domain: default + rgw_keystone_accepted_roles: 'Member, _member_, admin, swiftoperator' + rgw_s3_auth_use_keystone: true + rgw_enable_apis: swift diff --git a/inventory/group_vars/haproxy_all/haproxy.yml b/inventory/group_vars/haproxy_all/haproxy.yml index 1f44fb976b..8454561347 100644 --- a/inventory/group_vars/haproxy_all/haproxy.yml +++ b/inventory/group_vars/haproxy_all/haproxy.yml @@ -353,3 +353,15 @@ haproxy_default_services: haproxy_timeout_server: 5000s haproxy_whitelist_networks: "{{ haproxy_opendaylight_whitelist_networks }}" haproxy_service_enabled: "{{ neutron_plugin_type == 'ml2.opendaylight' }}" + - service: + haproxy_service_name: ceph-rgw + haproxy_backend_nodes: "{{ groups['ceph-rgw'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_alg: source + haproxy_port: "{{ hostvars[(groups['ceph-rgw'] | default(['localhost']))[0] | default('localhost')]['radosgw_service_port'] | default(7980) }}" + haproxy_balance_type: http + haproxy_backend_options: + - httpchk HEAD / + haproxy_backend_httpcheck_options: + - expect status 405 + haproxy_service_enabled: "{{ groups['ceph-rgw'] is defined and groups['ceph-rgw'] | length > 0 }}" diff --git a/inventory/group_vars/utility_all.yml b/inventory/group_vars/utility_all.yml index 53fb715dfd..20f1acd64e 100644 --- a/inventory/group_vars/utility_all.yml +++ b/inventory/group_vars/utility_all.yml @@ -52,7 +52,7 @@ tempest_service_available_heat: "{{ groups['heat_all'] is defined and groups['he tempest_service_available_horizon: "{{ groups['horizon_all'] is defined and groups['horizon_all'] | length > 0 }}" tempest_service_available_neutron: "{{ groups['neutron_all'] is defined and groups['neutron_all'] | length > 0 }}" tempest_service_available_nova: "{{ groups['nova_all'] is defined and groups['nova_all'] | length > 0 }}" -tempest_service_available_swift: "{{ groups['swift_all'] is defined and groups['swift_all'] | length > 0 }}" +tempest_service_available_swift: "{{ (groups['swift_all'] is defined and groups['swift_all'] | length > 0) or (groups['ceph-rgw'] is defined and groups['ceph-rgw'] | length > 0) }}" tempest_log_dir: /var/log/utility tempest_venv_tag: "{{ openstack_release }}" diff --git a/playbooks/ceph-rgw-install.yml b/playbooks/ceph-rgw-install.yml new file mode 100644 index 0000000000..9100c14e0f --- /dev/null +++ b/playbooks/ceph-rgw-install.yml @@ -0,0 +1,175 @@ +--- +# Copyright 2017, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install ceph radosgw + hosts: ceph-rgw + user: root + pre_tasks: + - name: Ensure RGW service + keystone: + command: "ensure_service" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + service_name: "{{ radosgw_service_name }}" + service_type: "{{ radosgw_service_type }}" + description: "{{ radosgw_service_description }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + delegate_to: "{{ groups['utility_all'][0] }}" + run_once: true + when: radosgw_keystone | bool + register: add_service + until: add_service|success + retries: 5 + delay: 2 + tags: + - ceph-rgw-setup + - rgw-service-add + - name: Ensure RGW user + keystone: + command: "ensure_user" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + user_name: "{{ radosgw_admin_user }}" + tenant_name: "{{ radosgw_admin_tenant }}" + role_name: "{{ radosgw_role_name | default('service') }}" + password: "{{ radosgw_admin_password }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + delegate_to: "{{ groups['utility_all'][0] }}" + run_once: true + when: radosgw_keystone | bool + register: add_service + until: add_service|success + retries: 5 + delay: 10 + tags: + - ceph-rgw-setup + - rgw-service-add + - name: Ensure RGW user to admin role + keystone: + command: "ensure_user_role" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + user_name: "{{ radosgw_admin_user }}" + tenant_name: "{{ radosgw_admin_tenant }}" + role_name: "{{ radosgw_role_name | default('admin') }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + delegate_to: "{{ groups['utility_all'][0] }}" + run_once: true + register: add_admin_role + when: radosgw_keystone | bool + until: add_admin_role|success + retries: 5 + delay: 10 + tags: + - ceph-rgw-setup + - rgw-service-add + - name: Ensure swiftoperator role + keystone: + command: "ensure_role" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + role_name: "swiftoperator" + insecure: "{{ keystone_service_adminuri_insecure }}" + delegate_to: "{{ groups['utility_all'][0] }}" + register: add_swiftoperator_role + until: add_swiftoperator_role|success + retries: 5 + delay: 10 + tags: + - ceph-rgw-setup + - rgw-service-add + - name: Ensure RGW endpoint + keystone: + command: "ensure_endpoint" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + region_name: "{{ radosgw_service_region }}" + service_name: "{{ radosgw_service_name }}" + service_type: "{{ radosgw_service_type }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + endpoint_list: + - url: "{{ radosgw_service_publicurl }}" + interface: "public" + - url: "{{ radosgw_service_adminurl }}" + interface: "admin" + - url: "{{ radosgw_service_internalurl }}" + interface: "internal" + delegate_to: "{{ groups['utility_all'][0] }}" + run_once: true + register: add_service + when: radosgw_keystone | bool + until: add_service|success + retries: 5 + delay: 10 + tags: + - ceph-rgw-setup + - rgw-service-add + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-ceph" + dest: "/var/log/ceph" + - include: common-tasks/os-lxc-container-setup.yml + - name: Gather ceph-mon facts + action: setup + delegate_to: "{{ item }}" + delegate_facts: yes + with_items: "{{ groups[mon_group_name] }}" + when: + - inventory_hostname == ansible_play_hosts[0] + tags: + - ceph-mon-facts + roles: + - role: ceph-defaults + tags: + - skip_ansible_lint + - role: ceph-common + tags: + - skip_ansible_lint + - role: ceph-config + tags: + - skip_ansible_lint + - role: ceph-rgw + tags: + - skip_ansible_lint + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: ceph_log_rotate + rsyslog_client_log_dir: "/var/log/ceph" + rsyslog_client_config_name: "99-ceph-rsyslog-client.conf" + tags: + - "ceph-rsyslog-client" + - "rsyslog-client" + - role: "system_crontab_coordination" + tags: + - "system-crontab-coordination" + vars: + is_metal: "{{ properties.is_metal|default(false) }}" + radosgw_keystone: yes + # TODO(logan): Remove the following line once + # upstream issue https://github.com/ceph/ceph-ansible/issues/2111 is fixed + radosgw_ssl: no + radosgw_civetweb_port: "{{ radosgw_service_port }}" + tags: + - ceph-rgw diff --git a/playbooks/setup-openstack.yml b/playbooks/setup-openstack.yml index 9fb8c65b5a..33bfa73f3c 100644 --- a/playbooks/setup-openstack.yml +++ b/playbooks/setup-openstack.yml @@ -40,6 +40,11 @@ - include: os-molteniron-install.yml - include: os-octavia-install.yml - include: os-tacker-install.yml + +# This is not an OpenStack service, but integrates with Keystone and must be +# deployed afterward. +- include: ceph-rgw-install.yml + - include: os-tempest-install.yml when: (tempest_install | default(False)) | bool or (tempest_run | default(False)) | bool diff --git a/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 index a9cb5dfc5c..90d73da60c 100644 --- a/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 @@ -19,7 +19,7 @@ monitor_interface: eth1 # Management network in the AIO public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}" journal_size: 100 osd_scenario: collocated -pool_default_pg_num: 32 +pool_default_pg_num: 16 openstack_config: true # Ceph ansible automatically creates pools & keys cinder_ceph_client: cinder cinder_default_volume_type: RBD diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 29b3e908e6..b07bb6c134 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -161,6 +161,11 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase): 'ceph-osd_container', 'ceph-osd_hosts', 'ceph-osd', + 'ceph-rgw_all', + 'ceph-rgw_containers', + 'ceph-rgw_container', + 'ceph-rgw_hosts', + 'ceph-rgw', 'cinder_all', 'cinder_api', 'cinder_api_container',