Add support for Ceph RadosGW integration

* Register Swift-compatible endpoints in Keystone
* Load balance across RadosGW API servers using HAProxy

The support is exercised in the cephadm CI jobs, but since RGW is
not currently enabled via cephadm, it is not yet tested.

https://docs.ceph.com/en/latest/radosgw/keystone/

Implements: blueprint ceph-rgw

Change-Id: I891c3ed4ed93512607afe65a42dd99596fd4dbf9

ansible/group_vars/all.yml

@@ -285,6 +285,10 @@ barbican_api_listen_port: "{{ barbican_api_port }}"
 
 blazar_api_port: "1234"
 
+ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}"
+ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}"
+ceph_rgw_port: "6780"
+
 cinder_internal_fqdn: "{{ kolla_internal_fqdn }}"
 cinder_external_fqdn: "{{ kolla_external_fqdn }}"
 cinder_api_port: "8776"
@@ -601,6 +605,8 @@ enable_ceilometer: "no"
 enable_ceilometer_ipmi: "no"
 enable_cells: "no"
 enable_central_logging: "no"
+enable_ceph_rgw: "no"
+enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}"
 enable_chrony: "no"
 enable_cinder: "no"
 enable_cinder_backup: "yes"

ansible/roles/ceph-rgw/defaults/main.yml

@@ -0,0 +1,92 @@
+---
+project_name: "ceph-rgw"
+
+ceph_rgw_services:
+  # NOTE(mgoddard): There is no container deployment, this is used for load
+  # balancer configuration.
+  ceph-rgw:
+    group: "all"
+    enabled: "{{ enable_ceph_rgw | bool }}"
+    haproxy:
+      radosgw:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: false
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+      radosgw_external:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: true
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+
+####################
+# Load balancer
+####################
+
+# List of Ceph hosts to use as HAProxy backends. Each item should contain
+# 'host' and 'port'` keys. The 'ip' and 'port' keys are optional. If 'ip' is
+# not specified, the 'host' values should be resolvable from the host running
+# HAProxy. If the ``port`` is not specified, the default HTTP (80) or HTTPS
+# (443) port will be used.
+ceph_rgw_hosts: []
+ceph_rgw_haproxy_members: >-
+  {%- set members = [] -%}
+  {%- for host in ceph_rgw_hosts -%}
+  {%- set port = (":" ~ host.port) if host.port is defined else "" -%}
+  {%- set member = "server " ~ host.host ~ " " ~ host.ip | default(host.host) ~ port ~ " " ~ ceph_rgw_haproxy_healthcheck -%}
+  {%- set _ = members.append(member) -%}
+  {%- endfor -%}
+  {{ members }}
+ceph_rgw_haproxy_healthcheck: "check inter 2000 rise 2 fall 5"
+
+
+####################
+# OpenStack
+####################
+
+# Whether to register Ceph RadosGW swift-compatible endpoints in Keystone.
+enable_ceph_rgw_keystone: "{{ enable_ceph_rgw | bool }}"
+
+# Enable/disable ceph-rgw compatibility with OpenStack Swift.
+# This should match the configuration used by Ceph RadosGW.
+ceph_rgw_swift_compatibility: false
+
+# Enable/disable including the account (project) in the endpoint URL. This
+# allows for cross-project and public object access.
+# This should match the 'rgw_swift_account_in_url' config option used by Ceph
+# RadosGW.
+ceph_rgw_swift_account_in_url: false
+
+ceph_rgw_endpoint_path: "{{ '/' if ceph_rgw_swift_compatibility | bool else '/swift/' }}v1{% if ceph_rgw_swift_account_in_url | bool %}/AUTH_%(project_id)s{% endif %}"
+
+ceph_rgw_admin_endpoint: "{{ admin_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_internal_endpoint: "{{ internal_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_public_endpoint: "{{ public_protocol }}://{{ ceph_rgw_external_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+
+ceph_rgw_keystone_user: "ceph_rgw"
+
+openstack_ceph_rgw_auth: "{{ openstack_auth }}"
+
+
+####################
+# Keystone
+####################
+ceph_rgw_ks_services:
+  - name: "swift"
+    type: "object-store"
+    description: "Openstack Object Storage"
+    endpoints:
+      - {'interface': 'admin', 'url': '{{ ceph_rgw_admin_endpoint }}'}
+      - {'interface': 'internal', 'url': '{{ ceph_rgw_internal_endpoint }}'}
+      - {'interface': 'public', 'url': '{{ ceph_rgw_public_endpoint }}'}
+
+ceph_rgw_ks_users:
+  - project: "service"
+    user: "{{ ceph_rgw_keystone_user }}"
+    password: "{{ ceph_rgw_keystone_password }}"
+    role: "admin"
+
+ceph_rgw_ks_roles:
+  - "ResellerAdmin"

ansible/roles/ceph-rgw/tasks/check.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/config.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy-containers.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: register.yml

ansible/roles/ceph-rgw/tasks/loadbalancer.yml

@@ -0,0 +1,7 @@
+---
+- name: "Configure haproxy for {{ project_name }}"
+  import_role:
+    role: haproxy-config
+  vars:
+    project_services: "{{ ceph_rgw_services }}"
+  tags: always

ansible/roles/ceph-rgw/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/ceph-rgw/tasks/precheck.yml

@@ -0,0 +1,10 @@
+---
+- name: Fail if load balancer members not set
+  fail:
+    msg: >-
+      Ceph RadosGW load balancer configuration is enabled
+      (enable_ceph_rgw_loadbalancer) but no HAProxy members are configured.
+      Have you set ceph_rgw_hosts?
+  when:
+    - enable_ceph_rgw_loadbalancer | bool
+    - ceph_rgw_haproxy_members | length == 0

ansible/roles/ceph-rgw/tasks/pull.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/ceph-rgw/tasks/register.yml

@@ -0,0 +1,9 @@
+---
+- import_role:
+    name: service-ks-register
+  vars:
+    service_ks_register_auth: "{{ openstack_ceph_rgw_auth }}"
+    service_ks_register_services: "{{ ceph_rgw_ks_services }}"
+    service_ks_register_users: "{{ ceph_rgw_ks_users }}"
+    service_ks_register_roles: "{{ ceph_rgw_ks_roles }}"
+  when: enable_ceph_rgw_keystone | bool

ansible/roles/ceph-rgw/tasks/stop.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/upgrade.yml

@@ -0,0 +1 @@
+---

ansible/roles/loadbalancer/tasks/precheck.yml

@@ -208,6 +208,20 @@
     - haproxy_stat.find('blazar_api') == -1
     - haproxy_vip_prechecks
 
+- name: Checking free port for Ceph RadosGW HAProxy
+  wait_for:
+    host: "{{ kolla_internal_vip_address }}"
+    port: "{{ ceph_rgw_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - enable_ceph_rgw | bool
+    - enable_ceph_rgw_loadbalancer | bool
+    - inventory_hostname in groups['loadbalancer']
+    - haproxy_stat.find('radosgw') == -1
+    - haproxy_vip_prechecks
+
 - name: Checking free port for Cinder API HAProxy
   wait_for:
     host: "{{ kolla_internal_vip_address }}"

ansible/site.yml

@@ -22,6 +22,7 @@
         - enable_barbican_{{ enable_barbican | bool }}
         - enable_blazar_{{ enable_blazar | bool }}
         - enable_ceilometer_{{ enable_ceilometer | bool }}
+        - enable_ceph_rgw_{{ enable_ceph_rgw | bool }}
         - enable_chrony_{{ enable_chrony | bool }}
         - enable_cinder_{{ enable_cinder | bool }}
         - enable_cloudkitty_{{ enable_cloudkitty | bool }}
@@ -143,6 +144,11 @@
             tasks_from: loadbalancer
           tags: blazar
           when: enable_blazar | bool
+        - include_role:
+            name: ceph-rgw
+            tasks_from: loadbalancer
+          tags: ceph-rgw
+          when: enable_ceph_rgw | bool
         - include_role:
             name: cinder
             tasks_from: loadbalancer
@@ -603,6 +609,19 @@
         tags: swift,
         when: enable_swift | bool }
 
+- name: Apply role ceph-rgw
+  gather_facts: false
+  hosts:
+    # NOTE(mgoddard): This is only used to register Keystone services, and
+    # can run on any host running kolla-toolbox.
+    - kolla-toolbox
+    - '&enable_ceph_rgw_True'
+  serial: '{{ kolla_serial|default("0") }}'
+  roles:
+    - { role: ceph-rgw,
+        tags: ceph-rgw,
+        when: enable_ceph_rgw | bool }
+
 - name: Apply role glance
   gather_facts: false
   hosts:

doc/source/reference/storage/external-ceph-guide.rst

@@ -211,3 +211,74 @@ type ``default_share_type``, please see :doc:`Manila in Kolla <manila-guide>`.
 
 For more details on the CephFS Native driver, please see
 :manila-doc:`CephFS Native driver <admin/cephfs_driver.html>`.
+
+RadosGW
+-------
+
+As of the Xena 13.0.0 release, Kolla Ansible supports integration with Ceph
+RadosGW. This includes:
+
+* Registration of Swift-compatible endpoints in Keystone
+* Load balancing across RadosGW API servers using HAProxy
+
+See the `Ceph documentation
+<https://docs.ceph.com/en/latest/radosgw/keystone/>`__ for further information,
+including changes that must be applied to the Ceph cluster configuration.
+
+Enable Ceph RadosGW integration:
+
+.. code-block:: yaml
+
+   enable_ceph_rgw: true
+
+Keystone integration
+====================
+
+A Keystone user and endpoints are registered by default, however this may be
+avoided by setting ``enable_ceph_rgw_keystone`` to ``false``. If registration
+is enabled, the username is defined via ``ceph_rgw_keystone_user``, and this
+defaults to ``ceph_rgw``. The hostnames used by the endpoints default to
+``ceph_rgw_external_fqdn`` and ``ceph_rgw_internal_fqdn`` for the public and
+internal endpoints respectively. These default to ``kolla_external_fqdn`` and
+``kolla_internal_fqdn`` respectively. The port used by the endpoints is defined
+via ``ceph_rgw_port``, and defaults to 6780.
+
+By default RadosGW supports both Swift and S3 API, and it is not completely
+compatible with Swift API. The option ``ceph_rgw_swift_compatibility`` can
+enable/disable complete RadosGW compatibility with Swift API.  This should
+match the configuration used by Ceph RadosGW. After changing the value, run
+the ``kolla-ansible deploy`` command to enable.
+
+By default, the RadosGW endpoint URL does not include the project (account) ID.
+This prevents cross-project and public object access. This can be resolved by
+setting ``ceph_rgw_swift_account_in_url`` to ``true``. This should match the
+``rgw_swift_account_in_url`` configuration option in Ceph RadosGW.
+
+Load balancing
+==============
+
+.. warning::
+
+   Users of Ceph RadosGW can generate very high volumes of traffic. It is
+   advisable to use a separate load balancer for RadosGW for anything other
+   than small or lightly utilised RadosGW deployments, however this is
+   currently out of scope for Kolla Ansible.
+
+Load balancing is enabled by default, however this may be avoided by setting
+``enable_ceph_rgw_loadbalancer`` to ``false``. If using load balancing, the
+RadosGW hosts and ports must be configured. Each item should contain
+``host`` and ``port`` keys. The ``ip`` and ``port`` keys are optional. If
+``ip`` is not specified, the ``host`` values should be resolvable from the host
+running HAProxy. If the ``port`` is not specified, the default HTTP (80) or
+HTTPS (443) port will be used. For example:
+
+.. code-block:: yaml
+
+   ceph_rgw_hosts:
+     - host: rgw-host-1
+     - host: rgw-host-2
+       ip: 10.0.0.42
+       port: 8080
+
+The HAProxy frontend port is defined via ``ceph_rgw_port``, and defaults to
+6780.

etc/kolla/globals.yml

@@ -296,6 +296,8 @@
 #enable_ceilometer_ipmi: "no"
 #enable_cells: "no"
 #enable_central_logging: "no"
+#enable_ceph_rgw: "no"
+#enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}"
 #enable_chrony: "no"
 #enable_cinder: "no"
 #enable_cinder_backup: "yes"

etc/kolla/passwords.yml

@@ -246,3 +246,8 @@ prometheus_alertmanager_password:
 # OpenStack identity federation
 ###############################
 keystone_federation_openid_crypto_password:
+
+####################
+# Ceph RadosGW options
+####################
+ceph_rgw_keystone_password:

releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds support for integration with Ceph RadosGW.

tests/templates/globals-default.j2

@@ -132,6 +132,14 @@ nova_backend_ceph: "yes"
 # TODO(yoctozepto): Remove this in the Xena cycle.
 # cephadm doesn't support chrony in a container (checks for chrony.service)
 enable_chrony: "no"
+
+enable_ceph_rgw: {{ not is_upgrade or previous_release != 'wallaby' }}
+ceph_rgw_hosts:
+{% for host in hostvars %}
+  - host: {{ host }}
+    ip: {{ hostvars[host]['ansible_host'] }}
+    port: 6780
+{% endfor %}
 {% endif %}
 
 {% if tls_enabled %}