From b758883418d1ea728bfd2bf482824f83f4b82782 Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Thu, 26 Mar 2020 20:42:58 +0000
Subject: [PATCH] Support setting Kafka storage volume

Not everyone wants Kafka data stored on a Docker volume. This
change allows a user to flexibly control where the data is stored.

Change-Id: I2ba8c7a85c7bf2564f954a43c6e6dbb3257fe902
---
 ansible/group_vars/all.yml                    |  5 ++++
 ansible/roles/destroy/tasks/cleanup_host.yml  |  1 +
 ansible/roles/kafka/defaults/main.yml         |  2 +-
 .../logging-and-monitoring/index.rst          |  1 +
 .../logging-and-monitoring/kafka-guide.rst    | 25 +++++++++++++++++++
 ...kafka-datadir-volume-382015a42sc57636.yaml |  6 +++++
 tools/cleanup-host                            |  5 ++++
 7 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 doc/source/reference/logging-and-monitoring/kafka-guide.rst
 create mode 100644 releasenotes/notes/add-variable-kafka-datadir-volume-382015a42sc57636.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index ffefbf87aa..148112835f 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -1042,6 +1042,11 @@ enable_vitrage_prometheus_datasource: "{{ enable_prometheus | bool }}"
 influxdb_address: "{{ kolla_internal_fqdn }}"
 influxdb_datadir_volume: "influxdb"
 
+#################
+# Kafka options
+#################
+kafka_datadir_volume: "kafka"
+
 #########################
 # Internal Image options
 #########################
diff --git a/ansible/roles/destroy/tasks/cleanup_host.yml b/ansible/roles/destroy/tasks/cleanup_host.yml
index 17dba02cb7..314d8712c7 100644
--- a/ansible/roles/destroy/tasks/cleanup_host.yml
+++ b/ansible/roles/destroy/tasks/cleanup_host.yml
@@ -10,6 +10,7 @@
     nova_instance_datadir_volume: "{{ nova_instance_datadir_volume }}"
     gnocchi_metric_datadir_volume: "{{ gnocchi_metric_datadir_volume }}"
     influxdb_datadir_volume: "{{ influxdb_datadir_volume }}"
+    kafka_datadir_volume: "{{ kafka_datadir_volume }}"
     kolla_internal_vip_address: "{{ kolla_internal_vip_address }}"
     kolla_external_vip_address: "{{ kolla_external_vip_address }}"
     kolla_dev_repos_directory: "{{ kolla_dev_repos_directory }}"
diff --git a/ansible/roles/kafka/defaults/main.yml b/ansible/roles/kafka/defaults/main.yml
index f242cece41..51c248e5e5 100644
--- a/ansible/roles/kafka/defaults/main.yml
+++ b/ansible/roles/kafka/defaults/main.yml
@@ -34,6 +34,6 @@ kafka_dimensions: "{{ default_container_dimensions }}"
 kafka_default_volumes:
   - "{{ node_config_directory }}/kafka/:{{ container_config_directory }}/"
   - "/etc/localtime:/etc/localtime:ro"
-  - "kafka:/var/lib/kafka/data"
+  - "{{ kafka_datadir_volume }}:/var/lib/kafka/data"
   - "kolla_logs:/var/log/kolla/"
 kafka_extra_volumes: "{{ default_extra_volumes }}"
diff --git a/doc/source/reference/logging-and-monitoring/index.rst b/doc/source/reference/logging-and-monitoring/index.rst
index a2cfbd0c9f..4aa730c2b1 100644
--- a/doc/source/reference/logging-and-monitoring/index.rst
+++ b/doc/source/reference/logging-and-monitoring/index.rst
@@ -10,6 +10,7 @@ logging and monitoring services available in kolla.
 
    central-logging-guide
    influxdb-guide
+   kafka-guide
    monasca-guide
    osprofiler-guide
    prometheus-guide
diff --git a/doc/source/reference/logging-and-monitoring/kafka-guide.rst b/doc/source/reference/logging-and-monitoring/kafka-guide.rst
new file mode 100644
index 0000000000..2867e9f2fc
--- /dev/null
+++ b/doc/source/reference/logging-and-monitoring/kafka-guide.rst
@@ -0,0 +1,25 @@
+.. _kafka-guide:
+
+============
+Apache Kafka
+============
+
+Overview
+~~~~~~~~
+
+`Kafka <https://kafka.apache.org/intro>`_ is a distributed stream processing
+system. It forms the central component of Monasca and in an OpenStack context
+can also be used as an experimental messaging backend in `Oslo messaging
+<https://docs.openstack.org/oslo.messaging/latest/admin/kafka.html>`_.
+
+Kafka
+~~~~~
+
+A spinning disk array is normally sufficient for Kafka. The data directory
+defaults to a docker volume, ``kafka``. Since it can use a lot of disk space,
+you may wish to store the data on a dedicated device. This can be achieved by
+setting ``kafka_datadir_volume`` in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+    kafka_datadir_volume: /mnt/spinning_array/kafka/
diff --git a/releasenotes/notes/add-variable-kafka-datadir-volume-382015a42sc57636.yaml b/releasenotes/notes/add-variable-kafka-datadir-volume-382015a42sc57636.yaml
new file mode 100644
index 0000000000..3e552e7ae1
--- /dev/null
+++ b/releasenotes/notes/add-variable-kafka-datadir-volume-382015a42sc57636.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds a new variable, ``kafka_datadir_volume``. This allows you to control
+    where the Kafka data is stored. Generally you will want this to be a
+    spinning disk, or an array of spinning disks.
diff --git a/tools/cleanup-host b/tools/cleanup-host
index 25dfc176a8..0994c41934 100755
--- a/tools/cleanup-host
+++ b/tools/cleanup-host
@@ -68,6 +68,11 @@ if [[ "$influxdb_datadir_volume" != "influxdb" && -d "$influxdb_datadir_volume"
     rm -rfv  $influxdb_datadir_volume
 fi
 
+if [[ "$kafka_datadir_volume" != "kafka" && -d "$kafka_datadir_volume" ]]; then
+    echo "Removing kafka volume if it is customised"
+    rm -rfv  $kafka_datadir_volume
+fi
+
 FOLDER_PATH="/etc/kolla/"
 
 if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then