Add missing elasticsearch cloudkitty storage

and prometheus collector backend support.

* Fix various remaining typos.
* Fix trailing character on reno.
* Enable Elasticsearch when selected as cloudkitty backend.
* Add a check for ES index creation when ES required.
* Add a release note
* Fix release note line length issue.

Change-Id: I18f3d8f2e10a2996b2ebf92733a1770bef548bda
Closes-bug: #1895945
(cherry picked from commit 9e72c0cb4e)

ansible/group_vars/all.yml

@@ -739,7 +739,7 @@ skip_stop_containers: []
 ####################
 
 elasticsearch_address: "{{ kolla_internal_fqdn }}"
-enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}"
+enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') else 'no' }}"
 
 # If using Curator an actions file will need to be defined. Please see
 # the documentation.

ansible/roles/cloudkitty/defaults/main.yml

@@ -102,11 +102,11 @@ cloudkitty_custom_metrics_yaml_file: "metrics.yml"
 ####################
 # Storage backend
 ####################
-# Valid options are 'sqlalchemy' or 'influxdb'. The default value is
+# Valid options are 'sqlalchemy', 'influxdb' or 'elasticsearch'. The default value is
 # 'influxdb', which matches the default in Cloudkitty since the Stein release.
-# When the backend is "influxdb", we also enable Influxdb.
-# Also, when using 'influxdb' as the backend, we trigger the configuration/use
-# of Cloudkitty storage backend version 2.
+# When the backend is "influxdb" or "elasticsearch", we also enable the required service
+# accordingly.
+# Additionally, we use cloudkitty API v2 for any backend but sqlalchemy.
 cloudkitty_storage_backend: "influxdb"
 
 # InfluxDB retention policy to use (defaults to autogen).
@@ -116,7 +116,7 @@ cloudkitty_storage_backend: "influxdb"
 # cloudkitty_influxdb_use_ssl: false
 
 # Path of the CA certificate to trust for HTTPS connections.
-# cloudkitty_influxdb_cafile: "/full/qualified/path/to/CAs/certificates"
+# cloudkitty_influxdb_cafile: "{{ openstack_cacert }}"
 
 # Set to true to authorize insecure HTTPS connections to InfluxDB.
 # This means, HTTPS connections without validating the certificate used by InfluxDB
@@ -124,6 +124,39 @@ cloudkitty_storage_backend: "influxdb"
 
 cloudkitty_influxdb_name: "cloudkitty"
 
+# Set the elasticsearch index name.
+cloudkitty_elasticsearch_index_name: "cloudkitty"
+
+# Set the elasticsearch host URL.
+cloudkitty_elasticsearch_url: "{{ internal_protocol }}://{{ elasticsearch_address }}:{{ elasticsearch_port }}"
+
+# Path of the CA certificate to trust for HTTPS connections.
+# cloudkitty_elasticsearch_cafile: "{{ openstack_cacert }}"
+
+# Set to true to authorize insecure HTTPS connections to Elasticsearch.
+# This means, HTTPS connections without validating the certificate used by elasticsearch
+cloudkitty_elasticsearch_insecure_connections: false
+
+####################
+# Collector
+####################
+# Valid options are 'gnocchi', 'monasca' or 'prometheus'. The default value is
+# 'gnocchi', which matches the default in Cloudkitty.
+cloudkitty_collector_backend: "gnocchi"
+
+# Set Monasca interface used for keystone URL discovery.
+cloudkitty_monasca_interface: "internal"
+
+# Set prometheus collector URL.
+cloudkitty_prometheus_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }}"
+
+# Path of the CA certificate to trust for HTTPS connections.
+# cloudkitty_prometheus_cafile: "{{ openstack_cacert }}"
+
+# Set to true to authorize insecure HTTPS connections to Prometheus.
+# This means, HTTPS connections without validating the certificate used by prometheus.
+cloudkitty_prometheus_insecure_connections: false
+
 ####################
 # Keystone
 ####################

ansible/roles/cloudkitty/tasks/bootstrap.yml

@@ -45,4 +45,34 @@
   delegate_to: "{{ groups['cloudkitty-api'][0] }}"
   when: cloudkitty_storage_backend == 'influxdb'
 
+- name: Checking if Cloudkitty elasticsearch index exists
+  become: true
+  kolla_toolbox:
+    module_name: uri
+    module_args:
+      url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}"
+      status_code: 200, 404
+  run_once: true
+  delegate_to: "{{ groups['cloudkitty-api'][0] }}"
+  register: cloudkitty_index
+  when: cloudkitty_storage_backend == 'elasticsearch'
+
+- name: Creating Cloudkitty elasticsearch index
+  become: true
+  kolla_toolbox:
+    module_name: uri
+    module_args:
+      url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}"
+      method: PUT
+      status_code: 200
+      return_content: yes
+      body: |
+        {}
+      body_format: json
+  run_once: True
+  delegate_to: "{{ groups['cloudkitty-api'][0] }}"
+  when:
+    - cloudkitty_storage_backend == 'elasticsearch'
+    - cloudkitty_index.get('status') != 200
+
 - import_tasks: bootstrap_service.yml

ansible/roles/cloudkitty/templates/cloudkitty.conf.j2

@@ -60,6 +60,23 @@ auth_section = keystone_authtoken
 region_name = {{ openstack_region_name }}
 {% endif %}
 
+{% if cloudkitty_collector_backend == "monasca" %}
+[collector_monasca]
+monasca_service_name = monasca
+interface = {{ cloudkitty_monasca_interface }}
+{% endif %}
+
+{% if cloudkitty_collector_backend == "prometheus" %}
+[collector_prometheus]
+prometheus_url = {{ cloudkitty_prometheus_url }}
+
+{% if cloudkitty_prometheus_cafile is defined %}
+cafile = {{ cloudkitty_prometheus_cafile }}
+{% endif %}
+
+insecure = {{ cloudkitty_prometheus_insecure_connections }}
+{% endif %}
+
 [api]
 host_ip = {{ api_interface_address }}
 port = {{ cloudkitty_api_port }}
@@ -70,11 +87,10 @@ max_workers = {{ openstack_service_workers }}
 
 [storage]
 backend = {{ cloudkitty_storage_backend }}
-{% if cloudkitty_storage_backend == 'influxdb' %}
-version = 2
-{% endif %}
 {% if cloudkitty_storage_backend == 'sqlalchemy' %}
 version = 1
+{% else %}
+version = 2
 {% endif %}
 
 {% if cloudkitty_storage_backend == 'influxdb' %}
@@ -100,3 +116,15 @@ cafile = {{ cloudkitty_influxdb_cafile }}
 {% endif %}
 
 {% endif %}
+
+{% if cloudkitty_storage_backend == 'elasticsearch' %}
+[storage_elasticsearch]
+host = {{ cloudkitty_elasticsearch_url }}
+index_name = {{ cloudkitty_elasticsearch_index_name }}
+insecure = {{ cloudkitty_elasticsearch_insecure_connections }}
+
+{% if cloudkitty_elasticsearch_cafile is defined %}
+cafile = {{ cloudkitty_elasticsearch_cafile }}
+{% endif %}
+
+{% endif %}

etc/kolla/globals.yml

@@ -257,7 +257,7 @@
 #enable_cyborg: "no"
 #enable_designate: "no"
 #enable_destroy_images: "no"
-#enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}"
+#enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') else 'no' }}"
 #enable_elasticsearch_curator: "no"
 #enable_etcd: "no"
 #enable_fluentd: "yes"

releasenotes/notes/add_elasticsearch_and_prometheus_support_on_cloudkitty-774e13e363e15a4b.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Adds support for elasticsearch storage backend with cloudkitty:
+    That feature let you store cloudkitty rating documents directly within
+    your elasticsearch cluster.
+
+    If you already have an elasticsearch cluster running for logging it create
+    a new cloudkitty specific index. That let you use kibana, grafana or any
+    other interface to browse your rating data and create appropriate
+    dashboard or build an appropriate billing service over it.
+
+    Adds support for prometheus as a fetcher/collector for cloudkitty:
+    That feature let you use prometheus metrics as your source of rating.
+    Using prometheus let you rate pretty much any openstack object directly
+    from the kolla provided exporters (Openstack_exporter) or your own
+    customs exporters.