From 9e72c0cb4e2206cc72104648e507237977258d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20THEROND=20=28Fl1nt=29?= Date: Mon, 8 Feb 2021 09:29:08 +0100 Subject: [PATCH] Add missing elasticsearch cloudkitty storage and prometheus collector backend support. * Fix various remaining typos. * Fix trailing character on reno. * Enable Elasticsearch when selected as cloudkitty backend. * Add a check for ES index creation when ES required. * Add a release note * Fix release note line length issue. Change-Id: I18f3d8f2e10a2996b2ebf92733a1770bef548bda Closes-bug: #1895945 --- ansible/group_vars/all.yml | 2 +- ansible/roles/cloudkitty/defaults/main.yml | 43 ++++++++++++++++--- ansible/roles/cloudkitty/tasks/bootstrap.yml | 30 +++++++++++++ .../cloudkitty/templates/cloudkitty.conf.j2 | 34 +++++++++++++-- etc/kolla/globals.yml | 2 +- ...upport_on_cloudkitty-774e13e363e15a4b.yaml | 17 ++++++++ 6 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 releasenotes/notes/add_elasticsearch_and_prometheus_support_on_cloudkitty-774e13e363e15a4b.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 3dca393799..8025595ca2 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -737,7 +737,7 @@ skip_stop_containers: [] #################### elasticsearch_address: "{{ kolla_internal_fqdn }}" -enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}" +enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') else 'no' }}" # If using Curator an actions file will need to be defined. Please see # the documentation. diff --git a/ansible/roles/cloudkitty/defaults/main.yml b/ansible/roles/cloudkitty/defaults/main.yml index a40f0eb798..e34cd82a67 100644 --- a/ansible/roles/cloudkitty/defaults/main.yml +++ b/ansible/roles/cloudkitty/defaults/main.yml @@ -130,11 +130,11 @@ cloudkitty_custom_metrics_yaml_file: "metrics.yml" #################### # Storage backend #################### -# Valid options are 'sqlalchemy' or 'influxdb'. The default value is +# Valid options are 'sqlalchemy', 'influxdb' or 'elasticsearch'. The default value is # 'influxdb', which matches the default in Cloudkitty since the Stein release. -# When the backend is "influxdb", we also enable Influxdb. -# Also, when using 'influxdb' as the backend, we trigger the configuration/use -# of Cloudkitty storage backend version 2. +# When the backend is "influxdb" or "elasticsearch", we also enable the required service +# accordingly. +# Additionally, we use cloudkitty API v2 for any backend but sqlalchemy. cloudkitty_storage_backend: "influxdb" # InfluxDB retention policy to use (defaults to autogen). @@ -144,7 +144,7 @@ cloudkitty_storage_backend: "influxdb" # cloudkitty_influxdb_use_ssl: false # Path of the CA certificate to trust for HTTPS connections. -# cloudkitty_influxdb_cafile: "/full/qualified/path/to/CAs/certificates" +# cloudkitty_influxdb_cafile: "{{ openstack_cacert }}" # Set to true to authorize insecure HTTPS connections to InfluxDB. # This means, HTTPS connections without validating the certificate used by InfluxDB @@ -152,6 +152,39 @@ cloudkitty_storage_backend: "influxdb" cloudkitty_influxdb_name: "cloudkitty" +# Set the elasticsearch index name. +cloudkitty_elasticsearch_index_name: "cloudkitty" + +# Set the elasticsearch host URL. +cloudkitty_elasticsearch_url: "{{ internal_protocol }}://{{ elasticsearch_address }}:{{ elasticsearch_port }}" + +# Path of the CA certificate to trust for HTTPS connections. +# cloudkitty_elasticsearch_cafile: "{{ openstack_cacert }}" + +# Set to true to authorize insecure HTTPS connections to Elasticsearch. +# This means, HTTPS connections without validating the certificate used by elasticsearch +cloudkitty_elasticsearch_insecure_connections: false + +#################### +# Collector +#################### +# Valid options are 'gnocchi', 'monasca' or 'prometheus'. The default value is +# 'gnocchi', which matches the default in Cloudkitty. +cloudkitty_collector_backend: "gnocchi" + +# Set Monasca interface used for keystone URL discovery. +cloudkitty_monasca_interface: "internal" + +# Set prometheus collector URL. +cloudkitty_prometheus_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }}" + +# Path of the CA certificate to trust for HTTPS connections. +# cloudkitty_prometheus_cafile: "{{ openstack_cacert }}" + +# Set to true to authorize insecure HTTPS connections to Prometheus. +# This means, HTTPS connections without validating the certificate used by prometheus. +cloudkitty_prometheus_insecure_connections: false + #################### # Keystone #################### diff --git a/ansible/roles/cloudkitty/tasks/bootstrap.yml b/ansible/roles/cloudkitty/tasks/bootstrap.yml index f066333eff..2ba2906564 100644 --- a/ansible/roles/cloudkitty/tasks/bootstrap.yml +++ b/ansible/roles/cloudkitty/tasks/bootstrap.yml @@ -45,4 +45,34 @@ delegate_to: "{{ groups['cloudkitty-api'][0] }}" when: cloudkitty_storage_backend == 'influxdb' +- name: Checking if Cloudkitty elasticsearch index exists + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}" + status_code: 200, 404 + run_once: true + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + register: cloudkitty_index + when: cloudkitty_storage_backend == 'elasticsearch' + +- name: Creating Cloudkitty elasticsearch index + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}" + method: PUT + status_code: 200 + return_content: yes + body: | + {} + body_format: json + run_once: True + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + when: + - cloudkitty_storage_backend == 'elasticsearch' + - cloudkitty_index.get('status') != 200 + - import_tasks: bootstrap_service.yml diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 index 16293a79a6..efa54250e0 100644 --- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 +++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 @@ -66,6 +66,23 @@ auth_section = keystone_authtoken region_name = {{ openstack_region_name }} {% endif %} +{% if cloudkitty_collector_backend == "monasca" %} +[collector_monasca] +monasca_service_name = monasca +interface = {{ cloudkitty_monasca_interface }} +{% endif %} + +{% if cloudkitty_collector_backend == "prometheus" %} +[collector_prometheus] +prometheus_url = {{ cloudkitty_prometheus_url }} + +{% if cloudkitty_prometheus_cafile is defined %} +cafile = {{ cloudkitty_prometheus_cafile }} +{% endif %} + +insecure = {{ cloudkitty_prometheus_insecure_connections }} +{% endif %} + [api] host_ip = {{ api_interface_address }} port = {{ cloudkitty_api_port }} @@ -76,11 +93,10 @@ max_workers = {{ openstack_service_workers }} [storage] backend = {{ cloudkitty_storage_backend }} -{% if cloudkitty_storage_backend == 'influxdb' %} -version = 2 -{% endif %} {% if cloudkitty_storage_backend == 'sqlalchemy' %} version = 1 +{% else %} +version = 2 {% endif %} {% if cloudkitty_storage_backend == 'influxdb' %} @@ -106,3 +122,15 @@ cafile = {{ cloudkitty_influxdb_cafile }} {% endif %} {% endif %} + +{% if cloudkitty_storage_backend == 'elasticsearch' %} +[storage_elasticsearch] +host = {{ cloudkitty_elasticsearch_url }} +index_name = {{ cloudkitty_elasticsearch_index_name }} +insecure = {{ cloudkitty_elasticsearch_insecure_connections }} + +{% if cloudkitty_elasticsearch_cafile is defined %} +cafile = {{ cloudkitty_elasticsearch_cafile }} +{% endif %} + +{% endif %} diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 198cf87e90..5ad5ee46b9 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -287,7 +287,7 @@ #enable_cyborg: "no" #enable_designate: "no" #enable_destroy_images: "no" -#enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}" +#enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') else 'no' }}" #enable_elasticsearch_curator: "no" #enable_etcd: "no" #enable_fluentd: "yes" diff --git a/releasenotes/notes/add_elasticsearch_and_prometheus_support_on_cloudkitty-774e13e363e15a4b.yaml b/releasenotes/notes/add_elasticsearch_and_prometheus_support_on_cloudkitty-774e13e363e15a4b.yaml new file mode 100644 index 0000000000..9d5686b95d --- /dev/null +++ b/releasenotes/notes/add_elasticsearch_and_prometheus_support_on_cloudkitty-774e13e363e15a4b.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Adds support for elasticsearch storage backend with cloudkitty: + That feature let you store cloudkitty rating documents directly within + your elasticsearch cluster. + + If you already have an elasticsearch cluster running for logging it create + a new cloudkitty specific index. That let you use kibana, grafana or any + other interface to browse your rating data and create appropriate + dashboard or build an appropriate billing service over it. + + Adds support for prometheus as a fetcher/collector for cloudkitty: + That feature let you use prometheus metrics as your source of rating. + Using prometheus let you rate pretty much any openstack object directly + from the kolla provided exporters (Openstack_exporter) or your own + customs exporters.