Add meter-sink as an option

If the user wants to add a meter-sink to the pipeline.yaml
at the moment this cannot be done. Previously, pipeline-yaml
config option was allowed and could replace the whole config.

This will now allow to also add meter-sink as an option to
the event_pipeline.yaml, and allow a new source of collecting
all meters, and pushing to the value of meter-sink URLs.

Closes-Bug: 1938807
Change-Id: If6cd82e4153b29c1d8fdb6df150d0d4bb11d4e49

config.yaml

@@ -259,12 +259,24 @@ options:
       .
       'udp://<host>:<port>/'
       'prometheus://pushgateway-host:9091/metrics/job/openstack-telemetry'
+  meter-sink:
+    type: string
+    default:
+    description: |
+      Space delimited list of meter consumers of Ceilometer meter reporting
+      which reside outside of the deployed model. Only supported for Mitaka
+      and later clouds. e.g.
+      .
+      'udp://<host>:<port>/'
+      'http://telegraf-host-host:8080/telegraf?timeout=10&max_retries=2&batch=False'
   pipeline-yaml:
     type: string
     default: ""
     description: |
       When set the content is written to /etc/ceilometer/pipeline.yaml, by
       default the package's file is used.
+
+      This setting is not supported from Mitaka onwards.
   polling-interval:
     type: int
     default: 300

lib/ceilometer_contexts.py

@@ -156,7 +156,7 @@ class HAProxyContext(OSContextGenerator):
         return ctxt
 
 
-class RemoteSinksContext(OSContextGenerator):
+class CustomSinksContext(OSContextGenerator):
     interfaces = ['event-service']
 
     def __call__(self):
@@ -166,6 +166,8 @@ class RemoteSinksContext(OSContextGenerator):
         ctxt = {}
         if config('remote-sink'):
             ctxt['remote_sinks'] = config('remote-sink').split(' ')
+        if config('meter-sink'):
+            ctxt['meter_sinks'] = config('meter-sink').split(' ')
         for relid in relation_ids('event-service'):
             for unit in related_units(relid):
                 publisher = relation_get('publisher', unit=unit, rid=relid)

lib/ceilometer_utils.py

@@ -32,7 +32,7 @@ from ceilometer_contexts import (
     HAProxyContext,
     MetricServiceContext,
     CEILOMETER_PORT,
-    RemoteSinksContext,
+    CustomSinksContext,
     AMQPListenersContext,
 )
 from charmhelpers.contrib.openstack.utils import (
@@ -170,7 +170,7 @@ QUEENS_CONFIG_FILES = OrderedDict([
         'services': QUEENS_SERVICES
     }),
     (PIPELINE_CONF, {
-        'hook_contexts': [RemoteSinksContext()],
+        'hook_contexts': [CustomSinksContext()],
         'services': QUEENS_SERVICES,
     }),
 ])
@@ -216,7 +216,7 @@ CONFIG_FILES = OrderedDict([
         'services': ['ceilometer-api', 'apache2'],
     }),
     (PIPELINE_CONF, {
-        'hook_contexts': [RemoteSinksContext()],
+        'hook_contexts': [CustomSinksContext()],
         'services': ['ceilometer-collector'],
     }),
 ])

templates/mitaka/event_pipeline.yaml

@@ -13,6 +13,13 @@ sources:
      - {{ item }}
 {% endfor -%}
 {% endif %}
+{%- if meter_sinks %}
+  - name: meter_source
+    meters:
+     - "*"
+    sinks:
+     - meter_sink
+{% endif %}
 sinks:
 {%- if remote_sinks %}
   - name: remote_sink
@@ -22,6 +29,14 @@ sinks:
      - {{ item }}
      {% endfor %}
 {%- endif -%}
+{%- if meter_sinks %}
+  - name: meter_sink
+    transformers:
+    publishers:
+     {% for item in meter_sinks -%}
+     - {{ item }}
+     {% endfor %}
+{%- endif -%}
 {%- if internal_sinks %}
 {%- for item, target in internal_sinks.items() -%}
   - name: {{ item }}

templates/queens/event_pipeline.yaml

@@ -13,6 +13,13 @@ sources:
      - {{ item }}
 {% endfor -%}
 {% endif %}
+{%- if meter_sinks %}
+  - name: meter_source
+    meters:
+     - "*"
+    sinks:
+     - meter_sink
+{% endif %}
 sinks:
 {%- if remote_sinks %}
   - name: remote_sink
@@ -22,6 +29,14 @@ sinks:
      - {{ item }}
      {% endfor %}
 {%- endif -%}
+{%- if meter_sinks %}
+  - name: meter_sink
+    transformers:
+    publishers:
+     {% for item in meter_sinks -%}
+     - {{ item }}
+     {% endfor %}
+{%- endif -%}
 {%- if internal_sinks %}
 {%- for item, target in internal_sinks.items() -%}
   - name: {{ item }}

unit_tests/test_ceilometer_contexts.py

@@ -231,20 +231,20 @@ class CeilometerContextsTest(CharmTestCase):
         self.assertEqual(contexts.HAProxyContext()(), expected)
 
     @patch.object(contexts, 'get_os_codename_package')
-    def test_remote_sink_context_no_config(self, mock_get_rel):
+    def test_custom_sink_context_no_config(self, mock_get_rel):
         mock_get_rel.return_value = 'mitaka'
         self.relation_ids.return_value = []
         self.os_release.return_value = 'mitaka'
-        self.assertEqual(contexts.RemoteSinksContext()(), {
+        self.assertEqual(contexts.CustomSinksContext()(), {
             'event_sink_publisher': None})
 
         mock_get_rel.return_value = 'queens'
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'event_sink_publisher':
                           'notifier://?topic=alarm.all'})
 
     @patch.object(contexts, 'get_os_codename_package')
-    def test_remote_sink_context_event_service_relation(self, mock_get_rel):
+    def test_custom_sink_context_event_service_relation(self, mock_get_rel):
         mock_get_rel.return_value = 'mitaka'
         self.relation_ids.return_value = ['event-service:0', 'meter-service:0']
         self.related_units.return_value = ['panko/0']
@@ -253,63 +253,71 @@ class CeilometerContextsTest(CharmTestCase):
             'publisher': 'panko://'
         }
         self.test_relation.set(data)
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'internal_sinks': {'panko': 'panko://'},
                           'event_sink_publisher': None})
 
         mock_get_rel.return_value = 'queens'
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'internal_sinks': {'panko': 'panko://'},
                           'event_sink_publisher':
                           'notifier://?topic=alarm.all'})
 
         self.test_config.set('events-publisher', 'gnocchi')
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'internal_sinks': {'panko': 'panko://'},
                           'event_sink_publisher':
                           'gnocchi://'})
 
     @patch.object(contexts, 'get_os_codename_package')
-    def test_remote_sink_context_with_single_config(self, mock_get_rel):
+    def test_custom_sink_context_with_single_config(self, mock_get_rel):
         mock_get_rel.return_value = 'mitaka'
         self.relation_ids.return_value = ['meter-service:0']
         self.os_release.return_value = 'mitaka'
         self.test_config.set('remote-sink', 'http://foo')
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.test_config.set('meter-sink', 'http://foo1')
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo'],
+                          'meter_sinks': ['http://foo1'],
                           'event_sink_publisher': None})
 
         mock_get_rel.return_value = 'queens'
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo'],
+                          'meter_sinks': ['http://foo1'],
                           'event_sink_publisher':
                           'notifier://?topic=alarm.all'})
 
         self.test_config.set('events-publisher', 'gnocchi')
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo'],
+                          'meter_sinks': ['http://foo1'],
                           'event_sink_publisher':
                           'gnocchi://'})
 
     @patch.object(contexts, 'get_os_codename_package')
-    def test_remote_sink_context_with_multiple_config(self, mock_get_rel):
+    def test_custom_sink_context_with_multiple_config(self, mock_get_rel):
         mock_get_rel.return_value = 'mitaka'
         self.relation_ids.return_value = ['meter-service:0']
         self.os_release.return_value = 'mitaka'
         self.test_config.set('remote-sink', 'http://foo http://bar')
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.test_config.set('meter-sink', 'http://foo1 http://bar1')
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo', 'http://bar'],
+                          'meter_sinks': ['http://foo1', 'http://bar1'],
                           'event_sink_publisher': None})
 
         mock_get_rel.return_value = 'queens'
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo', 'http://bar'],
+                          'meter_sinks': ['http://foo1', 'http://bar1'],
                           'event_sink_publisher':
                           'notifier://?topic=alarm.all'})
 
         self.test_config.set('events-publisher', 'gnocchi')
-        self.assertEqual(contexts.RemoteSinksContext()(),
+        self.assertEqual(contexts.CustomSinksContext()(),
                          {'remote_sinks': ['http://foo', 'http://bar'],
+                          'meter_sinks': ['http://foo1', 'http://bar1'],
                           'event_sink_publisher':
                           'gnocchi://'})