Ability to define batch size off polled samples.

Currently if batch_polled_samples=True (default) then it will batch
up all samples of a poller into one message. This can be problematic
if the poller returns a large number of samples.

Change-Id: Ifc60a0cb81876f6d9054cae82ba6579707a02f67
Closes-bug: #1770310

ceilometer/polling/manager.py

@@ -45,9 +45,13 @@ LOG = log.getLogger(__name__)
 OPTS = [
     cfg.BoolOpt('batch_polled_samples',
                 default=True,
+                deprecated_for_removal=True,
                 help='To reduce polling agent load, samples are sent to the '
                      'notification agent in a batch. To gain higher '
-                     'throughput at the cost of load set this to False.'),
+                     'throughput at the cost of load set this to False. '
+                     'This option is deprecated, to disable batching set '
+                     'batch_size = 0 in the polling group.'
+                ),
 ]
 
 POLLING_OPTS = [
@@ -62,6 +66,10 @@ POLLING_OPTS = [
                     'config files. For each sub-group of the agent '
                     'pool with the same partitioning_group_prefix a disjoint '
                     'subset of pollsters should be loaded.'),
+    cfg.IntOpt('batch_size',
+               default=50,
+               help='Batch size of samples to send to notification agent, '
+                    'Set to 0 to disable'),
 ]
 
 
@@ -132,6 +140,12 @@ class PollingTask(object):
         self.resources = collections.defaultdict(resource_factory)
 
         self._batch = self.manager.conf.batch_polled_samples
+        self._batch_size = self.manager.conf.polling.batch_size
+
+        if not self._batch:
+            # Support deprecated way of disabling baching
+            self._batch_size = 0
+
         self._telemetry_secret = self.manager.conf.publisher.telemetry_secret
 
     def add(self, pollster, source):
@@ -194,7 +208,10 @@ class PollingTask(object):
                             publisher_utils.meter_message_from_counter(
                                 sample, self._telemetry_secret
                             ))
-                        if self._batch:
+                        if self._batch_size:
+                            if len(sample_batch) >= self._batch_size:
+                                self._send_notification(sample_batch)
+                                sample_batch = []
                             sample_batch.append(sample_dict)
                         else:
                             self._send_notification([sample_dict])

ceilometer/tests/unit/polling/test_manager.py

@@ -791,13 +791,17 @@ class TestPollingAgent(BaseAgent):
                  res_list="[<NovaLikeServer: unknown-name>]",
                  source=source_name))
 
-    def test_batching_polled_samples_false(self):
+    def test_batching_polled_samples_false_deprecated(self):
         self.CONF.set_override('batch_polled_samples', False)
         self._batching_samples(4, 4)
 
-    def test_batching_polled_samples_true(self):
+    def test_batching_polled_samples_disable_batch(self):
-        self.CONF.set_override('batch_polled_samples', True)
+        self.CONF.set_override('batch_size', 0, group='polling')
-        self._batching_samples(4, 1)
+        self._batching_samples(4, 4)
+
+    def test_batching_polled_samples_batch_size(self):
+        self.CONF.set_override('batch_size', 2, group='polling')
+        self._batching_samples(4, 2)
 
     def test_batching_polled_samples_default(self):
         self._batching_samples(4, 1)

releasenotes/notes/polling-batch-size-7fe11925df8d1221.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - >
+    Add support for configuring the size of samples the poller will send in
+    each batch.
+upgrade:
+  - >
+    batch_size option added to [polling] section of configuration.
+    Use batch_size=0 to disable batching of samples.
+deprecations:
+  - >
+    The option batch_polled_samples in the [DEFAULT] section is deprecated.
+    Use batch_size option in [polling] to configure and/or disable batching.