Add splay option to offset polling intervals

When the os-collect-config process is started on multiple systems at the
same time, the polling intervals can line up to cause performance
problems against the configuration source. To reduce the impact, this
change adds a splay option to allow the operator to configure a random
delay prior to the polling to attempt to offset the polling
syncronization.

Change-Id: I1a8be3345d783da9014eca7ea26da19d57e767c0
Closes-Bug: #1677314

os_collect_config/collect.py

@@ -16,6 +16,7 @@
 import hashlib
 import json
 import os
+import random
 import shutil
 import signal
 import subprocess
@@ -89,7 +90,17 @@ opts = [
                     help='Key(s) to explode into multiple collected outputs. '
                     'Parsed according to the expected Metadata created by '
                     'OS::Heat::StructuredDeployment. Only exploded if seen at '
-                    'the root of the Metadata.')
+                    'the root of the Metadata.'),
+    cfg.FloatOpt('splay',
+                 default=0,
+                 help='Use this option to sleep for a random amount of time '
+                      'prior to starting the collect process. Takes a maximum '
+                      'number of seconds to wait before beginning collection '
+                      'as an argument. Disabled when set to 0. This option '
+                      'can help ensure that multiple collect processes '
+                      '(on different hosts) do not attempt to poll at the '
+                      'exact same time if they were all started at the same '
+                      'time. Ignored if --one-time or --force is used.'),
 ]
 
 CONF = cfg.CONF
@@ -253,6 +264,11 @@ def __main__(args=sys.argv, collector_kwargs_map=None):
     if CONF.force:
         CONF.set_override('one_time', True)
 
+    if CONF.splay > 0 and not CONF.one_time:
+        # sleep splay seconds in the beginning to prevent multiple collect
+        # processes from all running at the same time
+        time.sleep(random.randrange(0, CONF.splay))
+
     exitval = 0
     config_files = CONF.config_file
     config_hash = getfilehash(config_files)

os_collect_config/tests/test_collect.py

@@ -344,6 +344,16 @@ class TestCollect(testtools.TestCase):
                           ['os-collect-config', 'heat_local', '-i', '10',
                            '--min-polling-interval', '20', '-c', 'true'])
 
+    @mock.patch('time.sleep')
+    @mock.patch('random.randrange')
+    def test_main_with_splay(self, randrange_mock, sleep_mock):
+        randrange_mock.return_value = 4
+        collect.__main__(args=['os-collect-config', 'heat_local', '-i', '10',
+                               '--min-polling-interval', '20', '-c', 'true',
+                               '--print', '--splay', '29'])
+        randrange_mock.assert_called_with(0, 29)
+        sleep_mock.assert_called_with(4)
+
 
 class TestCollectAll(testtools.TestCase):