From 0762fcb0b5872f4fbbf6ce8e5086140c65edf185 Mon Sep 17 00:00:00 2001
From: Alex Schultz <aschultz@redhat.com>
Date: Wed, 29 Mar 2017 10:25:13 -0600
Subject: [PATCH] Add splay option to offset polling intervals

When the os-collect-config process is started on multiple systems at the
same time, the polling intervals can line up to cause performance
problems against the configuration source. To reduce the impact, this
change adds a splay option to allow the operator to configure a random
delay prior to the polling to attempt to offset the polling
syncronization.

Change-Id: I1a8be3345d783da9014eca7ea26da19d57e767c0
Closes-Bug: #1677314
---
 os_collect_config/collect.py            | 18 +++++++++++++++++-
 os_collect_config/tests/test_collect.py | 10 ++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/os_collect_config/collect.py b/os_collect_config/collect.py
index c353b2c..e9e4389 100644
--- a/os_collect_config/collect.py
+++ b/os_collect_config/collect.py
@@ -16,6 +16,7 @@
 import hashlib
 import json
 import os
+import random
 import shutil
 import signal
 import subprocess
@@ -89,7 +90,17 @@ opts = [
                     help='Key(s) to explode into multiple collected outputs. '
                     'Parsed according to the expected Metadata created by '
                     'OS::Heat::StructuredDeployment. Only exploded if seen at '
-                    'the root of the Metadata.')
+                    'the root of the Metadata.'),
+    cfg.FloatOpt('splay',
+                 default=0,
+                 help='Use this option to sleep for a random amount of time '
+                      'prior to starting the collect process. Takes a maximum '
+                      'number of seconds to wait before beginning collection '
+                      'as an argument. Disabled when set to 0. This option '
+                      'can help ensure that multiple collect processes '
+                      '(on different hosts) do not attempt to poll at the '
+                      'exact same time if they were all started at the same '
+                      'time. Ignored if --one-time or --force is used.'),
 ]
 
 CONF = cfg.CONF
@@ -253,6 +264,11 @@ def __main__(args=sys.argv, collector_kwargs_map=None):
     if CONF.force:
         CONF.set_override('one_time', True)
 
+    if CONF.splay > 0 and not CONF.one_time:
+        # sleep splay seconds in the beginning to prevent multiple collect
+        # processes from all running at the same time
+        time.sleep(random.randrange(0, CONF.splay))
+
     exitval = 0
     config_files = CONF.config_file
     config_hash = getfilehash(config_files)
diff --git a/os_collect_config/tests/test_collect.py b/os_collect_config/tests/test_collect.py
index b735770..3b09425 100644
--- a/os_collect_config/tests/test_collect.py
+++ b/os_collect_config/tests/test_collect.py
@@ -344,6 +344,16 @@ class TestCollect(testtools.TestCase):
                           ['os-collect-config', 'heat_local', '-i', '10',
                            '--min-polling-interval', '20', '-c', 'true'])
 
+    @mock.patch('time.sleep')
+    @mock.patch('random.randrange')
+    def test_main_with_splay(self, randrange_mock, sleep_mock):
+        randrange_mock.return_value = 4
+        collect.__main__(args=['os-collect-config', 'heat_local', '-i', '10',
+                               '--min-polling-interval', '20', '-c', 'true',
+                               '--print', '--splay', '29'])
+        randrange_mock.assert_called_with(0, 29)
+        sleep_mock.assert_called_with(4)
+
 
 class TestCollectAll(testtools.TestCase):