Add wait_time param for logscraper and logsender

In big deployment like Openstack CI sleep parameter with lower value
before next iteration is required.

Change-Id: Icdec8fe6dd872a15774b78c0bcf4d9eb7859e0bd

ansible/playbooks/deploy-services.yml

@@ -18,8 +18,9 @@
         es_insecure: false
         es_index_prefix: 'logstash-logs-'
         max_skipped: 1000
-        logscraper_workers: 8
-        logsender_workers: 2
+        logscraper_workers: 4
+        logsender_workers: 10
+        logsender_wait_time: 10
   roles:
     - logscraper
     - logsender

ansible/roles/logscraper/defaults/main.yml

@@ -28,4 +28,5 @@ container_images:
 #     logstash_url: https://somelogstash.com:9999
 #     max_skipped: 100
 #     debug: true
+#     logscraper_wait_time: 120
 tenant_builds: []

ansible/roles/logscraper/templates/logscraper.sh.j2

@@ -45,4 +45,7 @@
     {% if 'debug' in item %}
     --debug \
     {% endif %}
+    {% if 'logscraper_wait_time' in item %}
+    --wait-time {{ item['logscraper_wait_time'] }} \
+    {% endif %}
     --follow

ansible/roles/logsender/defaults/main.yml

@@ -25,4 +25,5 @@ container_images:
 #     chunk_size: 1500
 #     keep: true
 #     ignore_es_status: false
+#     logsender_wait_time: 60
 tenant_builds: []

ansible/roles/logsender/templates/logsender.sh.j2

@@ -51,4 +51,7 @@
     {% if 'ignore_es_status' in item and item['ignore_es_status'] %}
     --ignore-es-status \
     {% endif %}
+    {% if 'logsender_wait_time' in item %}
+    --wait-time {{ item['logsender_wait_time'] }} \
+    {% endif %}
     --follow

logscraper/logscraper.py

@@ -145,6 +145,10 @@ def get_arguments():
     parser.add_argument("--directory", help="Directory, where the logs will "
                         "be stored. Defaults to: /tmp/logscraper",
                         default="/tmp/logscraper")
+    parser.add_argument("--wait-time", help="Pause time for the next "
+                        "iteration",
+                        type=int,
+                        default=120)
     args = parser.parse_args()
     return args
 
@@ -646,7 +650,7 @@ def main():
         run(args)
         if not args.follow:
             break
-        time.sleep(120)
+        time.sleep(args.wait_time)
 
 
 if __name__ == "__main__":

logscraper/logsender.py

@@ -84,6 +84,10 @@ def get_arguments():
                         action="store_true")
     parser.add_argument("--debug", help="Be more verbose",
                         action="store_true")
+    parser.add_argument("--wait-time", help="Pause time for the next "
+                        "iteration",
+                        type=int,
+                        default=120)
     args = parser.parse_args()
     return args
 
@@ -426,7 +430,7 @@ def main():
         run(args)
         if not args.follow:
             break
-        time.sleep(60)
+        time.sleep(args.wait_time)
 
 
 if __name__ == "__main__":

logscraper/tests/test_logscraper.py

@@ -148,7 +148,7 @@ class FakeArgs(object):
                  checkpoint_file=None, ignore_checkpoint=None,
                  logstash_url=None, workers=None, max_skipped=None,
                  job_name=None, download=None, directory=None,
-                 config=None):
+                 config=None, wait_time=None):
 
         self.zuul_api_url = zuul_api_url
         self.gearman_server = gearman_server
@@ -164,6 +164,7 @@ class FakeArgs(object):
         self.download = download
         self.directory = directory
         self.config = config
+        self.wait_time = wait_time
 
 
 class TestScraper(base.TestCase):

logscraper/tests/test_logsender.py

@@ -271,7 +271,7 @@ class FakeArgs(object):
     def __init__(self, config=None, directory=None, host=None, port=None,
                  username=None, password=None, index_prefix=None, index=None,
                  doc_type=None, insecure=None, follow=None, workers=None,
-                 chunk_size=None, keep=None, debug=None):
+                 chunk_size=None, keep=None, debug=None, wait_time=None):
 
         self.config = config
         self.directory = directory
@@ -288,6 +288,7 @@ class FakeArgs(object):
         self.chunk_size = chunk_size
         self.keep = keep
         self.debug = debug
+        self.wait_time = wait_time
 
 
 class TestSender(base.TestCase):