diff --git a/ansible/playbooks/deploy-services.yml b/ansible/playbooks/deploy-services.yml index 3229377..333e6f9 100644 --- a/ansible/playbooks/deploy-services.yml +++ b/ansible/playbooks/deploy-services.yml @@ -18,8 +18,9 @@ es_insecure: false es_index_prefix: 'logstash-logs-' max_skipped: 1000 - logscraper_workers: 8 - logsender_workers: 2 + logscraper_workers: 4 + logsender_workers: 10 + logsender_wait_time: 10 roles: - logscraper - logsender diff --git a/ansible/roles/logscraper/defaults/main.yml b/ansible/roles/logscraper/defaults/main.yml index 4e5a82a..d426414 100644 --- a/ansible/roles/logscraper/defaults/main.yml +++ b/ansible/roles/logscraper/defaults/main.yml @@ -28,4 +28,5 @@ container_images: # logstash_url: https://somelogstash.com:9999 # max_skipped: 100 # debug: true +# logscraper_wait_time: 120 tenant_builds: [] diff --git a/ansible/roles/logscraper/templates/logscraper.sh.j2 b/ansible/roles/logscraper/templates/logscraper.sh.j2 index 101752c..e3dbb62 100644 --- a/ansible/roles/logscraper/templates/logscraper.sh.j2 +++ b/ansible/roles/logscraper/templates/logscraper.sh.j2 @@ -45,4 +45,7 @@ {% if 'debug' in item %} --debug \ {% endif %} + {% if 'logscraper_wait_time' in item %} + --wait-time {{ item['logscraper_wait_time'] }} \ + {% endif %} --follow diff --git a/ansible/roles/logsender/defaults/main.yml b/ansible/roles/logsender/defaults/main.yml index 6bceb57..e928053 100644 --- a/ansible/roles/logsender/defaults/main.yml +++ b/ansible/roles/logsender/defaults/main.yml @@ -25,4 +25,5 @@ container_images: # chunk_size: 1500 # keep: true # ignore_es_status: false +# logsender_wait_time: 60 tenant_builds: [] diff --git a/ansible/roles/logsender/templates/logsender.sh.j2 b/ansible/roles/logsender/templates/logsender.sh.j2 index 87cf0c5..0c447c7 100644 --- a/ansible/roles/logsender/templates/logsender.sh.j2 +++ b/ansible/roles/logsender/templates/logsender.sh.j2 @@ -51,4 +51,7 @@ {% if 'ignore_es_status' in item and item['ignore_es_status'] %} --ignore-es-status \ {% endif %} + {% if 'logsender_wait_time' in item %} + --wait-time {{ item['logsender_wait_time'] }} \ + {% endif %} --follow diff --git a/logscraper/logscraper.py b/logscraper/logscraper.py index 94c8637..4a6f482 100755 --- a/logscraper/logscraper.py +++ b/logscraper/logscraper.py @@ -145,6 +145,10 @@ def get_arguments(): parser.add_argument("--directory", help="Directory, where the logs will " "be stored. Defaults to: /tmp/logscraper", default="/tmp/logscraper") + parser.add_argument("--wait-time", help="Pause time for the next " + "iteration", + type=int, + default=120) args = parser.parse_args() return args @@ -646,7 +650,7 @@ def main(): run(args) if not args.follow: break - time.sleep(120) + time.sleep(args.wait_time) if __name__ == "__main__": diff --git a/logscraper/logsender.py b/logscraper/logsender.py index 6aca9fc..88d0bfa 100755 --- a/logscraper/logsender.py +++ b/logscraper/logsender.py @@ -84,6 +84,10 @@ def get_arguments(): action="store_true") parser.add_argument("--debug", help="Be more verbose", action="store_true") + parser.add_argument("--wait-time", help="Pause time for the next " + "iteration", + type=int, + default=120) args = parser.parse_args() return args @@ -426,7 +430,7 @@ def main(): run(args) if not args.follow: break - time.sleep(60) + time.sleep(args.wait_time) if __name__ == "__main__": diff --git a/logscraper/tests/test_logscraper.py b/logscraper/tests/test_logscraper.py index 53fc58b..56d0406 100644 --- a/logscraper/tests/test_logscraper.py +++ b/logscraper/tests/test_logscraper.py @@ -148,7 +148,7 @@ class FakeArgs(object): checkpoint_file=None, ignore_checkpoint=None, logstash_url=None, workers=None, max_skipped=None, job_name=None, download=None, directory=None, - config=None): + config=None, wait_time=None): self.zuul_api_url = zuul_api_url self.gearman_server = gearman_server @@ -164,6 +164,7 @@ class FakeArgs(object): self.download = download self.directory = directory self.config = config + self.wait_time = wait_time class TestScraper(base.TestCase): diff --git a/logscraper/tests/test_logsender.py b/logscraper/tests/test_logsender.py index 9d2f219..c8159a8 100755 --- a/logscraper/tests/test_logsender.py +++ b/logscraper/tests/test_logsender.py @@ -271,7 +271,7 @@ class FakeArgs(object): def __init__(self, config=None, directory=None, host=None, port=None, username=None, password=None, index_prefix=None, index=None, doc_type=None, insecure=None, follow=None, workers=None, - chunk_size=None, keep=None, debug=None): + chunk_size=None, keep=None, debug=None, wait_time=None): self.config = config self.directory = directory @@ -288,6 +288,7 @@ class FakeArgs(object): self.chunk_size = chunk_size self.keep = keep self.debug = debug + self.wait_time = wait_time class TestSender(base.TestCase):