From d4d3714adf91ad6e56e1ab60e64d87f336043286 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Fri, 28 Sep 2018 16:44:07 -0400 Subject: [PATCH] Blacklist logstash indexing of some very large screen logs This blacklists three screen logs that aren't actually using oslo formatting so rather than just index INFO+ level logs we're indexing the entire file which is huge and causes OOM errors on our logstash workers. Change-Id: Ie7f6b11f4b4fa7c69bdedd070910cd4240eacb46 Story: 2003911 Related-Bug: #1795067 Related-Bug: #1795068 Related-Bug: #1795069 --- roles/submit-logstash-jobs/defaults/main.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/submit-logstash-jobs/defaults/main.yaml b/roles/submit-logstash-jobs/defaults/main.yaml index 194add07f4..261f89335b 100644 --- a/roles/submit-logstash-jobs/defaults/main.yaml +++ b/roles/submit-logstash-jobs/defaults/main.yaml @@ -76,7 +76,13 @@ logstash_processor_config: tags: - console - console.html - - name: screen-(?!(peakmem_tracker|dstat|karaf)).*\.txt + # NOTE(mriedem): Logs that are known logstash index OOM killers are + # blacklisted here until fixed. + # screen-kubelet.txt: https://bugs.launchpad.net/kuryr-kubernetes/+bug/1795067 + # screen-mistral-engine.txt: https://bugs.launchpad.net/mistral/+bug/1795068 + # screen-monasca-persister.txt: https://storyboard.openstack.org/#!/story/2003911 + # screen-ovn-northd.txt: https://bugs.launchpad.net/networking-ovn/+bug/1795069 + - name: screen-(?!(peakmem_tracker|dstat|karaf|kubelet|mistral-engine|monasca-persister|ovn-northd)).*\.txt tags: - screen - oslofmt