From dd4fe190d0b7a92c72565091c99baba5b4ec6f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20M=C3=A1gr?= <mmagr@redhat.com>
Date: Tue, 17 Oct 2017 13:52:44 +0200
Subject: [PATCH] Clean fluentd_plugins_install block

fluentd_plugins_install should contain only plugin installation operations,
so that the override does not have to handle irrelevant operations

Change-Id: I8f810d194a1e638683df1f445c187b8d98629730
---
 docker/fluentd/Dockerfile.j2 | 56 ++++++++++++++++++++++++------------
 docker/macros.j2             | 11 +++++++
 2 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/docker/fluentd/Dockerfile.j2 b/docker/fluentd/Dockerfile.j2
index 5b486b2f0c..132508014c 100644
--- a/docker/fluentd/Dockerfile.j2
+++ b/docker/fluentd/Dockerfile.j2
@@ -33,34 +33,54 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {{ macros.install_packages(fluentd_packages | customizable("packages")) }}
 
-{% block fluentd_plugins_install %}
-
+# Distro specific files and operations
 {% if base_distro in  ['centos', 'oraclelinux', 'rhel'] %}
 COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
-RUN ulimit -n 65536 \
-    && gem install --minimal-deps activesupport:4.2.9 public_suffix:2.0.5 fluent-plugin-parser fluent-plugin-kubernetes_metadata_filter fluent-plugin-elasticsearch:1.10.2 fluent-plugin-grep fluent-plugin-grok-parser:0.3.1 fluent-plugin-rewrite-tag-filter:1.6.0 fluent-plugin-secure-forward \
-    && chmod -R 440 /etc/sudoers.d/kolla_fluentd_sudoers  \
-    && chmod 755 /usr/local/bin/kolla_extend_start \
+RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
     && mkdir -p /var/run/fluentd \
     && chown -R fluentd: /etc/fluentd /var/run/fluentd
-
 {% elif base_distro in ['debian', 'ubuntu'] %}
-RUN ulimit -n 65536 \
-    && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
-    && td-agent-gem install fluent-plugin-parser fluent-plugin-kubernetes_metadata_filter fluent-plugin-elasticsearch fluent-plugin-grep fluent-plugin-grok-parser:2.1.4 fluent-plugin-rewrite-tag-filter:2.0.0 fluent-plugin-secure-forward \
-    && rm -f /etc/td-agent/td-agent.conf
-
 COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
-COPY extend_start.sh /usr/local/bin/kolla_extend_start
-
 RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
-    && chmod 755 /usr/local/bin/kolla_extend_start \
+    && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
+    && rm -f /etc/td-agent/td-agent.conf \
     && chown -R td-agent: /etc/td-agent
-
 {% endif %}
 
+
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
+{% block fluentd_plugins_install %}
+
+
+{% set fluentd_plugins = [
+    'fluent-plugin-parser',
+    'fluent-plugin-kubernetes_metadata_filter',
+    'fluent-plugin-grep',
+    'fluent-plugin-secure-forward'
+] %}
+
+{% if base_distro in  ['centos', 'oraclelinux', 'rhel'] %}
+    # NOTE: We use fluentd version is v12.0 so fluent-plugin-grok-parse version should be < 1.0.0.
+    # https://github.com/fluent/fluent-plugin-grok-parser
+    {% set fluentd_plugins = [
+        'activesupport:4.2.9',
+        'public_suffix:2.0.5',
+        'fluent-plugin-elasticsearch:1.10.2',
+        'fluent-plugin-grok-parser:0.3.1',
+        'fluent-plugin-rewrite-tag-filter:1.6.0'
+    ] + fluentd_plugins  %}
+{% elif base_distro in ['debian', 'ubuntu'] %}
+    {% set fluentd_plugins = [
+        'fluent-plugin-elasticsearch',
+        'fluent-plugin-grok-parser:2.1.4',
+        'fluent-plugin-rewrite-tag-filter:2.0.0'
+    ] + fluentd_plugins %}
+{% endif %}
+
+{{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
+
 {% endblock %}
 
 {% block fluentd_footer %}{% endblock %}
diff --git a/docker/macros.j2 b/docker/macros.j2
index 5294ddacfa..dfb85447b1 100644
--- a/docker/macros.j2
+++ b/docker/macros.j2
@@ -80,3 +80,14 @@ RUN apt-get update \
     RUN curl -o /usr/bin/kubectl http://storage.googleapis.com/kubernetes-release/release/v1.5.4/bin/linux/${KUBE_ARCH}/kubectl \
         && chmod 755 /usr/bin/kubectl
 {% endmacro %}
+
+
+{% macro install_fluent_plugins(plugins) -%}
+{% if plugins is defined and plugins|length > 0 -%}
+    {% if base_distro in ['centos', 'oraclelinux', 'rhel'] -%}
+        RUN ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
+    {%- elif base_distro in ['debian', 'ubuntu'] -%}
+        RUN ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
+    {%- endif %}
+{%- endif %}
+{%- endmacro %}