Merge "Change fluentd to td-agent on CentOS"

docker/base/td.repo

@@ -1,5 +1,5 @@
 [treasuredata]
 name=TreasureData
-baseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch
+baseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch
 gpgcheck=1
 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent

docker/fluentd/Dockerfile.j2

@@ -1,108 +1,108 @@
 FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
 LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
-{% if base_package_type == 'rpm' or base_arch != 'x86_64' %}
+{% if base_package_type == 'rpm' and base_arch != 'x86_64' %}
 LABEL fluentd_version="0.12" fluentd_binary="fluentd"
+    {% set fluentd_user = 'fluentd' %}
+{% elif base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
+LABEL fluentd_version="0.14" fluentd_binary="fluentd"
+    {% set fluentd_user = 'fluentd' %}
 {% else %}
 LABEL fluentd_version="0.14" fluentd_binary="td-agent"
+    {% set fluentd_user = 'td-agent' %}
 {% endif %}
 
 {% block fluentd_header %}{% endblock %}
 
 {% import "macros.j2" as macros with context %}
 
-{% set fluentd_user = 'fluentd' %}
-
 {% if base_package_type == 'rpm' %}
 
-    {{ macros.configure_user(name='fluentd', groups='mysql') }}
-# TODO:remove hostname package as it should be installed by the
-# fluentd dependencies.
-# hostname package is installed by default on CentOS/RHEL but not on
-# oraclelinux.
-
     {% set fluentd_packages = [
-        'fluentd',
         'gcc-c++',
-        'hostname',
+        'make'
-        'make',
-        'ruby-devel',
-        'rubygem-fluent-plugin-elasticsearch',
-        'rubygem-fluent-plugin-grok-parser',
-        'rubygem-fluent-plugin-rewrite-tag-filter',
-        'rubygem-fluent-plugin-secure-forward'
     ] %}
 
-{% elif base_package_type == 'deb' %}
+    {% if base_arch == 'x86_64' %}
-    {% set fluentd_packages = [
-        'g++',
-        'make',
-        'ruby',
-        'ruby-dev',
-    ] %}
-    {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
-        {% set fluentd_user = 'td-agent' %}
-        {{ macros.configure_user(name='td-agent', groups='mysql') }}
         {% set fluentd_packages = fluentd_packages + [
             'td-agent'
         ] %}
     {% else %}
-        {{ macros.configure_user(name='fluentd', groups='mysql') }}
+        {% set fluentd_packages = fluentd_packages + [
+            'fluentd',
+            'ruby-devel',
+            'rubygem-fluent-plugin-elasticsearch',
+            'rubygem-fluent-plugin-grok-parser',
+            'rubygem-fluent-plugin-rewrite-tag-filter'
+        ] %}
+    {% endif %}
+
+{% elif base_package_type == 'deb' %}
+
+    {% set fluentd_packages = [
+        'g++',
+        'make'
+    ] %}
+    {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
+        {% set fluentd_packages = fluentd_packages + [
+            'td-agent'
+        ] %}
+    {% else %}
+        {% set fluentd_packages = fluentd_packages + [
+            'ruby',
+            'ruby-dev'
+        ] %}
     {% endif %}
 
 {% endif %}
 
+{{ macros.configure_user(name=fluentd_user, groups='mysql') }}
 {{ macros.install_packages(fluentd_packages | customizable("packages")) }}
 
 # Distro specific files and operations
-{% if base_package_type == 'rpm' %}
+# Fluentd on CentOS non x86_64 is installed from rpm
-COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
+{% if base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
-    && mkdir -p /var/run/fluentd \
-    && chown -R fluentd: /etc/fluentd /var/run/fluentd
-{% elif base_package_type == 'deb' %}
-    {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
-COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
-    && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
-    && rm -f /etc/td-agent/td-agent.conf \
-    && chown -R td-agent: /etc/td-agent
-    {% else %}
 RUN /usr/bin/gem install fluentd --no-rdoc --no-ri
-COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
-    && mkdir -p /etc/fluentd \
-    && mkdir -p /var/run/fluentd \
-    && chown -R fluentd: /etc/fluentd /var/run/fluentd
-    {% endif %}
 {% endif %}
 
+COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
+
+RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
+    && mkdir -p /etc/{{ fluentd_user }} \
+    && mkdir -p /var/run/{{ fluentd_user }} \
+    && chown -R {{ fluentd_user }}: /etc/{{ fluentd_user }} /var/run/{{ fluentd_user }}
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 RUN chmod 755 /usr/local/bin/kolla_extend_start
 
 {% block fluentd_plugins_install %}
 
-{% if base_package_type == 'deb' %}
+{% if (base_package_type == 'rpm' and base_arch == 'x86_64') or base_package_type == 'deb' %}
     {% set fluentd_plugins = [
         'fluent-plugin-elasticsearch',
         'fluent-plugin-grep',
-        'fluent-plugin-grok-parser:2.1.4',
+        'fluent-plugin-grok-parser',
         'fluent-plugin-parser',
-        'fluent-plugin-rewrite-tag-filter:2.0.0',
+        'fluent-plugin-rewrite-tag-filter',
-        'fluent-plugin-secure-forward',
     ] %}
 {% endif %}
 
 {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
 
+{% endblock %}
+
+{% block fluentd_monasca_plugin_install %}
 # Build and install Fluentd output plugin for Monasca Log API
 ARG monasca_output_plugin_tag=0.1.1
 ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz
-ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz
+
-RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
+RUN curl -sSL $monasca_output_plugin_url -o /tmp/fluentd-monasca.tar.gz \
+    && tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
     && cd /tmp/fluentd-monasca-* \
+{% if fluentd_user == 'fluentd' %}
     && gem build fluentd-monasca-output.gemspec \
-    && gem install fluentd-monasca-output-*.gem \
+{% else %}
+    && td-agent-gem build fluentd-monasca-output.gemspec \
+{% endif %}
     && {{ macros.install_fluent_plugins(['fluentd-monasca-output-*.gem'], chain=True) }} \
     && rm -rf /tmp/fluentd*
 

docker/fluentd/extend_start.sh

@@ -6,7 +6,7 @@
 # Also set the setgid permission on the /var/log/kolla directory so that new
 # files and sub-directories in that directory inherit its group id ("kolla").
 
-if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu && "${KOLLA_BASE_ARCH}" == "x86_64" ]]; then
+if [[ "${KOLLA_BASE_ARCH}" == "x86_64" && "${KOLLA_BASE_DISTRO}" != "debian" ]]; then
     USERGROUP="td-agent:kolla"
     FLUENTD="td-agent"
 else
@@ -23,6 +23,6 @@ fi
 if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then
     sudo chmod 2775 /var/log/kolla
 fi
-if [[ $(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}" ]]; then
+if [[ (-d /var/lib/${FLUENTD}) && ($(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}") ]]; then
     sudo chown ${USERGROUP} /var/lib/${FLUENTD}
 fi

docker/fluentd/fluentd_sudoers

@@ -1,4 +1,7 @@
+%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/log/kolla, /usr/bin/chown td-agent\:kolla /var/log/kolla
+%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/lib/td-agent, /usr/bin/chown td-agent\:kolla /var/lib/td-agent
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
+%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/td-agent, /usr/bin/chmod 2775 /var/lib/td-agent

docker/fluentd/rpm_fluentd_sudoers

@@ -1,4 +0,0 @@
-%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
-%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
-%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
-%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd

docker/macros.j2

@@ -108,14 +108,10 @@ RUN apt-get update \
 {% macro install_fluent_plugins(plugins, chain=False) -%}
 {% if plugins is defined and plugins|length > 0 -%}
     {% if not chain -%} RUN {% endif -%}
-    {%- if base_distro in ['centos', 'oraclelinux', 'rhel'] -%}
+    {%- if base_arch == 'x86_64' and base_distro in ['centos', 'oraclelinux', 'rhel', 'ubuntu'] -%}
+        ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
+    {%- else -%}
         ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
-    {%- elif base_distro in ['debian', 'ubuntu'] -%}
-        {%- if base_arch == 'x86_64' and base_distro == 'ubuntu' -%}
-            ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
-        {%- else -%}
-            ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
-        {%- endif %}
     {%- endif %}
 {%- endif %}
 {%- endmacro %}