fluentd: migrate to td-agent 4 where possible

td-agent 4 got released for all distributions we support. For both
x86-64 and aarch64 architectures.

Change-Id: I86ecdc1ac45dfd76b94a34d1b837f7c0bc975dcd

docker/base/Dockerfile.j2

@@ -92,6 +92,7 @@ COPY dnf.conf /etc/dnf/dnf.conf
     {% set base_yum_repo_files = [
         'grafana.repo',
         'rabbitmq_rabbitmq-server.repo',
+        'td.repo',
     ] %}
     # FIXME(mgoddard): Not available for CentOS 8 yet.
     #

docker/base/td.repo

@@ -1,6 +1,6 @@
 [treasuredata]
 name=TreasureData
-baseurl=http://packages.treasuredata.com/3/redhat/$releasever/$basearch
+baseurl=http://packages.treasuredata.com/4/redhat/$releasever/$basearch
 gpgcheck=1
 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent
 enabled=0

docker/fluentd/Dockerfile.j2

@@ -2,10 +2,10 @@ FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
 {% block labels %}
 LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
 {% endblock %}
-{% if base_package_type == 'rpm' and base_arch != 'x86_64' %}
+{% if base_package_type == 'rpm' and base_arch not in ['aarch64', 'x86_64'] %}
 LABEL fluentd_version="0.12" fluentd_binary="fluentd"
     {% set fluentd_user = 'fluentd' %}
-{% elif base_package_type == 'deb' and base_arch != 'x86_64' %}
+{% elif base_package_type == 'deb' and base_arch not in ['aarch64', 'x86_64'] %}
 LABEL fluentd_version="0.14" fluentd_binary="fluentd"
     {% set fluentd_user = 'fluentd' %}
 {% else %}
@@ -26,7 +26,7 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent"
         'make'
     ] %}
 
-    {% if base_arch == 'x86_64' %}
+    {% if base_arch in ['aarch64', 'x86_64'] %}
         {% set fluentd_packages = fluentd_packages + [
             'td-agent'
         ] %}
@@ -47,7 +47,7 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent"
         'make'
     ] %}
 
-    {% if base_arch == 'x86_64' %}
+    {% if base_arch in ['aarch64', 'x86_64'] %}
         {% set fluentd_packages = fluentd_packages + [
             'td-agent'
         ] %}
@@ -65,7 +65,7 @@ LABEL fluentd_version="0.14" fluentd_binary="td-agent"
 
 # Distro specific files and operations
 # Fluentd on rpm-based non-x86_64 is installed from rpm
-{% if base_package_type == 'deb' and base_arch != 'x86_64' %}
+{% if base_package_type == 'deb' and fluentd_binary == 'fluentd' %}
 RUN /usr/bin/gem install fluentd --no-rdoc --no-ri
 {% endif %}
 

kolla/image/build.py

@@ -149,7 +149,6 @@ UNBUILDABLE_IMAGES = {
 
     'centos+aarch64': {
         "elasticsearch",  # no binary package
-        "fluentd",        # no binary package
         "hacluster-pcs",  # no binary package
         "influxdb",       # no binary package
         "kibana",         # no binary package

kolla/template/repos.yaml

@@ -26,6 +26,7 @@ centos-aarch64:
   hacluster: "HighAvailability"
   opstools: "centos-opstools"
   rabbitmq: "rabbitmq_rabbitmq-server"
+  td-agent: "treasuredata"
 
 centos-ppc64le:
   ceph: "centos-ceph-nautilus"
@@ -50,7 +51,7 @@ debian:
   influxdb: "deb https://repos.influxdata.com/debian buster stable"
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
-  td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib"
+  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
 
 debian-aarch64:
   elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
@@ -60,6 +61,7 @@ debian-aarch64:
   kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
   libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main"
+  td-agent: "deb http://packages.treasuredata.com/4/debian/buster buster contrib"
 
 ubuntu:
   elasticsearch: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
@@ -70,7 +72,7 @@ ubuntu:
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   qdrouterd: "deb http://ppa.launchpad.net/qpid/released/ubuntu/ bionic main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
-  td-agent: "deb http://packages.treasuredata.com/3/ubuntu/bionic/ bionic contrib"
+  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/bionic/ bionic contrib"
 
 ubuntu-aarch64:
   elasticsearch: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main"
@@ -79,6 +81,7 @@ ubuntu-aarch64:
   influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable"
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"
   rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main"
+  td-agent: "deb http://packages.treasuredata.com/4/ubuntu/bionic/ bionic contrib"
 
 ubuntu-ppc64le:
   mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main"

releasenotes/notes/td-agent-4-b2c7fc574bff0a42.yaml

@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    ``td-agent`` has been upgraded to version 4. It will be now used for both
+    ``x86-64`` and ``aarch64`` architectures on all distributions Kolla is
+    supporting. Users before upgrade should analyse changed config file syntax
+    for their custom fluentd configuration files.