bump td-agent lts from v4 to v5

notice that the td-agent command got renamed to fluentd.
see also the upgrade notes at[0].

for more details, refer to the changelog[1].

[0]: https://web.archive.org/web/20231006084137/https://www.fluentd.org/blog/upgrade-td-agent-v4-to-v5
[1]: b4b7017c09/CHANGELOG.md (release-v500---20230729)

Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/901509
Change-Id: I5e95363c6c180e330945fe3d98e97c7049bbd6f7
Signed-off-by: Sven Kieske <kieske@osism.tech>

docker/base/td.repo

@@ -1,6 +1,6 @@
-[treasuredata]
+[fluent-package-lts]
-name=TreasureData
+name=Fluentd Project
-baseurl=http://packages.treasuredata.com/4/redhat/$releasever/$basearch
+baseurl=https://packages.treasuredata.com/lts/5/redhat/$releasever/$basearch
 gpgcheck=1
 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent
 enabled=0

docker/fluentd/Dockerfile.j2

@@ -1,23 +1,23 @@
 FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
 
-{% set fluentd_user = 'td-agent' %}
+{% set fluentd_user = 'fluentd' %}
 
 {% block labels %}
-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" fluentd_binary="td-agent" fluentd_user="{{ fluentd_user }}"
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}" fluentd_binary="fluentd" fluentd_user="{{ fluentd_user }}"
 {% endblock %}
 
 {% block fluentd_header %}{% endblock %}
 
 {% import "macros.j2" as macros with context %}
 
-{{ macros.enable_extra_repos(['td-agent']) }}
+{{ macros.enable_extra_repos(['fluentd']) }}
 
 {% if base_package_type == 'rpm' %}
 
     {% set fluentd_packages = [
         'gcc-c++',
         'make',
-        'td-agent',
+        'fluent-package',
     ] %}
 
 {% elif base_package_type == 'deb' %}
@@ -25,7 +25,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% set fluentd_packages = [
         'g++',
         'make',
-        'td-agent',
+        'fluent-package',
     ] %}
 
 {% endif %}

docker/fluentd/extend_start.sh

@@ -6,23 +6,15 @@
 # Also set the setgid permission on the /var/log/kolla directory so that new
 # files and sub-directories in that directory inherit its group id ("kolla").
 
-if [[ "${KOLLA_BASE_ARCH}" == "x86_64" && "${KOLLA_BASE_DISTRO}" != "debian" ]]; then
-    USERGROUP="td-agent:kolla"
-    FLUENTD="td-agent"
-else
-    USERGROUP="fluentd:kolla"
-    FLUENTD="fluentd"
-fi
-
 if [ ! -d /var/log/kolla ]; then
     mkdir -p /var/log/kolla
 fi
-if [[ $(stat -c %U:%G /var/log/kolla) != "${USERGROUP}" ]]; then
+if [[ $(stat -c %U:%G /var/log/kolla) != "fluentd:kolla" ]]; then
-    sudo chown ${USERGROUP} /var/log/kolla
+    sudo chown fluentd:kolla /var/log/kolla
 fi
 if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then
     sudo chmod 2775 /var/log/kolla
 fi
-if [[ (-d /var/lib/${FLUENTD}) && ($(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}") ]]; then
+if [[ (-d /var/lib/fluentd) && ($(stat -c %U:%G /var/lib/fluentd) != "fluentd:kolla") ]]; then
-    sudo chown ${USERGROUP} /var/lib/${FLUENTD}
+    sudo chown fluentd:kolla /var/lib/fluentd
 fi

docker/fluentd/fluentd_sudoers

@@ -2,6 +2,7 @@
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/log/kolla, /usr/bin/chown td-agent\:kolla /var/log/kolla
 %kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/lib/td-agent, /usr/bin/chown td-agent\:kolla /var/lib/td-agent
+%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/td-agent, /usr/bin/chown fluentd\:kolla /var/lib/td-agent
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/td-agent, /usr/bin/chmod 2775 /var/lib/td-agent

docker/macros.j2

@@ -61,7 +61,7 @@ RUN usermod --append --home {{ homedir }} --groups kolla {{ name }} \
 {% macro install_fluent_plugins(plugins, chain=False) -%}
 {% if plugins is defined and plugins|length > 0 -%}
     {% if not chain -%} RUN {% endif -%}
-    ulimit -n 1024 && td-agent-gem install {{ plugins | join(' ') }}
+    ulimit -n 1024 && fluent-gem install {{ plugins | join(' ') }}
 {%- endif %}
 {%- endmacro %}
 

kolla/template/repos.yaml

@@ -5,6 +5,7 @@ centos:
   epel: "epel"
   erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
+  fluentd: "fluent-package-lts"
   grafana: "grafana"
   hacluster: "highavailability"
   influxdb: "influxdb"
@@ -15,7 +16,6 @@ centos:
   opstools: "centos-opstools"
   proxysql: "proxysql"
   rabbitmq: "rabbitmq_rabbitmq-server"
-  td-agent: "treasuredata"
 
 centos-aarch64:
   ceph: "centos-ceph-reef"
@@ -23,6 +23,7 @@ centos-aarch64:
   epel: "epel"
   erlang: "copr-hrw-erlang-for-rabbitmq"
   extras: "extras"
+  fluentd: "fluent-package-lts"
   grafana: "grafana"
   hacluster: "highavailability"
   influxdb: "influxdb"
@@ -33,7 +34,6 @@ centos-aarch64:
   opstools: "centos-opstools"
   proxysql: "proxysql"
   rabbitmq: "rabbitmq_rabbitmq-server"
-  td-agent: "treasuredata"
 
 # NOTE(mnasiadka): For RabbitMQ Debuntu suite names is following:
 # https://www.rabbitmq.com/install-debian.html#apt-cloudsmith
@@ -44,6 +44,11 @@ debian:
     suite: "jammy"
     component: "main"
     gpg_key: "erlang-ppa.gpg"
+  fluentd:
+    url: "https://packages.treasuredata.com/lts/5/debian/bookworm"
+    suite: "bookworm"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
   grafana:
     url: "https://apt.grafana.com"
     suite: "stable"
@@ -79,11 +84,6 @@ debian:
     suite: "bullseye"
     component: "main"
     gpg_key: "rabbitmq.gpg"
-  td-agent:
-    url: "http://packages.treasuredata.com/4/ubuntu/jammy"
-    suite: "jammy"
-    component: "contrib"
-    gpg_key: "treasuredata.asc"
 
 debian-aarch64:
   erlang:
@@ -91,6 +91,11 @@ debian-aarch64:
     suite: "jammy"
     component: "main"
     gpg_key: "erlang-ppa.gpg"
+  fluentd:
+    url: "https://packages.treasuredata.com/lts/5/debian/bookworm"
+    suite: "bookworm"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
   grafana:
     url: "https://apt.grafana.com"
     suite: "stable"
@@ -129,11 +134,6 @@ debian-aarch64:
     # syncing binary-aarch64 - we're using amd64 here.
     arch: "amd64"
     gpg_key: "rabbitmq.gpg"
-  td-agent:
-    url: "http://packages.treasuredata.com/4/ubuntu/jammy"
-    suite: "jammy"
-    component: "contrib"
-    gpg_key: "treasuredata.asc"
 
 rocky:
   ceph: "centos-ceph-reef"
@@ -141,6 +141,7 @@ rocky:
   epel: "epel"
   erlang: "rabbitmq_rabbitmq-erlang"
   extras: "extras"
+  fluentd: "fluent-package-lts"
   grafana: "grafana"
   hacluster: "highavailability"
   influxdb: "influxdb"
@@ -151,7 +152,6 @@ rocky:
   opstools: "centos-opstools"
   proxysql: "proxysql"
   rabbitmq: "rabbitmq_rabbitmq-server"
-  td-agent: "treasuredata"
 
 rocky-aarch64:
   ceph: "centos-ceph-reef"
@@ -159,6 +159,7 @@ rocky-aarch64:
   epel: "epel"
   erlang: "copr-hrw-erlang-for-rabbitmq"
   extras: "extras"
+  fluentd: "fluent-package-lts"
   grafana: "grafana"
   influxdb: "influxdb"
   hacluster: "highavailability"
@@ -169,7 +170,6 @@ rocky-aarch64:
   opstools: "centos-opstools"
   proxysql: "proxysql"
   rabbitmq: "rabbitmq_rabbitmq-server"
-  td-agent: "treasuredata"
 
 ubuntu:
   erlang:
@@ -177,6 +177,11 @@ ubuntu:
     suite: "jammy"
     component: "main"
     gpg_key: "erlang-ppa.gpg"
+  fluentd:
+    url: "https://packages.treasuredata.com/lts/5/ubuntu/jammy/"
+    suite: "jammy"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
   grafana:
     url: "https://apt.grafana.com"
     suite: "stable"
@@ -212,11 +217,6 @@ ubuntu:
     suite: "jammy"
     component: "main"
     gpg_key: "rabbitmq.gpg"
-  td-agent:
-    url: "http://packages.treasuredata.com/4/ubuntu/jammy/"
-    suite: "jammy"
-    component: "contrib"
-    gpg_key: "treasuredata.asc"
 
 ubuntu-aarch64:
   erlang:
@@ -224,6 +224,11 @@ ubuntu-aarch64:
     suite: "jammy"
     component: "main"
     gpg_key: "erlang-ppa.gpg"
+  fluentd:
+    url: "https://packages.treasuredata.com/lts/5/ubuntu/jammy/"
+    suite: "jammy"
+    component: "contrib"
+    gpg_key: "treasuredata.asc"
   grafana:
     url: "https://apt.grafana.com"
     suite: "stable"
@@ -260,8 +265,3 @@ ubuntu-aarch64:
     component: "main"
     arch: "amd64"
     gpg_key: "rabbitmq.gpg"
-  td-agent:
-    url: "http://packages.treasuredata.com/4/ubuntu/jammy/"
-    suite: "jammy"
-    component: "contrib"
-    gpg_key: "treasuredata.asc"