Browse Source

Merge "Change fluentd to td-agent on CentOS"

changes/57/681057/1
Zuul 2 weeks ago
parent
commit
481843b80c

+ 1
- 1
docker/base/td.repo View File

@@ -1,5 +1,5 @@
1 1
 [treasuredata]
2 2
 name=TreasureData
3
-baseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch
3
+baseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch
4 4
 gpgcheck=1
5 5
 gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent

+ 49
- 49
docker/fluentd/Dockerfile.j2 View File

@@ -1,108 +1,108 @@
1 1
 FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
2 2
 LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
3
-{% if base_package_type == 'rpm' or base_arch != 'x86_64' %}
3
+{% if base_package_type == 'rpm' and base_arch != 'x86_64' %}
4 4
 LABEL fluentd_version="0.12" fluentd_binary="fluentd"
5
+    {% set fluentd_user = 'fluentd' %}
6
+{% elif base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
7
+LABEL fluentd_version="0.14" fluentd_binary="fluentd"
8
+    {% set fluentd_user = 'fluentd' %}
5 9
 {% else %}
6 10
 LABEL fluentd_version="0.14" fluentd_binary="td-agent"
11
+    {% set fluentd_user = 'td-agent' %}
7 12
 {% endif %}
8 13
 
9 14
 {% block fluentd_header %}{% endblock %}
10 15
 
11 16
 {% import "macros.j2" as macros with context %}
12 17
 
13
-{% set fluentd_user = 'fluentd' %}
14
-
15 18
 {% if base_package_type == 'rpm' %}
16 19
 
17
-    {{ macros.configure_user(name='fluentd', groups='mysql') }}
18
-# TODO:remove hostname package as it should be installed by the
19
-# fluentd dependencies.
20
-# hostname package is installed by default on CentOS/RHEL but not on
21
-# oraclelinux.
22
-
23 20
     {% set fluentd_packages = [
24
-        'fluentd',
25 21
         'gcc-c++',
26
-        'hostname',
27
-        'make',
28
-        'ruby-devel',
29
-        'rubygem-fluent-plugin-elasticsearch',
30
-        'rubygem-fluent-plugin-grok-parser',
31
-        'rubygem-fluent-plugin-rewrite-tag-filter',
32
-        'rubygem-fluent-plugin-secure-forward'
22
+        'make'
33 23
     ] %}
34 24
 
25
+    {% if base_arch == 'x86_64' %}
26
+        {% set fluentd_packages = fluentd_packages + [
27
+            'td-agent'
28
+        ] %}
29
+    {% else %}
30
+        {% set fluentd_packages = fluentd_packages + [
31
+            'fluentd',
32
+            'ruby-devel',
33
+            'rubygem-fluent-plugin-elasticsearch',
34
+            'rubygem-fluent-plugin-grok-parser',
35
+            'rubygem-fluent-plugin-rewrite-tag-filter'
36
+        ] %}
37
+    {% endif %}
38
+
35 39
 {% elif base_package_type == 'deb' %}
40
+
36 41
     {% set fluentd_packages = [
37 42
         'g++',
38
-        'make',
39
-        'ruby',
40
-        'ruby-dev',
43
+        'make'
41 44
     ] %}
42 45
     {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
43
-        {% set fluentd_user = 'td-agent' %}
44
-        {{ macros.configure_user(name='td-agent', groups='mysql') }}
45 46
         {% set fluentd_packages = fluentd_packages + [
46 47
             'td-agent'
47 48
         ] %}
48 49
     {% else %}
49
-        {{ macros.configure_user(name='fluentd', groups='mysql') }}
50
+        {% set fluentd_packages = fluentd_packages + [
51
+            'ruby',
52
+            'ruby-dev'
53
+        ] %}
50 54
     {% endif %}
51 55
 
52 56
 {% endif %}
53 57
 
58
+{{ macros.configure_user(name=fluentd_user, groups='mysql') }}
54 59
 {{ macros.install_packages(fluentd_packages | customizable("packages")) }}
55 60
 
56 61
 # Distro specific files and operations
57
-{% if base_package_type == 'rpm' %}
58
-COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
59
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
60
-    && mkdir -p /var/run/fluentd \
61
-    && chown -R fluentd: /etc/fluentd /var/run/fluentd
62
-{% elif base_package_type == 'deb' %}
63
-    {% if base_arch == 'x86_64' and base_distro == 'ubuntu' %}
64
-COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
65
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
66
-    && sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent \
67
-    && rm -f /etc/td-agent/td-agent.conf \
68
-    && chown -R td-agent: /etc/td-agent
69
-    {% else %}
62
+# Fluentd on CentOS non x86_64 is installed from rpm
63
+{% if base_distro == 'debian' or (base_distro == 'ubuntu' and base_arch != 'x86_64') %}
70 64
 RUN /usr/bin/gem install fluentd --no-rdoc --no-ri
71
-COPY rpm_fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
72
-RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
73
-    && mkdir -p /etc/fluentd \
74
-    && mkdir -p /var/run/fluentd \
75
-    && chown -R fluentd: /etc/fluentd /var/run/fluentd
76
-    {% endif %}
77 65
 {% endif %}
78 66
 
67
+COPY fluentd_sudoers /etc/sudoers.d/kolla_fluentd_sudoers
68
+
69
+RUN chmod 440 /etc/sudoers.d/kolla_fluentd_sudoers \
70
+    && mkdir -p /etc/{{ fluentd_user }} \
71
+    && mkdir -p /var/run/{{ fluentd_user }} \
72
+    && chown -R {{ fluentd_user }}: /etc/{{ fluentd_user }} /var/run/{{ fluentd_user }}
79 73
 
80 74
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
81 75
 RUN chmod 755 /usr/local/bin/kolla_extend_start
82 76
 
83 77
 {% block fluentd_plugins_install %}
84 78
 
85
-{% if base_package_type == 'deb' %}
79
+{% if (base_package_type == 'rpm' and base_arch == 'x86_64') or base_package_type == 'deb' %}
86 80
     {% set fluentd_plugins = [
87 81
         'fluent-plugin-elasticsearch',
88 82
         'fluent-plugin-grep',
89
-        'fluent-plugin-grok-parser:2.1.4',
83
+        'fluent-plugin-grok-parser',
90 84
         'fluent-plugin-parser',
91
-        'fluent-plugin-rewrite-tag-filter:2.0.0',
92
-        'fluent-plugin-secure-forward',
85
+        'fluent-plugin-rewrite-tag-filter',
93 86
     ] %}
94 87
 {% endif %}
95 88
 
96 89
 {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
97 90
 
91
+{% endblock %}
92
+
93
+{% block fluentd_monasca_plugin_install %}
98 94
 # Build and install Fluentd output plugin for Monasca Log API
99 95
 ARG monasca_output_plugin_tag=0.1.1
100 96
 ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz
101
-ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz
102
-RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
97
+
98
+RUN curl -sSL $monasca_output_plugin_url -o /tmp/fluentd-monasca.tar.gz \
99
+    && tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
103 100
     && cd /tmp/fluentd-monasca-* \
101
+{% if fluentd_user == 'fluentd' %}
104 102
     && gem build fluentd-monasca-output.gemspec \
105
-    && gem install fluentd-monasca-output-*.gem \
103
+{% else %}
104
+    && td-agent-gem build fluentd-monasca-output.gemspec \
105
+{% endif %}
106 106
     && {{ macros.install_fluent_plugins(['fluentd-monasca-output-*.gem'], chain=True) }} \
107 107
     && rm -rf /tmp/fluentd*
108 108
 

+ 2
- 2
docker/fluentd/extend_start.sh View File

@@ -6,7 +6,7 @@
6 6
 # Also set the setgid permission on the /var/log/kolla directory so that new
7 7
 # files and sub-directories in that directory inherit its group id ("kolla").
8 8
 
9
-if [[ "${KOLLA_BASE_DISTRO}" =~ debian|ubuntu && "${KOLLA_BASE_ARCH}" == "x86_64" ]]; then
9
+if [[ "${KOLLA_BASE_ARCH}" == "x86_64" && "${KOLLA_BASE_DISTRO}" != "debian" ]]; then
10 10
     USERGROUP="td-agent:kolla"
11 11
     FLUENTD="td-agent"
12 12
 else
@@ -23,6 +23,6 @@ fi
23 23
 if [[ $(stat -c %a /var/log/kolla) != "2775" ]]; then
24 24
     sudo chmod 2775 /var/log/kolla
25 25
 fi
26
-if [[ $(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}" ]]; then
26
+if [[ (-d /var/lib/${FLUENTD}) && ($(stat -c %U:%G /var/lib/${FLUENTD}) != "${USERGROUP}") ]]; then
27 27
     sudo chown ${USERGROUP} /var/lib/${FLUENTD}
28 28
 fi

+ 3
- 0
docker/fluentd/fluentd_sudoers View File

@@ -1,4 +1,7 @@
1
+%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
1 2
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/log/kolla, /usr/bin/chown td-agent\:kolla /var/log/kolla
3
+%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
2 4
 %kolla ALL=(root) NOPASSWD: /bin/chown td-agent\:kolla /var/lib/td-agent, /usr/bin/chown td-agent\:kolla /var/lib/td-agent
3 5
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
6
+%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd
4 7
 %kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/td-agent, /usr/bin/chmod 2775 /var/lib/td-agent

+ 0
- 4
docker/fluentd/rpm_fluentd_sudoers View File

@@ -1,4 +0,0 @@
1
-%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/log/kolla, /usr/bin/chown fluentd\:kolla /var/log/kolla
2
-%kolla ALL=(root) NOPASSWD: /bin/chown fluentd\:kolla /var/lib/fluentd, /usr/bin/chown fluentd\:kolla /var/lib/fluentd
3
-%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/log/kolla, /usr/bin/chmod 2775 /var/log/kolla
4
-%kolla ALL=(root) NOPASSWD: /bin/chmod 2775 /var/lib/fluentd, /usr/bin/chmod 2775 /var/lib/fluentd

+ 3
- 7
docker/macros.j2 View File

@@ -108,14 +108,10 @@ RUN apt-get update \
108 108
 {% macro install_fluent_plugins(plugins, chain=False) -%}
109 109
 {% if plugins is defined and plugins|length > 0 -%}
110 110
     {% if not chain -%} RUN {% endif -%}
111
-    {%- if base_distro in ['centos', 'oraclelinux', 'rhel'] -%}
111
+    {%- if base_arch == 'x86_64' and base_distro in ['centos', 'oraclelinux', 'rhel', 'ubuntu'] -%}
112
+        ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
113
+    {%- else -%}
112 114
         ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
113
-    {%- elif base_distro in ['debian', 'ubuntu'] -%}
114
-        {%- if base_arch == 'x86_64' and base_distro == 'ubuntu' -%}
115
-            ulimit -n 65536 && td-agent-gem install {{ plugins | join(' ') }}
116
-        {%- else -%}
117
-            ulimit -n 65536 && gem install --minimal-deps {{ plugins | join(' ') }}
118
-        {%- endif %}
119 115
     {%- endif %}
120 116
 {%- endif %}
121 117
 {%- endmacro %}

Loading…
Cancel
Save