diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 54b6268661..f8729b0118 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -224,6 +224,8 @@ grafana_server_port: "3000" tacker_server_port: "9890" +fluentd_syslog_port: "5140" + public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" internal_protocol: "http" admin_protocol: "http" diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index 8e93160512..11db34b922 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -10,10 +10,6 @@ kolla_toolbox_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d kolla_toolbox_tag: "{{ openstack_release }}" kolla_toolbox_image_full: "{{ kolla_toolbox_image }}:{{ kolla_toolbox_tag }}" -heka_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-heka" -heka_tag: "{{ openstack_release }}" -heka_image_full: "{{ heka_image }}:{{ heka_tag }}" - cron_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cron" cron_tag: "{{ openstack_release }}" cron_image_full: "{{ cron_image }}:{{ cron_tag }}" diff --git a/ansible/roles/common/tasks/clean_heka.yml b/ansible/roles/common/tasks/clean_heka.yml new file mode 100644 index 0000000000..94191031ff --- /dev/null +++ b/ansible/roles/common/tasks/clean_heka.yml @@ -0,0 +1,13 @@ +--- +- name: Get container facts + kolla_container_facts: + name: + - heka + register: container_facts + +- name: Removing heka container + kolla_docker: + action: "remove_container" + name: "heka" + when: + - container_facts['heka'] is defined diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 4fb093683d..dfaee19e75 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -5,7 +5,11 @@ state: "directory" recurse: yes with_items: - - "heka" + - "fluentd" + - "fluentd/input" + - "fluentd/output" + - "fluentd/format" + - "fluentd/filter" - "kolla-toolbox" - "cron" - "cron/logrotate" @@ -15,56 +19,50 @@ src: "{{ item }}.json.j2" dest: "{{ node_config_directory }}/{{ item }}/config.json" with_items: - - "heka" + - "fluentd" - "kolla-toolbox" - "cron" -- name: Copying over heka config files +- name: Copying over fluentd input config files template: - src: "heka-{{ item.src|default(item.name) }}.toml.j2" - dest: "{{ node_config_directory }}/heka/heka-{{ item.name }}.toml" - when: item.enabled | bool + src: "conf/input/{{ item }}.conf.j2" + dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf" with_items: - - { name: "aodh", enabled: "{{ enable_aodh }}" } - - { name: "barbican", enabled: "{{ enable_barbican }}"} - - { name: "ceilometer", enabled: "{{ enable_ceilometer }}" } - - { name: "elasticsearch", enabled: "{{ enable_elasticsearch }}" } - - { name: "global", enabled: "yes" } - - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" } - - { name: "grafana", enabled: "{{ enable_grafana }}" } - - { name: "haproxy", enabled: "{{ enable_haproxy }}" } - - { name: "horizon", enabled: "{{ enable_horizon }}" } - - { name: "keepalived", enabled: "{{ enable_haproxy }}" } - - { name: "keystone", enabled: "{{ enable_keystone }}" } - - { name: "mariadb", enabled: "{{ enable_mariadb }}" } - - { name: "openstack", enabled: "yes" } - - { name: "panko", enabled: "{{ enable_panko }}" } - - { name: "rabbitmq", enabled: "{{ enable_rabbitmq }}" } - - { name: "swift-account-auditor", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-account-reaper", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-account-replicator", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-account-server", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-container-auditor", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-container-replicator", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-container-server", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-container-updater", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-object-auditor", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-object-expirer", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-object-replicator", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-object-server", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-object-updater", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-proxy-server", src: "swift", enabled: "{{ enable_swift }}" } - - { name: "swift-rsyncd", src: "swift", enabled: "{{ enable_swift }}" } + - "00-global" + - "01-syslog" + - "02-mariadb" + - "03-rabbitmq" -- name: Heka custom config file exists - local_action: stat path="{{ node_custom_config }}/heka/heka-custom.toml" - register: heka_custom_stat_result - -- name: Copying over heka custom config file +- name: Copying over fluentd ouput config files template: - src: "{{ node_custom_config }}/heka/heka-custom.toml" - dest: "{{ node_config_directory }}/heka/heka-custom.toml" - when: heka_custom_stat_result.stat.exists == true + src: "conf/output/{{ item }}.conf.j2" + dest: "{{ node_config_directory }}/fluentd/output/{{ item }}.conf" + with_items: + - "00-local" + - "01-es" +- name: Copying over fluentd format config files + template: + src: "conf/format/{{ item }}.conf.j2" + dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf" + with_items: + - "apache_access" + - "wsgi_access" + - "wsgi_python" + +- name: Copying over fluentd filter config files + template: + src: "conf/filter/{{ item }}.conf.j2" + dest: "{{ node_config_directory }}/fluentd/filter/{{ item }}.conf" + with_items: + - "00-record_transformer" + - "01-rewrite" + +- name: Copying over tg-agent.conf + template: + src: "td-agent.conf.j2" + dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf" + with_items: + - "fluentd" - name: Copying over cron logrotate config files template: diff --git a/ansible/roles/common/tasks/pull.yml b/ansible/roles/common/tasks/pull.yml index e19971ce68..3cf41e2357 100644 --- a/ansible/roles/common/tasks/pull.yml +++ b/ansible/roles/common/tasks/pull.yml @@ -5,11 +5,11 @@ common_options: "{{ docker_common_options }}" image: "{{ kolla_toolbox_image_full }}" -- name: Pulling heka image +- name: Pulling fluentd image kolla_docker: action: "pull_image" common_options: "{{ docker_common_options }}" - image: "{{ heka_image_full }}" + image: "{{ fluentd_image_full }}" - name: Pulling cron image kolla_docker: diff --git a/ansible/roles/common/tasks/reconfigure.yml b/ansible/roles/common/tasks/reconfigure.yml index 961c656fa3..84e5c86c2b 100644 --- a/ansible/roles/common/tasks/reconfigure.yml +++ b/ansible/roles/common/tasks/reconfigure.yml @@ -1,28 +1,28 @@ --- -- name: Ensuring the heka container is up +- name: Ensuring the fluentd container is up kolla_docker: - name: "heka" + name: "fluentd" action: "get_container_state" register: container_state failed_when: container_state.Running == false - include: config.yml -- name: Checking the heka config - command: docker exec heka /usr/local/bin/kolla_set_configs --check +- name: Checking the fluentd config + command: docker exec fluentd /usr/local/bin/kolla_set_configs --check changed_when: false failed_when: false register: check_result -- name: Getting the heka container config strategy +- name: Getting the fluentd container config strategy kolla_docker: - name: "heka" + name: "fluentd" action: "get_container_env" register: container_env -- name: Removing the heka container +- name: Removing the fluentd container kolla_docker: - name: "heka" + name: "fluentd" action: "remove_container" register: remove_container when: @@ -32,9 +32,9 @@ - include: start.yml when: remove_container.changed -- name: Restarting the heka container +- name: Restarting the fluentd container kolla_docker: - name: "heka" + name: "fluentd" action: "restart_container" when: - config_strategy == "COPY_ALWAYS" diff --git a/ansible/roles/common/tasks/start.yml b/ansible/roles/common/tasks/start.yml index 963cd5de1c..cfba855032 100644 --- a/ansible/roles/common/tasks/start.yml +++ b/ansible/roles/common/tasks/start.yml @@ -1,19 +1,17 @@ --- -- name: Starting heka container +- name: Starting fluentd container kolla_docker: action: "start_container" common_options: "{{ docker_common_options }}" environment: KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" SKIP_LOG_SETUP: "true" - image: "{{ heka_image_full }}" - name: "heka" + image: "{{ fluentd_image_full }}" + name: "fluentd" volumes: - - "{{ node_config_directory }}/heka/:{{ container_config_directory }}/:ro" + - "{{ node_config_directory }}/fluentd/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" - - "heka:/var/cache/hekad" - - "heka_socket:/var/lib/kolla/heka/" - name: Starting kolla-toolbox container kolla_docker: @@ -45,5 +43,4 @@ volumes: - "{{ node_config_directory }}/cron/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" - "kolla_logs:/var/log/kolla/" diff --git a/ansible/roles/common/tasks/upgrade.yml b/ansible/roles/common/tasks/upgrade.yml index 1f16915ad9..b31527f661 100644 --- a/ansible/roles/common/tasks/upgrade.yml +++ b/ansible/roles/common/tasks/upgrade.yml @@ -1,4 +1,6 @@ --- - include: config.yml +- include: clean_heka.yml + - include: start.yml diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 new file mode 100644 index 0000000000..1dbfaf5ecb --- /dev/null +++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 @@ -0,0 +1,8 @@ +<filter *.var.log.kolla.*.*.log> + @type record_transformer + <record> + Hostname ${hostname} + Logger openstack.${tag_parts[4]} + programname ${tag_parts[5]} + </record> +</filter> diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2 new file mode 100644 index 0000000000..3e9f0053af --- /dev/null +++ b/ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2 @@ -0,0 +1,17 @@ +<match kolla.var.log.kolla.*.*.log> + @type rewrite_tag_filter + capitalize_regex_backreference yes + rewriterule1 programname ^(horizon-access|ceilometer-api-access|keystone-apache-admin-access|keystone-apache-public-access)$ apache_access + rewriterule2 programname ^aodh_wsgi_access$ wsgi_access + rewriterule3 programname ^(nova-api|nova-compute|nova-conductor|nova-consoleauth|nova-manage|nova-novncproxy|nova-scheduler|privsep-helper).* openstack_python + rewriterule4 programname ^(sahara-api|sahara-engine).* openstack_python + rewriterule5 programname ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent).* openstack_python + rewriterule6 programname ^(magnum-conductor|magnum-api).* openstack_python + rewriterule7 programname ^(keystone).* openstack_python + rewriterule8 programname ^(heat-engine|heat-api|heat-api-cfn).* openstack_python + rewriterule9 programname ^(registry|api).* openstack_python + rewriterule10 programname ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api).* openstack_python + rewriterule11 programname ^(ceilometer-polling|ceilometer-collector|ceilometer-agent-notification|loadwsgi).* openstack_python + rewriterule12 programname ^(barbican-worker|barbican-keystone-listener|barbican-db-manage|barbican-api|app).* openstack_python + rewriterule13 programname ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync).* openstack_python +</match> diff --git a/ansible/roles/common/templates/conf/format/apache_access.conf.j2 b/ansible/roles/common/templates/conf/format/apache_access.conf.j2 new file mode 100644 index 0000000000..631b0180c2 --- /dev/null +++ b/ansible/roles/common/templates/conf/format/apache_access.conf.j2 @@ -0,0 +1,20 @@ +<filter apache_access> + @type parser + reserve_data true + key_name message + <parse> + @type "grok" + grok_pattern \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:\d+|-) + time_key Timestamp + time_format %d/%b/%Y:%H:%M:%S %z + keep_time_key true + </parse> +</filter> + +<filter apache_access> + @type record_transformer + <record> + severity_label INFO + Severity 6 + </record> +</filter> diff --git a/ansible/roles/common/templates/conf/format/wsgi_access.conf.j2 b/ansible/roles/common/templates/conf/format/wsgi_access.conf.j2 new file mode 100644 index 0000000000..52ac0fab10 --- /dev/null +++ b/ansible/roles/common/templates/conf/format/wsgi_access.conf.j2 @@ -0,0 +1,12 @@ +<filter wsgi_access> + @type parser + reserve_data true + key_name message + <parse> + @type "grok" + grok_pattern %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) %{QS:referrer} %{QS:agent} + time_key Timestamp + time_format %d/%b/%Y:%H:%M:%S %z + keep_time_key true + </parse> +</filter> diff --git a/ansible/roles/common/templates/conf/format/wsgi_python.conf.j2 b/ansible/roles/common/templates/conf/format/wsgi_python.conf.j2 new file mode 100644 index 0000000000..4f215f22ec --- /dev/null +++ b/ansible/roles/common/templates/conf/format/wsgi_python.conf.j2 @@ -0,0 +1,12 @@ +<filter openstack_python> + @type parser + reserve_data true + key_name message + <parse> + @type "grok" + grok_pattern %{TIMESTAMP_ISO8601:Timestamp} %{NUMBER:Pid} %{LOGLEVEL:severity_label} %{NOTSPACE:python_module} \[req-(?:%{NOTSPACE:request_id}) (?:%{NOTSPACE:user_id}) (?:%{NOTSPACE:tenant_id}) .*\] %{GREEDYDATA:Payload} + time_format "%Y-%m-%d %H:%M:%S.%L" + time_key Timestamp + keep_time_key true + </parse> +</filter> diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2 new file mode 100644 index 0000000000..6bc0d5eb4f --- /dev/null +++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2 @@ -0,0 +1,8 @@ +<source> + @type tail + path /var/log/kolla/*/*.log + exclude_path ["/var/log/kolla/rabbitmq/*.log", "/var/log/kolla/mariadb/mariadb.log", "/var/log/kolla/haproxy/*.log", "/var/log/kolla/swift/*.log"] + pos_file /var/run/td-agent/kolla.pos + tag kolla.* + format /^(?<message>.*)$/ +</source> diff --git a/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 b/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 new file mode 100644 index 0000000000..d82208b591 --- /dev/null +++ b/ansible/roles/common/templates/conf/input/01-syslog.conf.j2 @@ -0,0 +1,7 @@ +<source> + @type syslog + port {{ fluentd_syslog_port }} + bind {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} + tag syslog + format /^(?<Payload>.*)$/ +</source> diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 new file mode 100644 index 0000000000..d2ef3b1cc6 --- /dev/null +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -0,0 +1,10 @@ +<source> + @type tail + path /var/log/kolla/mariadb/mariadb.log + pos_file /var/run/td-agent/mariadb.pos + tag mariadb.* + format multiline + format_firstline /^\d{6}/ + format1 /^(?<time>\d{6} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<severity_label>\S+)\] (?<Payload>.*)/ + time_format %y%m%d %k:%M:%S +</source> diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 new file mode 100644 index 0000000000..ac631877d4 --- /dev/null +++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 @@ -0,0 +1,10 @@ +<source> + @type tail + path /var/log/kolla/rabbitmq/rabbit.log + pos_file /var/run/td-agent/rabbit.pos + tag rabbit.* + format multiline + format_firstline /^=/ + format1 /^.*\n/ + format2 /^(?<Payload>.*)\n/ +</source> diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 new file mode 100644 index 0000000000..650f5730ba --- /dev/null +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -0,0 +1,41 @@ +<match syslog.local0.**> + @type copy + <store> + @type file + path /var/log/kolla/swift/swift_latest.*.log + symlink_path /var/log/kolla/swift/swift_latest.log + utc + append true + compress gzip + </store> + <store> + type elasticsearch + host {{ kolla_external_vip_address }} + port {{ elasticsearch_port }} + logstash_format true + logstash_prefix flog + flush_interval 15s + </store> +</match> + +<match syslog.local1.**> + @type copy + <store> + @type file + path /var/log/kolla/haproxy/haproxy_latest.*.log + symlink_path /var/log/kolla/haproxy/haproxy_latest.log + output_tag false + output_time false + utc + append true + compress gzip + </store> + <store> + type elasticsearch + host {{ kolla_external_vip_address }} + port {{ elasticsearch_port }} + logstash_format true + logstash_prefix flog + flush_interval 15s + </store> +</match> diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 new file mode 100644 index 0000000000..2367b8a7fa --- /dev/null +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -0,0 +1,11 @@ +<match *.**> + type copy + <store> + type elasticsearch + host {{ kolla_external_vip_address }} + port {{ elasticsearch_port }} + logstash_format true + logstash_prefix flog + flush_interval 15s + </store> +</match> diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 new file mode 100644 index 0000000000..1e555de92c --- /dev/null +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -0,0 +1,84 @@ +{ + "command": "/usr/sbin/td-agent", + "config_files": [ + { + "source": "{{ container_config_directory }}/td-agent.conf", + "dest": "/etc/td-agent/td-agent.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/input/00-global.conf", + "dest": "/etc/td-agent/input/00-global.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/input/02-mariadb.conf", + "dest": "/etc/td-agent/input/02-mariadb.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/input/03-rabbitmq.conf", + "dest": "/etc/td-agent/input/03-rabbitmq.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/input/01-syslog.conf", + "dest": "/etc/td-agent/input/01-syslog.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/filter/00-record_transformer.conf", + "dest": "/etc/td-agent/filter/00-record_transformer.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/filter/01-rewrite.conf", + "dest": "/etc/td-agent/filter/01-rewrite.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/format/apache_access.conf", + "dest": "/etc/td-agent/format/apache_access.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/format/wsgi_access.conf", + "dest": "/etc/td-agent/format/wsgi_access.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/format/wsgi_python.conf", + "dest": "/etc/td-agent/format/wsgi_python.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/output/01-es.conf", + "dest": "/etc/td-agent/output/01-es.conf", + "owner": "td-agent", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/output/00-local.conf", + "dest": "/etc/td-agent/output/00-local.conf", + "owner": "td-agent", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla", + "owner": "td-agent:td-agent", + "recurse": true + } + ] +} diff --git a/ansible/roles/common/templates/heka-aodh.toml.j2 b/ansible/roles/common/templates/heka-aodh.toml.j2 deleted file mode 100644 index 419d923e98..0000000000 --- a/ansible/roles/common/templates/heka-aodh.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[aodh_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_aodh_apache_log.lua" - [aodh_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[aodh_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "aodh_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'aodh/aodh-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["aodh-apache-", "Service"] diff --git a/ansible/roles/common/templates/heka-barbican.toml.j2 b/ansible/roles/common/templates/heka-barbican.toml.j2 deleted file mode 100644 index 9ec0e8c6b6..0000000000 --- a/ansible/roles/common/templates/heka-barbican.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[barbican_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_barbican_apache_log.lua" - [barbican_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[barbican_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "barbican_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'barbican/barbican-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["barbican-apache-", "Service"] diff --git a/ansible/roles/common/templates/heka-ceilometer.toml.j2 b/ansible/roles/common/templates/heka-ceilometer.toml.j2 deleted file mode 100644 index ea37e53ce9..0000000000 --- a/ansible/roles/common/templates/heka-ceilometer.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[ceilometer_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_ceilometer_apache_log.lua" - [ceilometer_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[ceilometer_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "ceilometer_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'ceilometer/(?P<Service>ceilometer-.*)\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["Service"] diff --git a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 b/ansible/roles/common/templates/heka-elasticsearch.toml.j2 deleted file mode 100644 index f3d4b13465..0000000000 --- a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 +++ /dev/null @@ -1,16 +0,0 @@ -[elasticsearch_json_encoder] -type = "ESJsonEncoder" -index = {{'"%{Type}-%{%Y.%m.%d}"'}} -es_index_from_timestamp = true -fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"] - -[elasticsearch_output] -type = "ElasticSearchOutput" -server = "{{ elasticsearch_protocol }}://{{ elasticsearch_address }}:{{ elasticsearch_port }}" -message_matcher = "Type == 'log'" -encoder = "elasticsearch_json_encoder" -use_buffering = true - [elasticsearch_output.buffering] - max_buffer_size = 1073741824 # 1024 * 1024 * 1024 - max_file_size = 134217728 # 128 * 1024 * 1024 - full_action = "drop" diff --git a/ansible/roles/common/templates/heka-global.toml.j2 b/ansible/roles/common/templates/heka-global.toml.j2 deleted file mode 100644 index 5e325026ae..0000000000 --- a/ansible/roles/common/templates/heka-global.toml.j2 +++ /dev/null @@ -1,30 +0,0 @@ -[hekad] -maxprocs = {{ ansible_processor_count * ansible_processor_cores }} - -[syslog_encoder] -type = "SandboxEncoder" -filename = "lua_encoders/os_syslog.lua" - -[syslog_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_syslog.lua" - [syslog_log_decoder.config] - hostname = "{{ ansible_hostname }}" - -[swift_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_swift_log.lua" - [swift_log_decoder.config] - hostname = "{{ ansible_hostname }}" - -[multi_log_decoder] -type = "MultiDecoder" -subs = ["syslog_log_decoder", "swift_log_decoder"] -cascade_strategy = "first-wins" - -[syslog_input] -type = "UdpInput" -net = "unixgram" -address = "/var/lib/kolla/heka/log" -decoder = "multi_log_decoder" -splitter = "NullSplitter" diff --git a/ansible/roles/common/templates/heka-gnocchi.toml.j2 b/ansible/roles/common/templates/heka-gnocchi.toml.j2 deleted file mode 100644 index e2596762d8..0000000000 --- a/ansible/roles/common/templates/heka-gnocchi.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[gnocchi_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_gnocchi_apache_log.lua" - [gnocchi_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[gnocchi_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "gnocchi_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'gnocchi/gnocchi-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["gnocchi-apache-", "Service"] diff --git a/ansible/roles/common/templates/heka-grafana.toml.j2 b/ansible/roles/common/templates/heka-grafana.toml.j2 deleted file mode 100644 index 7565c083e1..0000000000 --- a/ansible/roles/common/templates/heka-grafana.toml.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[grafana_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_grafana_log.lua" - -[grafana_log_splitter] -type = "RegexSplitter" -delimiter = '\n\n(=[^=]+====)' -delimiter_eol = false -deliver_incomplete_final = true - -[grafana_logstreamer_input] -type = "LogstreamerInput" -decoder = "grafana_log_decoder" -splitter = "grafana_log_splitter" -log_directory = "/var/log/kolla" -file_match = 'grafana/(?P<Service>grafana.*)\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["Service"] diff --git a/ansible/roles/common/templates/heka-haproxy.toml.j2 b/ansible/roles/common/templates/heka-haproxy.toml.j2 deleted file mode 100644 index 49338fd0a3..0000000000 --- a/ansible/roles/common/templates/heka-haproxy.toml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# HAProxy writes its logs to Syslog, so the generic -# Syslog input set in heka-global.toml.j2 is used. - -[haproxy_file_output] -type = "FileOutput" -message_matcher = "Fields[programname] =~ /(?i:haproxy)/" -path = "/var/log/kolla/haproxy/haproxy.log" -encoder = "syslog_encoder" -folder_perm = "755" diff --git a/ansible/roles/common/templates/heka-horizon.toml.j2 b/ansible/roles/common/templates/heka-horizon.toml.j2 deleted file mode 100644 index b447e9938a..0000000000 --- a/ansible/roles/common/templates/heka-horizon.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[horizon_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_horizon_apache_log.lua" - [horizon_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[horizon_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "horizon_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'horizon/(?P<Service>horizon.*)\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["Service"] diff --git a/ansible/roles/common/templates/heka-keepalived.toml.j2 b/ansible/roles/common/templates/heka-keepalived.toml.j2 deleted file mode 100644 index 2159982343..0000000000 --- a/ansible/roles/common/templates/heka-keepalived.toml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# Keepalived writes its logs to Syslog, so the generic -# Syslog input set in heka-global.toml.j2 is used. - -[keepalived_file_output] -type = "FileOutput" -message_matcher = "Fields[programname] =~ /(?i:keepalived)/" -path = "/var/log/kolla/keepalived/keepalived.log" -encoder = "syslog_encoder" -folder_perm = "755" diff --git a/ansible/roles/common/templates/heka-keystone.toml.j2 b/ansible/roles/common/templates/heka-keystone.toml.j2 deleted file mode 100644 index 19cc69c6d9..0000000000 --- a/ansible/roles/common/templates/heka-keystone.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[keystone_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_keystone_apache_log.lua" - [keystone_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[keystone_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "keystone_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'keystone/keystone-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["keystone-apache-", "Service"] diff --git a/ansible/roles/common/templates/heka-mariadb.toml.j2 b/ansible/roles/common/templates/heka-mariadb.toml.j2 deleted file mode 100644 index 07d7105410..0000000000 --- a/ansible/roles/common/templates/heka-mariadb.toml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[mariadb_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_mysql_log.lua" - -[mariadb_logstreamer_input] -type = "LogstreamerInput" -decoder = "mariadb_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'mariadb/mariadb\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ['mariadb'] diff --git a/ansible/roles/common/templates/heka-openstack.toml.j2 b/ansible/roles/common/templates/heka-openstack.toml.j2 deleted file mode 100644 index 2715965089..0000000000 --- a/ansible/roles/common/templates/heka-openstack.toml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[openstack_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_openstack_log.lua" - -[openstack_logstreamer_input] -type = "LogstreamerInput" -decoder = "openstack_log_decoder" -log_directory = "/var/log/kolla" -file_match = '(?P<Service>cloudkitty|designate|nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila|octavia|searchlight|senlin|sahara|tacker)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["Service", "_", "Program"] diff --git a/ansible/roles/common/templates/heka-panko.toml.j2 b/ansible/roles/common/templates/heka-panko.toml.j2 deleted file mode 100644 index 367db41813..0000000000 --- a/ansible/roles/common/templates/heka-panko.toml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[panko_apache_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_panko_apache_log.lua" - [panko_apache_log_decoder.config] - apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"' - -[panko_apache_logstreamer_input] -type = "LogstreamerInput" -decoder = "panko_apache_log_decoder" -log_directory = "/var/log/kolla" -file_match = 'panko/panko-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["panko-apache-", "Service"] diff --git a/ansible/roles/common/templates/heka-rabbitmq.toml.j2 b/ansible/roles/common/templates/heka-rabbitmq.toml.j2 deleted file mode 100644 index d2c49aa1e7..0000000000 --- a/ansible/roles/common/templates/heka-rabbitmq.toml.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[rabbitmq_log_decoder] -type = "SandboxDecoder" -filename = "lua_decoders/os_rabbitmq_log.lua" - -[rabbitmq_log_splitter] -type = "RegexSplitter" -delimiter = '\n\n(=[^=]+====)' -delimiter_eol = false -deliver_incomplete_final = true - -[rabbitmq_logstreamer_input] -type = "LogstreamerInput" -decoder = "rabbitmq_log_decoder" -splitter = "rabbitmq_log_splitter" -log_directory = "/var/log/kolla" -file_match = 'rabbitmq/(?P<Service>rabbit.*)\.log\.?(?P<Seq>\d*)$' -priority = ["^Seq"] -differentiator = ["Service"] diff --git a/ansible/roles/common/templates/heka-swift.toml.j2 b/ansible/roles/common/templates/heka-swift.toml.j2 deleted file mode 100644 index c70b1631db..0000000000 --- a/ansible/roles/common/templates/heka-swift.toml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# Swift writes its logs to Syslog, so the generic Syslog input set in -# heka-global.toml.j2 is used. - -[{{ item.name }}_file_output] -type = "FileOutput" -message_matcher = "Fields[programname] == '{{ item.name }}'" -path = "/var/log/kolla/swift/{{ item.name }}.log" -encoder = "syslog_encoder" -folder_perm = "755" diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2 deleted file mode 100644 index f7b583f86b..0000000000 --- a/ansible/roles/common/templates/heka.json.j2 +++ /dev/null @@ -1,153 +0,0 @@ -{% set swift_services = ["swift-account-auditor", "swift-account-reaper", "swift-account-replicator", "swift-account-server", "swift-container-auditor", "swift-container-replicator", "swift-container-server", "swift-container-updater", "swift-object-auditor", "swift-object-expirer", "swift-object-replicator", "swift-object-server", "swift-object-updater", "swift-proxy-server", "swift-rsyncd"] %} -{ - "command": "/usr/bin/hekad -config=/etc/heka/", - "config_files": [ - { - "source": "{{ container_config_directory }}/heka-elasticsearch.toml", - "dest": "/etc/heka/heka-elasticsearch.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_central_logging | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-global.toml", - "dest": "/etc/heka/heka-global.toml", - "owner": "heka", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/heka-aodh.toml", - "dest": "/etc/heka/heka-aodh.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_aodh | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-gnocchi.toml", - "dest": "/etc/heka/heka-gnocchi.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_gnocchi | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-grafana.toml", - "dest": "/etc/heka/heka-grafana.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_grafana | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-barbican.toml", - "dest": "/etc/heka/heka-barbican.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_barbican | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-ceilometer.toml", - "dest": "/etc/heka/heka-ceilometer.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_ceilometer | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-custom.toml", - "dest": "/etc/heka/heka-custom.toml", - "owner": "heka", - "perm": "0600", - "optional": true - }, - { - "source": "{{ container_config_directory }}/heka-haproxy.toml", - "dest": "/etc/heka/heka-haproxy.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_haproxy | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-keepalived.toml", - "dest": "/etc/heka/heka-keepalived.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_haproxy | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-rabbitmq.toml", - "dest": "/etc/heka/heka-rabbitmq.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_rabbitmq | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-openstack.toml", - "dest": "/etc/heka/heka-openstack.toml", - "owner": "heka", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/heka-mariadb.toml", - "dest": "/etc/heka/heka-mariadb.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_mariadb | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-panko.toml", - "dest": "/etc/heka/heka-panko.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_panko | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-keystone.toml", - "dest": "/etc/heka/heka-keystone.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_keystone | bool) | string | lower }} - }, - { - "source": "{{ container_config_directory }}/heka-horizon.toml", - "dest": "/etc/heka/heka-horizon.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_horizon | bool) | string | lower }} - }, -{% for swift_service in swift_services %} - { - "source": "{{ container_config_directory }}/heka-{{ swift_service }}.toml", - "dest": "/etc/heka/heka-{{ swift_service }}.toml", - "owner": "heka", - "perm": "0600", - "optional": {{ (not enable_swift | bool) | string | lower }} - }{% if not loop.last %},{% endif %} -{% endfor %} - - ], - "permissions": [ - { - "path": "/var/cache/hekad", - "owner": "heka:heka", - "recurse": true - }, - { - "path": "/var/lib/kolla/heka", - "owner": "heka:heka", - "recurse": true - }, - { - "path": "/var/log/kolla/haproxy", - "owner": "heka:kolla", - "recurse": true - }, - { - "path": "/var/log/kolla/keepalived", - "owner": "heka:kolla", - "recurse": true - }, - { - "path": "/var/log/kolla/swift", - "owner": "heka:kolla", - "recurse": true - } - ] -} diff --git a/ansible/roles/common/templates/td-agent.conf.j2 b/ansible/roles/common/templates/td-agent.conf.j2 new file mode 100644 index 0000000000..42c8df0769 --- /dev/null +++ b/ansible/roles/common/templates/td-agent.conf.j2 @@ -0,0 +1,4 @@ +@include input/*.conf +@include filter/*.conf +@include format/*.conf +@include output/*.conf diff --git a/ansible/roles/haproxy/tasks/start.yml b/ansible/roles/haproxy/tasks/start.yml index 692de8591c..aa76e2b4a2 100644 --- a/ansible/roles/haproxy/tasks/start.yml +++ b/ansible/roles/haproxy/tasks/start.yml @@ -10,7 +10,6 @@ - "{{ node_config_directory }}/haproxy/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "haproxy_socket:/var/lib/kolla/haproxy/" - - "heka_socket:/var/lib/kolla/heka/" - name: Starting keepalived container kolla_docker: @@ -24,7 +23,6 @@ - "/etc/localtime:/etc/localtime:ro" - "/lib/modules:/lib/modules:ro" - "haproxy_socket:/var/lib/kolla/haproxy/" - - "heka_socket:/var/lib/kolla/heka/" - name: Ensuring latest haproxy config is used command: docker exec haproxy /usr/local/bin/kolla_ensure_haproxy_latest_config diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index ee83f3f37b..286cf48a0b 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -4,7 +4,7 @@ global user haproxy group haproxy daemon - log /var/lib/kolla/heka/log local0 + log {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ fluentd_syslog_port }} local1 maxconn 4000 stats socket /var/lib/kolla/haproxy/haproxy.sock {% if kolla_enable_tls_external | bool %} diff --git a/ansible/roles/swift/tasks/start.yml b/ansible/roles/swift/tasks/start.yml index 39b08f174b..b83b8c962d 100644 --- a/ansible/roles/swift/tasks/start.yml +++ b/ansible/roles/swift/tasks/start.yml @@ -41,7 +41,6 @@ - "{{ node_config_directory }}/swift-rsyncd/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server'] @@ -57,7 +56,6 @@ - "{{ node_config_directory }}/swift-account-server/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-auditor container @@ -71,7 +69,6 @@ - "{{ node_config_directory }}/swift-account-auditor/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-replicator container @@ -85,7 +82,6 @@ - "{{ node_config_directory }}/swift-account-replicator/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-reaper container @@ -99,7 +95,6 @@ - "{{ node_config_directory }}/swift-account-reaper/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-container-server container @@ -113,7 +108,6 @@ - "{{ node_config_directory }}/swift-container-server/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-auditor container @@ -127,7 +121,6 @@ - "{{ node_config_directory }}/swift-container-auditor/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-replicator container @@ -141,7 +134,6 @@ - "{{ node_config_directory }}/swift-container-replicator/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-updater container @@ -155,7 +147,6 @@ - "{{ node_config_directory }}/swift-container-updater/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-object-server container @@ -197,7 +188,6 @@ - "{{ node_config_directory }}/swift-object-replicator/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-updater container @@ -211,7 +201,6 @@ - "{{ node_config_directory }}/swift-object-updater/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-expirer container @@ -225,7 +214,6 @@ - "{{ node_config_directory }}/swift-object-expirer/:{{ container_config_directory }}/:ro" - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-proxy-server container @@ -238,5 +226,4 @@ - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-proxy-server/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - - "heka_socket:/var/lib/kolla/heka/" when: inventory_hostname in groups['swift-proxy-server'] diff --git a/ansible/roles/swift/templates/account.conf.j2 b/ansible/roles/swift/templates/account.conf.j2 index 37bae3e5d0..753423db96 100644 --- a/ansible/roles/swift/templates/account.conf.j2 +++ b/ansible/roles/swift/templates/account.conf.j2 @@ -3,7 +3,8 @@ bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4' bind_port = {{ swift_account_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false -log_address = /var/lib/kolla/heka/log +log_udp_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} +log_udp_port = {{ fluentd_syslog_port }} log_name = {{ service_name }} log_facility = LOG_LOCAL0 log_level = INFO diff --git a/ansible/roles/swift/templates/container.conf.j2 b/ansible/roles/swift/templates/container.conf.j2 index 563cdcc2f6..6867ca8499 100644 --- a/ansible/roles/swift/templates/container.conf.j2 +++ b/ansible/roles/swift/templates/container.conf.j2 @@ -3,7 +3,8 @@ bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4' bind_port = {{ swift_container_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false -log_address = /var/lib/kolla/heka/log +log_udp_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} +log_udp_port = {{ fluentd_syslog_port }} log_name = {{ service_name }} log_facility = LOG_LOCAL0 log_level = INFO diff --git a/ansible/roles/swift/templates/object.conf.j2 b/ansible/roles/swift/templates/object.conf.j2 index 2fdad13509..6fa1c77d1a 100644 --- a/ansible/roles/swift/templates/object.conf.j2 +++ b/ansible/roles/swift/templates/object.conf.j2 @@ -4,7 +4,8 @@ bind_port = {{ swift_object_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false -log_address = /var/lib/kolla/heka/log +log_udp_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} +log_udp_port = {{ fluentd_syslog_port }} log_name = {{ service_name }} log_facility = LOG_LOCAL0 log_level = INFO diff --git a/ansible/roles/swift/templates/proxy-server.conf.j2 b/ansible/roles/swift/templates/proxy-server.conf.j2 index 596d96cb43..6177700f89 100644 --- a/ansible/roles/swift/templates/proxy-server.conf.j2 +++ b/ansible/roles/swift/templates/proxy-server.conf.j2 @@ -2,7 +2,8 @@ bind_ip = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} bind_port = {{ swift_proxy_server_port }} -log_address = /var/lib/kolla/heka/log +log_udp_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} +log_udp_port = {{ fluentd_syslog_port }} log_name = {{ service_name }} log_facility = LOG_LOCAL0 log_level = INFO diff --git a/releasenotes/notes/add-fluentd-role-bc3f4976c807a50b.yaml b/releasenotes/notes/add-fluentd-role-bc3f4976c807a50b.yaml new file mode 100644 index 0000000000..1cea8af500 --- /dev/null +++ b/releasenotes/notes/add-fluentd-role-bc3f4976c807a50b.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - Heka is deprecated and is replaced with Fluentd +features: + - Add Fluentd role, Fluentd is an open source data collector for unified logging layer diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index ac9de5d8eb..d6d7473ef2 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -68,7 +68,7 @@ install_type = ${INSTALL_TYPE} profile = gate [profiles] -gate = cron,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,heka,horizon +gate = cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon EOF if [[ "${DISTRO}" == "Debian" ]]; then