Files
browbeat/ansible/install/roles/collectd/templates/controller.collectd.conf.j2
Sai Sindhur Malleni e2686f9a5f Add ovs flows monitoring
This commit

1. Adds a custom plugin to get number of ovs flows on br-int
2. Adds a row/panel to the OpenStack General System Performance
   dashboard to visualize this
3. Adds support for containerized/non-containerized collectd
4. In case of containerized, ovs is installed on container and
   /etc/openvswitch and /var/run/openvswitch from host are mounted
   into container
5. The custom collectd exec plugin cannot be run as root, hence stack
   user is created within container and the plugin is run as stack user.
   In the case of non-containerized collectd, heat-admin user existing on
   overcloud nodes is used
6. Controllers and Computes are monitored

Change-Id: I4687bd6aa0168c0b4bab72cf8eaca5f92c42b635
2020-04-06 18:56:21 +00:00

607 lines
22 KiB
Django/Jinja

# Installed by Browbeat Ansible Installer
# Config type: {{config_type}}
# Interval default is 10s
Interval {{collectd_interval}}
# Hostname for this machine, if not defined, use gethostname(2) system call
Hostname "{{inventory_hostname}}"
# Loaded Plugins:
LoadPlugin "logfile"
<Plugin "logfile">
File "/var/log/collectd.log"
LogLevel "info"
PrintSeverity true
Timestamp true
</Plugin>
LoadPlugin write_graphite
{% if apache_controller_collectd_plugin %}
LoadPlugin apache
{% endif %}
{% if ceph_controller_collectd_plugin %}
{% if inventory_hostname == groups['controller'][0] %}
LoadPlugin ceph
{% endif %}
{% endif %}
LoadPlugin cpu
LoadPlugin conntrack
{% if keystone_overcloud_collectd_plugin %}
{%if inventory_hostname == groups['controller'][0] %}
LoadPlugin dbi
{% endif %}
{% endif %}
LoadPlugin df
LoadPlugin disk
LoadPlugin exec
LoadPlugin interface
LoadPlugin irq
LoadPlugin load
LoadPlugin match_regex
LoadPlugin memory
LoadPlugin mysql
LoadPlugin numa
LoadPlugin processes
<LoadPlugin python>
Globals true
</LoadPlugin>
LoadPlugin swap
LoadPlugin tail
LoadPlugin uptime
{% if ping_plugin %}
LoadPlugin ping
{% endif %}
PreCacheChain "PreCache"
<Chain "PreCache">
<Rule "ignore_tap">
<Match "regex">
Plugin "^interface$"
PluginInstance "^tap*"
</Match>
Target "stop"
</Rule>
<Rule "ignore_interfaces_q">
<Match "regex">
Plugin "^interface$"
PluginInstance "^q.*"
</Match>
Target "stop"
</Rule>
Target "return"
</Chain>
# Graphite Host Configuration
<Plugin write_graphite>
<Carbon>
Host "{{graphite_host}}"
Port "{{collectd_write_graphite_port}}"
Prefix "{{graphite_prefix}}."
Protocol "tcp"
LogSendErrors true
StoreRates true
AlwaysAppendDS false
EscapeCharacter "_"
</Carbon>
</Plugin>
{% if apache_controller_collectd_plugin %}
<Plugin apache>
<Instance "status">
URL "http://127.0.0.1:{{apache_controller_mod_status_port}}/mod_status?auto"
</Instance>
</Plugin>
{% endif %}
{% if ceph_controller_collectd_plugin %}
{% if inventory_hostname == groups['controller'][0] %}
<Plugin ceph>
LongRunAvgLatency false
ConvertSpecialMetricTypes true
<Daemon "mon.{{inventory_hostname}}">
SocketPath "/var/run/ceph/ceph-mon.{{inventory_hostname}}.asok"
</Daemon>
</Plugin>
{% endif %}
{% endif %}
{% if ceph_controller_collectd_radosbench_plugin or ceph_controller_collectd_mon_plugin or ceph_controller_collectd_osd_plugin or ceph_controller_collectd_pg_plugin or ceph_controller_collectd_pool_plugin %}
{% if inventory_hostname == groups['controller'][0] %}
<Plugin python>
LogTraces true
Interactive false
ModulePath "/usr/local/bin/"
Import "collectd_ceph_storage"
<Module collectd_ceph_storage>
CephCluster "ceph"
CephRadosBench {{ceph_controller_collectd_radosbench_plugin}}
CephRadosBenchInterval {{ceph_controller_collectd_radosbench_interval}}
CephMONStats {{ceph_controller_collectd_mon_plugin}}
CephMONStatsInterval {{ceph_controller_collectd_mon_interval}}
CephOSDStats {{ceph_controller_collectd_osd_plugin}}
CephOSDStatsInterval {{ceph_controller_collectd_osd_interval}}
CephPGStats {{ceph_controller_collectd_pg_plugin}}
CephPGStatsInterval {{ceph_controller_collectd_pg_interval}}
CephPoolStats {{ceph_controller_collectd_pool_plugin}}
CephPoolStatsInterval {{ceph_controller_collectd_pool_interval}}
</Module>
</Plugin>
{% endif %}
{% endif %}
{% if keystone_overcloud_collectd_plugin %}
{%if inventory_hostname == groups['controller'][0] %}
<Plugin dbi>
<Query token_count>
Statement "select count(*) as count from token"
<Result>
Type gauge
InstancePrefix "token"
ValuesFrom "count"
</Result>
</Query>
<Database "keystone">
Driver "mysql"
DriverOption "host" "localhost"
DriverOption "dbname" "keystone"
DriverOption "username" "root"
DriverOption "password" "{{mysql_root_password.stdout}}"
DriverOption "mysql_unix_socket" "/var/lib/mysql/mysql.sock"
Query token_count
</Database>
</Plugin>
{% else %}
# Token Count plugin installed and enabled on {{groups['controller'][0]}}
{% endif %}
{% endif %}
<Plugin df>
FSType anon_inodefs
FSType bdev
FSType cgroup
FSType cpuset
FSType debugfs
FSType devpts
FSType devtmpfs
FSType ecryptfs
FSType fuse
FSType fusectl
FSType hugetlbfs
FSType mqueue
FSType nfs
FSType nfs4
FSType nfsd
FSType overlay
FSType pipefs
FSType proc
FSType pstore
FSType ramfs
#FSType rootfs
FSType rpc_pipefs
FSType securityfs
FSType sockfs
FSType sysfs
FSType tmpfs
FSType vboxsf
MountPoint "/^/var/lib/docker/.*/"
IgnoreSelected true
ValuesPercentage true
ReportInodes true
</Plugin>
{%if disk_controller_collectd_plugin %}
<Plugin disk>
Disk "/^[hsv]d[a-z]+[0-9]?$/"
Disk "/^nvm/"
IgnoreSelected false
</Plugin>
{% endif %}
{% if ovs_flows_monitoring %}
<Plugin exec>
Exec stack "/usr/local/bin/ovs_flows.sh"
</Plugin>
{% endif %}
{%if iostat_controller_collectd_plugin %}
<Plugin python>
ModulePath "/usr/local/bin/"
Import "collectd_iostat_python"
<Module collectd_iostat_python>
Path "/usr/bin/iostat"
Interval {{iostat_controller_collectd_interval}}
IostatInterval 2
Count 2
Verbose false
NiceNames false
PluginName collectd_iostat_python
</Module>
</Plugin>
{% endif %}
{%if gnocchi_status_controller_collectd_plugin %}
{%if inventory_hostname == groups['controller'][0] %}
<Plugin python>
ModulePath "/usr/local/bin/"
LogTraces true
Interactive false
Import "collectd_gnocchi_status"
<Module collectd_gnocchi_status>
interval {{gnocchi_status_controller_collectd_interval}}
</Module>
</Plugin>
{% else %}
# Gnocchi status plugin installed and enabled on {{groups['controller'][0]}}
{% endif %}
{% endif %}
<Plugin mysql>
<Database "overcloud">
Host "localhost"
User "root"
Password "{{mysql_root_password.stdout}}"
Socket "/var/lib/mysql/mysql.sock"
InnodbStats true
</Database>
</Plugin>
# (akrzos) Including the version of OpenStack that the process was verified as running after
# OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
# A Minus before the version means the process was not found in that version. (Ex -10)
<Plugin processes>
# Aodh (OpenStack Installed)
ProcessMatch "aodh-evaluator" "aodh-evaluator" # 10,11,12,13
ProcessMatch "aodh-listener" "aodh-listener" # 10,11,12,13
ProcessMatch "aodh-notifier" "aodh-notifier" # 10,11,12,13
ProcessMatch "aodh_wsgi" "aodh_wsgi.*-DFOREGROUND" # 11,12,13
# Barbican (OpenStack Installed)
ProcessMatch "barbican_wsgi" "barbican_wsgi.*-DFOREGROUND" # 13
ProcessMatch "barbican-keystone-listener" "python.+barbican-keystone-listener" # 13
ProcessMatch "barbican-worker" "python.+barbican-worker" # 13
# Ceilometer (OpenStack Installed)
ProcessMatch "ceilometer-agent-notification" "ceilometer-agent-notification" # 10,11,12,13
ProcessMatch "ceilometer-collector" "ceilometer-collector" # 10,11,-12,-13
ProcessMatch "ceilometer-polling" "ceilometer-polling" # 10,11,12,13
ProcessMatch "ceilometer_wsgi" "ceilometer_wsgi.*-DFOREGROUND" # 11,-12,-13
# Ceph (OpenStack Installed)
# When CephStorage nodes deployed
ProcessMatch "ceph-mon" "^/usr/bin/ceph-mon" # -10,-11,-12,-13
# Cinder (OpenStack Installed)
ProcessMatch "cinder-api" "python.+cinder-api" # 10,-11,-12
ProcessMatch "cinder-scheduler" "python.+cinder-scheduler" # 10,11,12,13
ProcessMatch "cinder-volume" "python.+cinder-volume" # 10,11,12,13
ProcessMatch "cinder_wsgi" "cinder_wsgi.*-DFOREGROUND" # 11,12,13
# Collectd (Browbeat Installed)
ProcessMatch "collectd" "/usr/sbin/collectd"
# Docker (OpenStack Installed)
ProcessMatch "dockerd-current" "dockerd-current" # 12,13
# Pacemaker / Corosync (OpenStack Installed)
ProcessMatch "attrd" "/usr/libexec/pacemaker/attrd" # 10,11,12,13
ProcessMatch "cib" "/usr/libexec/pacemaker/cib" # 10,11,12,13
Process "corosync" # 10,11,12,13
ProcessMatch "crmd" "/usr/libexec/pacemaker/crmd" # 10,11,12,13
ProcessMatch "lrmd" "/usr/libexec/pacemaker/lrmd" # 10,11,12,13
ProcessMatch "pacemakerd" "/usr/sbin/pacemakerd" # 10,11,12,13
ProcessMatch "pcsd" "^/usr/bin/ruby.+/usr/lib/pcsd" # 10,11,12,13
ProcessMatch "pengine" "/usr/libexec/pacemaker/pengine" # 10,11,12,13
ProcessMatch "stonithd" "/usr/libexec/pacemaker/stonithd" # 10,11,12,13
# Everything Else (OpenStack Installed)
# (Processes displayed under "Everything Else" on Grafana Dashboards)
ProcessMatch "dnsmasq" "^dnsmasq.+" # 10,11,12
ProcessMatch "haproxy" "/usr/sbin/haproxy.+/etc/haproxy/haproxy.cfg" # 10,11,12,13
Process "httpd" # 10,11,12,13
Process "keepalived" # 10,11,12
Process "memcached" # 10,11,12,13
Process "mongod" # 10,11,-12,-13
ProcessMatch "mysqld" "/usr/libexec/mysqld" # 10,11,12,13
ProcessMatch "rabbitmq" "/usr/lib64/erlang/erts-.+/bin/beam.smp" # 10,11,12,13
Process "redis-server" # 10,11,12,13
ProcessMatch "karaf" "java.+karaf" # ODL Specific
# Glance (OpenStack Installed)
ProcessMatch "glance-api" "python.+glance-api" # 10,11,12,13
ProcessMatch "glance-registry" "python.+glance-registry" # 10,-11,-12,-13
# Gnocchi (OpenStack Installed)
ProcessMatch "gnocchi-metricd-master" "gnocchi-metricd.*master" # 11,12,13
ProcessMatch "gnocchi-metricd-scheduler" "gnocchi-metricd.*scheduler" # 10,11,-12,-13
ProcessMatch "gnocchi-metricd-processing" "gnocchi-metricd.*processing" # 10,11,12,13
ProcessMatch "gnocchi-metricd-reporting" "gnocchi-metricd.*reporting" # 10,11,12,13
ProcessMatch "gnocchi-metricd-janitor" "gnocchi-metricd.*janitor" # 10,11,12,13
ProcessMatch "gnocchi-metricd" "gnocchi-metricd " # 10(Old proctitle)
ProcessMatch "gnocchi-statsd" "python.+gnocchi-statsd" # 10,11,12,13
ProcessMatch "gnocchi_wsgi" "gnocchi_wsgi.*-DFOREGROUND" # 11,12,13
# Heat (OpenStack Installed)
ProcessMatch "heat-api" "python.+heat-api --config-file" # 10,11,-12,-13
ProcessMatch "heat-api-cfn" "python.+heat-api-cfn" # 10,11,-12,-13
ProcessMatch "heat-api-cloudwatch" "python.+heat-api-cloudwatch" # 10,11,-12,-123
ProcessMatch "heat_api_cfn" "heat_api_cfn_ws" # 12,13
ProcessMatch "heat_api_cloudwatch" "heat_api_cloudw" # 12,-13
ProcessMatch "heat_api_wsgi" "heat_api_wsgi" # 12,13
ProcessMatch "heat-engine" "python.+heat-engine" # 10,11,12,13
# Horizon (OpenStack Installed)
ProcessMatch "horizon" "horizon" # 13
# Keystone (OpenStack Installed)
ProcessMatch "keystone-admin" "keystone-admin.*-DFOREGROUND" # 10,11,12,13
ProcessMatch "keystone-main" "keystone-main.*-DFOREGROUND" # 10,11,12,13
ProcessMatch "keystone" "keystone.*-DFOREGROUND" # 16
# Starting Pike, fernet tokens are default thus token_flush not needed
ProcessMatch "keystone-token-flush" "keystone-manage.*token_flush" # 10,11,-12,-13
# Neutron (OpenStack Installed)
ProcessMatch "neutron-dhcp-agent" "python.+neutron-dhcp-agent" # 10,11,12,13
ProcessMatch "neutron-l3-agent" "python.+neutron-l3-agent" # 10,11,12,13
ProcessMatch "neutron-metadata-agent" "python.+neutron-metadata-agent" # 10,11,12,13
ProcessMatch "neutron-ns-metadata-proxy" "python.+neutron-ns-metadata-proxy" # 10,11
ProcessMatch "neutron-openvswitch-agent" "python.+neutron-openvswitch-agent" # 10,11,12,13
ProcessMatch "neutron-rootwrap-daemon" "python.+neutron-rootwrap-daemon" # 10,11,12,13
ProcessMatch "neutron-server" "python.+neutron-server" # 10,11,12,13
ProcessMatch "neutron-keepalived-state-change" "python.+neutron-keepalived-state-change" #For HA router
# Nova (OpenStack Installed)
ProcessMatch "nova-api" "python.+nova-api$" # 10,11,-12,-13
ProcessMatch "nova-api-metadata" "python.+nova-api-metadata" # 12,13
ProcessMatch "nova_api_wsgi" "nova_api_wsgi" # 12,13
ProcessMatch "nova-conductor" "python.+nova-conductor" # 10,11,12,13
ProcessMatch "nova-consoleauth" "python.+nova-consoleauth" # 10,11,12,13
ProcessMatch "nova-novncproxy" "python.+nova-novncproxy" # 10,11,12,13
ProcessMatch "nova-scheduler" "python.+nova-scheduler" # 10,11,12,13
ProcessMatch "placement_wsgi" "placement_wsgi.*-DFOREGROUND" # 11,12,13
ProcessMatch "octavia-housekeeping" "python.+octavia-housekeeping" # 13
ProcessMatch "octavia-health-manager" "python.+octavia-health-manager" # 13
ProcessMatch "octavia-api" "python.+octavia-api" # 13
ProcessMatch "octavia-worker" "octavia-worker --config-file" # 13
# OVS (OpenStack Installed)
ProcessMatch "ovs-vswitchd" "ovs-vswitchd.+openvswitch" # 10,11,12,13
ProcessMatch "ovsdb-client" "ovsdb-client" # 10,11,12,13
ProcessMatch "ovsdb-server" "ovsdb-server.+openvswitch" # 10,11,12,13
ProcessMatch "ovn-northd" "ovn-northd.+openvswitch" # 9,10
ProcessMatch "ovn-controller" "ovn-controller.+openvswitch" # 9,10
ProcessMatch "ovn-controller-vtep" "ovn-controller-vtep.+openvswitch" # 9,10
# Panko (OpenStack Installed)
ProcessMatch "panko_wsgi" "panko_wsgi.*-DFOREGROUND" # 11,12,13
# Swift (OpenStack Installed)
ProcessMatch "swift-account-auditor" "python.+swift-account-auditor" # 10,11,12,13
ProcessMatch "swift-account-reaper" "python.+swift-account-reaper" # 10,11,12,13
ProcessMatch "swift-account-replicator" "python.+swift-account-replicator" # 10,11,12,13
ProcessMatch "swift-account-server" "python.+swift-account-server" # 10,11,12,13
ProcessMatch "swift-container-auditor" "python.+swift-container-auditor" # 10,11,12,13
ProcessMatch "swift-container-replicator" "python.+swift-container-replicator" # 10,11,12,13
ProcessMatch "swift-container-server" "python.+swift-container-server" # 10,11,12,13
ProcessMatch "swift-container-updater" "python.+swift-container-updater" # 10,11,12,13
ProcessMatch "swift-object-auditor" "python.+swift-object-auditor" # 10,11,12,13
ProcessMatch "swift-object-expirer" "python.+swift-object-expirer" # 11,12,13
ProcessMatch "swift-object-replicator" "python.+swift-object-replicator" # 10,11,12,13
ProcessMatch "swift-object-server" "python.+swift-object-server" # 10,11,12,13
ProcessMatch "swift-object-updater" "python.+swift-object-updater" # 10,11,12,13
ProcessMatch "swift-proxy-server" "python.+swift-proxy-server" # 10,11,12,13
</Plugin>
{%if rabbitmq_controller_collectd_plugin %}
{%if inventory_hostname == groups['controller'][0] %}
<Plugin python>
ModulePath "/usr/local/bin/"
LogTraces true
Interactive false
Import "collectd_rabbitmq_monitoring"
<Module collectd_rabbitmq_monitoring>
interval {{rabbitmq_controller_collectd_interval}}
host "{{inventory_hostname}}.internalapi.localdomain"
port 15672
username {{controller0_rabbitmq_username.stdout}}
password {{controller0_rabbitmq_password.stdout}}
message_count {% for a_queue in controller_monitored_queues %}"{{a_queue}}" {% endfor %}
</Module>
</Plugin>
{% else %}
# Rabbitmq plugin installed and enabled on {{groups['controller'][0]}}
{% endif %}
{% endif %}
{%if swift_stat_controller_collectd_plugin %}
{%if inventory_hostname == groups['controller'][0] %}
<Plugin python>
ModulePath "/usr/local/bin/"
LogTraces true
Interactive false
Import "collectd_swift_stat"
<Module collectd_swift_stat>
Interval {{swift_stat_controller_collectd_interval}}
Prefix "gnocchi"
AuthURL "{{controller0_gnocchi_swift_authurl.stdout}}"
AuthVersion "{{controller0_gnocchi_swift_authversion.stdout}}"
User "{{controller0_gnocchi_swift_user.stdout}}"
Password "{{controller0_gnocchi_swift_auth_key.stdout}}"
Project "service"
</Module>
</Plugin>
{% else %}
# swift_stat plugin installed and enabled on {{groups['controller'][0]}}
{% endif %}
{% endif %}
<Plugin swap>
ReportBytes true
ValuesPercentage true
</Plugin>
# ping plugin
{% if ping_plugin %}
{% if groups['controller'] | length > 1 %}
<Plugin "ping">
{% if inventory_hostname == groups['controller'][0] %}
Host "{{groups['controller'][1]}}"
Host "{{groups['controller'][2]}}"
{% elif inventory_hostname == groups['controller'][1] %}
Host "{{groups['controller'][0]}}"
Host "{{groups['controller'][2]}}"
{% elif inventory_hostname == groups['controller'][2] %}
Host "{{groups['controller'][0]}}"
Host "{{groups['controller'][1]}}"
{% endif %}
Interval {{ping_interval}}
</Plugin>
{% endif %}
{% endif %}
# Tail plugin configuration
<Plugin "tail">
{% if 'rabbitmq-server' in collectd_logs[config_type] | items2dict(key_name='instance',value_name='log_path') %}
<File "{{ collectd_logs[config_type] | items2dict(key_name='instance',value_name='log_path') | json_query('"rabbitmq-server"') }}">
Instance "rabbitmq"
<Match>
Regex "ERROR REPORT"
DSType "CounterInc"
Type "counter"
Instance "error"
</Match>
{%if regex_warn %}
<Match>
Regex "WARNING REPORT"
DSType "CounterInc"
Type "counter"
Instance "warn"
</Match>
{% endif %}
{%if regex_info %}
<Match>
Regex "INFO REPORT"
DSType "CounterInc"
Type "counter"
Instance "info"
</Match>
{% endif %}
</File>
{% endif %}
{% for item in collectd_logs[config_type] | rejectattr('instance', 'match', 'rabbitmq-server') | rejectattr('instance', 'match', 'swift')%}
<File "{{ item.log_path }}">
Instance "{{ item.instance }}"
<Match>
Regex " ERROR "
DSType "CounterInc"
Type "counter"
Instance "error"
</Match>
{%if regex_warn %}
<Match>
Regex " WARNING "
DSType "CounterInc"
Type "counter"
Instance "warn"
</Match>
{% endif %}
{%if regex_info %}
<Match>
Regex " INFO "
DSType "CounterInc"
Type "counter"
Instance "info"
</Match>
{% endif %}
</File>
{% endfor %}
{% if 'swift' in collectd_logs[config_type] | items2dict(key_name='instance',value_name='log_path') %}
<File "{{ collectd_logs[config_type] | items2dict(key_name='instance',value_name='log_path') | json_query('"swift"') }}">
Instance "swift"
<Match>
Regex "account-server: ERROR "
DSType "CounterInc"
Type "counter"
Instance "account-server-error"
</Match>
<Match>
Regex "container-server: ERROR "
DSType "CounterInc"
Type "counter"
Instance "container-server-error"
</Match>
<Match>
Regex "object-server: ERROR "
DSType "CounterInc"
Type "counter"
Instance "object-server-error"
</Match>
{%if regex_warn %}
<Match>
Regex "account-server: WARNING "
DSType "CounterInc"
Type "counter"
Instance "account-server-warn"
</Match>
<Match>
Regex "container-server: WARNING "
DSType "CounterInc"
Type "counter"
Instance "container-server-warn"
</Match>
<Match>
Regex "object-server: WARNING "
DSType "CounterInc"
Type "counter"
Instance "object-server-warn"
</Match>
{% endif %}
{%if regex_info %}
<Match>
Regex "account-server: INFO "
DSType "CounterInc"
Type "counter"
Instance "account-server-info"
</Match>
<Match>
Regex "container-server-info
</Match>
<Match>
Regex "object-server: INFO "
DSType "CounterInc"
Type "counter"
Instance "object-server-info"
</Match>
{% endif %}
</File>
{% endif %}
</Plugin>
{% if ovsagent_controller_monitor %}
<Plugin python>
ModulePath "/usr/local/bin/"
LogTraces true
Interactive false
Import "collectd_ovsagent"
<Module collectd_ovsagent>
interval 10
prefix ovsagent
interfaces {% for int in controller_monitored_ints %} {{int}} {% endfor %}
namespaces {% for ns in controller_monitored_ns %} {{ns}} {% endfor %}
</Module>
</Plugin>
{% endif %}
# Include other collectd configuration files
Include "/etc/collectd.d"