Tail other log files for errors in collectd

Count Errors in log files for: * Glance * Keystone (httpd) * Ceilometer * Gnocchi * Swift * L3 Agent and Metadata Agent for DVR Computes Also modifies how grafana presents tail data to deal with how Swift uses single log file for all processes/services. Logs collectd to its own log too. Co-Authored-By: Sai Sindhur Malleni <smalleni@redhat.com> Change-Id: I2fd19c3fb802e7f73d9ae7219e5a59e6eb61aa5b
2016-09-26 20:41:41 -04:00
parent 050acccf32
commit 825dda6e91
8 changed files with 231 additions and 22 deletions
--- a/ansible/install/roles/collectd-generic/templates/baremetal.collectd.conf.j2
+++ b/ansible/install/roles/collectd-generic/templates/baremetal.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df
--- a/ansible/install/roles/collectd-generic/templates/graphite.collectd.conf.j2
+++ b/ansible/install/roles/collectd-generic/templates/graphite.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df
--- a/ansible/install/roles/collectd-generic/templates/guest.collectd.conf.j2
+++ b/ansible/install/roles/collectd-generic/templates/guest.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df
--- a/ansible/install/roles/collectd-openstack/templates/ceph.collectd.conf.j2
+++ b/ansible/install/roles/collectd-openstack/templates/ceph.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
--- a/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2
+++ b/ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
@@ -134,6 +138,33 @@ PreCacheChain "PreCache"
      Instance "total"
    </Match>
  </File>
+  <File "/var/log/neutron/l3-agent.log">
+    Instance "neutron-l3-agent"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/neutron/metadata-agent.log">
+    Instance "neutron-metadata-agent"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/compute.log">
+    Instance "ceilometer-compute"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
 </Plugin>

 # Include other collectd configuration files
--- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2
+++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
@@ -228,6 +232,25 @@ PreCacheChain "PreCache"
      Instance "total"
    </Match>
  </File>
+  <File "/var/log/httpd/keystone_wsgi_admin_error.log">
+    Instance "keystone-wsgi-admin"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/keystone_wsgi_main_error.log">
+    Instance "keystone-wsgi-main"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+

  <File "/var/log/nova/nova-api.log">
    Instance "nova-api"
@@ -247,6 +270,24 @@ PreCacheChain "PreCache"
      Instance "total"
    </Match>
  </File>
+  <File "/var/log/nova/nova-consoleauth.log">
+    Instance "nova-consoleauth"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/nova/nova-novncproxy.log">
+    Instance "nova-novncproxy"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
  <File "/var/log/nova/nova-scheduler.log">
    Instance "nova-scheduler"
    <Match>
@@ -330,6 +371,123 @@ PreCacheChain "PreCache"
      Instance "total"
    </Match>
  </File>
+
+  <File "/var/log/glance/api.log">
+    Instance "glance-api"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/glance/registry.log">
+    Instance "glance-registry"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
+  # Swift logs all into the same file
+  <File "/var/log/swift/swift.log">
+    Instance "swift"
+    <Match>
+      Regex "account-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "account-server"
+    </Match>
+    <Match>
+      Regex "container-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "container-server"
+    </Match>
+    <Match>
+      Regex "object-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "object-server"
+    </Match>
+  </File>
+
+  <File "/var/log/ceilometer/agent-notification.log">
+    Instance "ceilometer-agent-notification"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/central.log">
+    Instance "ceilometer-central"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/collector.log">
+    Instance "ceilometer-collector"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/ceilometer_wsgi_error.log">
+    Instance "ceilometer-wsgi-api"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
+  <File "/var/log/gnocchi/app.log">
+    Instance "gnocchi-app"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/gnocchi/metricd.log">
+    Instance "gnocchi-metricd"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/gnocchi/statsd.log">
+    Instance "gnocchi-statsd"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/gnocchi_wsgi_error.log">
+    Instance "gnocchi-wsgi-api"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
 </Plugin>

 # Include other collectd configuration files
--- a/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2
+++ b/ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2
@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"

-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
--- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2
+++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2
@@ -3787,7 +3787,7 @@
            "targets": [
              {
                "refId": "A",
-                "target": "aliasByNode($Cloud.$Node.tail-*.counter-total, 2)"
+                "target": "aliasByNode(aliasSub(aliasSub(aliasSub($Cloud.$Node.tail-*.*, 'counter-', ''), 'total', ''), 'tail-', ''), 2, 3)"
              }
            ],
            "timeFrom": null,