Tail other log files for errors in collectd

Count Errors in log files for:
* Glance
* Keystone (httpd)
* Ceilometer
* Gnocchi
* Swift
* L3 Agent and Metadata Agent for DVR Computes

Also modifies how grafana presents tail data to deal with how Swift
uses single log file for all processes/services.

Logs collectd to its own log too.

Co-Authored-By: Sai Sindhur Malleni <smalleni@redhat.com>
Change-Id: I2fd19c3fb802e7f73d9ae7219e5a59e6eb61aa5b

ansible/install/roles/collectd-generic/templates/baremetal.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df

ansible/install/roles/collectd-generic/templates/graphite.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df

ansible/install/roles/collectd-generic/templates/guest.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin df

ansible/install/roles/collectd-openstack/templates/ceph.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack

ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
@@ -134,6 +138,33 @@ PreCacheChain "PreCache"
       Instance "total"
     </Match>
   </File>
+  <File "/var/log/neutron/l3-agent.log">
+    Instance "neutron-l3-agent"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/neutron/metadata-agent.log">
+    Instance "neutron-metadata-agent"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/compute.log">
+    Instance "ceilometer-compute"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
 </Plugin>
 
 # Include other collectd configuration files

ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack
@@ -228,6 +232,25 @@ PreCacheChain "PreCache"
       Instance "total"
     </Match>
   </File>
+  <File "/var/log/httpd/keystone_wsgi_admin_error.log">
+    Instance "keystone-wsgi-admin"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/keystone_wsgi_main_error.log">
+    Instance "keystone-wsgi-main"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
 
   <File "/var/log/nova/nova-api.log">
     Instance "nova-api"
@@ -247,6 +270,24 @@ PreCacheChain "PreCache"
       Instance "total"
     </Match>
   </File>
+  <File "/var/log/nova/nova-consoleauth.log">
+    Instance "nova-consoleauth"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/nova/nova-novncproxy.log">
+    Instance "nova-novncproxy"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
   <File "/var/log/nova/nova-scheduler.log">
     Instance "nova-scheduler"
     <Match>
@@ -330,6 +371,123 @@ PreCacheChain "PreCache"
       Instance "total"
     </Match>
   </File>
+
+  <File "/var/log/glance/api.log">
+    Instance "glance-api"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/glance/registry.log">
+    Instance "glance-registry"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
+  # Swift logs all into the same file
+  <File "/var/log/swift/swift.log">
+    Instance "swift"
+    <Match>
+      Regex "account-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "account-server"
+    </Match>
+    <Match>
+      Regex "container-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "container-server"
+    </Match>
+    <Match>
+      Regex "object-server: ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "object-server"
+    </Match>
+  </File>
+
+  <File "/var/log/ceilometer/agent-notification.log">
+    Instance "ceilometer-agent-notification"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/central.log">
+    Instance "ceilometer-central"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/ceilometer/collector.log">
+    Instance "ceilometer-collector"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/ceilometer_wsgi_error.log">
+    Instance "ceilometer-wsgi-api"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
+  <File "/var/log/gnocchi/app.log">
+    Instance "gnocchi-app"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/gnocchi/metricd.log">
+    Instance "gnocchi-metricd"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/gnocchi/statsd.log">
+    Instance "gnocchi-statsd"
+    <Match>
+      Regex " ERROR "
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+  <File "/var/log/httpd/gnocchi_wsgi_error.log">
+    Instance "gnocchi-wsgi-api"
+    <Match>
+      Regex ":error"
+      DSType "CounterInc"
+      Type "counter"
+      Instance "total"
+    </Match>
+  </File>
+
 </Plugin>
 
 # Include other collectd configuration files

ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2

@@ -7,10 +7,14 @@ Interval {{collectd_interval}}
 # Hostname for this machine, if not defined, use gethostname(2) system call
 Hostname "{{inventory_hostname}}"
 
-# Allow collectd to log
-LoadPlugin syslog
-
 # Loaded Plugins:
+LoadPlugin "logfile"
+<Plugin "logfile">
+  LogLevel "info"
+  File "/var/log/collectd.log"
+  Timestamp true
+</Plugin>
+
 LoadPlugin write_graphite
 LoadPlugin cpu
 LoadPlugin conntrack

ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2

@@ -3787,7 +3787,7 @@
             "targets": [
               {
                 "refId": "A",
-                "target": "aliasByNode($Cloud.$Node.tail-*.counter-total, 2)"
+                "target": "aliasByNode(aliasSub(aliasSub(aliasSub($Cloud.$Node.tail-*.*, 'counter-', ''), 'total', ''), 'tail-', ''), 2, 3)"
               }
             ],
             "timeFrom": null,