Read Keystone Apache logs

This change addresses a functional regression introduced in
https://review.openstack.org/#/c/255418/ where no HTTP metrics were generated
anymore for Keystone. To fix the issue we read Keystone logs from
/var/log/keystone/*.log files as well as from
/var/log/apache2/keystone_wsgi_*_access.log files. The Keystone Apache logs
include the "request time" values we need to derive HTTP metrics.

Change-Id: Idf90ec99350b264fad9c667a46076dc49d91b509

deployment_scripts/puppet/manifests/controller.pp

@@ -62,6 +62,7 @@ lma_collector::logs::openstack { 'cinder': }
 lma_collector::logs::openstack { 'glance': }
 lma_collector::logs::openstack { 'heat': }
 lma_collector::logs::openstack { 'keystone': }
+class {'lma_collector::logs::keystone_wsgi': }
 lma_collector::logs::openstack { 'horizon': }
 if $murano['enabled'] {
   lma_collector::logs::openstack { 'murano': }

deployment_scripts/puppet/modules/lma_collector/README.md

@@ -61,6 +61,14 @@ class { 'lma_collector::logs::swift':
 }
 ```
 
+For Keystone, in addition to declaring the `lma_collector::logs::openstack`
+define, the `lma_collector::logs::keystone_wsgi` class should be declared
+to read Keystone logs stored from Apache log files:
+
+```puppet
+class { 'lma_collector::logs::keystone_wsgi': }
+```
+
 ### Collect libvirt logs
 
 To make the collector collect logs created by libvirt declare the
@@ -125,6 +133,7 @@ Public Classes:
 
 * [`lma_collector`](#class-lma_collector)
 * [`lma_collector::elasticsearch`](#class-lma_collectorelasticsearch)
+* [`lma_collector::logs::keystone_wsgi`](#class-lma_collectorlogskeystone_wsgi)
 * [`lma_collector::logs::libvirt`](#class-lma_collectorlogslibvirt)
 * [`lma_collector::logs::mysql`](#class-lma_collectorlogsmysql)
 * [`lma_collector::logs::ovs`](#class-lma_collectorlogsovs)
@@ -161,6 +170,25 @@ Elasticsearch for indexing.
 * `server`: *Required*. Elasticsearch server name. Valid options: a string.
 * `port`: *Optional*. Elasticsearch service port. Valid options: a string. Default: "9200".
 
+#### Class: `lma_collector::logs::keystone_wsgi`
+
+Declare this class to create an Heka `logstreamer` that reads Keystone Apache
+logs from `/var/log/apache2/keystone_wsgi_*_access.log`.
+
+This class currently assumes the following log configuration in Apache:
+
+```
+CustomLog "/var/log/apache2/keystone_wsgi_main_access.log" "%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\""
+```
+
+for Keystone main and:
+
+```
+CustomLog "/var/log/apache2/keystone_wsgi_admin_access.log" "%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\""
+```
+
+for Keystone admin.
+
 #### Class: `lma_collector::logs::libvirt`
 
 Declare this class to create an Heka `logstreamer` that reads libvirt logs

deployment_scripts/puppet/modules/lma_collector/files/plugins/common/patterns.lua

@@ -105,6 +105,11 @@ local http_method = l.Cg(l.R"AZ"^3, "http_method")
 local url = l.Cg( (1 - sp)^1, "http_url")
 local http_version = l.Cg(l.digit * dot * l.digit, "http_version")
 
+-- Pattern for the "<http_method> <http_url> HTTP/<http_version>" format found
+-- found in both OpenStack and Apache log files.
+-- Example : OPTIONS /example.com HTTP/1.0
+http_request = http_method * sp * url * sp * l.P'HTTP/' * http_version
+
 -- Patterns for HTTP status, HTTP response size and HTTP response time in
 -- OpenLayers logs.
 --
@@ -123,7 +128,7 @@ local openstack_response_time = l.P"time: "^-1 * l.Cg(l.digit^1 * dot^0 * l.digi
 -- Capture for OpenStack HTTP producing six values: http_method, http_url,
 -- http_version, http_status, http_response_size and http_response_time.
 openstack_http = anywhere(l.Ct(
-    quote * http_method * sp * url * sp * l.P'HTTP/' * http_version * quote * sp *
+    quote * http_request * quote * sp *
     openstack_http_status * sp * openstack_response_size * sp *
     openstack_response_time
 ))

deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/keystone_wsgi_log.lua

@@ -0,0 +1,73 @@
+-- Copyright 2015 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+local l      = require 'lpeg'
+l.locale(l)
+
+local common_log_format = require 'common_log_format'
+local patt = require 'patterns'
+local utils  = require 'lma_utils'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+local severity_label = utils.severity_to_label_map[msg.Severity]
+
+local apache_log_pattern = read_config("apache_log_pattern") or error(
+    "apache_log_pattern configuration must be specificed")
+local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
+local request_grammar = l.Ct(patt.http_request)
+
+function process_message ()
+
+    -- logger is either "keystone-wsgi-main" or "keystone-wsgi-admin"
+    local logger = read_message("Logger")
+
+    local log = read_message("Payload")
+
+    local m
+
+    m = apache_grammar:match(log)
+    if m then
+        msg.Logger = 'openstack.keystone'
+        msg.Payload = log
+        msg.Timestamp = m.time
+
+        msg.Fields = {}
+        msg.Fields.http_status = m.status
+        msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
+        msg.Fields.programname = logger
+        msg.Fields.severity_label = severity_label
+
+        local request = m.request
+        m = request_grammar:match(request)
+        if m then
+            msg.Fields.http_method = m.http_method
+            msg.Fields.http_url = m.http_url
+            msg.Fields.http_version = m.http_version
+        end
+
+        utils.inject_tags(msg)
+        return utils.safe_inject_message(msg)
+    end
+
+    return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+end

deployment_scripts/puppet/modules/lma_collector/manifests/logs/keystone_wsgi.pp

@@ -0,0 +1,42 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+#
+# == Class lma_collector::logs::keystone_wsgi
+#
+class lma_collector::logs::keystone_wsgi (
+  $log_directory = $lma_collector::params::apache_log_directory,
+) inherits lma_collector::params {
+  include lma_collector::service
+
+  heka::decoder::sandbox { 'keystone_wsgi':
+    config_dir => $lma_collector::params::config_dir,
+    filename   => "${lma_collector::params::plugins_dir}/decoders/keystone_wsgi_log.lua",
+    config     => {
+      apache_log_pattern => $lma_collector::params::apache_log_pattern,
+    },
+    notify     => Class['lma_collector::service'],
+  }
+
+  heka::input::logstreamer { 'keystone_wsgi':
+    config_dir     => $lma_collector::params::config_dir,
+    decoder        => 'keystone_wsgi',
+    log_directory  => $log_directory,
+    file_match     => 'keystone_wsgi_(?P<Service>.+)_access\.log',
+    differentiator => "['keystone-wsgi-', 'Service']",
+    require        => Heka::Decoder::Sandbox['keystone_wsgi'],
+    notify         => Class['lma_collector::service'],
+  }
+}

deployment_scripts/puppet/modules/lma_collector/manifests/params.pp

@@ -46,6 +46,9 @@ class lma_collector::params {
   # same pattern except the <PRI> tag
   $fallback_syslog_pattern = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%\n'
 
+  $apache_log_directory = '/var/log/apache2'
+  $apache_log_pattern = '%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
   # required to read the log files
   case $::osfamily {
     'Debian': {