Add support for collecting apache logs

Depends-On: Ib62ace1db43fb2c9125a890575fb901646b76e53
Depends-On: Iac338a9e3183428c07357d02d3a597d78249b155

Change-Id: I662a12cfcc195cf457f85cdd2fee66f754a2acf2

docker/heka/plugins/decoders/os_horizon_apache_log.lua

@@ -15,6 +15,7 @@
 local l      = require 'lpeg'
 l.locale(l)
 
+local dt     = require "date_time"
 local common_log_format = require 'common_log_format'
 local patt = require 'os_patterns'
 local utils  = require 'os_utils'
@@ -26,47 +27,74 @@ local msg = {
     Payload     = nil,
     Pid         = nil,
     Fields      = nil,
-    Severity    = 6,
+    Severity    = nil,
 }
 
 local severity_label = utils.severity_to_label_map[msg.Severity]
 
-local apache_log_pattern = read_config("apache_log_pattern") or error(
-    "apache_log_pattern configuration must be specificed")
-local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
+local access_log_pattern = read_config("access_log_pattern") or error(
+    "access_log_pattern configuration must be specificed")
+local access_log_grammar = common_log_format.build_apache_grammar(access_log_pattern)
 local request_grammar = l.Ct(patt.http_request)
 
+-- Since "common_log_format.build_apache_grammar", doesnt support ErrorLogFormat,
+-- we have to create error log grammar by ourself. Example error log string:
+-- 2016-08-15 10:46:27.679999 wsgi:error 340:140359488239360 Not Found: /favicon.ico
+
+local sp = patt.sp
+local colon = patt.colon
+local p_timestamp = l.Cg(l.Ct(dt.rfc3339_full_date * (sp + l.P"T") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1), "Timestamp")
+local p_module = l.Cg(l.R("az")^0, "Module")
+local p_errtype = l.Cg(l.R("az")^0, "ErrorType")
+local p_pid = l.Cg(l.digit^-5, "Pid")
+local p_tid = l.Cg(l.digit^-15, "TreadID")
+local p_mess = l.Cg(patt.Message, "Message")
+local error_log_grammar = l.Ct(p_timestamp * sp * p_module * colon * p_errtype * sp * p_pid * colon * p_tid * sp * p_mess)
+
+function prepare_message (timestamp, pid, severity, severity_label, programname, payload)
+    msg.Logger = 'openstack.horizon-apache'
+    msg.Payload = payload
+    msg.Timestamp = timestamp
+    msg.Pid = pid
+    msg.Severity = severity
+    msg.Fields = {}
+    msg.Fields.programname = programname
+    msg.Fields.severity_label = severity_label
+end
+
 function process_message ()
 
-    -- logger is either "horizon-apache-public" or "horizon-apache-admin"
+    -- logger is either "horizon-access" or "horizon-error"
     local logger = read_message("Logger")
-
     local log = read_message("Payload")
-
     local m
 
-    m = apache_grammar:match(log)
-    if m then
-        msg.Logger = 'openstack.horizon'
-        msg.Payload = log
-        msg.Timestamp = m.time
-
-        msg.Fields = {}
-        msg.Fields.http_status = m.status
-        msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
-        msg.Fields.programname = logger
-        msg.Fields.severity_label = severity_label
-
-        local request = m.request
-        m = request_grammar:match(request)
+    if logger == "horizon-access" then
+        m = access_log_grammar:match(log)
         if m then
-            msg.Fields.http_method = m.http_method
-            msg.Fields.http_url = m.http_url
-            msg.Fields.http_version = m.http_version
+            prepare_message(m.Timestamp, m.Pid, "6", "INFO", logger, log)
+            msg.Fields.http_status = m.status
+            msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
+            local request = m.request
+            r = request_grammar:match(request)
+            if r then
+                msg.Fields.http_method = r.http_method
+                msg.Fields.http_url = r.http_url
+                msg.Fields.http_version = r.http_version
+            end
+        else
+            return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
         end
-
-        return utils.safe_inject_message(msg)
+    elseif logger == "horizon-error" then
+        m = error_log_grammar:match(log)
+        if m then
+            prepare_message(m.Timestamp, m.Pid, "3", "ERROR", logger, m.Message)
+        else
+            return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+        end
+    else
+        error("Logger unknown")
     end
 
-    return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    return utils.safe_inject_message(msg)
 end

docker/heka/plugins/decoders/os_keystone_apache_log.lua

@@ -47,7 +47,7 @@ function process_message ()
 
     m = apache_grammar:match(log)
     if m then
-        msg.Logger = 'openstack.keystone'
+        msg.Logger = 'openstack.keystone-apache'
         msg.Payload = log
         msg.Timestamp = m.time
 

service/files/heka-horizon.toml.j2

@@ -0,0 +1,13 @@
+[horizon_apache_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_horizon_apache_log.lua"
+    [horizon_apache_log_decoder.config]
+    access_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
+[horizon_apache_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "horizon_apache_log_decoder"
+log_directory = "/var/log/ccp/horizon"
+file_match = 'horizon-(?P<Service>.+)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["horizon-", "Service"]

service/files/heka-keystone.toml.j2

@@ -0,0 +1,13 @@
+[keystone_apache_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_keystone_apache_log.lua"
+    [keystone_apache_log_decoder.config]
+    apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
+[keystone_apache_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "keystone_apache_log_decoder"
+log_directory = "/var/log/ccp/keystone"
+file_match = 'keystone-(?P<Service>.+)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["keystone-", "Service"]

service/heka.yaml

@@ -19,6 +19,14 @@ service:
           path: "{{ rabbitmq_log_base }}"
           type: host
           readOnly: True
+        - name: keystone-logs
+          path: "/var/log/ccp/keystone"
+          type: host
+          readOnly: True
+        - name: horizon-logs
+          path: "/var/log/ccp/horizon"
+          type: host
+          readOnly: True
       daemon:
         command: hekad --config=/etc/heka
         dependencies:
@@ -31,6 +39,8 @@ service:
           - heka-rabbitmq.toml
           - heka-ovs.toml
           - heka-dockerlogs.toml
+          - heka-keystone.toml
+          - heka-horizon.toml
 files:
   heka-global.toml:
     path: /etc/heka/heka-global.toml
@@ -53,3 +63,9 @@ files:
   heka-dockerlogs.toml:
     path: /etc/heka/heka-dockerlogs.toml
     content: heka-dockerlogs.toml
+  heka-keystone.toml:
+    path: /etc/heka/heka-keystone.toml
+    content: heka-keystone.toml.j2
+  heka-horizon.toml:
+    path: /etc/heka/heka-horizon.toml
+    content: heka-horizon.toml.j2