logstash-filters/indexer.conf.erb

input {
  tcp {
    host => "localhost"
    port => 9999
    codec => line {}
    type => "jenkins"
  }
}

# You can check grok patterns at http://grokdebug.herokuapp.com/
filter {
  # This is a work around for a bug. We should be able to set the tcp
  # input codec to json, but that codec doesn't support streaming.
  # Convert to json here instead.
  json {
    source => "message"
  }
  if "screen" in [tags] and [message] =~ "^\+ " {
    drop {}
  }
  if "console.html" in [tags] {
    if [message] == "<pre>" or [message] == "</pre>" {
      drop {}
    }
    multiline {
      negate => true
      pattern => "^%{TIMESTAMP_ISO8601} \|"
      what => "previous"
      stream_identity => "%{host}.%{filename}"
    }
    grok {
      # Do multiline matching as the above mutliline filter may add newlines
      # to the log messages.
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate} \| %{GREEDYDATA:logmessage}" }
      add_field => { "received_at" => "%{@timestamp}" }
    }
  } else if "oslofmt" in [tags] {
    multiline {
      negate => true
      pattern => "^%{TIMESTAMP_ISO8601} "
      what => "previous"
      stream_identity => "%{host}.%{filename}"
    }
    multiline {
      negate => false
      pattern => "^%{TIMESTAMP_ISO8601}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
      what => "previous"
      stream_identity => "%{host}.%{filename}"
    }
    grok {
      # Do multiline matching as the above mutliline filter may add newlines
      # to the log messages.
      # TODO move the LOGLEVELs into a proper grok pattern.
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
      add_field => { "received_at" => "%{@timestamp}" }
    }
  } else if "keystonefmt" in [tags] {
    if [message] == "" {
      drop {}
    }
    multiline {
      negate => true
      pattern => "^\(\b%{NOTSPACE}\b\):"
      what => "previous"
      stream_identity => "%{host}.%{filename}"
    }
    grok {
      # Do multiline matching as the above mutliline filter may add newlines
      # to the log messages.
      # TODO move the LOGLEVELs into a proper grok pattern.
      match => { "message" => "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" }
      add_field => { "received_at" => "%{@timestamp}" }
    }
  } else if "apachecombined" in [tags] {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
      add_field => { "received_at" => "%{@timestamp}" }
      add_field => { "logdate" => "%{timestamp}" }
      add_field => { "logmessage" => "%{verb} %{request} %{response}" }
    }
  } else if "libvirtd" in [tags] {
    grok {
      # libvirtd grok filter adapted from
      # https://github.com/OpenStratus/openstack-logstash/blob/master/agent.conf
      match => { "message" => "%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:pid}:%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}:%{SPACE}%{GREEDYDATA:logmessage}" }
      add_field => { "received_at" => "%{@timestamp}" }
    }
  } else if "syslog" in [tags] {
    grok {
      # Syslog grok filter adapted from
      # http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf
      match => { "message" => "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" }
      add_field => { "received_at" => "%{@timestamp}" }
    }
  }

  # Filters below here should be consistent for all Jenkins log formats.
  # Remove DEBUG logs to reduce the amount of data that needs to be processed.
  if [loglevel] == "DEBUG" {
    drop {}
  }

  if ! ("_grokparsefailure" in [tags]) {
    date {
      match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "dd/MMM/yyyy:HH:mm:ss Z" ]
      timezone => "UTC"
    }
    mutate {
      replace => { "message" => "%{logmessage}" }
    }
    mutate {
      remove_field => [ "logdate", "logmessage" ]
    }
  }
}

output {
  elasticsearch {
    host => "<%= scope.lookupvar("::openstack_project::logstash_worker::discover_node") %>"
    node_name => "<%= scope.lookupvar("::hostname") %>"
    max_inflight_requests => 512
  }
}