What's wrong with my logstash filter grok syntax?

107 Views Asked by At

I'm trying to use OpenSearch over Logstash logs. I have two logs, but the second log does not apply to OpenSearch available fields.

Here is my two logs and each has a different log4j pattern. It is a third-party solution, so these logs pattern can't change.

Log1
%d %5p [%c] [%X{txIp}] [%X{mbrNo}] %m%n

Log2
[%-5p][%d{yyyyMMdd HH:mm:ss.SSS}][%t]%c{4}.%M(%L) - %X{reqCommand}|%X{svcTrId}|%m%n

and this is my logstash.conf filter:

filter {
  if [fields][index] == "log1"{
     grok {
       match => {
         "message" => "%{TIMESTAMP_ISO8601:date} %{LOGLEVEL:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
       }
     }
  } else if [fields][index] == "log2" {
     grok {
       match => {
         "message" => "\[%{LOGLEVEL:logLevel}\]\[%{TIMESTAMP_ISO8601:date}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
       }
     }
  }


  date {
    match => ["date", "ISO8601"]
  }
}
1

There are 1 best solutions below

0
neeson.lee On BEST ANSWER

Finally It works.

This is my Filter.

filter {
  if [fields][index] == "log1" {
     grok {
       match => {
         "message" => "%{TIMESTAMP_ISO8601:logTimestamp} %{DATA:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
       }
     }
     date {
       match => ["logTimestamp", "ISO8601"]
       target => "@timestamp"
       timezone => "Asia/Seoul"
     }
  } else if [fields][index] == "log2" {
     grok {
       match => {
         "message" => "\[%{DATA:logLevel}]\[%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day} %{TIME:time}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
       }
     }
     mutate {
       add_field => {
         "logtimestamp" => "%{year}-%{month}-%{day}T%{time}"
       }
     }
     date {
       match => ["logtimestamp", "ISO8601"]
       target =>  "@timestamp"
       timezone => "Asia/Seoul"
     }

   }
}