Merge pull request #1540 from dmwm/update_aps_logstash

Update aps logstash for prod
dmwm · Aug 21, 2024 · 4f92b33 · 4f92b33
2 parents da8ba52 + dd7a1a7
commit 4f92b33
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/kubernetes/cmsweb/monitoring/logstash.conf b/kubernetes/cmsweb/monitoring/logstash.conf
@@ -78,7 +78,7 @@ filter {
   if "aps" in [tags] {
       mutate { replace => { "type" => "aps" } }
       grok {
-        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}:%{NUMBER:fe_port}\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: %{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int}\] \[X-Forwarded-Host: %{HOSTNAME:x_forwarded_host}\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
+        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}(?::%{NUMBER:fe_port})?\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: (%{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int})?\] \[X-Forwarded-Host: (%{HOSTNAME:x_forwarded_host})?\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
       }
       grok {
         match => {
@@ -124,7 +124,6 @@ filter {
       }
       if ![api] {
           mutate { replace => { "api" => "%{request}" } }
-          mutate { replace => { "system" => "%{request}" } }
       }
       if [client] {
           grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
@@ -240,8 +239,6 @@ filter {
 
   # common filters
 
-  # drop failed records
-  if "_grokparsefailure" in [tags] { drop { } }
   # remove quotes from message entry since it will break the JSON
   mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }