20-openstack-filter.conf

filter {
  if "oslofmt" in [tags] {
    mutate {
      add_tag      => [ "in_20" ]
    }
    grok {
      break_on_match => true
      patterns_dir => ["/etc/logstash/patterns"]
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:timestamp} %{POSINT:pid} %{AUDITLOGLEVEL:loglevel} %{DATA:module} \[%{DATA:request_id} %{DATA} %{DATA}\] %{GREEDYDATA:content}" } 
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:timestamp} %{POSINT:pid} %{AUDITLOGLEVEL:loglevel} %{DATA:module} %{GREEDYDATA:content}" } 
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:timestamp} %{POSINT:pid} %{AUDITLOGLEVEL:loglevel} %{GREEDYDATA:content}" } 
      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:timestamp} %{POSINT:pid} %{AUDITLOGLEVEL:loglevel} %{GREEDYDATA:content} %{IP:ip_client" } 
      match => { "message" => "\[%{HTTPDERROR_DATE:timestamp}\] \[%{AUDITLOGLEVEL:loglevel}\] \[client %{IP:client_ip}\] %{GREEDYDATA:content}" }
      match => { "message" => "\[%{HTTPDERROR_DATE:timestamp}\] \[%{AUDITLOGLEVEL:loglevel}\] %{GREEDYDATA:content}" }
      add_tag => [ "oslo_processed" ]
      tag_on_failure => ["_gpf_olsofmt"]
    }
    
   
    if "oslo_processed" in [tags] {
      mutate {
       replace      => { message => "%{content}" }
       remove_field   => [ "content" ]
      }
    }

    if "nova.compute.resource_tracker"  in [module] or
       "nova.openstack.common.periodic_task" in [module] or
       "nova.openstack.common.lockutils" in [module] or
       "nova.openstack.common.loopingcall" in [module] or
       "nova.openstack.common.processutils" in [module] or
       "SchedulerManager._expire_reservations" in [message] or
       "SchedulerManager._expire_reservations" in [message] or
       "nova/openstack/common/lockutils.py" in [message] or
       "nova/openstack/common/periodic_task.py" in [message] or
       "nova/openstack/common/loopingcall" in [message] or
       "^combined" in [message] or
       "Result was 0 execute /usr/lib/python2.6/site-packages/nova/openstack/common/processutils.py" in [message] or
       "cache _heal_instance_info_cache" in [message]
      {
      mutate {
        add_tag => [ "drop" ]
      }
    }
    
    if [mesage] =~ /[Tt]race/ {
      mutate {
        add_tag => [ "trace" ]
      }
    }
  
    if "Reconnecting to AMQP server" in [message] or [message] =~ /AMQP server on [0-9]+.[0-9]+.[0-9]+.[0-9]:5671 is unreachable/ {
      mutate {
        add_tag => [ "amqp" ]
      }
    }
  
    if "Starting instance" in [message] {
      grok {
        break_on_match => false
        patterns_dir => [ "/etc/logstash/patterns" ]
        match => { "message" => "\[instance: %{UUID:instance_UUIDs}\] Starting instance..." }
        add_tag => [ "instance_start" ]
      }
    }
  
    if "scheduler" in [tags] {
      mutate {
      add_tag => [ "in_scheduler" ]
      }
      if "Choosing host WeightedHost" in [message] {
        metrics {
          meter => [ "novascheduler.schedule_host" ]
          add_tag => [ "metric" ]
          clear_interval => 3600 # This needs to be multiples of 5
        }
      }
    }
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd'THH:mm:ss:S+ZZ" ]
      add_tag => [ "oslofmt_date_match" ]
      tag_on_failure  => [ "oslofmt_date_fail" ]
    }
    mutate {
      #remove_field => [ "hostname", "timestamp" ]
    }
  }
 
}