Add IIS logging

apps/iis.go

@@ -15,7 +15,10 @@
 package apps
 
 import (
+	"fmt"
+
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
 )
 
@@ -97,3 +100,118 @@ func (r MetricsReceiverIis) Pipelines() []otel.Pipeline {
 func init() {
 	confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.Component { return &MetricsReceiverIis{} }, "windows")
 }
+
+type LoggingProcessorIisAccess struct {
+	confgenerator.ConfigComponent `yaml:",inline"`
+}
+
+func (*LoggingProcessorIisAccess) Type() string {
+	return "iis_access"
+}
+
+const (
+	iisMergeRecordFieldsLuaFunction       = `iis_merge_fields`
+	iisMergeRecordFieldsLuaScriptContents = `
+	function iis_merge_fields(tag, timestamp, record)
+	  record["http_request_serverIp"] = table.concat({record["http_request_serverIp"], ":", record["s_port"]})
+	  if (record["cs_uri_query"] == nil or record["cs_uri_query"] == '') then
+		record["http_request_requestUrl"] = record["cs_uri_stem"]
+	  else
+		record["http_request_requestUrl"] = table.concat({record["cs_uri_stem"], "?", record["cs_uri_query"]})
+	  end
+
+	  record["cs_uri_query"] = nil
+	  record["cs_uri_stem"] = nil
+	  record["s_port"] = nil
+	  return 2, timestamp, record 
+	end
+	`
+)
+
+func (p *LoggingProcessorIisAccess) Components(tag, uid string) []fluentbit.Component {
+	c := confgenerator.LoggingProcessorParseRegex{
+		// Documentation:
+		// https://docs.microsoft.com/en-us/windows/win32/http/w3c-logging
+		// sample line: 2022-03-10 17:26:30 ::1 GET /iisstart.png - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko http://localhost/ 200 0 0 18
+		// sample line: 2022-03-10 17:26:30 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - 200 0 0 352
+		// sample line: 2022-03-10 17:26:32 ::1 GET /favicon.ico - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - 404 0 2 49
+		Regex: `^(?<timestamp>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})\s(?<http_request_serverIp>[^\s]+)\s(?<http_request_requestMethod>[^\s]+)\s(?<cs_uri_stem>\/[^\s]*)\s(?<cs_uri_query>[^\s]*)\s(?<s_port>\d*)\s(?<user>[^\s]+)\s(?<http_request_remoteIp>[^\s]+)\s(?<http_request_userAgent>[^\s]+)\s(?<http_request_referer>[^\s]+)\s(?<http_request_status>\d{3})\s(?<sc_substatus>\d+)\s(?<sc_win32_status>\d+)\s(?<time_taken>\d+)$`,
+		ParserShared: confgenerator.ParserShared{
+			TimeKey:    "timestamp",
+			TimeFormat: " %Y-%m-%d %H:%M:%S",
+			Types: map[string]string{
+				"http_request_status": "integer",
+			},
+		},
+	}.Components(tag, uid)
+	// iis logs "-" when a field does not have a value. Remove the field entirely when this happens.
+	for _, field := range []string{
+		"cs_uri_query",
+		"http_request_referer",
+		"user",
+	} {
+		c = append(c, fluentbit.Component{
+			Kind: "FILTER",
+			Config: map[string]string{
+				"Name":      "modify",
+				"Match":     tag,
+				"Condition": fmt.Sprintf("Key_Value_Equals %s -", field),
+				"Remove":    field,
+			},
+		})
+	}
+
+	c = append(c, fluentbit.LuaFilterComponents(tag, iisMergeRecordFieldsLuaFunction, iisMergeRecordFieldsLuaScriptContents)...)
+
+	c = append(c, []fluentbit.Component{
+		// This is used to exlude the header lines above the logs
+
+		// EXAMPLE LINES:
+		// #Software: Microsoft Internet Information Services 10.0
+		// #Version: 1.0
+		// #Date: 2022-04-11 12:53:50
+		{
+			Kind: "FILTER",
+			Config: map[string]string{
+				"Name":    "grep",
+				"Match":   tag,
+				"Exclude": "message ^#(?:Fields|Date|Version|Software):",
+			},
+		},
+
+		// Generate the httpRequest structure.
+		{
+			Kind: "FILTER",
+			Config: map[string]string{
+				"Name":          "nest",
+				"Match":         tag,
+				"Operation":     "nest",
+				"Wildcard":      "http_request_*",
+				"Nest_under":    "logging.googleapis.com/http_request",
+				"Remove_prefix": "http_request_",
+			},
+		},
+	}...)
+	return c
+}
+
+type LoggingReceiverIisAccess struct {
+	LoggingProcessorIisAccess               `yaml:",inline"`
+	confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
+}
+
+func (r LoggingReceiverIisAccess) Components(tag string) []fluentbit.Component {
+	if len(r.IncludePaths) == 0 {
+		r.IncludePaths = []string{
+			`C:\inetpub\logs\LogFiles\W3SVC1\u_ex*`,
+		}
+	}
+	c := r.LoggingReceiverFilesMixin.Components(tag)
+	c = append(c, r.LoggingProcessorIisAccess.Components(tag, "iis_access")...)
+	return c
+}
+
+func init() {
+	confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.Component { return &LoggingReceiverIisAccess{} }, "windows")
+	confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.Component { return &LoggingProcessorIisAccess{} }, "windows")
+}

confgenerator/testdata/invalid/windows/logging-receiver_unsupported_type_systemd/golden_error

@@ -1 +1 @@
-logging receiver with type "systemd" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, fluent_forward, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, rabbitmq, redis, solr_system, syslog, tcp, tomcat_access, tomcat_system, wildfly_system, windows_event_log, zookeeper_general].
+logging receiver with type "systemd" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, fluent_forward, hadoop, hbase_system, iis_access, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, rabbitmq, redis, solr_system, syslog, tcp, tomcat_access, tomcat_system, wildfly_system, windows_event_log, zookeeper_general].

confgenerator/testdata/valid/windows/logging-receiver_iis/golden_0b33cca8c5c059f785e2c91ddc91d27f.lua

@@ -0,0 +1,15 @@
+
+	function iis_merge_fields(tag, timestamp, record)
+	  record["http_request_serverIp"] = table.concat({record["http_request_serverIp"], ":", record["s_port"]})
+	  if (record["cs_uri_query"] == nil or record["cs_uri_query"] == '') then
+		record["http_request_requestUrl"] = record["cs_uri_stem"]
+	  else
+		record["http_request_requestUrl"] = table.concat({record["cs_uri_stem"], "?", record["cs_uri_query"]})
+	  end
+
+	  record["cs_uri_query"] = nil
+	  record["cs_uri_stem"] = nil
+	  record["s_port"] = nil
+	  return 2, timestamp, record 
+	end
+

confgenerator/testdata/valid/windows/logging-receiver_iis/golden_fluent_bit_main.conf

@@ -0,0 +1,164 @@
+@SET buffers_dir=C:\ProgramData\Google\Cloud Operations\Ops Agent\run/buffers
+@SET logs_dir=C:\ProgramData\Google\Cloud Operations\Ops Agent\log
+
+[SERVICE]
+    Daemon                    off
+    Flush                     1
+    Log_Level                 info
+    dns.resolver              legacy
+    storage.backlog.mem_limit 50M
+    storage.checksum          on
+    storage.max_chunks_up     128
+    storage.metrics           on
+    storage.sync              normal
+
+[INPUT]
+    Name            fluentbit_metrics
+    Scrape_Interval 60
+    Scrape_On_Start True
+
+[INPUT]
+    Channels     System,Application,Security
+    DB           ${buffers_dir}/default_pipeline_windows_event_log
+    Interval_Sec 1
+    Name         winlog
+    Tag          default_pipeline.windows_event_log
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/iis_iis_access
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              C:\inetpub\logs\LogFiles\W3SVC1\u_ex*
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               iis.iis_access
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/ops-agent-fluent-bit
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/logging-module.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-fluent-bit
+    storage.type      filesystem
+
+[FILTER]
+    Add       logging.googleapis.com/severity ERROR
+    Condition Key_Value_Equals EventType Error
+    Match     default_pipeline.windows_event_log
+    Name      modify
+
+[FILTER]
+    Add       logging.googleapis.com/severity INFO
+    Condition Key_Value_Equals EventType Information
+    Match     default_pipeline.windows_event_log
+    Name      modify
+
+[FILTER]
+    Add       logging.googleapis.com/severity WARNING
+    Condition Key_Value_Equals EventType Warning
+    Match     default_pipeline.windows_event_log
+    Name      modify
+
+[FILTER]
+    Add       logging.googleapis.com/severity NOTICE
+    Condition Key_Value_Equals EventType SuccessAudit
+    Match     default_pipeline.windows_event_log
+    Name      modify
+
+[FILTER]
+    Add       logging.googleapis.com/severity NOTICE
+    Condition Key_Value_Equals EventType FailureAudit
+    Match     default_pipeline.windows_event_log
+    Name      modify
+
+[FILTER]
+    Add   logging.googleapis.com/logName windows_event_log
+    Match default_pipeline.windows_event_log
+    Name  modify
+
+[FILTER]
+    Key_Name message
+    Match    iis.iis_access
+    Name     parser
+    Parser   iis.iis_access.iis_access
+
+[FILTER]
+    Condition Key_Value_Equals cs_uri_query -
+    Match     iis.iis_access
+    Name      modify
+    Remove    cs_uri_query
+
+[FILTER]
+    Condition Key_Value_Equals http_request_referer -
+    Match     iis.iis_access
+    Name      modify
+    Remove    http_request_referer
+
+[FILTER]
+    Condition Key_Value_Equals user -
+    Match     iis.iis_access
+    Name      modify
+    Remove    user
+
+[FILTER]
+    Match  iis.iis_access
+    Name   lua
+    call   iis_merge_fields
+    script 0b33cca8c5c059f785e2c91ddc91d27f.lua
+
+[FILTER]
+    Exclude message ^#(?:Fields|Date|Version|Software):
+    Match   iis.iis_access
+    Name    grep
+
+[FILTER]
+    Match         iis.iis_access
+    Name          nest
+    Nest_under    logging.googleapis.com/http_request
+    Operation     nest
+    Remove_prefix http_request_
+    Wildcard      http_request_*
+
+[FILTER]
+    Add   logging.googleapis.com/logName iis_access
+    Match iis.iis_access
+    Name  modify
+
+[OUTPUT]
+    Match_Regex                   ^(default_pipeline\.windows_event_log|iis\.iis_access)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match_Regex                   ^(ops-agent-fluent-bit)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match *
+    Name  prometheus_exporter
+    host  0.0.0.0
+    port  20202

confgenerator/testdata/valid/windows/logging-receiver_iis/golden_fluent_bit_parser.conf

@@ -0,0 +1,7 @@
+[PARSER]
+    Format      regex
+    Name        iis.iis_access.iis_access
+    Regex       ^(?<timestamp>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})\s(?<http_request_serverIp>[^\s]+)\s(?<http_request_requestMethod>[^\s]+)\s(?<cs_uri_stem>\/[^\s]*)\s(?<cs_uri_query>[^\s]*)\s(?<s_port>\d*)\s(?<user>[^\s]+)\s(?<http_request_remoteIp>[^\s]+)\s(?<http_request_userAgent>[^\s]+)\s(?<http_request_referer>[^\s]+)\s(?<http_request_status>\d{3})\s(?<sc_substatus>\d+)\s(?<sc_win32_status>\d+)\s(?<time_taken>\d+)$
+    Time_Format  %Y-%m-%d %H:%M:%S
+    Time_Key    timestamp
+    Types       http_request_status:integer