RabbitMQ Logging Receiver

apps/rabbitmq.go

@@ -16,9 +16,84 @@ package apps
 
 import (
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
 )
 
+type LoggingProcessorRabbitmq struct {
+	confgenerator.ConfigComponent `yaml:",inline"`
+}
+
+func (*LoggingProcessorRabbitmq) Type() string {
+	return "rabbitmq"
+}
+
+func (p *LoggingProcessorRabbitmq) Components(tag, uid string) []fluentbit.Component {
+	c := []fluentbit.Component{}
+	regexParser := confgenerator.LoggingProcessorParseRegex{
+		// Sample log line:
+		// 2022-01-31 18:01:20.441571+00:00 [erro] <0.692.0> ** Connection attempt from node 'rabbit_ctl_17@keith-testing-rabbitmq' rejected. Invalid challenge reply. **
+		Regex: `^(?<timestamp>\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+) \[(?<severity>\w+)\] \<(?<process_id>\d+\.\d+\.\d+)\> (?<message>.*)$`,
+		ParserShared: confgenerator.ParserShared{
+			TimeKey:    "timestamp",
+			TimeFormat: "%Y-%m-%d %H:%M:%S.%L+%Z",
+		},
+	}
+	c = append(c, regexParser.Components(tag, uid)...)
+
+	// severities documented here: https://www.rabbitmq.com/logging.html#log-levels
+	c = append(c, fluentbit.TranslationComponents(tag, "severity", "logging.googleapis.com/severity", true, []struct {
+		SrcVal  string
+		DestVal string
+	}{
+		{"debug", "DEBUG"},
+		{"error", "ERROR"},
+		{"info", "INFO"},
+		{"noti", "DEFAULT"},
+	})...)
+
+	return c
+}
+
+type LoggingReceiverRabbitmq struct {
+	LoggingProcessorRabbitmq                `yaml:",inline"`
+	confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
+}
+
+func (r LoggingReceiverRabbitmq) Components(tag string) []fluentbit.Component {
+	if len(r.IncludePaths) == 0 {
+		r.IncludePaths = []string{
+			"/var/log/rabbitmq/rabbit*.log",
+		}
+	}
+	// Some multiline entries related to crash logs are important to capture and end in
+	//
+	// 2022-01-31 18:07:43.557042+00:00 [erro] <0.130.0>
+	// BOOT FAILED
+	// ===========
+	// ERROR: could not bind to distribution port 25672, it is in use by another node: rabbit@keith-testing-rabbitmq
+	//
+	r.MultilineRules = []confgenerator.MultilineRule{
+		{
+			StateName: "start_state",
+			NextState: "cont",
+			Regex:     `\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+`,
+		},
+		{
+			StateName: "cont",
+			NextState: "cont",
+			Regex:     `^(?!\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+)`,
+		},
+	}
+	c := r.LoggingReceiverFilesMixin.Components(tag)
+	c = append(c, r.LoggingProcessorRabbitmq.Components(tag, "rabbitmq")...)
+	return c
+}
+
+func init() {
+	confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.Component { return &LoggingReceiverRabbitmq{} })
+}
+
 type MetricsReceiverRabbitmq struct {
 	confgenerator.ConfigComponent `yaml:",inline"`
 

confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/golden_error

@@ -1 +1 @@
-logging receiver with type "windows_event_log" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, redis, solr_system, syslog, systemd_journald, tcp, tomcat_access, tomcat_system, wildfly_system, zookeeper_general].
+logging receiver with type "windows_event_log" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, rabbitmq, redis, solr_system, syslog, systemd_journald, tcp, tomcat_access, tomcat_system, wildfly_system, zookeeper_general].

confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error

@@ -1 +1 @@
-logging receiver with type "unsupported_type" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, redis, solr_system, syslog, systemd_journald, tcp, tomcat_access, tomcat_system, wildfly_system, zookeeper_general].
+logging receiver with type "unsupported_type" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, rabbitmq, redis, solr_system, syslog, systemd_journald, tcp, tomcat_access, tomcat_system, wildfly_system, zookeeper_general].

confgenerator/testdata/invalid/windows/logging-receiver_unsupported_type_systemd/golden_error

@@ -1 +1 @@
-logging receiver with type "systemd" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, redis, solr_system, syslog, tcp, tomcat_access, tomcat_system, wildfly_system, windows_event_log, zookeeper_general].
+logging receiver with type "systemd" is not supported. Supported logging receiver types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, files, hadoop, hbase_system, kafka, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, postgresql_general, rabbitmq, redis, solr_system, syslog, tcp, tomcat_access, tomcat_system, wildfly_system, windows_event_log, zookeeper_general].

confgenerator/testdata/valid/linux/logging-receiver_rabbitmq/golden_fluent_bit_main.conf

@@ -0,0 +1,138 @@
+@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers
+@SET logs_dir=/var/log/google-cloud-ops-agent/subagents
+
+[SERVICE]
+    Daemon                    off
+    Flush                     1
+    Log_Level                 info
+    dns.resolver              legacy
+    storage.backlog.mem_limit 50M
+    storage.checksum          on
+    storage.max_chunks_up     128
+    storage.metrics           on
+    storage.sync              normal
+
+[INPUT]
+    Name            fluentbit_metrics
+    Scrape_Interval 60
+    Scrape_On_Start True
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/default_pipeline_syslog
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              /var/log/messages,/var/log/syslog
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               default_pipeline.syslog
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/rabbitmq_rabbitmq
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              /var/log/rabbitmq/rabbit*.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               rabbitmq.rabbitmq
+    multiline.parser  multiline.rabbitmq.rabbitmq
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   5M
+    DB                ${buffers_dir}/ops-agent-fluent-bit
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/logging-module.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-fluent-bit
+    storage.type      filesystem
+
+[FILTER]
+    Add   logging.googleapis.com/logName syslog
+    Match default_pipeline.syslog
+    Name  modify
+
+[FILTER]
+    Match  rabbitmq.rabbitmq
+    Name   modify
+    Rename log message
+
+[FILTER]
+    Key_Name message
+    Match    rabbitmq.rabbitmq
+    Name     parser
+    Parser   rabbitmq.rabbitmq.rabbitmq
+
+[FILTER]
+    Add       logging.googleapis.com/severity DEBUG
+    Condition Key_Value_Equals severity debug
+    Match     rabbitmq.rabbitmq
+    Name      modify
+    Remove    severity
+
+[FILTER]
+    Add       logging.googleapis.com/severity ERROR
+    Condition Key_Value_Equals severity error
+    Match     rabbitmq.rabbitmq
+    Name      modify
+    Remove    severity
+
+[FILTER]
+    Add       logging.googleapis.com/severity INFO
+    Condition Key_Value_Equals severity info
+    Match     rabbitmq.rabbitmq
+    Name      modify
+    Remove    severity
+
+[FILTER]
+    Add       logging.googleapis.com/severity DEFAULT
+    Condition Key_Value_Equals severity noti
+    Match     rabbitmq.rabbitmq
+    Name      modify
+    Remove    severity
+
+[FILTER]
+    Add   logging.googleapis.com/logName rabbitmq
+    Match rabbitmq.rabbitmq
+    Name  modify
+
+[OUTPUT]
+    Match_Regex                   ^(default_pipeline\.syslog|rabbitmq\.rabbitmq)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match_Regex                   ^(ops-agent-fluent-bit)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match *
+    Name  prometheus_exporter
+    host  0.0.0.0
+    port  20202

confgenerator/testdata/valid/linux/logging-receiver_rabbitmq/golden_fluent_bit_parser.conf

@@ -0,0 +1,13 @@
+[PARSER]
+    Format      regex
+    Name        rabbitmq.rabbitmq.rabbitmq
+    Regex       ^(?<timestamp>\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+) \[(?<severity>\w+)\] \<(?<process_id>\d+\.\d+\.\d+)\> (?<message>.*)$
+    Time_Format %Y-%m-%d %H:%M:%S.%L+%Z
+    Time_Key    timestamp
+
+[MULTILINE_PARSER]
+    flush_timeout 5000
+    name          multiline.rabbitmq.rabbitmq
+    type          regex
+    rule          "start_state"    "\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+"    "cont"
+    rule          "cont"    "^(?!\d+-\d+-\d+ \d+:\d+:\d+\.\d+\+\d+:\d+)"    "cont"