Skip to content

Commit

Permalink
Adjust log name via logging.googleapis.com/logName instead of using a…
Browse files Browse the repository at this point in the history 
… rewrite_tag. (#269)
  • Loading branch information
@qingling128
qingling128 authored Nov 16, 2021
1 parent 3bcf519 commit dc09e78
Show file tree
Hide file tree
Showing 53 changed files with 159 additions and 1,402 deletions.
10 changes: 5 additions & 5 deletions confgenerator/confgenerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (l *Logging) generateFluentbitComponents(userAgent string, hostInfo *host.I
components []fluentbit.Component
}
var sources []fbSource
var logNames []string
var tags []string
for pID, p := range l.Service.Pipelines {
for _, rID := range p.ReceiverIDs {
receiver, ok := l.Receivers[rID]
Expand All @@ -157,18 +157,18 @@ func (l *Logging) generateFluentbitComponents(userAgent string, hostInfo *host.I
components = append(components, processor.Components(tag, strconv.Itoa(i))...)
}
components = append(components, setLogNameComponents(tag, rID)...)
logNames = append(logNames, regexp.QuoteMeta(rID))
tags = append(tags, regexp.QuoteMeta(tag))
sources = append(sources, fbSource{tag, components})
}
}
sort.Slice(sources, func(i, j int) bool { return sources[i].tag < sources[j].tag })
sort.Strings(logNames)
sort.Strings(tags)

for _, s := range sources {
out = append(out, s.components...)
}
if len(logNames) > 0 {
out = append(out, stackdriverOutputComponent(strings.Join(logNames, "|"), userAgent))
if len(tags) > 0 {
out = append(out, stackdriverOutputComponent(strings.Join(tags, "|"), userAgent))
}
}
out = append(out, LoggingReceiverFilesMixin{
Expand Down
21 changes: 1 addition & 20 deletions confgenerator/logging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,34 +22,15 @@ import (

// setLogNameComponents generates a series of components that rewrites the tag on log entries tagged `tag` to be `logName`.
func setLogNameComponents(tag, logName string) []fluentbit.Component {
// TODO: Can we just set log_name_key in the output plugin and avoid this mess?
return []fluentbit.Component{
{
Kind: "FILTER",
Config: map[string]string{
"Match": tag,
"Add": fmt.Sprintf("logName %s", logName),
"Add": fmt.Sprintf("logging.googleapis.com/logName %s", logName),
"Name": "modify",
},
},
{
Kind: "FILTER",
Config: map[string]string{
"Emitter_Mem_Buf_Limit": "10M",
"Emitter_Storage.type": "filesystem",
"Match": tag,
"Name": "rewrite_tag",
"Rule": "$logName .* $logName false",
},
},
{
Kind: "FILTER",
Config: map[string]string{
"Match": logName,
"Name": "modify",
"Remove": "logName",
},
},
}
}

Expand Down
16 changes: 2 additions & 14 deletions ...a/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,12 @@
storage.type filesystem

[FILTER]
Add logName syslog
Add logging.googleapis.com/logName syslog
Match default_pipeline.syslog
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match default_pipeline.syslog
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match syslog
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(syslog)$
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
16 changes: 2 additions & 14 deletions confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,12 @@
storage.type filesystem

[FILTER]
Add logName syslog
Add logging.googleapis.com/logName syslog
Match default_pipeline.syslog
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match default_pipeline.syslog
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match syslog
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(syslog)$
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
16 changes: 2 additions & 14 deletions confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,12 @@
storage.type filesystem

[FILTER]
Add logName syslog
Add logging.googleapis.com/logName syslog
Match default_pipeline.syslog
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match default_pipeline.syslog
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match syslog
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(syslog)$
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
16 changes: 2 additions & 14 deletions confgenerator/testdata/valid/linux/logging-custom_log_level/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,12 @@
storage.type filesystem

[FILTER]
Add logName syslog
Add logging.googleapis.com/logName syslog
Match default_pipeline.syslog
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match default_pipeline.syslog
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match syslog
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(syslog)$
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
58 changes: 5 additions & 53 deletions ...ator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,87 +79,39 @@
storage.type filesystem

[FILTER]
Add logName log_source_id1
Add logging.googleapis.com/logName log_source_id1
Match pipeline1.log_source_id1
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline1.log_source_id1
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match log_source_id1
Name modify
Remove logName

[FILTER]
Add logName log_source_id2
Add logging.googleapis.com/logName log_source_id2
Match pipeline2.log_source_id2
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline2.log_source_id2
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match log_source_id2
Name modify
Remove logName

[FILTER]
Key_Name message
Match pipeline3.test_syslog_source_id_tcp
Name parser
Parser pipeline3.test_syslog_source_id_tcp.0

[FILTER]
Add logName test_syslog_source_id_tcp
Add logging.googleapis.com/logName test_syslog_source_id_tcp
Match pipeline3.test_syslog_source_id_tcp
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline3.test_syslog_source_id_tcp
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match test_syslog_source_id_tcp
Name modify
Remove logName

[FILTER]
Key_Name message
Match pipeline4.test_syslog_source_id_udp
Name parser
Parser pipeline4.test_syslog_source_id_udp.0

[FILTER]
Add logName test_syslog_source_id_udp
Add logging.googleapis.com/logName test_syslog_source_id_udp
Match pipeline4.test_syslog_source_id_udp
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline4.test_syslog_source_id_udp
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match test_syslog_source_id_udp
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(log_source_id1|log_source_id2|test_syslog_source_id_tcp|test_syslog_source_id_udp)$
Match_Regex ^(pipeline1\.log_source_id1|pipeline2\.log_source_id2|pipeline3\.test_syslog_source_id_tcp|pipeline4\.test_syslog_source_id_udp)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
44 changes: 4 additions & 40 deletions confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,22 +71,10 @@
storage.type filesystem

[FILTER]
Add logName syslog
Add logging.googleapis.com/logName syslog
Match default_pipeline.syslog
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match default_pipeline.syslog
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match syslog
Name modify
Remove logName

[FILTER]
Key_Name message
Match pipeline1.sample_logs
Expand All @@ -100,22 +88,10 @@
Parser pipeline1.sample_logs.1

[FILTER]
Add logName sample_logs
Add logging.googleapis.com/logName sample_logs
Match pipeline1.sample_logs
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline1.sample_logs
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match sample_logs
Name modify
Remove logName

[FILTER]
Key_Name message
Match pipeline2.sample_logs
Expand All @@ -129,24 +105,12 @@
Parser pipeline2.sample_logs.1

[FILTER]
Add logName sample_logs
Add logging.googleapis.com/logName sample_logs
Match pipeline2.sample_logs
Name modify

[FILTER]
Emitter_Mem_Buf_Limit 10M
Emitter_Storage.type filesystem
Match pipeline2.sample_logs
Name rewrite_tag
Rule $logName .* $logName false

[FILTER]
Match sample_logs
Name modify
Remove logName

[OUTPUT]
Match_Regex ^(sample_logs|sample_logs|syslog)$
Match_Regex ^(default_pipeline\.syslog|pipeline1\.sample_logs|pipeline2\.sample_logs)$
Name stackdriver
Retry_Limit 3
resource gce_instance
Expand Down
Loading

0 comments on commit dc09e78

Please sign in to comment.