Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update log collection #3017

Merged
merged 2 commits into from
Apr 8, 2021
Merged

Update log collection #3017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
43a3283
Update log collection
djaglowski Apr 7, 2021
1a8a440
Fix k8s testbed
djaglowski Apr 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions examples/kubernetes/otel-collector-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ receivers:
id: get-format
routes:
- output: parser-docker
expr: '$$record matches "^\\{"'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please also remember to update the helm chart.

expr: '$$body matches "^\\{"'
- output: parser-crio
expr: '$$record matches "^[^ Z]+ "'
expr: '$$body matches "^[^ Z]+ "'
- output: parser-containerd
expr: '$$record matches "^[^ Z]+Z"'
expr: '$$body matches "^[^ Z]+Z"'
# Parse CRI-O format
- type: regex_parser
id: parser-crio
Expand Down Expand Up @@ -57,9 +57,9 @@ receivers:
k8s.pod.name: 'EXPR($.pod_name)'
run_id: 'EXPR($.run_id)'
k8s.pod.uid: 'EXPR($.uid)'
# Clean up log record
# Clean up log body
- type: restructure
id: clean-up-log-record
id: clean-up-log-body
ops:
- move:
from: log
Expand Down
10 changes: 5 additions & 5 deletions examples/kubernetes/otel-collector.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ data:
id: get-format
routes:
- output: parser-docker
expr: '$$record matches "^\\{"'
expr: '$$body matches "^\\{"'
- output: parser-crio
expr: '$$record matches "^[^ Z]+ "'
expr: '$$body matches "^[^ Z]+ "'
- output: parser-containerd
expr: '$$record matches "^[^ Z]+Z"'
expr: '$$body matches "^[^ Z]+Z"'
# Parse CRI-O format
- type: regex_parser
id: parser-crio
Expand Down Expand Up @@ -64,9 +64,9 @@ data:
k8s.pod.name: 'EXPR($.pod_name)'
run_id: 'EXPR($.run_id)'
k8s.pod.uid: 'EXPR($.uid)'
# Clean up log record
# Clean up log body
- type: restructure
id: clean-up-log-record
id: clean-up-log-body
ops:
- move:
from: log
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,8 @@ github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuK
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31 h1:FruNezloXnyrkbsQhprOfk0i1M5ftEOZ3sG0+lpRJP4=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0 h1:B4J6pUlpMf7GXTb0Tt9NpxXjRJPedd4RrP/zkBl9eCQ=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
Expand Down
2 changes: 1 addition & 1 deletion internal/stanza/converter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func Convert(obsLog *entry.Entry) pdata.Logs {
}
}

insertToAttributeVal(obsLog.Record, lr.Body())
insertToAttributeVal(obsLog.Body, lr.Body())

ills.Logs().Append(lr)

Expand Down
54 changes: 27 additions & 27 deletions internal/stanza/converter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func complexEntry() *entry.Entry {
e.AddResourceKey("type", "global")
e.AddAttribute("one", "two")
e.AddAttribute("two", "three")
e.Record = map[string]interface{}{
e.Body = map[string]interface{}{
"bool": true,
"int": 123,
"double": 12.34,
Expand Down Expand Up @@ -78,7 +78,7 @@ func TestConvertMetadata(t *testing.T) {
e.Severity = entry.Error
e.AddResourceKey("type", "global")
e.AddAttribute("one", "two")
e.Record = true
e.Body = true

result := Convert(e)

Expand Down Expand Up @@ -110,30 +110,30 @@ func TestConvertMetadata(t *testing.T) {

func TestConvertSimpleBody(t *testing.T) {

require.True(t, recordToBody(true).BoolVal())
require.False(t, recordToBody(false).BoolVal())
require.True(t, anyToBody(true).BoolVal())
require.False(t, anyToBody(false).BoolVal())

require.Equal(t, "string", recordToBody("string").StringVal())
require.Equal(t, "bytes", recordToBody([]byte("bytes")).StringVal())
require.Equal(t, "string", anyToBody("string").StringVal())
require.Equal(t, "bytes", anyToBody([]byte("bytes")).StringVal())

require.Equal(t, int64(1), recordToBody(1).IntVal())
require.Equal(t, int64(1), recordToBody(int8(1)).IntVal())
require.Equal(t, int64(1), recordToBody(int16(1)).IntVal())
require.Equal(t, int64(1), recordToBody(int32(1)).IntVal())
require.Equal(t, int64(1), recordToBody(int64(1)).IntVal())
require.Equal(t, int64(1), anyToBody(1).IntVal())
require.Equal(t, int64(1), anyToBody(int8(1)).IntVal())
require.Equal(t, int64(1), anyToBody(int16(1)).IntVal())
require.Equal(t, int64(1), anyToBody(int32(1)).IntVal())
require.Equal(t, int64(1), anyToBody(int64(1)).IntVal())

require.Equal(t, int64(1), recordToBody(uint(1)).IntVal())
require.Equal(t, int64(1), recordToBody(uint8(1)).IntVal())
require.Equal(t, int64(1), recordToBody(uint16(1)).IntVal())
require.Equal(t, int64(1), recordToBody(uint32(1)).IntVal())
require.Equal(t, int64(1), recordToBody(uint64(1)).IntVal())
require.Equal(t, int64(1), anyToBody(uint(1)).IntVal())
require.Equal(t, int64(1), anyToBody(uint8(1)).IntVal())
require.Equal(t, int64(1), anyToBody(uint16(1)).IntVal())
require.Equal(t, int64(1), anyToBody(uint32(1)).IntVal())
require.Equal(t, int64(1), anyToBody(uint64(1)).IntVal())

require.Equal(t, float64(1), recordToBody(float32(1)).DoubleVal())
require.Equal(t, float64(1), recordToBody(float64(1)).DoubleVal())
require.Equal(t, float64(1), anyToBody(float32(1)).DoubleVal())
require.Equal(t, float64(1), anyToBody(float64(1)).DoubleVal())
}

func TestConvertMapBody(t *testing.T) {
structuredRecord := map[string]interface{}{
structuredBody := map[string]interface{}{
"true": true,
"false": false,
"string": "string",
Expand All @@ -152,7 +152,7 @@ func TestConvertMapBody(t *testing.T) {
"float64": float64(1),
}

result := recordToBody(structuredRecord).MapVal()
result := anyToBody(structuredBody).MapVal()

v, _ := result.Get("true")
require.True(t, v.BoolVal())
Expand All @@ -174,7 +174,7 @@ func TestConvertMapBody(t *testing.T) {
}

func TestConvertArrayBody(t *testing.T) {
structuredRecord := []interface{}{
structuredBody := []interface{}{
true,
false,
"string",
Expand All @@ -195,7 +195,7 @@ func TestConvertArrayBody(t *testing.T) {
map[string]interface{}{"one": 1, "yes": true},
}

result := recordToBody(structuredRecord).ArrayVal()
result := anyToBody(structuredBody).ArrayVal()

require.True(t, result.At(0).BoolVal())
require.False(t, result.At(1).BoolVal())
Expand Down Expand Up @@ -229,20 +229,20 @@ func TestConvertArrayBody(t *testing.T) {

func TestConvertUnknownBody(t *testing.T) {
unknownType := map[string]int{"0": 0, "1": 1}
require.Equal(t, fmt.Sprintf("%v", unknownType), recordToBody(unknownType).StringVal())
require.Equal(t, fmt.Sprintf("%v", unknownType), anyToBody(unknownType).StringVal())
}

func TestConvertNestedMapBody(t *testing.T) {

unknownType := map[string]int{"0": 0, "1": 1}

structuredRecord := map[string]interface{}{
structuredBody := map[string]interface{}{
"array": []interface{}{0, 1},
"map": map[string]interface{}{"0": 0, "1": "one"},
"unknown": unknownType,
}

result := recordToBody(structuredRecord).MapVal()
result := anyToBody(structuredBody).MapVal()

arrayAttVal, _ := result.Get("array")
a := arrayAttVal.ArrayVal()
Expand All @@ -260,9 +260,9 @@ func TestConvertNestedMapBody(t *testing.T) {
require.Equal(t, fmt.Sprintf("%v", unknownType), unknownAttVal.StringVal())
}

func recordToBody(record interface{}) pdata.AttributeValue {
func anyToBody(body interface{}) pdata.AttributeValue {
entry := entry.New()
entry.Record = record
entry.Body = body
return convertAndDrill(entry).Body()
}

Expand Down
2 changes: 1 addition & 1 deletion internal/stanza/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/stanza
go 1.14

require (
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31
github.com/open-telemetry/opentelemetry-log-collection v0.17.0
github.com/stretchr/testify v1.7.0
go.opentelemetry.io/collector v0.24.0
go.uber.org/zap v1.16.0
Expand Down
2 changes: 2 additions & 0 deletions internal/stanza/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31 h1:FruNezloXnyrkbsQhprOfk0i1M5ftEOZ3sG0+lpRJP4=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0 h1:B4J6pUlpMf7GXTb0Tt9NpxXjRJPedd4RrP/zkBl9eCQ=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
Expand Down
2 changes: 1 addition & 1 deletion receiver/filelogreceiver/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Supported pipeline types: logs
| `include` | required | A list of file glob patterns that match the file paths to be read |
| `exclude` | [] | A list of file glob patterns to exclude from reading |
| `start_at` | `end` | At startup, where to start reading logs from the file. Options are `beginning` or `end` |
| `write_to` | $record | The record [field](https://github.com/open-telemetry/opentelemetry-log-collection/blob/main/docs/types/field.md) written to when creating a new log entry |
| `write_to` | $body | The body [field](https://github.com/open-telemetry/opentelemetry-log-collection/blob/main/docs/types/field.md) written to when creating a new log entry |
| `multiline` | | A `multiline` configuration block. See below for more details |
| `encoding` | `nop` | The encoding of the file being read. See the list of supported encodings below for available options |
| `include_file_name` | `true` | Whether to add the file name as the label `file_name` |
Expand Down
8 changes: 4 additions & 4 deletions receiver/filelogreceiver/filelog_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,19 +90,19 @@ func TestReadStaticFile(t *testing.T) {
e1 := entry.New()
e1.Timestamp = expectedTimestamp
e1.Severity = entry.Info
e1.Set(entry.NewRecordField("msg"), "Something routine")
e1.Set(entry.NewBodyField("msg"), "Something routine")
e1.AddAttribute("file_name", "simple.log")

e2 := entry.New()
e2.Timestamp = expectedTimestamp
e2.Severity = entry.Error
e2.Set(entry.NewRecordField("msg"), "Something bad happened!")
e2.Set(entry.NewBodyField("msg"), "Something bad happened!")
e2.AddAttribute("file_name", "simple.log")

e3 := entry.New()
e3.Timestamp = expectedTimestamp
e3.Severity = entry.Debug
e3.Set(entry.NewRecordField("msg"), "Some details...")
e3.Set(entry.NewBodyField("msg"), "Some details...")
e3.AddAttribute("file_name", "simple.log")

expectedLogs := []pdata.Logs{
Expand Down Expand Up @@ -189,7 +189,7 @@ func (rt *rotationTest) Run(t *testing.T) {

e := entry.New()
e.Timestamp = expectedTimestamp
e.Set(entry.NewRecordField("msg"), msg)
e.Set(entry.NewBodyField("msg"), msg)
expectedLogs[i] = stanza.Convert(e)
}

Expand Down
2 changes: 1 addition & 1 deletion receiver/filelogreceiver/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.14
require (
github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc
github.com/open-telemetry/opentelemetry-collector-contrib/internal/stanza v0.0.0-00010101000000-000000000000
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31
github.com/open-telemetry/opentelemetry-log-collection v0.17.0
github.com/stretchr/testify v1.7.0
go.opentelemetry.io/collector v0.24.0
go.uber.org/zap v1.16.0
Expand Down
2 changes: 2 additions & 0 deletions receiver/filelogreceiver/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31 h1:FruNezloXnyrkbsQhprOfk0i1M5ftEOZ3sG0+lpRJP4=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0 h1:B4J6pUlpMf7GXTb0Tt9NpxXjRJPedd4RrP/zkBl9eCQ=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
Expand Down
2 changes: 1 addition & 1 deletion receiver/syslogreceiver/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.14

require (
github.com/open-telemetry/opentelemetry-collector-contrib/internal/stanza v0.0.0-00010101000000-000000000000
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31
github.com/open-telemetry/opentelemetry-log-collection v0.17.0
github.com/stretchr/testify v1.7.0
go.opentelemetry.io/collector v0.24.0
go.uber.org/zap v1.16.0
Expand Down
2 changes: 2 additions & 0 deletions receiver/syslogreceiver/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31 h1:FruNezloXnyrkbsQhprOfk0i1M5ftEOZ3sG0+lpRJP4=
github.com/open-telemetry/opentelemetry-log-collection v0.16.1-0.20210315170618-aa8d82509c31/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0 h1:B4J6pUlpMf7GXTb0Tt9NpxXjRJPedd4RrP/zkBl9eCQ=
github.com/open-telemetry/opentelemetry-log-collection v0.17.0/go.mod h1:9E7qy5/82Mr4FysOjfbxcHxk2+AXyOukDhdmezEjNdo=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
Expand Down
14 changes: 7 additions & 7 deletions testbed/datasenders/k8s.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,11 @@ func NewKubernetesContainerWriter() *FileLogK8sWriter {
id: get-format
routes:
- output: parser-docker
expr: '$$record matches "^\\{"'
expr: '$$body matches "^\\{"'
- output: parser-crio
expr: '$$record matches "^[^ Z]+ "'
expr: '$$body matches "^[^ Z]+ "'
- output: parser-containerd
expr: '$$record matches "^[^ Z]+Z"'
expr: '$$body matches "^[^ Z]+Z"'
# Parse CRI-O format
- type: regex_parser
id: parser-crio
Expand Down Expand Up @@ -211,9 +211,9 @@ func NewKubernetesContainerWriter() *FileLogK8sWriter {
k8s.pod.name: 'EXPR($.pod_name)'
run_id: 'EXPR($.run_id)'
k8s.pod.uid: 'EXPR($.uid)'
# Clean up log record
# Clean up log body
- type: restructure
id: clean-up-log-record
id: clean-up-log-body
ops:
- move:
from: log
Expand Down Expand Up @@ -253,9 +253,9 @@ func NewKubernetesCRIContainerdWriter() *FileLogK8sWriter {
k8s.pod.name: 'EXPR($.pod_name)'
run_id: 'EXPR($.run_id)'
k8s.pod.uid: 'EXPR($.uid)'
# Clean up log record
# Clean up log body
- type: restructure
id: clean-up-log-record
id: clean-up-log-body
ops:
- move:
from: log
Expand Down