Skip to content

Commit

Permalink
[pkg/stanza] Consolidate attribute code into single package (open-tel…
Browse files Browse the repository at this point in the history
…emetry#30449)

This change simplifies the reader package by pulling all file resolution
logic into a single package. It also moves related config and attribute
name constants into the same package.
  • Loading branch information
djaglowski authored and cparkins committed Feb 1, 2024
1 parent 013366a commit 90fbca9
Show file tree
Hide file tree
Showing 10 changed files with 264 additions and 283 deletions.
27 changes: 27 additions & 0 deletions .chloggen/pkg-stanza-reader-attributes-refactor.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: deprecation

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: pkg/stanza

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Deprecate pkg/stanza/attrs package in favor of pkg/stanza/fileconsumer/attrs

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [30449]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [api]
21 changes: 17 additions & 4 deletions pkg/stanza/attrs/attrs.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,22 @@

package attrs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/attrs"

import fca "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/attrs"

const (
LogFileName = "log.file.name"
LogFilePath = "log.file.path"
LogFileNameResolved = "log.file.name_resolved"
LogFilePathResolved = "log.file.path_resolved"
// Deprecated: [v0.93.0] Use pkg/stanza/fileconsumer/attrs.LogFileName instead.
// Will be removed in v0.94.0.
LogFileName = fca.LogFileName

// Deprecated: [v0.92.0] Use pkg/stanza/fileconsumer/attrs.LogFilePath instead.
// Will be removed in v0.94.0.
LogFilePath = fca.LogFilePath

// Deprecated: [v0.92.0] Use pkg/stanza/fileconsumer/attrs.LogFileNameResolved instead.
// Will be removed in v0.94.0.
LogFileNameResolved = fca.LogFileNameResolved

// Deprecated: [v0.92.0] Use pkg/stanza/fileconsumer/attrs.LogFilePathResolved instead.
// Will be removed in v0.94.0.
LogFilePathResolved = fca.LogFilePathResolved
)
60 changes: 60 additions & 0 deletions pkg/stanza/fileconsumer/attrs/attrs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

package attrs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/attrs"

import (
"fmt"
"path/filepath"
"runtime"
)

const (
LogFileName = "log.file.name"
LogFilePath = "log.file.path"
LogFileNameResolved = "log.file.name_resolved"
LogFilePathResolved = "log.file.path_resolved"
)

type Resolver struct {
IncludeFileName bool `mapstructure:"include_file_name,omitempty"`
IncludeFilePath bool `mapstructure:"include_file_path,omitempty"`
IncludeFileNameResolved bool `mapstructure:"include_file_name_resolved,omitempty"`
IncludeFilePathResolved bool `mapstructure:"include_file_path_resolved,omitempty"`
}

func (r *Resolver) Resolve(path string) (attributes map[string]any, err error) {
// size 2 is sufficient if not resolving symlinks. This optimizes for the most performant cases.
attributes = make(map[string]any, 2)
if r.IncludeFileName {
attributes[LogFileName] = filepath.Base(path)
}
if r.IncludeFilePath {
attributes[LogFilePath] = path
}
if !r.IncludeFileNameResolved && !r.IncludeFilePathResolved {
return attributes, nil
}

resolved := path
// Dirty solution, waiting for this permanent fix https://github.com/golang/go/issues/39786
// EvalSymlinks on windows is partially working depending on the way you use Symlinks and Junctions
if runtime.GOOS != "windows" {
resolved, err = filepath.EvalSymlinks(path)
if err != nil {
return nil, fmt.Errorf("resolve symlinks: %w", err)
}
}
abs, err := filepath.Abs(resolved)
if err != nil {
return nil, fmt.Errorf("resolve abs: %w", err)
}

if r.IncludeFileNameResolved {
attributes[LogFileNameResolved] = filepath.Base(abs)
}
if r.IncludeFilePathResolved {
attributes[LogFilePathResolved] = abs
}
return attributes, nil
}
73 changes: 73 additions & 0 deletions pkg/stanza/fileconsumer/attrs/attrs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

package attrs

import (
"fmt"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/filetest"
)

func TestResolver(t *testing.T) {
t.Parallel()

for i := 0; i < 16; i++ {

// Create a 4 bit string where each bit represents the value of a config option
bitString := fmt.Sprintf("%04b", i)

// Create a resolver with a config that matches the bit pattern of i
r := Resolver{
IncludeFileName: bitString[0] == '1',
IncludeFilePath: bitString[1] == '1',
IncludeFileNameResolved: bitString[2] == '1',
IncludeFilePathResolved: bitString[3] == '1',
}

t.Run(bitString, func(t *testing.T) {
// Create a file
tempDir := t.TempDir()
temp := filetest.OpenTemp(t, tempDir)

attributes, err := r.Resolve(temp.Name())
assert.NoError(t, err)

var expectLen int
if r.IncludeFileName {
expectLen++
assert.Equal(t, filepath.Base(temp.Name()), attributes[LogFileName])
} else {
assert.Empty(t, attributes[LogFileName])
}
if r.IncludeFilePath {
expectLen++
assert.Equal(t, temp.Name(), attributes[LogFilePath])
} else {
assert.Empty(t, attributes[LogFilePath])
}

// We don't have an independent way to resolve the path, so the only meangingful validate
// is to ensure that the resolver returns nothing vs something based on the config.
if r.IncludeFileNameResolved {
expectLen++
assert.NotNil(t, attributes[LogFileNameResolved])
assert.IsType(t, "", attributes[LogFileNameResolved])
} else {
assert.Empty(t, attributes[LogFileNameResolved])
}
if r.IncludeFilePathResolved {
expectLen++
assert.NotNil(t, attributes[LogFilePathResolved])
assert.IsType(t, "", attributes[LogFilePathResolved])
} else {
assert.Empty(t, attributes[LogFilePathResolved])
}
assert.Equal(t, expectLen, len(attributes))
})
}
}
63 changes: 30 additions & 33 deletions pkg/stanza/fileconsumer/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"golang.org/x/text/encoding"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/decode"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/attrs"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/emit"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/fingerprint"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/header"
Expand Down Expand Up @@ -55,29 +56,28 @@ func NewConfig() *Config {
MaxLogSize: reader.DefaultMaxLogSize,
Encoding: defaultEncoding,
FlushPeriod: reader.DefaultFlushPeriod,
IncludeFileName: true,
Resolver: attrs.Resolver{
IncludeFileName: true,
},
}
}

// Config is the configuration of a file input operator
type Config struct {
matcher.Criteria `mapstructure:",squash"`
PollInterval time.Duration `mapstructure:"poll_interval,omitempty"`
MaxConcurrentFiles int `mapstructure:"max_concurrent_files,omitempty"`
MaxBatches int `mapstructure:"max_batches,omitempty"`
StartAt string `mapstructure:"start_at,omitempty"`
FingerprintSize helper.ByteSize `mapstructure:"fingerprint_size,omitempty"`
MaxLogSize helper.ByteSize `mapstructure:"max_log_size,omitempty"`
Encoding string `mapstructure:"encoding,omitempty"`
SplitConfig split.Config `mapstructure:"multiline,omitempty"`
TrimConfig trim.Config `mapstructure:",squash,omitempty"`
FlushPeriod time.Duration `mapstructure:"force_flush_period,omitempty"`
IncludeFileName bool `mapstructure:"include_file_name,omitempty"`
IncludeFilePath bool `mapstructure:"include_file_path,omitempty"`
IncludeFileNameResolved bool `mapstructure:"include_file_name_resolved,omitempty"`
IncludeFilePathResolved bool `mapstructure:"include_file_path_resolved,omitempty"`
Header *HeaderConfig `mapstructure:"header,omitempty"`
DeleteAfterRead bool `mapstructure:"delete_after_read,omitempty"`
matcher.Criteria `mapstructure:",squash"`
attrs.Resolver `mapstructure:",squash"`
PollInterval time.Duration `mapstructure:"poll_interval,omitempty"`
MaxConcurrentFiles int `mapstructure:"max_concurrent_files,omitempty"`
MaxBatches int `mapstructure:"max_batches,omitempty"`
StartAt string `mapstructure:"start_at,omitempty"`
FingerprintSize helper.ByteSize `mapstructure:"fingerprint_size,omitempty"`
MaxLogSize helper.ByteSize `mapstructure:"max_log_size,omitempty"`
Encoding string `mapstructure:"encoding,omitempty"`
SplitConfig split.Config `mapstructure:"multiline,omitempty"`
TrimConfig trim.Config `mapstructure:",squash,omitempty"`
FlushPeriod time.Duration `mapstructure:"force_flush_period,omitempty"`
Header *HeaderConfig `mapstructure:"header,omitempty"`
DeleteAfterRead bool `mapstructure:"delete_after_read,omitempty"`
}

type HeaderConfig struct {
Expand Down Expand Up @@ -150,21 +150,18 @@ func (c Config) buildManager(logger *zap.SugaredLogger, emit emit.Callback, spli
}

readerFactory := reader.Factory{
SugaredLogger: logger.With("component", "fileconsumer"),
FromBeginning: startAtBeginning,
FingerprintSize: int(c.FingerprintSize),
MaxLogSize: int(c.MaxLogSize),
Encoding: enc,
SplitFunc: splitFunc,
TrimFunc: trimFunc,
FlushTimeout: c.FlushPeriod,
EmitFunc: emit,
IncludeFileName: c.IncludeFileName,
IncludeFilePath: c.IncludeFilePath,
IncludeFileNameResolved: c.IncludeFileNameResolved,
IncludeFilePathResolved: c.IncludeFilePathResolved,
HeaderConfig: hCfg,
DeleteAtEOF: c.DeleteAfterRead,
SugaredLogger: logger.With("component", "fileconsumer"),
FromBeginning: startAtBeginning,
FingerprintSize: int(c.FingerprintSize),
MaxLogSize: int(c.MaxLogSize),
Encoding: enc,
SplitFunc: splitFunc,
TrimFunc: trimFunc,
FlushTimeout: c.FlushPeriod,
EmitFunc: emit,
Attributes: c.Resolver,
HeaderConfig: hCfg,
DeleteAtEOF: c.DeleteAfterRead,
}

return &Manager{
Expand Down
2 changes: 1 addition & 1 deletion pkg/stanza/fileconsumer/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/featuregate"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/attrs"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/attrs"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/emittest"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/filetest"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader"
Expand Down
Loading

0 comments on commit 90fbca9

Please sign in to comment.