diff --git a/.chloggen/builder-confmap-providers.yaml b/.chloggen/builder-confmap-providers.yaml new file mode 100755 index 00000000000..f617b1f058e --- /dev/null +++ b/.chloggen/builder-confmap-providers.yaml @@ -0,0 +1,31 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: cmd/builder + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Allow configuring `confmap.Provider`s in the builder. + +# One or more tracking issues or pull requests related to the change +issues: [4759] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + If no providers are specified, the defaults are used. + The default providers are: env, file, http, https, and yaml. + + To configure providers, use the `providers` key in your OCB build + manifest with a list of Go modules for your providers. + The modules will work the same as other Collector components. + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/cmd/builder/README.md b/cmd/builder/README.md index 18cffa052d9..9191daf8b6c 100644 --- a/cmd/builder/README.md +++ b/cmd/builder/README.md @@ -12,14 +12,20 @@ dist: description: Local OpenTelemetry Collector binary output_path: /tmp/dist exporters: - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.86.0 - - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.86.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.99.0 + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.99.0 receivers: - - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.86.0 + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.99.0 processors: - - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.86.0 + - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.99.0 + +providers: + - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.99.0 + +converters: + - gomod: go.opentelemetry.io/collector/confmap/converter/expandconverter v0.99.0 EOF $ builder --config=otelcol-builder.yaml $ cat > /tmp/otelcol.yaml <= 0.86.0") if err != nil { return err @@ -124,6 +133,20 @@ func (c *Config) SetRequireOtelColModule() error { } c.Distribution.RequireOtelColModule = constraint.Check(otelColVersion) + + // check whether confmap factories are supported + constraint, err = version.NewConstraint(">= 0.99.0") + if err != nil { + return err + } + + otelColVersion, err = version.NewVersion(c.Distribution.OtelColVersion) + if err != nil { + return err + } + + c.Distribution.SupportsConfmapFactories = constraint.Check(otelColVersion) + return nil } @@ -156,6 +179,36 @@ func (c *Config) ParseModules() error { return err } + if c.Providers != nil { + providers, err := parseModules(*c.Providers) + if err != nil { + return err + } + c.Providers = &providers + } else { + providers, err := parseModules([]Module{ + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/envprovider v" + c.Distribution.OtelColVersion, + }, + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/fileprovider v" + c.Distribution.OtelColVersion, + }, + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/httpprovider v" + c.Distribution.OtelColVersion, + }, + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/httpsprovider v" + c.Distribution.OtelColVersion, + }, + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/yamlprovider v" + c.Distribution.OtelColVersion, + }, + }) + if err != nil { + return err + } + c.Providers = &providers + } + return nil } diff --git a/cmd/builder/internal/builder/config_test.go b/cmd/builder/internal/builder/config_test.go index dccee464fe4..cd81b9f2887 100644 --- a/cmd/builder/internal/builder/config_test.go +++ b/cmd/builder/internal/builder/config_test.go @@ -220,8 +220,89 @@ func TestRequireOtelColModule(t *testing.T) { t.Run(tt.Version, func(t *testing.T) { cfg := NewDefaultConfig() cfg.Distribution.OtelColVersion = tt.Version - require.NoError(t, cfg.SetRequireOtelColModule()) + require.NoError(t, cfg.SetBackwardsCompatibility()) assert.Equal(t, tt.ExpectedRequireOtelColModule, cfg.Distribution.RequireOtelColModule) }) } } + +func TestConfmapFactoryVersions(t *testing.T) { + testCases := []struct { + version string + supported bool + err bool + }{ + { + version: "x.0.0", + supported: false, + err: true, + }, + { + version: "0.x.0", + supported: false, + err: true, + }, + { + version: "0.0.0", + supported: false, + }, + { + version: "0.98.0", + supported: false, + }, + { + version: "0.98.1", + supported: false, + }, + { + version: "0.99.0", + supported: true, + }, + { + version: "0.99.7", + supported: true, + }, + { + version: "0.100.0", + supported: true, + }, + { + version: "0.100.1", + supported: true, + }, + { + version: "1.0", + supported: true, + }, + { + version: "1.0.0", + supported: true, + }, + } + + for _, tt := range testCases { + t.Run(tt.version, func(t *testing.T) { + cfg := NewDefaultConfig() + cfg.Distribution.OtelColVersion = tt.version + if !tt.err { + require.NoError(t, cfg.SetBackwardsCompatibility()) + assert.Equal(t, tt.supported, cfg.Distribution.SupportsConfmapFactories) + } else { + require.Error(t, cfg.SetBackwardsCompatibility()) + } + }) + } +} + +func TestAddsDefaultProviders(t *testing.T) { + cfg := NewDefaultConfig() + cfg.Providers = nil + assert.NoError(t, cfg.ParseModules()) + assert.Len(t, *cfg.Providers, 5) +} + +func TestSkipsNilFieldValidation(t *testing.T) { + cfg := NewDefaultConfig() + cfg.Providers = nil + assert.NoError(t, cfg.Validate()) +} diff --git a/cmd/builder/internal/builder/main.go b/cmd/builder/internal/builder/main.go index d8a2d4d07a8..8663574f16c 100644 --- a/cmd/builder/internal/builder/main.go +++ b/cmd/builder/internal/builder/main.go @@ -234,7 +234,8 @@ func (c *Config) allComponents() []Module { append(c.Receivers, append(c.Processors, append(c.Extensions, - c.Connectors...)...)...)...) + append(c.Connectors, + *c.Providers...)...)...)...)...) } func (c *Config) readGoModFile() (string, map[string]string, error) { diff --git a/cmd/builder/internal/builder/main_test.go b/cmd/builder/internal/builder/main_test.go index 721b27b1456..884ed8f99d9 100644 --- a/cmd/builder/internal/builder/main_test.go +++ b/cmd/builder/internal/builder/main_test.go @@ -33,12 +33,23 @@ require ( )`) ) +func newInitializedConfig(t *testing.T) Config { + cfg := NewDefaultConfig() + // Validate and ParseModules will be called before the config is + // given to Generate. + assert.NoError(t, cfg.Validate()) + assert.NoError(t, cfg.SetBackwardsCompatibility()) + assert.NoError(t, cfg.ParseModules()) + + return cfg +} + func TestGenerateDefault(t *testing.T) { - require.NoError(t, Generate(NewDefaultConfig())) + require.NoError(t, Generate(newInitializedConfig(t))) } func TestGenerateInvalidOutputPath(t *testing.T) { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.OutputPath = "/:invalid" err := Generate(cfg) require.Error(t, err) @@ -54,7 +65,7 @@ func TestVersioning(t *testing.T) { { description: "defaults", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.Go = "go" return cfg }, @@ -63,7 +74,7 @@ func TestVersioning(t *testing.T) { { description: "require otelcol", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.Go = "go" cfg.Distribution.RequireOtelColModule = true return cfg @@ -73,7 +84,7 @@ func TestVersioning(t *testing.T) { { description: "only gomod file, skip generate", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) tempDir := t.TempDir() err := makeModule(tempDir, goModTestFile) require.NoError(t, err) @@ -87,7 +98,7 @@ func TestVersioning(t *testing.T) { { description: "old otel version", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Verbose = true cfg.Distribution.Go = "go" cfg.Distribution.OtelColVersion = "0.97.0" @@ -105,6 +116,12 @@ func TestVersioning(t *testing.T) { }, }) require.NoError(t, err) + *cfg.Providers, err = parseModules([]Module{ + { + GoMod: "go.opentelemetry.io/collector/confmap/provider/envprovider v0.97.0", + }, + }) + require.NoError(t, err) return cfg }, expectedErr: nil, @@ -112,7 +129,7 @@ func TestVersioning(t *testing.T) { { description: "old component version", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.Go = "go" cfg.Exporters = []Module{ { @@ -126,7 +143,7 @@ func TestVersioning(t *testing.T) { { description: "old component version without strict mode", cfgBuilder: func() Config { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.Go = "go" cfg.SkipStrictVersioning = true cfg.Exporters = []Module{ @@ -142,7 +159,7 @@ func TestVersioning(t *testing.T) { for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { cfg := tc.cfgBuilder() - require.NoError(t, cfg.SetRequireOtelColModule()) + require.NoError(t, cfg.SetBackwardsCompatibility()) require.NoError(t, cfg.Validate()) require.NoError(t, cfg.ParseModules()) err := GenerateAndCompile(cfg) @@ -152,7 +169,7 @@ func TestVersioning(t *testing.T) { } func TestSkipGenerate(t *testing.T) { - cfg := NewDefaultConfig() + cfg := newInitializedConfig(t) cfg.Distribution.OutputPath = t.TempDir() cfg.SkipGenerate = true err := Generate(cfg) @@ -241,6 +258,38 @@ func TestGenerateAndCompile(t *testing.T) { return cfg }, }, + { + testCase: "No providers", + cfgBuilder: func(t *testing.T) Config { + cfg := NewDefaultConfig() + cfg.Distribution.OutputPath = t.TempDir() + cfg.Replaces = append(cfg.Replaces, replaces...) + cfg.Providers = &[]Module{} + return cfg + }, + }, + { + testCase: "Pre-confmap factories", + cfgBuilder: func(t *testing.T) Config { + cfg := NewDefaultConfig() + cfg.Distribution.OutputPath = t.TempDir() + cfg.Replaces = append(cfg.Replaces, replaces...) + cfg.Distribution.OtelColVersion = "0.98.0" + cfg.SkipStrictVersioning = true + return cfg + }, + }, + { + testCase: "With confmap factories", + cfgBuilder: func(t *testing.T) Config { + cfg := NewDefaultConfig() + cfg.Distribution.OutputPath = t.TempDir() + cfg.Replaces = append(cfg.Replaces, replaces...) + cfg.Distribution.OtelColVersion = "0.99.0" + cfg.SkipStrictVersioning = true + return cfg + }, + }, } for _, tt := range testCases { @@ -248,6 +297,7 @@ func TestGenerateAndCompile(t *testing.T) { cfg := tt.cfgBuilder(t) assert.NoError(t, cfg.Validate()) assert.NoError(t, cfg.SetGoPath()) + assert.NoError(t, cfg.ParseModules()) require.NoError(t, GenerateAndCompile(cfg)) }) } diff --git a/cmd/builder/internal/builder/templates/go.mod.tmpl b/cmd/builder/internal/builder/templates/go.mod.tmpl index f70306ae95b..ad6fd3f78f6 100644 --- a/cmd/builder/internal/builder/templates/go.mod.tmpl +++ b/cmd/builder/internal/builder/templates/go.mod.tmpl @@ -5,6 +5,12 @@ module {{.Distribution.Module}} go 1.21 require ( + {{if .Distribution.SupportsConfmapFactories -}} + go.opentelemetry.io/collector/confmap/converter/expandconverter v{{.Distribution.OtelColVersion}} + {{- range .Providers}} + {{if .GoMod}}{{.GoMod}}{{end}} + {{- end}} + {{- end}} {{- range .Connectors}} {{if .GoMod}}{{.GoMod}}{{end}} {{- end}} diff --git a/cmd/builder/internal/builder/templates/main.go.tmpl b/cmd/builder/internal/builder/templates/main.go.tmpl index 64a81295bcb..f9964c5d5ec 100644 --- a/cmd/builder/internal/builder/templates/main.go.tmpl +++ b/cmd/builder/internal/builder/templates/main.go.tmpl @@ -7,6 +7,13 @@ import ( "log" "go.opentelemetry.io/collector/component" + {{- if .Distribution.SupportsConfmapFactories}} + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/confmap/converter/expandconverter" + {{- range .Providers}} + {{.Name}} "{{.Import}}" + {{- end}} + {{- end}} "go.opentelemetry.io/collector/otelcol" ) @@ -17,7 +24,26 @@ func main() { Version: "{{ .Distribution.Version }}", } - if err := run(otelcol.CollectorSettings{BuildInfo: info, Factories: components}); err != nil { + set := otelcol.CollectorSettings{ + BuildInfo: info, + Factories: components, + {{- if .Distribution.SupportsConfmapFactories}} + ConfigProviderSettings: otelcol.ConfigProviderSettings{ + ResolverSettings: confmap.ResolverSettings{ + ProviderFactories: []confmap.ProviderFactory{ + {{- range .Providers}} + {{.Name}}.NewFactory(), + {{- end}} + }, + ConverterFactories: []confmap.ConverterFactory{ + expandconverter.NewFactory(), + }, + }, + }, + {{- end}} + } + + if err := run(set); err != nil { log.Fatal(err) } } diff --git a/cmd/builder/internal/command.go b/cmd/builder/internal/command.go index 9ced5aa6c28..b6cc41a9ba5 100644 --- a/cmd/builder/internal/command.go +++ b/cmd/builder/internal/command.go @@ -66,7 +66,7 @@ configuration is provided, ocb will generate a default Collector. return fmt.Errorf("go not found: %w", err) } - if err := cfg.SetRequireOtelColModule(); err != nil { + if err := cfg.SetBackwardsCompatibility(); err != nil { return fmt.Errorf("unable to compare otelcol version: %w", err) } @@ -170,6 +170,7 @@ func applyCfgFromFile(flags *flag.FlagSet, cfgFromFile builder.Config) { cfg.Receivers = cfgFromFile.Receivers cfg.Processors = cfgFromFile.Processors cfg.Connectors = cfgFromFile.Connectors + cfg.Providers = cfgFromFile.Providers cfg.Replaces = cfgFromFile.Replaces cfg.Excludes = cfgFromFile.Excludes diff --git a/cmd/builder/internal/config/default.yaml b/cmd/builder/internal/config/default.yaml index 68aa48b8211..978eaea6775 100644 --- a/cmd/builder/internal/config/default.yaml +++ b/cmd/builder/internal/config/default.yaml @@ -24,3 +24,10 @@ processors: connectors: - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.98.0 +providers: + - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v0.95.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.95.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v0.95.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.95.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.95.0 + diff --git a/cmd/otelcorecol/builder-config.yaml b/cmd/otelcorecol/builder-config.yaml index 11170523329..9946db772de 100644 --- a/cmd/otelcorecol/builder-config.yaml +++ b/cmd/otelcorecol/builder-config.yaml @@ -24,6 +24,13 @@ processors: connectors: - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.98.0 +providers: + - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v0.98.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.98.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v0.98.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.98.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.98.0 + replaces: - go.opentelemetry.io/collector => ../../ - go.opentelemetry.io/collector/otelcol => ../../otelcol diff --git a/cmd/otelcorecol/main.go b/cmd/otelcorecol/main.go index ad081f185cd..27ebe7aaa11 100644 --- a/cmd/otelcorecol/main.go +++ b/cmd/otelcorecol/main.go @@ -17,7 +17,12 @@ func main() { Version: "0.98.0-dev", } - if err := run(otelcol.CollectorSettings{BuildInfo: info, Factories: components}); err != nil { + set := otelcol.CollectorSettings{ + BuildInfo: info, + Factories: components, + } + + if err := run(set); err != nil { log.Fatal(err) } }