Add a default build configuration to ocb. (#5752)

Embed the build configuration that is used to build otelcorecol into ocb
so that end users can easily generate a useful collector. This makes the
`--config` flag optional again.

Signed-off-by: James Peach <[email protected]>

Signed-off-by: James Peach <[email protected]>

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 - Fix reading resource attributes for trace JSON, remove duplicate code. (#6023)
 - Add support to unmarshalls bytes into plogs.Logs with `jsoniter` in jsonUnmarshaler(#5935)
+- Instead of exiting, `ocb` now generates a default Collector when no build configuration is supplied (#5752)
 
 
 ### 🛑 Breaking changes 🛑

Makefile

@@ -205,6 +205,7 @@ genotelcorecol:
 
 .PHONY: ocb
 ocb:
+	$(MAKE) -C cmd/builder config
 	$(MAKE) -C cmd/builder ocb
 
 DEPENDABOT_PATH=".github/dependabot.yml"
@@ -397,6 +398,7 @@ endif
 	sed -i.bak 's/$(PREVIOUS_VERSION)/$(RELEASE_CANDIDATE)/g' examples/k8s/otel-config.yaml
 	find . -name "*.bak" -type f -delete
 	# regenerate files
+	$(MAKE) -C cmd/builder config
 	$(MAKE) genotelcorecol
 	# commit changes before running multimod
 	git checkout -b opentelemetry-collector-bot/release-$(RELEASE_CANDIDATE)

cmd/builder/Makefile

@@ -3,3 +3,12 @@ include ../../Makefile.Common
 .PHONY: ocb
 ocb:
 	GO111MODULE=on CGO_ENABLED=0 $(GOCMD) build -trimpath -o ../../bin/ocb_$(GOOS)_$(GOARCH) .
+
+# Generate the default build config from otelcorecol, by removing the
+# "replaces" stanza, which is assumed to be at the end of the file.
+#
+# The default config file is checked in so that `go install` will work
+# and so that non-unix builds don't need sed to be installed.
+.PHONY: config
+config: internal/config/default.yaml
+	sed '-e/replaces:/,$$d' <../otelcorecol/builder-config.yaml > internal/config/default.yaml

cmd/builder/internal/command.go

@@ -27,6 +27,7 @@ import (
 	"go.uber.org/zap"
 
 	"go.opentelemetry.io/collector/cmd/builder/internal/builder"
+	"go.opentelemetry.io/collector/cmd/builder/internal/config"
 )
 
 const (
@@ -56,7 +57,8 @@ func Command() (*cobra.Command, error) {
 		Long: fmt.Sprintf("OpenTelemetry Collector Builder (%s)", version) + `
 
 ocb generates a custom OpenTelemetry Collector binary using the
-build configuration given by the "--config" argument.
+build configuration given by the "--config" argument. If no build
+configuration is provided, ocb will generate a default Collector.
 `,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -77,13 +79,6 @@ build configuration given by the "--config" argument.
 
 	cmd.Flags().StringVar(&cfgFile, "config", "", "build configuration file")
 
-	// A build configuration file is always required, and there's no
-	// default. We can relax this in future by embedding the default
-	// config that is used to build otelcorecol.
-	if err := cmd.MarkFlagRequired("config"); err != nil {
-		panic(err) // Only fails if the usage message is empty, which is a programmer error.
-	}
-
 	// the distribution parameters, which we accept as CLI flags as well
 	cmd.Flags().BoolVar(&cfg.SkipCompilation, skipCompilationFlag, false, "Whether builder should only generate go code with no compile of the collector (default false)")
 	cmd.Flags().BoolVar(&cfg.SkipGetModules, skipGetModulesFlag, false, "Whether builder should skip updating go.mod and retrieve Go module list (default false)")
@@ -126,8 +121,18 @@ func initConfig(flags *flag.FlagSet) error {
 	cfg.Logger.Info("OpenTelemetry Collector Builder",
 		zap.String("version", version), zap.String("date", date))
 
-	// load the config file
-	if err := k.Load(file.Provider(cfgFile), yaml.Parser()); err != nil {
+	var provider koanf.Provider
+
+	if cfgFile != "" {
+		// load the config file
+		provider = file.Provider(cfgFile)
+	} else {
+		// or the default if the config isn't provided
+		provider = config.DefaultProvider()
+		cfg.Logger.Info("Using default build configuration")
+	}
+
+	if err := k.Load(provider, yaml.Parser()); err != nil {
 		return fmt.Errorf("failed to load configuration file: %w", err)
 	}
 
@@ -145,7 +150,10 @@ func initConfig(flags *flag.FlagSet) error {
 
 	applyCfgFromFile(flags, cfgFromFile)
 
-	cfg.Logger.Info("Using config file", zap.String("path", cfgFile))
+	if cfgFile != "" {
+		cfg.Logger.Info("Using config file", zap.String("path", cfgFile))
+	}
+
 	return nil
 }
 

cmd/builder/internal/config/default.go

@@ -0,0 +1,32 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config // import "go.opentelemetry.io/collector/cmd/builder/internal/config"
+
+import (
+	"embed"
+
+	"github.com/knadh/koanf"
+	"github.com/knadh/koanf/providers/fs"
+)
+
+//go:embed *.yaml
+var configs embed.FS
+
+// DefaultProvider returns a koanf.Provider that provides the default build
+// configuration file. This is the same configuration that otelcorecol is
+// built with.
+func DefaultProvider() koanf.Provider {
+	return fs.Provider(configs, "default.yaml")
+}

cmd/builder/internal/config/default.yaml

@@ -0,0 +1,28 @@
+dist:
+  module: go.opentelemetry.io/collector/cmd/otelcorecol
+  name: otelcorecol
+  description: Local OpenTelemetry Collector binary, testing only.
+  version: 0.59.0-dev
+  otelcol_version: 0.59.0
+
+receivers:
+  - import: go.opentelemetry.io/collector/receiver/otlpreceiver
+    gomod: go.opentelemetry.io/collector v0.59.0
+exporters:
+  - import: go.opentelemetry.io/collector/exporter/loggingexporter
+    gomod: go.opentelemetry.io/collector v0.59.0
+  - import: go.opentelemetry.io/collector/exporter/otlpexporter
+    gomod: go.opentelemetry.io/collector v0.59.0
+  - import: go.opentelemetry.io/collector/exporter/otlphttpexporter
+    gomod: go.opentelemetry.io/collector v0.59.0
+extensions:
+  - import: go.opentelemetry.io/collector/extension/ballastextension
+    gomod: go.opentelemetry.io/collector v0.59.0
+  - import: go.opentelemetry.io/collector/extension/zpagesextension
+    gomod: go.opentelemetry.io/collector v0.59.0
+processors:
+  - import: go.opentelemetry.io/collector/processor/batchprocessor
+    gomod: go.opentelemetry.io/collector v0.59.0
+  - import: go.opentelemetry.io/collector/processor/memorylimiterprocessor
+    gomod: go.opentelemetry.io/collector v0.59.0
+

cmd/builder/test/default.otel.yaml

@@ -0,0 +1,22 @@
+extensions:
+  zpages:
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+processors:
+
+exporters:
+  logging:
+
+service:
+  extensions: [zpages]
+  pipelines:
+    traces:
+      receivers:
+        - otlp
+      processors: []
+      exporters:
+        - logging

cmd/builder/test/test.sh

@@ -12,17 +12,10 @@ fi
 
 echo "Using ${GOBIN} to compile the distributions."
 
-# each attempt pauses for 100ms before retrying
-max_retries=50
-
-tests="core"
-
-base=`mktemp -d`
-echo "Running the tests in ${base}"
+test_build_config() {
+    local test="$1"
+    local build_config="$2"
 
-failed=false
-for test in $tests
-do 
     out="${base}/${test}"
     mkdir -p "${out}"
     if [ $? != 0 ]; then
@@ -32,22 +25,33 @@ do
 
     echo "Starting test '${test}' at `date`" >> "${out}/test.log"
 
-    go run . --go "${GOBIN}" --config "./test/${test}.builder.yaml" --output-path "${out}" --name otelcol-built-test > "${out}/builder.log" 2>&1
+    if [ -z "$build_config" ] ; then
+        go run . --go "${GOBIN}" --output-path "${out}" --name otelcol-built-test > "${out}/builder.log" 2>&1
+    else
+        go run . --go "${GOBIN}" --config "$build_config" --output-path "${out}" --name otelcol-built-test > "${out}/builder.log" 2>&1
+    fi
+
     if [ $? != 0 ]; then
         echo "❌ FAIL ${test}. Failed to compile the test ${test}. Build logs:"
         cat "${out}/builder.log"
         failed=true
-        continue
+        return
     fi
 
     if [ ! -f "${out}/otelcol-built-test" ]; then
         echo "❌ FAIL ${test}. Binary not found for ${test} at '${out}/otelcol-built-test'. Build logs:"
         cat "${out}/builder.log"
         failed=true
-        continue
+        return
     fi
 
     # start the distribution
+    if [ ! -f "./test/${test}.otel.yaml" ]; then
+        echo "❌ FAIL ${test}. Config file for ${test} not found at './test/${test}.otel.yaml'"
+        failed=true
+        return
+    fi
+
     "${out}/otelcol-built-test" --config "./test/${test}.otel.yaml" > "${out}/otelcol.log" 2>&1 &
     pid=$!
 
@@ -62,7 +66,10 @@ do
             break
         fi
 
-        curl -s http://localhost:55679/debug/servicez | grep Uptime > /dev/null
+        # Since the content of the servicez page depend on which extensions are
+        # built into the collector, we depend only on the zpages extension
+        # being present and serving something.
+        curl --fail --silent --output /dev/null http://localhost:55679/debug/servicez
         if [ $? == 0 ]; then
             echo "✅ PASS ${test}"
 
@@ -92,7 +99,25 @@ do
     done
 
     echo "Stopping server for '${test}' (pid: ${pid})" >> "${out}/test.log"
-    wait ${pid}
+
+}
+
+# each attempt pauses for 100ms before retrying
+max_retries=50
+
+tests="core"
+
+base=`mktemp -d`
+echo "Running the tests in ${base}"
+
+failed=false
+
+# Test that an empty build configuration builds a working default collector.
+test_build_config "default"
+
+for test in $tests
+do
+    test_build_config "$test" "./test/${test}.builder.yaml"
 done
 
 if [[ "$failed" == "true" ]]; then