diff --git a/Dockerfile b/Dockerfile index 990245e61e..4663d65651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,10 +156,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM centos7-build-golang-base AS centos7-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -257,10 +255,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM centos8-build-golang-base AS centos8-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -361,10 +357,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM rockylinux9-build-golang-base AS rockylinux9-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -460,10 +454,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM bookworm-build-golang-base AS bookworm-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -559,10 +551,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM bullseye-build-golang-base AS bullseye-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -660,10 +650,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM buster-build-golang-base AS buster-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -778,10 +766,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM sles12-build-golang-base AS sles12-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -882,10 +868,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM sles15-build-golang-base AS sles15-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -981,10 +965,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM focal-build-golang-base AS focal-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1080,10 +1062,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM jammy-build-golang-base AS jammy-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1179,10 +1159,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM lunar-build-golang-base AS lunar-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1278,10 +1256,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM mantic-build-golang-base AS mantic-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang diff --git a/cmd/agent_wrapper/main_linux_test.go b/cmd/agent_wrapper/main_linux_test.go index be7840c833..3adcecc473 100644 --- a/cmd/agent_wrapper/main_linux_test.go +++ b/cmd/agent_wrapper/main_linux_test.go @@ -17,11 +17,12 @@ package main import ( + "fmt" "os/exec" "strings" ) // Get a command that will write the given number of bytes func getCommand(writeBytes int) *exec.Cmd { - return exec.Command("echo", strings.Repeat("a", int(writeBytes-1))) + return exec.Command("/bin/sh", "-c", fmt.Sprintf("echo %s", strings.Repeat("a", writeBytes-1))) } diff --git a/confgenerator/config.go b/confgenerator/config.go index a6fe67ccb8..e559749c2b 100644 --- a/confgenerator/config.go +++ b/confgenerator/config.go @@ -419,9 +419,9 @@ func (r *componentTypeRegistry[CI, M]) RegisterType(constructor func() CI, platf r.TypeMap[name] = &componentFactory[CI]{constructor, platformsValue} } -// unmarshalComponentYaml is the custom unmarshaller for reading a component's configuration from the config file. +// UnmarshalComponentYaml is the custom unmarshaller for reading a component's configuration from the config file. // It first unmarshals into a struct containing only the "type" field, then looks up the config struct with the full set of fields for that type, and finally unmarshals into an instance of that struct. -func (r *componentTypeRegistry[CI, M]) unmarshalComponentYaml(ctx context.Context, inner *CI, unmarshal func(interface{}) error) error { +func (r *componentTypeRegistry[CI, M]) UnmarshalComponentYaml(ctx context.Context, inner *CI, unmarshal func(interface{}) error) error { c := ConfigComponent{} unmarshal(&c) // Get the type; ignore the error var o interface{} @@ -482,7 +482,7 @@ func (r *componentTypeRegistry[CI, M]) unmarshalToMap(ctx context.Context, m *M, // Step 2: Unmarshal into the destination map for k, u := range um { var inner CI - if err := r.unmarshalComponentYaml(ctx, &inner, u.unmarshal); err != nil { + if err := r.UnmarshalComponentYaml(ctx, &inner, u.unmarshal); err != nil { return err } (*m)[k] = inner diff --git a/confgenerator/feature_tracking_test.go b/confgenerator/feature_tracking_test.go index cf109be405..99f1f4c918 100644 --- a/confgenerator/feature_tracking_test.go +++ b/confgenerator/feature_tracking_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/google/go-cmp/cmp" @@ -794,7 +794,6 @@ func TestPrometheusFeatureMetrics(t *testing.T) { } func TestGolden(t *testing.T) { - _ = apps.BuiltInConfStructs components := confgenerator.LoggingReceiverTypes.GetComponentsFromRegistry() components = append(components, confgenerator.LoggingProcessorTypes.GetComponentsFromRegistry()...) components = append(components, confgenerator.MetricsReceiverTypes.GetComponentsFromRegistry()...) diff --git a/confgenerator/files.go b/confgenerator/files.go index 9e7120d16f..1816491cc1 100644 --- a/confgenerator/files.go +++ b/confgenerator/files.go @@ -57,7 +57,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l return fmt.Errorf("can't parse configuration: %w", err) } for name, contents := range files { - if err = writeConfigFile([]byte(contents), filepath.Join(outDir, name)); err != nil { + if err = WriteConfigFile([]byte(contents), filepath.Join(outDir, name)); err != nil { return err } } @@ -73,7 +73,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l if err != nil { return fmt.Errorf("can't parse configuration: %w", err) } - if err = writeConfigFile([]byte(otelConfig), filepath.Join(outDir, "otel.yaml")); err != nil { + if err = WriteConfigFile([]byte(otelConfig), filepath.Join(outDir, "otel.yaml")); err != nil { return err } default: @@ -82,7 +82,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l return nil } -func writeConfigFile(content []byte, path string) error { +func WriteConfigFile(content []byte, path string) error { // Make sure the directory exists before writing the file. if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return fmt.Errorf("failed to create directory for %q: %w", path, err) diff --git a/debian/rules b/debian/rules index 164d4bc16e..67f1f3882a 100755 --- a/debian/rules +++ b/debian/rules @@ -8,7 +8,7 @@ override_dh_auto_configure: override_dh_auto_build: true override_dh_auto_test: - true + PATH="/usr/local/go/bin:$$PATH" FLB="../cache/opt/google-cloud-ops-agent/subagents/fluent-bit/bin/fluent-bit" go test ./... override_dh_strip_nondeterminism: dh_strip_nondeterminism -X.jar override_dh_auto_install: diff --git a/dev-docs/create-gce-windows-build-vm.md b/dev-docs/create-gce-windows-build-vm.md index 0c5dea7d4f..612671cc03 100644 --- a/dev-docs/create-gce-windows-build-vm.md +++ b/dev-docs/create-gce-windows-build-vm.md @@ -12,7 +12,7 @@ Note: Using `ssd` speeds up the build. 1. Create the VM: ```shell # VM image family. - export WIN_BUILD_VM_OS_IMAGE_FAMILY=windows-2019-for-containers + export WIN_BUILD_VM_OS_IMAGE_FAMILY=windows-2019 # Your dev GCP project ID. export VM_PROJECT_ID=${USER}-sandbox @@ -97,4 +97,4 @@ Current available images: Note that the default user for the VM is `opsagentdev`. Record the password to use for connecting to this VM as `$WIN_BUILD_VM_PASSWORD`. - Skip the steps in [Set up a Windows VM](set-up-windows-vm.md); follow the steps in [Connect to a Windows VM](connect-to-windows-vm.md) to connect via RDP. \ No newline at end of file + Skip the steps in [Set up a Windows VM](set-up-windows-vm.md); follow the steps in [Connect to a Windows VM](connect-to-windows-vm.md) to connect via RDP. diff --git a/dev-docs/dev.md b/dev-docs/dev.md index eb093d7f72..ee11c7d648 100644 --- a/dev-docs/dev.md +++ b/dev-docs/dev.md @@ -290,6 +290,10 @@ See [Connect to a Windows VM](connect-to-windows-vm.md). See [Set up a Windows VM](set-up-windows-vm.md). +#### Install Docker + +See [Get started: Prep Windows for containers](https://learn.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=dockerce#windows-server-1). + #### Trigger the build Note: Make sure you have finished the 3 steps above including some one-time diff --git a/dockerfiles/compile_test.go b/dockerfiles/compile_test.go index dfd135bfe6..717b14f958 100644 --- a/dockerfiles/compile_test.go +++ b/dockerfiles/compile_test.go @@ -7,7 +7,9 @@ import ( func TestDockerfileMatches(t *testing.T) { gotDockerfile, err := os.ReadFile(getDockerfilePath()) - if err != nil { + if os.IsNotExist(err) { + t.Skip("Dockerfile not found") + } else if err != nil { t.Error(err) } expectedDockerfile, err := getDockerfile() diff --git a/dockerfiles/template b/dockerfiles/template index 79f986c9db..ef5a2c8897 100644 --- a/dockerfiles/template +++ b/dockerfiles/template @@ -71,10 +71,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM {target_name}-build-golang-base AS {target_name}-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang diff --git a/transformation_test/testdata/modify_fields/config.yaml b/transformation_test/testdata/modify_fields/config.yaml new file mode 100644 index 0000000000..abf77a2051 --- /dev/null +++ b/transformation_test/testdata/modify_fields/config.yaml @@ -0,0 +1,4 @@ +- type: modify_fields + fields: + jsonPayload.hello: + static_value: world diff --git a/transformation_test/testdata/modify_fields/input.log b/transformation_test/testdata/modify_fields/input.log new file mode 100644 index 0000000000..69b115ab22 --- /dev/null +++ b/transformation_test/testdata/modify_fields/input.log @@ -0,0 +1,3 @@ +foo +goo +yes maybe no diff --git a/transformation_test/testdata/modify_fields/output_fluentbit.yaml b/transformation_test/testdata/modify_fields/output_fluentbit.yaml new file mode 100644 index 0000000000..e3bbc6ce3a --- /dev/null +++ b/transformation_test/testdata/modify_fields/output_fluentbit.yaml @@ -0,0 +1,9 @@ +- date: now + hello: world + message: foo +- date: now + hello: world + message: goo +- date: now + hello: world + message: yes maybe no diff --git a/transformation_test/testdata/parse_json/config.yaml b/transformation_test/testdata/parse_json/config.yaml new file mode 100644 index 0000000000..fb66d6d204 --- /dev/null +++ b/transformation_test/testdata/parse_json/config.yaml @@ -0,0 +1,6 @@ +- type: parse_json + field: message + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S.%L%z" +- type: parse_json + field: log diff --git a/transformation_test/testdata/parse_json/input.log b/transformation_test/testdata/parse_json/input.log new file mode 100644 index 0000000000..fe63e056c1 --- /dev/null +++ b/transformation_test/testdata/parse_json/input.log @@ -0,0 +1 @@ +{"log":"{\"level\":\"info\",\"message\":\"start\"}\n","time":"2023-10-17T14:55:59.683530289Z"} diff --git a/transformation_test/testdata/parse_json/output_fluentbit.yaml b/transformation_test/testdata/parse_json/output_fluentbit.yaml new file mode 100644 index 0000000000..e0901e621f --- /dev/null +++ b/transformation_test/testdata/parse_json/output_fluentbit.yaml @@ -0,0 +1,3 @@ +- date: 2023-10-17T14:55:59.68353Z + level: info + message: start diff --git a/transformation_test/testdata/parse_json_upstream/config.yaml b/transformation_test/testdata/parse_json_upstream/config.yaml new file mode 100644 index 0000000000..fe50911c9f --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/config.yaml @@ -0,0 +1,4 @@ +- type: parse_json + field: message + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S %z" diff --git a/transformation_test/testdata/parse_json_upstream/input.log b/transformation_test/testdata/parse_json_upstream/input.log new file mode 100644 index 0000000000..f3c3aa24b9 --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/input.log @@ -0,0 +1,2 @@ +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04+01:00"} diff --git a/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml b/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml new file mode 100644 index 0000000000..359dbb6bd4 --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml @@ -0,0 +1,6 @@ +- date: 2006-07-28T13:22:04Z + key1: 12345 + key2: abc +- date: 2006-07-28T12:22:04Z + key1: 12345 + key2: abc diff --git a/transformation_test/testdata/test_timezone/config.yaml b/transformation_test/testdata/test_timezone/config.yaml new file mode 100644 index 0000000000..662470751c --- /dev/null +++ b/transformation_test/testdata/test_timezone/config.yaml @@ -0,0 +1,6 @@ +# When a timezone is not within a log Fluent-bit assumes UTC instead of system timezone +# https://github.com/fluent/fluent-bit/issues/593#issuecomment-811183471 +- type: parse_json + field: message + time_key: time + time_format: "%Y-%m-%dT%H:%M" diff --git a/transformation_test/testdata/test_timezone/input.log b/transformation_test/testdata/test_timezone/input.log new file mode 100644 index 0000000000..2327f46633 --- /dev/null +++ b/transformation_test/testdata/test_timezone/input.log @@ -0,0 +1 @@ +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22"} diff --git a/transformation_test/testdata/test_timezone/output_fluentbit.yaml b/transformation_test/testdata/test_timezone/output_fluentbit.yaml new file mode 100644 index 0000000000..0124b1ec10 --- /dev/null +++ b/transformation_test/testdata/test_timezone/output_fluentbit.yaml @@ -0,0 +1,3 @@ +- date: 2006-07-28T13:22:00Z + key1: 12345 + key2: abc diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go new file mode 100644 index 0000000000..616ab5d7c3 --- /dev/null +++ b/transformation_test/transformation_test.go @@ -0,0 +1,216 @@ +package transformation_test + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/GoogleCloudPlatform/ops-agent/confgenerator" + "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" + "github.com/goccy/go-yaml" + "github.com/google/go-cmp/cmp" + "golang.org/x/sync/errgroup" + "gotest.tools/v3/golden" +) + +const ( + flbMainConf = "fluent_bit_main.conf" + flbParserConf = "fluent_bit_parser.conf" + transformationInput = "input.log" + transformationOutput = "output_fluentbit.yaml" + flbTag = "transformation_test" +) + +var ( + flbPath = flag.String("flb", os.Getenv("FLB"), "Path to Fluent-bit binary") +) + +type transformationTest []loggingProcessor +type loggingProcessor struct { + confgenerator.LoggingProcessor +} + +func (l *loggingProcessor) UnmarshalYAML(ctx context.Context, unmarshal func(interface{}) error) error { + return confgenerator.LoggingProcessorTypes.UnmarshalComponentYaml(ctx, &l.LoggingProcessor, unmarshal) +} + +func TestTransformationTests(t *testing.T) { + ctx := context.Background() + if len(*flbPath) == 0 { + t.Skip("--flb not supplied") + } + + allTests, err := os.ReadDir("testdata") + if err != nil { + t.Fatal(err) + } + + for _, dir := range allTests { + dir := dir + if !dir.IsDir() { + continue + } + t.Run(dir.Name(), func(t *testing.T) { + t.Parallel() + testStartTime := time.Now() + // Unmarshal transformation_config.yaml + var transformationConfig transformationTest + transformationConfig, err = readTransformationConfig(dir.Name()) + if err != nil { + t.Fatal("failed to unmarshal config:", err) + } + + // Generate config files + var genFiles map[string]string + genFiles, err = generateFluentBitConfigs(ctx, transformationConfig, dir.Name()) + + // Write config files in temp directory + tempPath := t.TempDir() + for k, v := range genFiles { + err := confgenerator.WriteConfigFile([]byte(v), filepath.Join(tempPath, k)) + + if err != nil { + t.Fatal(err) + } + t.Logf("generated file %q\n%s", k, v) + } + + // Start Fluent-bit + cmd := exec.Command( + fmt.Sprintf("%s", *flbPath), + "-v", + fmt.Sprintf("--config=%s", filepath.Join(tempPath, flbMainConf)), + fmt.Sprintf("--parser=%s", filepath.Join(filepath.Join(tempPath, flbParserConf)))) + + var stdout, stderr io.ReadCloser + stdout, err = cmd.StdoutPipe() + if err != nil { + t.Fatal("stdout pipe failure", err) + } + stderr, err = cmd.StderrPipe() + if err != nil { + t.Fatal("stderr pipe failure", err) + } + + if err := cmd.Start(); err != nil { + t.Fatal("Failed to start command:", err) + } + + // stderr and stdout need to be read in parallel to prevent deadlock + var eg errgroup.Group + eg.Go(func() error { + // read stderr + slurp, _ := io.ReadAll(stderr) + t.Logf("stderr: %s\n", slurp) + return nil + }) + + // read and unmarshal output + var data []map[string]any + out, _ := io.ReadAll(stdout) + _ = eg.Wait() + + err := yaml.Unmarshal(out, &data) + if err != nil { + t.Log(string(out)) + t.Fatal(err) + } + // transform timestamp of actual results + for i, d := range data { + if date, ok := d["date"].(float64); ok { + date := time.UnixMicro(int64(date * 1e6)).UTC() + if date.After(testStartTime) { + data[i]["date"] = "now" + } else { + data[i]["date"] = date.Format(time.RFC3339Nano) + } + } + } + checkOutput(t, filepath.Join(dir.Name(), transformationOutput), data) + }) + } +} + +func checkOutput(t *testing.T, name string, got []map[string]any) { + t.Helper() + wantBytes := golden.Get(t, name) + gotBytes, err := yaml.Marshal(got) + if err != nil { + t.Fatal(err) + } + t.Logf("%s", string(gotBytes)) + if golden.FlagUpdate() { + golden.AssertBytes(t, gotBytes, name) + return + } + var want []map[string]any + if err := yaml.Unmarshal(wantBytes, &want); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("got(-)/want(+):\n%s", diff) + } +} +func readTransformationConfig(dir string) (transformationTest, error) { + var transformationTestData []byte + var config transformationTest + + transformationTestData, err := os.ReadFile(filepath.Join("testdata", dir, "config.yaml")) + if err != nil { + return config, err + } + transformationTestData = bytes.ReplaceAll(transformationTestData, []byte("\r\n"), []byte("\n")) + + err = yaml.UnmarshalWithOptions(transformationTestData, &config, yaml.DisallowUnknownField()) + if err != nil { + return config, err + } + + return config, nil +} + +func generateFluentBitConfigs(ctx context.Context, transformationTest transformationTest, dirPath string) (map[string]string, error) { + abs, err := filepath.Abs(filepath.Join("testdata", dirPath, transformationInput)) + if err != nil { + return nil, err + } + var components []fluentbit.Component + input := fluentbit.Component{ + Kind: "INPUT", + Config: map[string]string{ + "Name": "tail", + "Tag": flbTag, + "Read_from_Head": "True", + "Exit_On_Eof": "True", + "Path": abs, + "Key": "message", + }, + } + + components = append(components, input) + + for i, t := range transformationTest { + components = append(components, t.Components(ctx, flbTag, strconv.Itoa(i))...) + } + + output := fluentbit.Component{ + Kind: "OUTPUT", + Config: map[string]string{ + "Name": "stdout", + "Match": "*", + "Format": "json", + }, + } + components = append(components, output) + return fluentbit.ModularConfig{ + Components: components, + }.Generate() +}