Merge branch 'master' into avilevy-fix-restart-diagnostic

Dockerfile

@@ -156,10 +156,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM centos7-build-golang-base AS centos7-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -257,10 +255,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM centos8-build-golang-base AS centos8-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -361,10 +357,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM rockylinux9-build-golang-base AS rockylinux9-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -460,10 +454,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM bookworm-build-golang-base AS bookworm-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -559,10 +551,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM bullseye-build-golang-base AS bullseye-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -660,10 +650,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM buster-build-golang-base AS buster-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -778,10 +766,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM sles12-build-golang-base AS sles12-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -882,10 +868,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM sles15-build-golang-base AS sles15-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -981,10 +965,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM focal-build-golang-base AS focal-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -1080,10 +1062,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM jammy-build-golang-base AS jammy-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -1179,10 +1159,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM lunar-build-golang-base AS lunar-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang
@@ -1278,10 +1256,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM mantic-build-golang-base AS mantic-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang

cmd/agent_wrapper/main_linux_test.go

@@ -17,11 +17,12 @@
 package main
 
 import (
+	"fmt"
 	"os/exec"
 	"strings"
 )
 
 // Get a command that will write the given number of bytes
 func getCommand(writeBytes int) *exec.Cmd {
-	return exec.Command("echo", strings.Repeat("a", int(writeBytes-1)))
+	return exec.Command("/bin/sh", "-c", fmt.Sprintf("echo %s", strings.Repeat("a", writeBytes-1)))
 }

confgenerator/config.go

@@ -419,9 +419,9 @@ func (r *componentTypeRegistry[CI, M]) RegisterType(constructor func() CI, platf
 	r.TypeMap[name] = &componentFactory[CI]{constructor, platformsValue}
 }
 
-// unmarshalComponentYaml is the custom unmarshaller for reading a component's configuration from the config file.
+// UnmarshalComponentYaml is the custom unmarshaller for reading a component's configuration from the config file.
 // It first unmarshals into a struct containing only the "type" field, then looks up the config struct with the full set of fields for that type, and finally unmarshals into an instance of that struct.
-func (r *componentTypeRegistry[CI, M]) unmarshalComponentYaml(ctx context.Context, inner *CI, unmarshal func(interface{}) error) error {
+func (r *componentTypeRegistry[CI, M]) UnmarshalComponentYaml(ctx context.Context, inner *CI, unmarshal func(interface{}) error) error {
 	c := ConfigComponent{}
 	unmarshal(&c) // Get the type; ignore the error
 	var o interface{}
@@ -482,7 +482,7 @@ func (r *componentTypeRegistry[CI, M]) unmarshalToMap(ctx context.Context, m *M,
 	// Step 2: Unmarshal into the destination map
 	for k, u := range um {
 		var inner CI
-		if err := r.unmarshalComponentYaml(ctx, &inner, u.unmarshal); err != nil {
+		if err := r.UnmarshalComponentYaml(ctx, &inner, u.unmarshal); err != nil {
 			return err
 		}
 		(*m)[k] = inner

confgenerator/feature_tracking_test.go

@@ -26,7 +26,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/GoogleCloudPlatform/ops-agent/apps"
+	_ "github.com/GoogleCloudPlatform/ops-agent/apps"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
 	"github.com/google/go-cmp/cmp"
@@ -794,7 +794,6 @@ func TestPrometheusFeatureMetrics(t *testing.T) {
 }
 
 func TestGolden(t *testing.T) {
-	_ = apps.BuiltInConfStructs
 	components := confgenerator.LoggingReceiverTypes.GetComponentsFromRegistry()
 	components = append(components, confgenerator.LoggingProcessorTypes.GetComponentsFromRegistry()...)
 	components = append(components, confgenerator.MetricsReceiverTypes.GetComponentsFromRegistry()...)

confgenerator/files.go

@@ -57,7 +57,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l
 			return fmt.Errorf("can't parse configuration: %w", err)
 		}
 		for name, contents := range files {
-			if err = writeConfigFile([]byte(contents), filepath.Join(outDir, name)); err != nil {
+			if err = WriteConfigFile([]byte(contents), filepath.Join(outDir, name)); err != nil {
 				return err
 			}
 		}
@@ -73,7 +73,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l
 		if err != nil {
 			return fmt.Errorf("can't parse configuration: %w", err)
 		}
-		if err = writeConfigFile([]byte(otelConfig), filepath.Join(outDir, "otel.yaml")); err != nil {
+		if err = WriteConfigFile([]byte(otelConfig), filepath.Join(outDir, "otel.yaml")); err != nil {
 			return err
 		}
 	default:
@@ -82,7 +82,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l
 	return nil
 }
 
-func writeConfigFile(content []byte, path string) error {
+func WriteConfigFile(content []byte, path string) error {
 	// Make sure the directory exists before writing the file.
 	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
 		return fmt.Errorf("failed to create directory for %q: %w", path, err)

debian/rules

@@ -8,7 +8,7 @@ override_dh_auto_configure:
 override_dh_auto_build:
 	true
 override_dh_auto_test:
-	true
+	PATH="/usr/local/go/bin:$$PATH" FLB="../cache/opt/google-cloud-ops-agent/subagents/fluent-bit/bin/fluent-bit" go test ./...
 override_dh_strip_nondeterminism:
 	dh_strip_nondeterminism -X.jar
 override_dh_auto_install:

dev-docs/create-gce-windows-build-vm.md

@@ -12,7 +12,7 @@ Note: Using `ssd` speeds up the build.
 1. Create the VM: 
     ```shell
     # VM image family.
-    export WIN_BUILD_VM_OS_IMAGE_FAMILY=windows-2019-for-containers
+    export WIN_BUILD_VM_OS_IMAGE_FAMILY=windows-2019
 
     # Your dev GCP project ID.
     export VM_PROJECT_ID=${USER}-sandbox
@@ -97,4 +97,4 @@ Current available images:
     Note that the default user for the VM is `opsagentdev`. 
     Record the password to use for connecting to this VM as `$WIN_BUILD_VM_PASSWORD`.
 
-    Skip the steps in [Set up a Windows VM](set-up-windows-vm.md); follow the steps in [Connect to a Windows VM](connect-to-windows-vm.md) to connect via RDP. 
+    Skip the steps in [Set up a Windows VM](set-up-windows-vm.md); follow the steps in [Connect to a Windows VM](connect-to-windows-vm.md) to connect via RDP. 

dev-docs/dev.md

@@ -290,6 +290,10 @@ See [Connect to a Windows VM](connect-to-windows-vm.md).
 
 See [Set up a Windows VM](set-up-windows-vm.md).
 
+#### Install Docker
+
+See [Get started: Prep Windows for containers](https://learn.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=dockerce#windows-server-1).
+
 #### Trigger the build
 
 Note: Make sure you have finished the 3 steps above including some one-time

dockerfiles/compile_test.go

@@ -7,7 +7,9 @@ import (
 
 func TestDockerfileMatches(t *testing.T) {
 	gotDockerfile, err := os.ReadFile(getDockerfilePath())
-	if err != nil {
+	if os.IsNotExist(err) {
+		t.Skip("Dockerfile not found")
+	} else if err != nil {
 		t.Error(err)
 	}
 	expectedDockerfile, err := getDockerfile()

dockerfiles/template

@@ -71,10 +71,8 @@ RUN ./agent_wrapper.sh /work/cache/
 
 FROM {target_name}-build-golang-base AS {target_name}-build
 WORKDIR /work
-COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine
-COPY VERSION build.sh ./
-COPY debian debian
-COPY pkg pkg
+COPY . /work
+
 # Run the build script once to build the ops agent engine to a cache
 RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang
 WORKDIR /tmp/cache_run/golang

transformation_test/testdata/modify_fields/config.yaml

@@ -0,0 +1,4 @@
+- type: modify_fields
+  fields:
+    jsonPayload.hello:
+      static_value: world

transformation_test/testdata/modify_fields/input.log

@@ -0,0 +1,3 @@
+foo
+goo
+yes maybe no

transformation_test/testdata/modify_fields/output_fluentbit.yaml

@@ -0,0 +1,9 @@
+- date: now
+  hello: world
+  message: foo
+- date: now
+  hello: world
+  message: goo
+- date: now
+  hello: world
+  message: yes maybe no

transformation_test/testdata/parse_json/config.yaml

@@ -0,0 +1,6 @@
+- type: parse_json
+  field: message
+  time_key: time
+  time_format: "%Y-%m-%dT%H:%M:%S.%L%z"
+- type: parse_json
+  field: log

transformation_test/testdata/parse_json/input.log

@@ -0,0 +1 @@
+{"log":"{\"level\":\"info\",\"message\":\"start\"}\n","time":"2023-10-17T14:55:59.683530289Z"}

transformation_test/testdata/parse_json/output_fluentbit.yaml

@@ -0,0 +1,3 @@
+- date: 2023-10-17T14:55:59.68353Z
+  level: info
+  message: start

transformation_test/testdata/parse_json_upstream/config.yaml

@@ -0,0 +1,4 @@
+- type: parse_json
+  field: message
+  time_key: time
+  time_format: "%Y-%m-%dT%H:%M:%S %z"

transformation_test/testdata/parse_json_upstream/input.log

@@ -0,0 +1,2 @@
+{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z"}
+{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04+01:00"}

transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml

@@ -0,0 +1,6 @@
+- date: 2006-07-28T13:22:04Z
+  key1: 12345
+  key2: abc
+- date: 2006-07-28T12:22:04Z
+  key1: 12345
+  key2: abc

transformation_test/testdata/test_timezone/config.yaml

@@ -0,0 +1,6 @@
+# When a timezone is not within a log Fluent-bit assumes UTC instead of system timezone
+# https://github.com/fluent/fluent-bit/issues/593#issuecomment-811183471
+- type: parse_json
+  field: message
+  time_key: time
+  time_format: "%Y-%m-%dT%H:%M"

transformation_test/testdata/test_timezone/input.log

@@ -0,0 +1 @@
+{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22"}

transformation_test/testdata/test_timezone/output_fluentbit.yaml

@@ -0,0 +1,3 @@
+- date: 2006-07-28T13:22:00Z
+  key1: 12345
+  key2: abc