Merge branch 'master' into igorpeshansky-gcloud-config-context

Dockerfile.windows

@@ -100,7 +100,79 @@ COPY submodules/fluent-bit /work/submodules/fluent-bit
 
 WORKDIR /work/submodules/fluent-bit/build
 
-RUN cmake -G "'Visual Studio 16 2019'" -DCMAKE_BUILD_TYPE=RELWITHDEBINFO -DFLB_OUT_KINESIS_STREAMS=OFF ../;
+RUN cmake -G "'Visual Studio 16 2019'" -DCMAKE_BUILD_TYPE=RELWITHDEBINFO `
+    -DFLB_WINDOWS_DEFAULTS=Off `
+    -DFLB_REGEX=On `
+    -DFLB_BACKTRACE=Off `
+    -DFLB_CONFIG_YAML=Off `
+    -DFLB_LUAJIT=On `
+    -DFLB_EXAMPLES=On `
+    -DFLB_PARSER=On `
+    -DFLB_TLS=On `
+    -DFLB_HTTP_SERVER=On `
+    -DFLB_METRICS=On `
+    -DFLB_WASM=Off `
+    -DFLB_WAMRC=Off `
+    -DFLB_IN_CPU=Off `
+    -DFLB_IN_DISK=Off `
+    -DFLB_IN_EXEC=On `
+    -DFLB_IN_EXEC_WASI=Off `
+    -DFLB_IN_FORWARD=On `
+    -DFLB_IN_HEALTH=Off `
+    -DFLB_IN_HTTP=On `
+    -DFLB_IN_MEM=Off `
+    -DFLB_IN_KAFKA=Off `
+    -DFLB_IN_KMSG=Off `
+    -DFLB_IN_LIB=On `
+    -DFLB_IN_RANDOM=On `
+    -DFLB_IN_SERIAL=Off `
+    -DFLB_IN_STDIN=Off `
+    -DFLB_IN_SYSLOG=On `
+    -DFLB_IN_TAIL=On `
+    -DFLB_IN_TCP=On `
+    -DFLB_IN_MQTT=Off `
+    -DFLB_IN_HEAD=Off `
+    -DFLB_IN_PROC=Off `
+    -DFLB_IN_SYSTEMD=Off `
+    -DFLB_IN_DUMMY=On `
+    -DFLB_IN_NETIF=Off `
+    -DFLB_IN_WINLOG=On `
+    -DFLB_IN_WINSTAT=On `
+    -DFLB_IN_WINEVTLOG=On `
+    -DFLB_IN_COLLECTD=Off `
+    -DFLB_IN_STATSD=On `
+    -DFLB_IN_STORAGE_BACKLOG=On `
+    -DFLB_IN_EMITTER=On `
+    -DFLB_IN_PODMAN_METRICS=Off `
+    -DFLB_OUT_BIGQUERY=Off `
+    -DFLB_OUT_COUNTER=On `
+    -DFLB_OUT_EXIT=Off `
+    -DFLB_OUT_FORWARD=On `
+    -DFLB_OUT_GELF=On `
+    -DFLB_OUT_HTTP=On `
+    -DFLB_OUT_NATS=Off `
+    -DFLB_OUT_PLOT=Off `
+    -DFLB_OUT_FILE=On `
+    -DFLB_OUT_TD=Off `
+    -DFLB_OUT_RETRY=Off `
+    -DFLB_OUT_STACKDRIVER=On `
+    -DFLB_OUT_STDOUT=On `
+    -DFLB_OUT_LIB=On `
+    -DFLB_OUT_NULL=On `
+    -DFLB_OUT_FLOWCOUNTER=On `
+    -DFLB_OUT_KAFKA=Off `
+    -DFLB_OUT_KAFKA_REST=Off `
+    -DFLB_FILTER_GREP=On `
+    -DFLB_FILTER_MODIFY=On `
+    -DFLB_FILTER_STDOUT=On `
+    -DFLB_FILTER_PARSER=On `
+    -DFLB_FILTER_THROTTLE=On `
+    -DFLB_FILTER_THROTTLE_SIZE=On `
+    -DFLB_FILTER_NEST=On `
+    -DFLB_FILTER_LUA=On `
+    -DFLB_FILTER_RECORD_MODIFIER=On `
+    -DFLB_FILTER_REWRITE_TAG=On `
+    -DFLB_FILTER_WASM=Off ../;
 
 RUN cmake --build . --config Release; `
     Copy-Item -Path bin/Release/fluent-bit.exe -Destination /work/out/bin/; `
@@ -136,7 +208,7 @@ WORKDIR /work/submodules/opentelemetry-operations-collector
 RUN $JarHash = (Get-FileHash /work/out/bin/opentelemetry-java-contrib-jmx-metrics.jar -Algorithm SHA256 | Select -Expand Hash).toLower(); `
     go build -o bin/google-cloud-metrics-agent_windows_amd64.exe `
     -ldflags \"-X github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver.MetricsGathererHash=$JarHash\" `
-     ./cmd/otelopscol; `
+    ./cmd/otelopscol; `
     Copy-Item -Path bin/google-cloud-metrics-agent_windows_amd64.exe -Destination /work/out/bin/;
 
 ###############################################################################
@@ -153,7 +225,7 @@ RUN Get-Content VERSION | Where-Object length | ForEach-Object { Invoke-Expressi
     go build -o bin/google-cloud-ops-agent.exe -ldflags \"-X github.com/GoogleCloudPlatform/ops-agent/internal/version.BuildDistro=$env:BUILD_DISTRO -X github.com/GoogleCloudPlatform/ops-agent/internal/version.Version=$env:PKG_VERSION\" ./cmd/ops_agent_windows; `
     Copy-Item -Path bin/google-cloud-ops-agent.exe -Destination /work/out/bin/; `
     Copy-Item -Path confgenerator/default-config.yaml -Destination /work/out/config/config.yaml;
- 
+
 ###############################################################################
 # Build diagnostic service
 ###############################################################################

VERSION

@@ -1 +1 @@
-PKG_VERSION="2.50.0"
+PKG_VERSION="2.51.0"

integration_test/cmd/generate_expected_metrics/generate_expected_metrics.go

@@ -178,16 +178,21 @@ func getAppName(metricType string) string {
 
 // toExpectedMetric converts from metric.MetricDescriptor to ExpectedMetric.
 func toExpectedMetric(metric *metric.MetricDescriptor) *metadata.ExpectedMetric {
-	labels := make(map[string]string)
+	labels := make([]*metadata.MetricLabel, len(metric.Labels))
 	for _, l := range metric.Labels {
-		labels[l.Key] = ".*"
+		labels = append(labels, &metadata.MetricLabel{
+			Name: l.Key,
+			ValueRegex: ".*",
+		})
 	}
 	return &metadata.ExpectedMetric{
-		Type:              metric.Type,
-		Kind:              metric.MetricKind.String(),
-		ValueType:         metric.ValueType.String(),
-		MonitoredResource: "gce_instance",
-		Labels:            labels,
+		MetricSpec: metadata.MetricSpec{
+			Type:               metric.Type,
+			Kind:               metric.MetricKind.String(),
+			ValueType:          metric.ValueType.String(),
+			MonitoredResources: []string{"gce_instance"},
+			Labels:             labels,
+		},
 	}
 }
 
@@ -265,20 +270,25 @@ func updateMetric(toUpdate *metadata.ExpectedMetric, withValuesFrom *metadata.Ex
 	result := toUpdate
 	result.Kind = withValuesFrom.Kind
 	result.ValueType = withValuesFrom.ValueType
-	result.MonitoredResource = withValuesFrom.MonitoredResource
-	result.Labels = make(map[string]string)
+	result.MonitoredResources = withValuesFrom.MonitoredResources
+	result.Labels = make([]*metadata.MetricLabel, len(withValuesFrom.Labels))
 
 	// TODO: Refactor to a simple map copy once we improve listMetrics to fetch
 	// label patterns automatically.
 
 	// The keys of result.Labels should be the same as withValuesFrom.Labels,
 	// except that existing values/patterns are preserved.
-	for k, v := range withValuesFrom.Labels {
-		existingPattern, ok := toUpdate.Labels[k]
+	existingLabels := make(map[string]*metadata.MetricLabel)
+	for _, l := range toUpdate.Labels {
+		existingLabels[l.Name] = l
+	}
+	for _, l := range withValuesFrom.Labels {
+		existingLabel, ok := existingLabels[l.Name]
 		if ok {
-			result.Labels[k] = existingPattern
+			// TODO: Merge the label specs.
+			result.Labels = append(result.Labels, existingLabel)
 		} else {
-			result.Labels[k] = v
+			result.Labels = append(result.Labels, l)
 		}
 	}
 

integration_test/metadata/integration_metadata.go

@@ -26,21 +26,45 @@ import (
 	"go.uber.org/multierr"
 )
 
-// ExpectedMetric encodes a series of assertions about what data we expect
-// to see in the metrics backend.
-type ExpectedMetric struct {
+// MetricLabel encodes a specification of a metric label in the metrics backend.
+type MetricLabel struct {
+	// The label name, for example state.
+	Name string `yaml:"name" validate:"required"`
+	// The label value pattern, as an RE2 regular expression.
+	ValueRegex string `yaml:"value_regex" validate:"required"`
+	// The description of the label.
+	Description string `yaml:"description" validate:"excludesall=‘’“”"`
+	// Annotations/footnotes about the label.
+	Notes []string `yaml:"notes,omitempty" validate:"omitempty,unique"`
+}
+
+// MetricSpec encodes a specification of a metric in the metrics backend.
+type MetricSpec struct {
 	// The metric type, for example workload.googleapis.com/apache.current_connections.
 	Type string `yaml:"type" validate:"required"`
 	// The value type, for example INT64.
 	ValueType string `yaml:"value_type" validate:"required,oneof=BOOL INT64 DOUBLE STRING DISTRIBUTION"`
 	// The kind, for example GAUGE.
 	Kind string `yaml:"kind" validate:"required,oneof=GAUGE DELTA CUMULATIVE"`
+	// The unit of the metric.
+	Unit string `yaml:"unit"`
+	// The description of the metric.
+	Description string `yaml:"description" validate:"excludesall=‘’“”"`
 	// The monitored resource, for example gce_instance.
 	// Currently we only test with gce_instance.
-	MonitoredResource string `yaml:"monitored_resource" validate:"required,oneof=gce_instance"`
-	// Mapping of expected label keys to value patterns.
-	// Patterns are RE2 regular expressions.
-	Labels map[string]string `yaml:"labels,omitempty" validate:"omitempty,gt=0"`
+	MonitoredResources []string `yaml:"monitored_resources,flow" validate:"required,gt=0,dive,oneof=gce_instance"`
+	// Mapping of expected label keys to label specs.
+	Labels []*MetricLabel `yaml:"labels,omitempty" validate:"omitempty,gt=0,unique=Name,dive"`
+	// Annotations/footnotes about the metric.
+	Notes []string `yaml:"notes,omitempty" validate:"omitempty,unique"`
+}
+
+// ExpectedMetric encodes a series of assertions about what data we expect
+// to see in the metrics backend.
+type ExpectedMetric struct {
+	// The metric being described.
+	MetricSpec `yaml:",inline"`
+
 	// If Optional is true, the test for this metric will be skipped.
 	Optional bool `yaml:"optional,omitempty" validate:"excluded_with=Representative"`
 	// Exactly one metric in each expected_metrics.yaml must
@@ -63,11 +87,15 @@ type LogField struct {
 	// A list of platforms that this field is not available on.
 	// Examples: debian-11.
 	UnavailableOn []string `yaml:"unavailable_on,omitempty"`
+	// Annotations/footnotes about the field.
+	Notes []string `yaml:"notes,omitempty" validate:"omitempty,unique"`
 }
 
 type ExpectedLog struct {
 	LogName string      `yaml:"log_name" validate:"required"`
 	Fields  []*LogField `yaml:"fields" validate:"required,unique=Name,dive"`
+	// Annotations/footnotes about the log.
+	Notes []string `yaml:"notes,omitempty" validate:"omitempty,unique"`
 }
 
 type MinimumSupportedAgentVersion struct {
@@ -109,7 +137,7 @@ type IntegrationMetadata struct {
 	Description                  string                        `yaml:"description" validate:"required,excludesall=‘’“”"`
 	ConfigurationOptions         *ConfigurationOptions         `yaml:"configuration_options" validate:"required"`
 	ConfigureIntegration         string                        `yaml:"configure_integration"`
-	ExpectedLogs                 []*ExpectedLog                `yaml:"expected_logs" validate:"dive"`
+	ExpectedLogs                 []*ExpectedLog                `yaml:"expected_logs" validate:"unique=LogName,dive"`
 	MinimumSupportedAgentVersion *MinimumSupportedAgentVersion `yaml:"minimum_supported_agent_version"`
 	SupportedAppVersion          []string                      `yaml:"supported_app_version" validate:"required,unique,min=1"`
 	SupportedOperatingSystems    string                        `yaml:"supported_operating_systems" validate:"required,oneof=linux windows linux_and_windows"`
@@ -198,8 +226,8 @@ func AssertMetric(metric *ExpectedMetric, series *monitoringpb.TimeSeries) error
 	if series.MetricKind.String() != metric.Kind {
 		err = multierr.Append(err, fmt.Errorf("kind: expected %s but got %s", metric.Kind, series.MetricKind.String()))
 	}
-	if series.Resource.Type != metric.MonitoredResource {
-		err = multierr.Append(err, fmt.Errorf("monitored_resource: expected %s but got %s", metric.MonitoredResource, series.Resource.Type))
+	if !SliceContains(metric.MonitoredResources, series.Resource.Type) {
+		err = multierr.Append(err, fmt.Errorf("unexpected monitored_resource: expected %v but got %s", metric.MonitoredResources, series.Resource.Type))
 	}
 	err = multierr.Append(err, assertMetricLabels(metric, series))
 	if err != nil {
@@ -209,31 +237,35 @@ func AssertMetric(metric *ExpectedMetric, series *monitoringpb.TimeSeries) error
 }
 
 func assertMetricLabels(metric *ExpectedMetric, series *monitoringpb.TimeSeries) error {
-	// Only expected labels must be present
 	var err error
+	// Only expected labels must be present
+	expectedLabels := make(map[string]bool)
+	for _, expectedLabel := range metric.Labels {
+		expectedLabels[expectedLabel.Name] = true
+	}
 	for actualLabel, actualValue := range series.Metric.Labels {
-		if _, ok := metric.Labels[actualLabel]; !ok {
+		if !expectedLabels[actualLabel] {
 			err = multierr.Append(err, fmt.Errorf("got unexpected label %q with value %q", actualLabel, actualValue))
 		}
 	}
 	// All expected labels must be present and match the given pattern
-	for expectedLabel, expectedPattern := range metric.Labels {
-		actualValue, ok := series.Metric.Labels[expectedLabel]
+	for _, expectedLabel := range metric.Labels {
+		actualValue, ok := series.Metric.Labels[expectedLabel.Name]
 		if !ok {
 			err = multierr.Append(err, fmt.Errorf("expected label not found: %s", expectedLabel))
 			continue
 		}
-		match, matchErr := regexp.MatchString(fmt.Sprintf("^(?:%s)$", expectedPattern), actualValue)
+		match, matchErr := regexp.MatchString(fmt.Sprintf("^(?:%s)$", expectedLabel.ValueRegex), actualValue)
 		if matchErr != nil {
 			err = multierr.Append(err, fmt.Errorf("error parsing pattern. label=%s, pattern=%s, err=%v",
-				expectedLabel,
-				expectedPattern,
+				expectedLabel.Name,
+				expectedLabel.ValueRegex,
 				matchErr,
 			))
 		} else if !match {
 			err = multierr.Append(err, fmt.Errorf("error: label value does not match pattern. label=%s, pattern=%s, value=%s",
-				expectedLabel,
-				expectedPattern,
+				expectedLabel.Name,
+				expectedLabel.ValueRegex,
 				actualValue,
 			))
 		}

integration_test/metadata/testdata/configuration-options_metrics-configuration_required/golden_error

@@ -1,2 +1,2 @@
-[43:8] Key: 'ConfigurationOptions.LogsConfiguration' Error:Field validation for 'LogsConfiguration' failed on the 'required_without' tag
+[45:8] Key: 'ConfigurationOptions.LogsConfiguration' Error:Field validation for 'LogsConfiguration' failed on the 'required_without' tag
 Key: 'ConfigurationOptions.MetricsConfiguration' Error:Field validation for 'MetricsConfiguration' failed on the 'required_without' tag

integration_test/metadata/testdata/configuration-options_metrics-configuration_required/input.yaml

@@ -26,10 +26,12 @@ expected_metrics:
   - type: example.googleapis.com/example.example1
     value_type: INT64
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
     labels:
-      broker: .*
-      destination: .*
+      - name: broker
+        value_regex: .*
+      - name: destination
+        value_regex: .*
     representative: true
 expected_logs:
   - log_name: examplelog

integration_test/metadata/testdata/expected-metric_optional_excluded_with/golden_error

@@ -1,2 +1,2 @@
-[37:15] Key: 'ExpectedMetric.Optional' Error:Field validation for 'Optional' failed on the 'excluded_with' tag
+[39:15] Key: 'ExpectedMetric.Optional' Error:Field validation for 'Optional' failed on the 'excluded_with' tag
 Key: 'ExpectedMetric.Representative' Error:Field validation for 'Representative' failed on the 'excluded_with' tag

integration_test/metadata/testdata/expected-metric_optional_excluded_with/input.yaml

@@ -26,14 +26,16 @@ expected_metrics:
   - type: example.googleapis.com/example.example0
     value_type: INT64
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
   - type: example.googleapis.com/example.example1
     value_type: INT64
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
     labels:
-      broker: .*
-      destination: .*
+      - name: broker
+        value_regex: .*
+      - name: destination
+        value_regex: .*
     optional: true
     representative: true
 expected_logs:

integration_test/metadata/testdata/expected-metric_value-type_one_of/golden_error

@@ -1 +1 @@
-[27:17] Key: 'ExpectedMetric.ValueType' Error:Field validation for 'ValueType' failed on the 'oneof' tag
+[27:17] Key: 'MetricSpec.ValueType' Error:Field validation for 'ValueType' failed on the 'oneof' tag

integration_test/metadata/testdata/expected-metric_value-type_one_of/input.yaml

@@ -26,10 +26,12 @@ expected_metrics:
   - type: example.googleapis.com/example.example1
     value_type: FOO
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
     labels:
-      broker: .*
-      destination: .*
+      - name: broker
+        value_regex: .*
+      - name: destination
+        value_regex: .*
     representative: true
 expected_logs:
   - log_name: examplelog

integration_test/metadata/testdata/integration-metadata_app-url_not-a-url/input.yaml

@@ -26,10 +26,12 @@ expected_metrics:
   - type: example.googleapis.com/example.example1
     value_type: INT64
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
     labels:
-      broker: .*
-      destination: .*
+      - name: broker
+        value_regex: .*
+      - name: destination
+        value_regex: .*
     representative: true
 expected_logs:
   - log_name: examplelog

integration_test/metadata/testdata/integration-metadata_app-url_required/input.yaml

@@ -25,10 +25,12 @@ expected_metrics:
   - type: example.googleapis.com/example.example1
     value_type: INT64
     kind: GAUGE
-    monitored_resource: gce_instance
+    monitored_resources: [gce_instance]
     labels:
-      broker: .*
-      destination: .*
+      - name: broker
+        value_regex: .*
+      - name: destination
+        value_regex: .*
     representative: true
 expected_logs:
   - log_name: examplelog