[dbm] bump go-sqllexer to v0.0.15 with new KeepJsonPath option (#29655)

DataDog · Oct 2, 2024 · 81296d0 · 81296d0
1 parent 33fa5d4
commit 81296d0
Show file tree

Hide file tree

Showing 23 changed files with 90 additions and 27 deletions.
diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod
@@ -242,7 +242,7 @@ require (
 	github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect
 	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
 	github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect
 	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect

diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum
diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod
@@ -191,7 +191,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect
 	github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect
 	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.20.0 // indirect

diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum
diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/sketches-go v1.4.2 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect

diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum
diff --git a/go.mod b/go.mod
@@ -708,7 +708,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/winutil v0.57.0
 	github.com/DataDog/datadog-agent/pkg/version v0.57.0
 	github.com/DataDog/go-libddwaf/v3 v3.3.0
-	github.com/DataDog/go-sqllexer v0.0.14
+	github.com/DataDog/go-sqllexer v0.0.15
 	github.com/Datadog/dublin-traceroute v0.0.1
 	github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d
 	github.com/aws/aws-sdk-go-v2/service/kms v1.34.1

diff --git a/go.sum b/go.sum
diff --git a/pkg/collector/python/datadog_agent.go b/pkg/collector/python/datadog_agent.go
@@ -320,6 +320,11 @@ type sqlConfig struct {
 	// By default, identifier quotation is removed during normalization.
 	// This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize".
 	KeepIdentifierQuotation bool `json:"keep_identifier_quotation"`
+
+	// KeepJSONPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation.
+	// By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?".
+	// This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize".
+	KeepJSONPath bool `json:"keep_json_path" yaml:"keep_json_path"`
 }
 
 // ObfuscateSQL obfuscates & normalizes the provided SQL query, writing the error into errResult if the operation
@@ -354,6 +359,7 @@ func ObfuscateSQL(rawQuery, opts *C.char, errResult **C.char) *C.char {
 		KeepPositionalParameter:       sqlOpts.KeepPositionalParameter,
 		KeepTrailingSemicolon:         sqlOpts.KeepTrailingSemicolon,
 		KeepIdentifierQuotation:       sqlOpts.KeepIdentifierQuotation,
+		KeepJSONPath:                  sqlOpts.KeepJSONPath,
 	})
 	if err != nil {
 		// memory will be freed by caller

diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod
@@ -46,7 +46,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/cache v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
 	github.com/DataDog/go-libddwaf/v3 v3.3.0 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/sketches-go v1.4.5 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect

diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum
diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod
@@ -4,7 +4,7 @@ go 1.22.0
 
 require (
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/DataDog/go-sqllexer v0.0.14
+	github.com/DataDog/go-sqllexer v0.0.15
 	github.com/outcaste-io/ristretto v0.2.1
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/atomic v1.10.0

diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum
diff --git a/pkg/obfuscate/obfuscate.go b/pkg/obfuscate/obfuscate.go
@@ -185,6 +185,11 @@ type SQLConfig struct {
 	// This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize".
 	KeepIdentifierQuotation bool `json:"keep_identifier_quotation" yaml:"keep_identifier_quotation"`
 
+	// KeepJSONPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation.
+	// By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?".
+	// This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize".
+	KeepJSONPath bool `json:"keep_json_path" yaml:"keep_json_path"`
+
 	// Cache reports whether the obfuscator should use a LRU look-up cache for SQL obfuscations.
 	Cache bool
 }

diff --git a/pkg/obfuscate/sql.go b/pkg/obfuscate/sql.go
@@ -439,6 +439,7 @@ func (o *Obfuscator) ObfuscateWithSQLLexer(in string, opts *SQLConfig) (*Obfusca
 			sqllexer.WithReplacePositionalParameter(!opts.KeepPositionalParameter),
 			sqllexer.WithReplaceBoolean(!opts.KeepBoolean),
 			sqllexer.WithReplaceNull(!opts.KeepNull),
+			sqllexer.WithKeepJsonPath(opts.KeepJSONPath),
 		)
 	}
 

diff --git a/pkg/obfuscate/sql_test.go b/pkg/obfuscate/sql_test.go
@@ -2136,6 +2136,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) {
 		keepPositionalParameter       bool
 		keepTrailingSemicolon         bool
 		keepIdentifierQuotation       bool
+		KeepJSONPath                  bool
 		metadata                      SQLMetadata
 	}{
 		{
@@ -2417,6 +2418,50 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) {
 				Procedures: []string{},
 			},
 		},
+		{
+			name:     "select with json path not keep",
+			query:    "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'",
+			expected: "SELECT * FROM users WHERE id = ? AND name -> ? = ?",
+			metadata: SQLMetadata{
+				Size:      11,
+				TablesCSV: "users",
+				Commands: []string{
+					"SELECT",
+				},
+				Comments:   []string{},
+				Procedures: []string{},
+			},
+		},
+		{
+			name:         "select with json path ->",
+			query:        "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'",
+			expected:     "SELECT * FROM users WHERE id = ? AND name -> 'first' = ?",
+			KeepJSONPath: true,
+			metadata: SQLMetadata{
+				Size:      11,
+				TablesCSV: "users",
+				Commands: []string{
+					"SELECT",
+				},
+				Comments:   []string{},
+				Procedures: []string{},
+			},
+		},
+		{
+			name:         "select with json path ->>",
+			query:        "SELECT * FROM users WHERE id = 1 AND name->>2 = 'test'",
+			expected:     "SELECT * FROM users WHERE id = ? AND name ->> 2 = ?",
+			KeepJSONPath: true,
+			metadata: SQLMetadata{
+				Size:      11,
+				TablesCSV: "users",
+				Commands: []string{
+					"SELECT",
+				},
+				Comments:   []string{},
+				Procedures: []string{},
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -2437,6 +2482,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) {
 					RemoveSpaceBetweenParentheses: tt.removeSpaceBetweenParentheses,
 					KeepTrailingSemicolon:         tt.keepTrailingSemicolon,
 					KeepIdentifierQuotation:       tt.keepIdentifierQuotation,
+					KeepJSONPath:                  tt.KeepJSONPath,
 				},
 			}).ObfuscateSQLString(tt.query)
 			require.NoError(t, err)

diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod
@@ -53,7 +53,7 @@ require (
 )
 
 require (
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum
diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/sketches-go v1.4.2 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect

diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum
diff --git a/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml b/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    DBM: Add configuration options to SQL obfuscator to customize the obfuscation of SQL statements:
+    - ``KeepJSONPath`` - option to control whether JSON paths following JSON operators in SQL statements should be obfuscated. This option is only valid when ``ObfuscationMode`` is ``obfuscate_and_normalize``.
diff --git a/test/otel/go.mod b/test/otel/go.mod
@@ -177,7 +177,7 @@ require (
 	github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect
 	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
 	github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect
-	github.com/DataDog/go-sqllexer v0.0.14 // indirect
+	github.com/DataDog/go-sqllexer v0.0.15 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect
 	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect

diff --git a/test/otel/go.sum b/test/otel/go.sum