From 81296d09e31cf063855f83f78f8af7b68c3dc8fe Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Wed, 2 Oct 2024 11:05:14 -0400 Subject: [PATCH] [dbm] bump go-sqllexer to v0.0.15 with new KeepJsonPath option (#29655) --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 +- .../exporter/datadogexporter/go.mod | 2 +- .../exporter/datadogexporter/go.sum | 4 +- .../otlp/components/statsprocessor/go.mod | 2 +- .../otlp/components/statsprocessor/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- pkg/collector/python/datadog_agent.go | 6 +++ pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 +- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 +- pkg/obfuscate/obfuscate.go | 5 ++ pkg/obfuscate/sql.go | 1 + pkg/obfuscate/sql_test.go | 46 +++++++++++++++++++ pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 +- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 +- ...p-go-sqllexer-0.0.15-18432a15bec8d683.yaml | 5 ++ test/otel/go.mod | 2 +- test/otel/go.sum | 4 +- 23 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index fa3f1abcce0743..76390a61a90af0 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -242,7 +242,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 667f46a92da6eb..b52bd4c77197db 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -66,8 +66,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c58e356e1ecab5..ae077e73651769 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -191,7 +191,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.20.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index 0f46315566de3a..1b9f4ba2789ad6 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index ca6cee287f497a..bec1bac721a173 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -38,7 +38,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 8c6d9031f59ae7..a08668d36b2f8d 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/go.mod b/go.mod index 8429be04f73698..c6e87294efc673 100644 --- a/go.mod +++ b/go.mod @@ -708,7 +708,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.57.0 github.com/DataDog/datadog-agent/pkg/version v0.57.0 github.com/DataDog/go-libddwaf/v3 v3.3.0 - github.com/DataDog/go-sqllexer v0.0.14 + github.com/DataDog/go-sqllexer v0.0.15 github.com/Datadog/dublin-traceroute v0.0.1 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.34.1 diff --git a/go.sum b/go.sum index b6c767077c43a2..3fba94836c5265 100644 --- a/go.sum +++ b/go.sum @@ -714,8 +714,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.3.0 h1:jS72fuQpFgJZEdEJDmHJCPAgNTEMZoz1EUvimPUOiJ4= github.com/DataDog/go-libddwaf/v3 v3.3.0/go.mod h1:Bz/0JkpGf689mzbUjKJeheJINqsyyhM8p9PDuHdK2Ec= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/collector/python/datadog_agent.go b/pkg/collector/python/datadog_agent.go index c64eaa5f35b12a..1c1cf0fefd1402 100644 --- a/pkg/collector/python/datadog_agent.go +++ b/pkg/collector/python/datadog_agent.go @@ -320,6 +320,11 @@ type sqlConfig struct { // By default, identifier quotation is removed during normalization. // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". KeepIdentifierQuotation bool `json:"keep_identifier_quotation"` + + // KeepJSONPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation. + // By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?". + // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". + KeepJSONPath bool `json:"keep_json_path" yaml:"keep_json_path"` } // ObfuscateSQL obfuscates & normalizes the provided SQL query, writing the error into errResult if the operation @@ -354,6 +359,7 @@ func ObfuscateSQL(rawQuery, opts *C.char, errResult **C.char) *C.char { KeepPositionalParameter: sqlOpts.KeepPositionalParameter, KeepTrailingSemicolon: sqlOpts.KeepTrailingSemicolon, KeepIdentifierQuotation: sqlOpts.KeepIdentifierQuotation, + KeepJSONPath: sqlOpts.KeepJSONPath, }) if err != nil { // memory will be freed by caller diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 241aca51f7c21c..4cd1b9833f550e 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -46,7 +46,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/cache v0.56.0-rc.3 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.3.0 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/sketches-go v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index bd75efb7f9d964..270cfceba1d7bd 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -50,8 +50,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.3.0 h1:jS72fuQpFgJZEdEJDmHJCPAgNTEMZoz1EUvimPUOiJ4= github.com/DataDog/go-libddwaf/v3 v3.3.0/go.mod h1:Bz/0JkpGf689mzbUjKJeheJINqsyyhM8p9PDuHdK2Ec= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index e25602108c4fb2..0d0fb90ced4972 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.5.0 - github.com/DataDog/go-sqllexer v0.0.14 + github.com/DataDog/go-sqllexer v0.0.15 github.com/outcaste-io/ristretto v0.2.1 github.com/stretchr/testify v1.9.0 go.uber.org/atomic v1.10.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index e6d91aba3fc146..b62d4da45dfe33 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= diff --git a/pkg/obfuscate/obfuscate.go b/pkg/obfuscate/obfuscate.go index fd5ac9ddd3f239..3032fb54f38a4b 100644 --- a/pkg/obfuscate/obfuscate.go +++ b/pkg/obfuscate/obfuscate.go @@ -185,6 +185,11 @@ type SQLConfig struct { // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". KeepIdentifierQuotation bool `json:"keep_identifier_quotation" yaml:"keep_identifier_quotation"` + // KeepJSONPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation. + // By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?". + // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". + KeepJSONPath bool `json:"keep_json_path" yaml:"keep_json_path"` + // Cache reports whether the obfuscator should use a LRU look-up cache for SQL obfuscations. Cache bool } diff --git a/pkg/obfuscate/sql.go b/pkg/obfuscate/sql.go index 18fc120a73b142..8674173c8fdc37 100644 --- a/pkg/obfuscate/sql.go +++ b/pkg/obfuscate/sql.go @@ -439,6 +439,7 @@ func (o *Obfuscator) ObfuscateWithSQLLexer(in string, opts *SQLConfig) (*Obfusca sqllexer.WithReplacePositionalParameter(!opts.KeepPositionalParameter), sqllexer.WithReplaceBoolean(!opts.KeepBoolean), sqllexer.WithReplaceNull(!opts.KeepNull), + sqllexer.WithKeepJsonPath(opts.KeepJSONPath), ) } diff --git a/pkg/obfuscate/sql_test.go b/pkg/obfuscate/sql_test.go index f6b7005c46342c..c0922af69a085a 100644 --- a/pkg/obfuscate/sql_test.go +++ b/pkg/obfuscate/sql_test.go @@ -2136,6 +2136,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { keepPositionalParameter bool keepTrailingSemicolon bool keepIdentifierQuotation bool + KeepJSONPath bool metadata SQLMetadata }{ { @@ -2417,6 +2418,50 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { Procedures: []string{}, }, }, + { + name: "select with json path not keep", + query: "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name -> ? = ?", + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, + { + name: "select with json path ->", + query: "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name -> 'first' = ?", + KeepJSONPath: true, + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, + { + name: "select with json path ->>", + query: "SELECT * FROM users WHERE id = 1 AND name->>2 = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name ->> 2 = ?", + KeepJSONPath: true, + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, } for _, tt := range tests { @@ -2437,6 +2482,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { RemoveSpaceBetweenParentheses: tt.removeSpaceBetweenParentheses, KeepTrailingSemicolon: tt.keepTrailingSemicolon, KeepIdentifierQuotation: tt.keepIdentifierQuotation, + KeepJSONPath: tt.KeepJSONPath, }, }).ObfuscateSQLString(tt.query) require.NoError(t, err) diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index ffbca4d83b1f83..dfa1f86ab5200c 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -53,7 +53,7 @@ require ( ) require ( - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 4ba7b5e0d92e4c..dae3a88e3d71e1 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 59c592dd08090d..c3f51342a5e791 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -26,7 +26,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 8c6d9031f59ae7..a08668d36b2f8d 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml b/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml new file mode 100644 index 00000000000000..aab369a281863d --- /dev/null +++ b/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + DBM: Add configuration options to SQL obfuscator to customize the obfuscation of SQL statements: + - ``KeepJSONPath`` - option to control whether JSON paths following JSON operators in SQL statements should be obfuscated. This option is only valid when ``ObfuscationMode`` is ``obfuscate_and_normalize``. diff --git a/test/otel/go.mod b/test/otel/go.mod index f34321698acb62..69222235a4400b 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -177,7 +177,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 0f46315566de3a..1b9f4ba2789ad6 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ=