From 09c76aeeb9cba76d6182276f440637444bf879e0 Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Mon, 30 Sep 2024 11:09:35 -0400 Subject: [PATCH] bump go-sqllexer to v0.0.15 with new KeepJsonPath option --- comp/otelcol/ddflareextension/impl/go.mod | 2 +- comp/otelcol/ddflareextension/impl/go.sum | 4 +- .../exporter/datadogexporter/go.mod | 2 +- .../exporter/datadogexporter/go.sum | 4 +- .../otlp/components/statsprocessor/go.mod | 2 +- .../otlp/components/statsprocessor/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- pkg/collector/python/datadog_agent.go | 6 +++ pkg/config/remote/go.mod | 2 +- pkg/config/remote/go.sum | 4 +- pkg/obfuscate/go.mod | 2 +- pkg/obfuscate/go.sum | 4 +- pkg/obfuscate/obfuscate.go | 5 ++ pkg/obfuscate/sql.go | 1 + pkg/obfuscate/sql_test.go | 46 +++++++++++++++++++ pkg/trace/go.mod | 2 +- pkg/trace/go.sum | 4 +- pkg/trace/stats/oteltest/go.mod | 2 +- pkg/trace/stats/oteltest/go.sum | 4 +- ...p-go-sqllexer-0.0.15-18432a15bec8d683.yaml | 39 ++++++++++++++++ ...-obfuscation-options-ca13fcbeb4c9b299.yaml | 5 +- test/otel/go.mod | 2 +- test/otel/go.sum | 4 +- 24 files changed, 126 insertions(+), 30 deletions(-) create mode 100644 releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 2364112c77e95..061dd8c0ebfb7 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -242,7 +242,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 667f46a92da6e..b52bd4c77197d 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -66,8 +66,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c58e356e1ecab..ae077e7365176 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -191,7 +191,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.20.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index 0f46315566de3..1b9f4ba2789ad 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index ca6cee287f497..bec1bac721a17 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -38,7 +38,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 8c6d9031f59ae..a08668d36b2f8 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/go.mod b/go.mod index c497f4d13ca5a..912323c78ccbd 100644 --- a/go.mod +++ b/go.mod @@ -708,7 +708,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.57.0 github.com/DataDog/datadog-agent/pkg/version v0.56.2 github.com/DataDog/go-libddwaf/v3 v3.3.0 - github.com/DataDog/go-sqllexer v0.0.14 + github.com/DataDog/go-sqllexer v0.0.15 github.com/Datadog/dublin-traceroute v0.0.1 github.com/aquasecurity/trivy v0.49.2-0.20240227072422-e1ea02c7b80d github.com/aws/aws-sdk-go-v2/service/kms v1.34.1 diff --git a/go.sum b/go.sum index b6c767077c43a..3fba94836c526 100644 --- a/go.sum +++ b/go.sum @@ -714,8 +714,8 @@ github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302 github.com/DataDog/go-grpc-bidirectional-streaming-example v0.0.0-20221024060302-b9cf785c02fe/go.mod h1:90sqV0j7E8wYCyqIp5d9HmYWLTFQttqPFFtNYDyAybQ= github.com/DataDog/go-libddwaf/v3 v3.3.0 h1:jS72fuQpFgJZEdEJDmHJCPAgNTEMZoz1EUvimPUOiJ4= github.com/DataDog/go-libddwaf/v3 v3.3.0/go.mod h1:Bz/0JkpGf689mzbUjKJeheJINqsyyhM8p9PDuHdK2Ec= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= diff --git a/pkg/collector/python/datadog_agent.go b/pkg/collector/python/datadog_agent.go index c64eaa5f35b12..5c78030688186 100644 --- a/pkg/collector/python/datadog_agent.go +++ b/pkg/collector/python/datadog_agent.go @@ -320,6 +320,11 @@ type sqlConfig struct { // By default, identifier quotation is removed during normalization. // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". KeepIdentifierQuotation bool `json:"keep_identifier_quotation"` + + // KeepJsonPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation. + // By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?". + // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". + KeepJsonPath bool `json:"keep_json_path" yaml:"keep_json_path"` } // ObfuscateSQL obfuscates & normalizes the provided SQL query, writing the error into errResult if the operation @@ -354,6 +359,7 @@ func ObfuscateSQL(rawQuery, opts *C.char, errResult **C.char) *C.char { KeepPositionalParameter: sqlOpts.KeepPositionalParameter, KeepTrailingSemicolon: sqlOpts.KeepTrailingSemicolon, KeepIdentifierQuotation: sqlOpts.KeepIdentifierQuotation, + KeepJsonPath: sqlOpts.KeepJsonPath, }) if err != nil { // memory will be freed by caller diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 241aca51f7c21..4cd1b9833f550 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -46,7 +46,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/cache v0.56.0-rc.3 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.3.0 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/sketches-go v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index bd75efb7f9d96..270cfceba1d7b 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -50,8 +50,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.3.0 h1:jS72fuQpFgJZEdEJDmHJCPAgNTEMZoz1EUvimPUOiJ4= github.com/DataDog/go-libddwaf/v3 v3.3.0/go.mod h1:Bz/0JkpGf689mzbUjKJeheJINqsyyhM8p9PDuHdK2Ec= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index e25602108c4fb..0d0fb90ced497 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.5.0 - github.com/DataDog/go-sqllexer v0.0.14 + github.com/DataDog/go-sqllexer v0.0.15 github.com/outcaste-io/ristretto v0.2.1 github.com/stretchr/testify v1.9.0 go.uber.org/atomic v1.10.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index e6d91aba3fc14..b62d4da45dfe3 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= diff --git a/pkg/obfuscate/obfuscate.go b/pkg/obfuscate/obfuscate.go index fd5ac9ddd3f23..d8cdb6eca1f88 100644 --- a/pkg/obfuscate/obfuscate.go +++ b/pkg/obfuscate/obfuscate.go @@ -185,6 +185,11 @@ type SQLConfig struct { // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". KeepIdentifierQuotation bool `json:"keep_identifier_quotation" yaml:"keep_identifier_quotation"` + // KeepJsonPath specifies whether to keep JSON paths following JSON operators in SQL statements in obfuscation. + // By default, JSON paths are treated as literals and are obfuscated to ?, e.g. "data::jsonb -> 'name'" -> "data::jsonb -> ?". + // This option is only valid when ObfuscationMode is "normalize_only" or "obfuscate_and_normalize". + KeepJsonPath bool `json:"keep_json_path" yaml:"keep_json_path"` + // Cache reports whether the obfuscator should use a LRU look-up cache for SQL obfuscations. Cache bool } diff --git a/pkg/obfuscate/sql.go b/pkg/obfuscate/sql.go index 18fc120a73b14..3963b12fdcbed 100644 --- a/pkg/obfuscate/sql.go +++ b/pkg/obfuscate/sql.go @@ -439,6 +439,7 @@ func (o *Obfuscator) ObfuscateWithSQLLexer(in string, opts *SQLConfig) (*Obfusca sqllexer.WithReplacePositionalParameter(!opts.KeepPositionalParameter), sqllexer.WithReplaceBoolean(!opts.KeepBoolean), sqllexer.WithReplaceNull(!opts.KeepNull), + sqllexer.WithKeepJsonPath(opts.KeepJsonPath), ) } diff --git a/pkg/obfuscate/sql_test.go b/pkg/obfuscate/sql_test.go index f6b7005c46342..59190a538bb36 100644 --- a/pkg/obfuscate/sql_test.go +++ b/pkg/obfuscate/sql_test.go @@ -2136,6 +2136,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { keepPositionalParameter bool keepTrailingSemicolon bool keepIdentifierQuotation bool + keepJsonPath bool metadata SQLMetadata }{ { @@ -2417,6 +2418,50 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { Procedures: []string{}, }, }, + { + name: "select with json path not keep", + query: "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name -> ? = ?", + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, + { + name: "select with json path ->", + query: "SELECT * FROM users WHERE id = 1 AND name->'first' = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name -> 'first' = ?", + keepJsonPath: true, + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, + { + name: "select with json path ->>", + query: "SELECT * FROM users WHERE id = 1 AND name->>2 = 'test'", + expected: "SELECT * FROM users WHERE id = ? AND name ->> 2 = ?", + keepJsonPath: true, + metadata: SQLMetadata{ + Size: 11, + TablesCSV: "users", + Commands: []string{ + "SELECT", + }, + Comments: []string{}, + Procedures: []string{}, + }, + }, } for _, tt := range tests { @@ -2437,6 +2482,7 @@ func TestSQLLexerObfuscationAndNormalization(t *testing.T) { RemoveSpaceBetweenParentheses: tt.removeSpaceBetweenParentheses, KeepTrailingSemicolon: tt.keepTrailingSemicolon, KeepIdentifierQuotation: tt.keepIdentifierQuotation, + KeepJsonPath: tt.keepJsonPath, }, }).ObfuscateSQLString(tt.query) require.NoError(t, err) diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index ffbca4d83b1f8..dfa1f86ab5200 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -53,7 +53,7 @@ require ( ) require ( - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 4ba7b5e0d92e4..dae3a88e3d71e 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 59c592dd08090..c3f51342a5e79 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -26,7 +26,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 8c6d9031f59ae..a08668d36b2f8 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= diff --git a/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml b/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml new file mode 100644 index 0000000000000..97be566b8eb88 --- /dev/null +++ b/releasenotes/notes/bump-go-sqllexer-0.0.15-18432a15bec8d683.yaml @@ -0,0 +1,39 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +upgrade: + - | + List upgrade notes here, or remove this section. + Upgrade notes should be rare: only list known/potential breaking changes, + or major behaviorial changes that require user action before the upgrade. + Notes here must include steps that users can follow to 1. know if they're + affected and 2. handle the change gracefully on their end. +features: + - | + List new features here, or remove this section. +enhancements: + - | + List enhancements (new behavior that is too small to be + considered a new feature), or remove this section. +issues: + - | + List known issues here, or remove this section. +deprecations: + - | + List deprecations notes here, or remove this section. +security: + - | + Add security notes here, or remove this section. +fixes: + - | + Add normal bug fixes here, or remove this section. +other: + - | + Add here every other information you want in the CHANGELOG that + don't fit in any other section. This section should rarely be + used. diff --git a/releasenotes/notes/bump-go-sqllexer-with-more-obfuscation-options-ca13fcbeb4c9b299.yaml b/releasenotes/notes/bump-go-sqllexer-with-more-obfuscation-options-ca13fcbeb4c9b299.yaml index de69a7ffb6411..21708cacfb2aa 100644 --- a/releasenotes/notes/bump-go-sqllexer-with-more-obfuscation-options-ca13fcbeb4c9b299.yaml +++ b/releasenotes/notes/bump-go-sqllexer-with-more-obfuscation-options-ca13fcbeb4c9b299.yaml @@ -1,6 +1,5 @@ --- features: - | - DBM: Add configuration options to SQL obfuscator to customize the normalization of SQL statements: - - ``KeepTrailingSemicolon`` - disable removing trailing semicolon. This option is only valid when ``ObfuscationMode`` is ``obfuscate_and_normalize``. - - ``KeepIdentifierQuotation`` - disable removing quotation marks around identifiers. This option is only valid when ``ObfuscationMode`` is ``obfuscate_and_normalize``. + DBM: Add configuration options to SQL obfuscator to customize the obfuscation of SQL statements: + - ``KeepJsonPath`` - option to control whether JSON paths following JSON operators in SQL statements should be obfuscated. This option is only valid when ``ObfuscationMode`` is ``obfuscate_and_normalize``. diff --git a/test/otel/go.mod b/test/otel/go.mod index f34321698acb6..69222235a4400 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -177,7 +177,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.14 // indirect + github.com/DataDog/go-sqllexer v0.0.15 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 0f46315566de3..1b9f4ba2789ad 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= -github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= +github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ=