From a2237db372c196d3339e66ed05f83a1a64d68fa4 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 7 Apr 2021 13:44:43 -0400 Subject: [PATCH 1/2] feat(telemetry): replace unique ids for resources with a slug --- kit/transport/http/middleware.go | 56 +++++++++++++++++++++++++++ kit/transport/http/middleware_test.go | 30 ++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/kit/transport/http/middleware.go b/kit/transport/http/middleware.go index 1643736a505..facbc6b8da0 100644 --- a/kit/transport/http/middleware.go +++ b/kit/transport/http/middleware.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "path" + "regexp" "strings" "time" @@ -111,17 +112,32 @@ func UserAgent(r *http.Request) string { return ua.Parse(header).Name } +// Constants used for normalizing paths +const ( + fileSlug = ":file_name" + shardSlug = ":shard_id" +) + func normalizePath(p string) string { + // Normalize any paths used during backup or restore processes + p = normalizeBackupAndRestore(p) + + // Go through each part of the path and normalize IDs and UI assets var parts []string for head, tail := shiftPath(p); ; head, tail = shiftPath(tail) { piece := head + + // Normalize any ID's in the path as the ":id" slug if len(piece) == platform.IDLength { if _, err := platform.IDFromString(head); err == nil { piece = ":id" } } parts = append(parts, piece) + if tail == "/" { + // Normalize UI asset file names. The UI asset file is the last part of the path. + parts[len(parts)-1] = normalizeAssetFile(parts[len(parts)-1]) break } } @@ -137,6 +153,46 @@ func shiftPath(p string) (head, tail string) { return p[1:i], p[i:] } +// Normalize the file name for a UI asset +// For example: 838442d56d.svg will return as :file_id.svg +// Files names that do not have one of the listed extensions will be returned unchanged +func normalizeAssetFile(f string) string { + exts := []string{ + ".js", + ".svg", + ".woff2", + ".wasm", + ".map", + ".LICENSE", + } + + for _, ext := range exts { + if strings.HasSuffix(f, ext) { + return fileSlug + ext + } + } + + return f +} + +// Normalize paths used during the backup and restore process. +// Paths not matching any of the patterns will be returned unchanged. +func normalizeBackupAndRestore(pth string) string { + patterns := map[string]string{ + `restore/shards/\d+`: path.Join("restore/shards", shardSlug), + `backup/shards/\d+`: path.Join("backup/shards", shardSlug), + } + + for p, s := range patterns { + re := regexp.MustCompile(p) + if re.MatchString(pth) { + return re.ReplaceAllString(pth, s) + } + } + + return pth +} + type OrgContext string const CtxOrgKey OrgContext = "orgID" diff --git a/kit/transport/http/middleware_test.go b/kit/transport/http/middleware_test.go index 5a500498692..2a6532e09cf 100644 --- a/kit/transport/http/middleware_test.go +++ b/kit/transport/http/middleware_test.go @@ -37,6 +37,36 @@ func Test_normalizePath(t *testing.T) { path: path.Join("/api/v2/organizations", platform.ID(2).String(), "users", platform.ID(3).String()), expected: "/api/v2/organizations/:id/users/:id", }, + { + name: "5", + path: "/838442d56d.svg", + expected: "/" + fileSlug + ".svg", + }, + { + name: "6", + path: "/838442d56d.svg/extra", + expected: "/838442d56d.svg/extra", + }, + { + name: "7", + path: "/api/v2/restore/shards/1001", + expected: path.Join("/api/v2/restore/shards/", shardSlug), + }, + { + name: "8", + path: "/api/v2/restore/shards/1001/extra", + expected: path.Join("/api/v2/restore/shards/", shardSlug, "extra"), + }, + { + name: "9", + path: "/api/v2/backup/shards/1005", + expected: path.Join("/api/v2/backup/shards/", shardSlug), + }, + { + name: "10", + path: "/api/v2/backup/shards/1005/extra", + expected: path.Join("/api/v2/backup/shards/", shardSlug, "extra"), + }, } for _, tt := range tests { From ef3aeac4b4b285ea16cb45df88f3e5e709701d8e Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 7 Apr 2021 15:40:18 -0400 Subject: [PATCH 2/2] chore: update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d4a1f4477c..b15017cf983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The prefix used for Prometheus metrics from the query controller has changed fro 1. [21006](https://github.com/influxdata/influxdb/pull/21006): Add `-p, --profilers` flag to `influx query` command. 1. [21090](https://github.com/influxdata/influxdb/pull/21090): Update UI to match InfluxDB Cloud. 1. [21127](https://github.com/influxdata/influxdb/pull/21127): Allow for disabling concurrency-limits in Flux controller. +1. [21158](https://github.com/influxdata/influxdb/pull/21158): Replace unique resource IDs (UI assets, backup shards) with slugs to reduce cardinality of telemetry data. ### Bug Fixes