From d175bbc0dd7648f5f9bae44926a32a4432aa40af Mon Sep 17 00:00:00 2001 From: Ashmita Date: Mon, 22 Nov 2021 02:56:18 +0800 Subject: [PATCH] Add adapter that implements http.FileSystem for reading gzip files (#3399) * Use "embed" to package UI assets Signed-off-by: Yuri Shkuro * makefile Signed-off-by: Yuri Shkuro * remove mjibson/esc completely Signed-off-by: Yuri Shkuro * comments Signed-off-by: Yuri Shkuro * fix submodules Signed-off-by: Yuri Shkuro * Add adapter that implements http.FileSystem for reading gzip files Signed-off-by: Ashmita152 Co-authored-by: Yuri Shkuro * Make build-ui target more robust Signed-off-by: Yuri Shkuro * rebuild-ui Signed-off-by: Yuri Shkuro * fix docs Signed-off-by: Yuri Shkuro * rename Signed-off-by: Yuri Shkuro * simplify tests Signed-off-by: Yuri Shkuro * fmt Signed-off-by: Yuri Shkuro * Feedbacks Signed-off-by: Ashmita Bohara * Add tests Signed-off-by: Yuri Shkuro * Add final test for 100% coverage of new package Signed-off-by: Yuri Shkuro * add comments Signed-off-by: Yuri Shkuro Co-authored-by: Yuri Shkuro Co-authored-by: Yuri Shkuro --- Makefile | 26 ++-- cmd/query/app/ui/.gitignore | 1 + cmd/query/app/ui/actual.go | 13 +- cmd/query/app/ui/actual/.gitignore | 1 - cmd/query/app/ui/actual/doc.go | 16 --- cmd/query/app/ui/doc.go | 6 +- cmd/query/app/ui/placeholder.go | 2 +- pkg/gzipfs/gzip.go | 118 +++++++++++++++++++ pkg/gzipfs/gzip_test.go | 150 ++++++++++++++++++++++++ pkg/gzipfs/testdata/foobar | 1 + pkg/gzipfs/testdata/foobar.gz | Bin 0 -> 38 bytes pkg/gzipfs/testdata/foobaz.gz | Bin 0 -> 38 bytes pkg/gzipfs/testdata/foobaz_truncated.gz | Bin 0 -> 36 bytes pkg/gzipfs/testdata/not_archive.gz | 1 + scripts/import-order-cleanup.sh | 1 - scripts/updateLicenses.sh | 1 - 16 files changed, 296 insertions(+), 41 deletions(-) create mode 100644 cmd/query/app/ui/.gitignore delete mode 100644 cmd/query/app/ui/actual/.gitignore delete mode 100644 cmd/query/app/ui/actual/doc.go create mode 100644 pkg/gzipfs/gzip.go create mode 100644 pkg/gzipfs/gzip_test.go create mode 100644 pkg/gzipfs/testdata/foobar create mode 100644 pkg/gzipfs/testdata/foobar.gz create mode 100644 pkg/gzipfs/testdata/foobaz.gz create mode 100644 pkg/gzipfs/testdata/foobaz_truncated.gz create mode 100644 pkg/gzipfs/testdata/not_archive.gz diff --git a/Makefile b/Makefile index d0395e5787b..faf7bcc3cd2 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ ALL_SRC := $(shell find . -name '*.go' \ -not -name 'doc.go' \ -not -name '_*' \ -not -name '.*' \ - -not -name 'gen_assets.go' \ -not -name 'mocks*' \ -not -name 'model.pb.go' \ -not -name 'model_test.pb.go' \ @@ -194,22 +193,20 @@ docker-hotrod: run-all-in-one: build-ui go run -tags ui ./cmd/all-in-one --log-level debug -.PHONY: build-ui -build-ui: cmd/query/app/ui/actual/gen_assets.go - # UI packaged assets are up-to-date. To force a rebuild, run `make clean`. +build-ui: cmd/query/app/ui/actual/index.html.gz + +cmd/query/app/ui/actual/index.html.gz: jaeger-ui/packages/jaeger-ui/build/index.html + rm -rf cmd/query/app/ui/actual + mkdir cmd/query/app/ui/actual + cp -r jaeger-ui/packages/jaeger-ui/build/* cmd/query/app/ui/actual/ + find cmd/query/app/ui/actual -type f | xargs gzip jaeger-ui/packages/jaeger-ui/build/index.html: - cd jaeger-ui && yarn install --frozen-lockfile && cd packages/jaeger-ui && yarn build + $(MAKE) rebuild-ui -cmd/query/app/ui/actual/gen_assets.go: jaeger-ui/packages/jaeger-ui/build/index.html - @if ! command -v esc > /dev/null 2>&1 ; then \ - echo "esc: Command not found" ; \ - echo "Check:" ; \ - echo "- esc is installed: 'make install-tools'" ; \ - echo "- add \$$GOPATH into \$$PATH: 'export PATH=\$$PATH:\$$(go env GOPATH)/bin'" ; \ - false ; \ - fi - esc -pkg assets -o cmd/query/app/ui/actual/gen_assets.go -prefix jaeger-ui/packages/jaeger-ui/build jaeger-ui/packages/jaeger-ui/build ; \ +.PHONY: rebuild-ui +rebuild-ui: + cd jaeger-ui && yarn install --frozen-lockfile && cd packages/jaeger-ui && yarn build .PHONY: build-all-in-one-linux build-all-in-one-linux: @@ -367,7 +364,6 @@ changelog: .PHONY: install-tools install-tools: go install github.com/vektra/mockery/v2@v2.9.4 - go install github.com/mjibson/esc@v0.2.0 go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.0 .PHONY: install-ci diff --git a/cmd/query/app/ui/.gitignore b/cmd/query/app/ui/.gitignore new file mode 100644 index 00000000000..b3696964687 --- /dev/null +++ b/cmd/query/app/ui/.gitignore @@ -0,0 +1 @@ +actual/ diff --git a/cmd/query/app/ui/actual.go b/cmd/query/app/ui/actual.go index 1b5880e1035..5a257f18e78 100644 --- a/cmd/query/app/ui/actual.go +++ b/cmd/query/app/ui/actual.go @@ -18,8 +18,15 @@ package ui import ( - "github.com/jaegertracing/jaeger/cmd/query/app/ui/actual" + "embed" + "net/http" + + "github.com/jaegertracing/jaeger/pkg/gzipfs" + "github.com/jaegertracing/jaeger/pkg/httpfs" ) -// StaticFiles provides http filesystem with static files for UI -var StaticFiles = assets.FS(false) +//go:embed actual/* +var assetsFS embed.FS + +// StaticFiles provides http filesystem with static files for UI. +var StaticFiles = httpfs.PrefixedFS("actual", http.FS(gzipfs.New(assetsFS))) diff --git a/cmd/query/app/ui/actual/.gitignore b/cmd/query/app/ui/actual/.gitignore deleted file mode 100644 index 9f178a6878d..00000000000 --- a/cmd/query/app/ui/actual/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gen_assets.go diff --git a/cmd/query/app/ui/actual/doc.go b/cmd/query/app/ui/actual/doc.go deleted file mode 100644 index f0c17b88413..00000000000 --- a/cmd/query/app/ui/actual/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package assets contains auto-generated UI asset files created during the build. -package assets diff --git a/cmd/query/app/ui/doc.go b/cmd/query/app/ui/doc.go index 5398f61d876..1863eafdd6d 100644 --- a/cmd/query/app/ui/doc.go +++ b/cmd/query/app/ui/doc.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ui installs UI assets packaged with github.com/rakyll/statik. +// Package ui bundles UI assets packaged with stdlib/embed. // By default it imports the placeholder, non-functional index.html. -// When building with "ui" build tag, it imports the real UI assets -// generated in build_ui Makefile target. +// When building with "-tags ui", it imports the real UI assets +// generated in build-ui Makefile target. package ui diff --git a/cmd/query/app/ui/placeholder.go b/cmd/query/app/ui/placeholder.go index ac2e59eeb80..e4ebd67bf39 100644 --- a/cmd/query/app/ui/placeholder.go +++ b/cmd/query/app/ui/placeholder.go @@ -27,5 +27,5 @@ import ( //go:embed placeholder/index.html var assetsFS embed.FS -// StaticFiles provides http filesystem with static files for UI +// StaticFiles provides http filesystem with static files for UI. var StaticFiles = httpfs.PrefixedFS("placeholder", http.FS(assetsFS)) diff --git a/pkg/gzipfs/gzip.go b/pkg/gzipfs/gzip.go new file mode 100644 index 00000000000..cc58883f94a --- /dev/null +++ b/pkg/gzipfs/gzip.go @@ -0,0 +1,118 @@ +// Copyright (c) 2021 The Jaeger Authors. +// Copyright 2021 The Prometheus Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gzipfs + +import ( + "compress/gzip" + "io" + "io/fs" + "time" +) + +const suffix = ".gz" + +type file struct { + file fs.File + content []byte + offset int +} + +type fileInfo struct { + info fs.FileInfo + size int64 +} + +type fileSystem struct { + fs fs.FS +} + +func (f file) Stat() (fs.FileInfo, error) { + stat, err := f.file.Stat() + if err != nil { + return nil, err + } + return fileInfo{ + info: stat, + size: int64(len(f.content)), + }, nil +} + +func (f *file) Read(buf []byte) (n int, err error) { + if len(buf) > len(f.content)-f.offset { + buf = buf[0:len(f.content[f.offset:])] + } + + n = copy(buf, f.content[f.offset:]) + if n == len(f.content)-f.offset { + return n, io.EOF + } + f.offset += n + return +} + +func (f file) Close() error { + return f.file.Close() +} + +func (fi fileInfo) Name() string { + name := fi.info.Name() + return name[:len(name)-len(suffix)] +} + +func (fi fileInfo) Size() int64 { return fi.size } + +func (fi fileInfo) Mode() fs.FileMode { return fi.info.Mode() } + +func (fi fileInfo) ModTime() time.Time { return fi.info.ModTime() } + +func (fi fileInfo) IsDir() bool { return fi.info.IsDir() } + +func (fi fileInfo) Sys() interface{} { return nil } + +// New wraps underlying fs that is expected to contain gzipped files +// and presents an unzipped view of it. +func New(fs fs.FS) fs.FS { + return fileSystem{fs} +} + +func (cfs fileSystem) Open(path string) (fs.File, error) { + var f fs.File + f, err := cfs.fs.Open(path) + if err == nil { + return f, nil + } + + f, err = cfs.fs.Open(path + suffix) + if err != nil { + return f, err + } + + gr, err := gzip.NewReader(f) + if err != nil { + return f, err + } + defer gr.Close() + + c, err := io.ReadAll(gr) + if err != nil { + return f, err + } + + return &file{ + file: f, + content: c, + }, nil +} diff --git a/pkg/gzipfs/gzip_test.go b/pkg/gzipfs/gzip_test.go new file mode 100644 index 00000000000..3883f334b95 --- /dev/null +++ b/pkg/gzipfs/gzip_test.go @@ -0,0 +1,150 @@ +// Copyright (c) 2021 The Jaeger Authors. +// Copyright 2021 The Prometheus Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gzipfs + +import ( + "embed" + "io/fs" + "io/ioutil" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//go:embed testdata +var EmbedFS embed.FS + +var testFS = New(EmbedFS) + +type mockFile struct { + err error +} + +func (f *mockFile) Stat() (fs.FileInfo, error) { + return nil, f.err +} +func (f *mockFile) Read([]byte) (int, error) { + return 0, f.err +} +func (f *mockFile) Close() error { + return f.err +} + +func TestFS(t *testing.T) { + cases := []struct { + name string + path string + expectedErr string + expectedName string + expectedMode fs.FileMode + expectedSize int64 + expectedContent string + expectedModTime time.Time + }{ + { + name: "uncompressed file", + path: "testdata/foobar", + expectedMode: 0444, + expectedName: "foobar", + expectedSize: 11, + expectedContent: "hello world", + expectedModTime: time.Date(1, 1, 1, 0, 0, 0, 0 /* nanos */, time.UTC), + }, + { + name: "compressed file", + path: "testdata/foobar.gz", + expectedMode: 0444, + expectedName: "foobar.gz", + expectedSize: 38, + expectedContent: "", // actual gzipped payload is returned + expectedModTime: time.Date(1, 1, 1, 0, 0, 0, 0 /* nanos */, time.UTC), + }, + { + name: "compressed file accessed without gz extension", + path: "testdata/foobaz", + expectedMode: 0444, + expectedName: "foobaz", + expectedSize: 11, + expectedContent: "hello world", + expectedModTime: time.Date(1, 1, 1, 0, 0, 0, 0 /* nanos */, time.UTC), + }, + { + name: "non-existing file", + path: "testdata/non-existing-file", + expectedErr: "file does not exist", + }, + { + name: "not gzipped file", + path: "testdata/not_archive", + expectedErr: "invalid header", + }, + { + // To provide coverage of the error from io.ReadAll function, we use a file + // that is a copy of proper gzipped file testdata/foobaz.gz but truncated + // to 36 bytes with: + // perl -e "truncate 'pkg/gzipfs/testdata/foobaz_truncated.gz', 36" + // This allows gzip.NewReader() to succeed because the file has a proper gz + // header, but subsequent read fails with unexpected EOF. + name: "compressed but truncated file accessed without gz extension", + path: "testdata/foobaz_truncated", + expectedErr: "unexpected EOF", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + f, err := testFS.Open(c.path) + if c.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), c.expectedErr) + return + } + require.NoError(t, err) + defer f.Close() + + stat, err := f.Stat() + require.NoError(t, err) + + assert.Equal(t, c.expectedName, stat.Name()) + assert.Equal(t, c.expectedMode, stat.Mode()) + assert.Equal(t, c.expectedSize, stat.Size()) + assert.Equal(t, c.expectedModTime, stat.ModTime()) + assert.False(t, stat.IsDir()) + assert.Nil(t, stat.Sys()) + content, err := ioutil.ReadAll(f) + require.NoError(t, err) + if c.expectedContent != "" { + assert.Equal(t, c.expectedContent, string(content)) + } + }) + } +} + +func TestFileStatError(t *testing.T) { + f := &file{file: &mockFile{assert.AnError}} + _, err := f.Stat() + assert.Equal(t, assert.AnError, err) +} + +func TestFileRead(t *testing.T) { + f := &file{content: ([]byte)("long content")} + buf := make([]byte, 5) // shorter buffer + n, err := f.Read(buf) + require.NoError(t, err) + assert.Equal(t, 5, n) +} diff --git a/pkg/gzipfs/testdata/foobar b/pkg/gzipfs/testdata/foobar new file mode 100644 index 00000000000..95d09f2b101 --- /dev/null +++ b/pkg/gzipfs/testdata/foobar @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/pkg/gzipfs/testdata/foobar.gz b/pkg/gzipfs/testdata/foobar.gz new file mode 100644 index 0000000000000000000000000000000000000000..dab8327ac5440841884b8b9ca6b572010aeb10b9 GIT binary patch literal 38 ucmb2|=HO8LHX)ILIW0dwDY1y*w8z<#C&M+)>!0#uWM~!i;^k&wU;qFHIt)4h literal 0 HcmV?d00001 diff --git a/pkg/gzipfs/testdata/foobaz.gz b/pkg/gzipfs/testdata/foobaz.gz new file mode 100644 index 0000000000000000000000000000000000000000..dab8327ac5440841884b8b9ca6b572010aeb10b9 GIT binary patch literal 38 ucmb2|=HO8LHX)ILIW0dwDY1y*w8z<#C&M+)>!0#uWM~!i;^k&wU;qFHIt)4h literal 0 HcmV?d00001 diff --git a/pkg/gzipfs/testdata/foobaz_truncated.gz b/pkg/gzipfs/testdata/foobaz_truncated.gz new file mode 100644 index 0000000000000000000000000000000000000000..cfb0d9b11ce2089b4a42720d93526622d9e9760c GIT binary patch literal 36 scmb2|=HO8LHX)ILIW0dwDY1y*w8z<#C&M+)>!0#uWM~!i;^k%l0PVL7IsgCw literal 0 HcmV?d00001 diff --git a/pkg/gzipfs/testdata/not_archive.gz b/pkg/gzipfs/testdata/not_archive.gz new file mode 100644 index 00000000000..cb2fd2ec342 --- /dev/null +++ b/pkg/gzipfs/testdata/not_archive.gz @@ -0,0 +1 @@ +this is not a gzipped file diff --git a/scripts/import-order-cleanup.sh b/scripts/import-order-cleanup.sh index b305ab5c0c0..41f2834e0cb 100755 --- a/scripts/import-order-cleanup.sh +++ b/scripts/import-order-cleanup.sh @@ -7,7 +7,6 @@ python scripts/import-order-cleanup.py -o $1 -t $(git ls-files "*\.go" | \ -e thrift-gen \ -e swagger-gen \ -e thrift-0.9.2 \ - -e gen_assets.go \ -e proto-gen \ -e model.pb.go \ -e model_test.pb.go \ diff --git a/scripts/updateLicenses.sh b/scripts/updateLicenses.sh index 4ab95dfffb8..0c857cbebdf 100755 --- a/scripts/updateLicenses.sh +++ b/scripts/updateLicenses.sh @@ -7,7 +7,6 @@ python scripts/updateLicense.py $(git ls-files "*\.go" | \ -e thrift-gen \ -e swagger-gen \ -e proto-gen \ - -e gen_assets.go \ -e model.pb.go \ -e model.pb.gw.go \ -e model_test.pb.go \