switch pkger to goembed (#340)

* switch pkger to goembed

* Update tests-go.yml

* changes the way errors are handled

* removes a comment that's no longer relevant

* makes it so that webapp/public directory is always present

* removes pkger

* changes go embed pattern slightly making it work with empty public directory

* upgrades webpack, makes it clean directory after builds

* removes pyroscope.go from dockerfiles

* updates webpack

* adds webpack bundle analyzer

* handles embedded assets via tags instead of build vars

Co-authored-by: Dmitry Filimonov <[email protected]>

.github/workflows/tests-go.yml

@@ -12,6 +12,9 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.3'
       - name: Cache go mod directories
         uses: actions/cache@v2
         with:

Dockerfile

@@ -85,11 +85,12 @@ COPY --from=phpspy-builder /var/www/html/third_party/phpspy/libphpspy.a /opt/pyr
 COPY --from=js-builder /opt/pyroscope/webapp/public ./webapp/public
 COPY Makefile ./
 COPY tools ./tools
-COPY go.mod go.sum pyroscope.go ./
+COPY go.mod go.sum ./
 RUN make install-dev-tools
 
 COPY pkg ./pkg
 COPY cmd ./cmd
+COPY webapp/assets_embedded.go ./webapp/assets_embedded.go
 COPY scripts ./scripts
 
 RUN EMBEDDED_ASSETS_DEPS="" EXTRA_LDFLAGS="-linkmode external -extldflags '-static'" make build-release

Dockerfile.windows

@@ -6,8 +6,9 @@ WORKDIR /opt/pyroscope/
 
 COPY pkg ./pkg
 COPY cmd ./cmd
+COPY webapp/assets.go ./webapp/assets.go
 COPY scripts ./scripts
-COPY go.mod go.sum pyroscope.go ./
+COPY go.mod go.sum ./
 
 # Generate .syso object file.
 RUN source scripts/packages/git-info && go run scripts/windows/generate-windows-version-info/main.go \

Makefile

@@ -26,7 +26,8 @@ else
 	THIRD_PARTY_DEPENDENCIES ?= "build-rust-dependencies"
 endif
 
-GO_TAGS = $(ENABLED_SPIES)
+EXTRA_GO_TAGS ?=
+GO_TAGS = $(ENABLED_SPIES)$(EXTRA_GO_TAGS)
 
 ifeq ("$(OS)", "Linux")
 	ifeq ("$(shell cat /etc/os-release | grep ^ID=)", "ID=alpine")
@@ -41,7 +42,6 @@ else
 	endif
 endif
 
-EMBEDDED_ASSETS ?= ""
 EMBEDDED_ASSETS_DEPS ?= "assets-release"
 EXTRA_LDFLAGS ?= ""
 
@@ -62,11 +62,11 @@ all: build ## Runs the build target
 
 .PHONY: build
 build: ## Builds the binary
-	$(GOBUILD) -tags "$(GO_TAGS)" -ldflags "$(EXTRA_LDFLAGS) $(shell scripts/generate-build-flags.sh $(EMBEDDED_ASSETS))" -o ./bin/pyroscope ./cmd/pyroscope
+	$(GOBUILD) -tags "$(GO_TAGS)" -ldflags "$(EXTRA_LDFLAGS) $(shell scripts/generate-build-flags.sh)" -o ./bin/pyroscope ./cmd/pyroscope
 
 .PHONY: build-release
 build-release: embedded-assets ## Builds the release build
-	EMBEDDED_ASSETS=true $(MAKE) build
+	EXTRA_GO_TAGS=,embedassets $(MAKE) build
 
 .PHONY: build-rust-dependencies
 build-rust-dependencies: ## Builds the rust dependency
@@ -104,7 +104,8 @@ assets-watch: install-web-dependencies ## Configure the assets with live reloadi
 
 .PHONY: assets
 assets-release: install-web-dependencies ## Configure the assets for release
-	rm -rf webapp/public
+	rm -rf webapp/public/assets
+	rm -rf webapp/public/*.html
 	NODE_ENV=production $(shell yarn bin webpack) --config scripts/webpack/webpack.prod.js
 
 .PHONY: assets-size-build
@@ -113,7 +114,6 @@ assets-size-build: assets-release ## Build assets for the size report
 
 .PHONY: embedded-assets
 embedded-assets: install-dev-tools $(shell echo $(EMBEDDED_ASSETS_DEPS)) ## Configure the assets along with dev tools
-	go run "$(shell scripts/pinned-tool.sh github.com/markbates/pkger)/cmd/pkger" -o pkg/server
 
 .PHONY: lint
 lint: ## Run the lint across the codebase
@@ -174,11 +174,11 @@ update-protobuf: ## Update the protobuf
 
 .PHONY: docker-dev
 docker-dev: ## Build the docker dev
-	docker build . --tag pyroscope/pyroscope:dev
+	docker build . --tag pyroscope/pyroscope:dev --progress=plain
 
 .PHONY: windows-dev
 windows-dev: ## Build the windows dev
-	docker build --platform linux/amd64 -f Dockerfile.windows --output type=local,dest=out .
+	docker build --platform linux/amd64 -f Dockerfile.windows --progress=plain --output type=local,dest=out .
 
 .PHONY: print-deps-error-message
 print-deps-error-message:

benchmark/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.15.6
 
 WORKDIR /go/src/app
 
-COPY go.mod go.sum pyroscope.go ./
+COPY go.mod go.sum ./
 RUN go get -d ./
 
 COPY ./pkg pkg

go.mod

@@ -1,6 +1,6 @@
 module github.com/pyroscope-io/pyroscope
 
-go 1.14
+go 1.16
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
@@ -30,7 +30,6 @@ require (
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kyoh86/richgo v0.3.3
 	github.com/kyoh86/xdg v1.2.0 // indirect
-	github.com/markbates/pkger v0.17.1
 	github.com/mattn/go-runewidth v0.0.10 // indirect
 	github.com/mattn/goreman v0.3.5
 	github.com/mgechev/revive v1.0.3

go.sum

@@ -94,7 +94,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deadcheat/goblet v1.3.1 h1:OPOYQjHlbPmG8fFCt8GNNGECLVhaJ0n7F7cP5Mh7G/A=
 github.com/deadcheat/goblet v1.3.1/go.mod h1:IrMNyAwyrVgB30HsND2WgleTUM4wHTS9m40yNY6NJQg=
-github.com/deadcheat/gonch v0.0.0-20180528124129-c2ff7a019863 h1:WiIagMEsLYiZCeD76SSLTJPdBdnmXkrFOFbI6Chf0xg=
 github.com/deadcheat/gonch v0.0.0-20180528124129-c2ff7a019863/go.mod h1:/5mH3gAuXUxGN3maOBAxBfB8RXvP9tBIX5fx2x1k0V0=
 github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k=
 github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
@@ -133,8 +132,6 @@ github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=
-github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
@@ -247,8 +244,6 @@ github.com/kyoh86/xdg v0.0.0-20171007020617-d28e4c5d7b81/go.mod h1:Z5mDqe0fxyxn3
 github.com/kyoh86/xdg v1.2.0 h1:CERuT/ShdTDj+A2UaX3hQ3mOV369+Sj+wyn2nIRIIkI=
 github.com/kyoh86/xdg v1.2.0/go.mod h1:/mg8zwu1+qe76oTFUBnyS7rJzk7LLC0VGEzJyJ19DHs=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno=
-github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -692,7 +687,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

package.json

@@ -37,9 +37,9 @@
     "eslint-plugin-react": "^7.21.5",
     "eslint-plugin-react-hooks": "4.0.0",
     "eslint-webpack-plugin": "^2.4.1",
-    "html-webpack-plugin": "^4.3.0",
+    "html-webpack-plugin": "^5.3.2",
     "jest": "^26.6.3",
-    "mini-css-extract-plugin": "^0.9.0",
+    "mini-css-extract-plugin": "^2.2.0",
     "monaco-editor-webpack-plugin": "^1.9.0",
     "postcss-browser-reporter": "^0.6.0",
     "postcss-loader": "^3.0.0",
@@ -50,9 +50,9 @@
     "sass-loader": "^9.0.2",
     "size-limit": "^5.0.3",
     "typescript": "3.9.3",
-    "webpack": "^4.42.0",
-    "webpack-bundle-analyzer": "^4.3.0",
-    "webpack-cli": "^3.3.12",
+    "webpack": "^5.51.1",
+    "webpack-bundle-analyzer": "^4.4.2",
+    "webpack-cli": "^4.8.0",
     "webpack-merge": "^5.0.9",
     "webpack-plugin-hash-output": "^3.2.1"
   },

pkg/build/build.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/pyroscope-io/pyroscope/pkg/agent/spy"
+	"github.com/pyroscope-io/pyroscope/webapp"
 )
 
 var (
@@ -19,14 +20,10 @@ var (
 	GitSHA      = "N/A"
 	GitDirtyStr = "-1"
 	GitDirty    int
-
-	UseEmbeddedAssetsStr = "false"
-	UseEmbeddedAssets    bool
 )
 
 func init() {
 	GitDirty, _ = strconv.Atoi(GitDirtyStr)
-	UseEmbeddedAssets = UseEmbeddedAssetsStr == "true"
 }
 
 const tmplt = `
@@ -55,7 +52,7 @@ func Summary() string {
 		Time,
 		GitSHA,
 		GitDirty,
-		UseEmbeddedAssets,
+		webapp.AssetsEmbedded,
 		spy.SupportedSpies,
 	)
 }
@@ -82,7 +79,7 @@ func generateBuildInfoJSON() buildInfoJSON {
 		Time:              Time,
 		GitSHA:            GitSHA,
 		GitDirty:          GitDirty,
-		UseEmbeddedAssets: UseEmbeddedAssets,
+		UseEmbeddedAssets: webapp.AssetsEmbedded,
 	}
 }
 

pkg/server/controller.go

@@ -14,16 +14,15 @@ import (
 	"time"
 
 	"github.com/golang-jwt/jwt"
-	"github.com/markbates/pkger"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
 
-	"github.com/pyroscope-io/pyroscope/pkg/build"
 	"github.com/pyroscope-io/pyroscope/pkg/config"
 	"github.com/pyroscope-io/pyroscope/pkg/storage"
 	"github.com/pyroscope-io/pyroscope/pkg/util/hyperloglog"
 	"github.com/pyroscope-io/pyroscope/pkg/util/updates"
+	"github.com/pyroscope-io/pyroscope/webapp"
 )
 
 const (
@@ -68,11 +67,9 @@ func New(c *config.Server, s *storage.Storage, i storage.Ingester, l *logrus.Log
 		appStats: appStats,
 	}
 
-	if build.UseEmbeddedAssets {
-		// for this to work you need to run `pkger` first. See Makefile for more information
-		ctrl.dir = pkger.Dir("/webapp/public")
-	} else {
-		ctrl.dir = http.Dir("./webapp/public")
+	ctrl.dir, err = webapp.Assets()
+	if err != nil {
+		return nil, err
 	}
 
 	return &ctrl, nil

scripts/generate-build-flags.sh

@@ -2,10 +2,6 @@
 
 set -e
 
-if [ "$1" = "true" ]; then
-  echo "-X github.com/pyroscope-io/pyroscope/pkg/build.UseEmbeddedAssetsStr=true"
-fi
-
 CURRENT_TIME="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
 echo "-X github.com/pyroscope-io/pyroscope/pkg/build.Time=$CURRENT_TIME"
 

scripts/webpack/webpack.common.js

@@ -46,6 +46,7 @@ module.exports = {
     publicPath: "",
     path: path.resolve(__dirname, "../../webapp/public/assets"),
     filename: "[name].[hash].js",
+    clean: true,
   },
 
   resolve: {

scripts/webpack/webpack.dev.js

@@ -1,7 +1,11 @@
 const { merge } = require("webpack-merge");
+const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 
 const common = require("./webpack.common.js");
 
 module.exports = merge(common, {
   mode: "development",
+  plugins: [
+    new BundleAnalyzerPlugin()
+  ]
 });

tools/tools.go

@@ -14,7 +14,6 @@ import (
 	_ "github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb"
 	_ "github.com/kisielk/godepgraph"
 	_ "github.com/kyoh86/richgo"
-	_ "github.com/markbates/pkger/cmd/pkger"
 	_ "github.com/mattn/goreman"
 	_ "github.com/mgechev/revive"
 	_ "github.com/onsi/ginkgo/ginkgo"

webapp/assets.go

@@ -0,0 +1,13 @@
+// +build !embedassets
+
+package webapp
+
+import (
+	"net/http"
+)
+
+var AssetsEmbedded = false
+
+func Assets() (http.FileSystem, error) {
+	return http.Dir("./webapp/public"), nil
+}

webapp/assets_embedded.go

@@ -0,0 +1,24 @@
+// +build embedassets
+
+package webapp
+
+import (
+	"embed"
+	"io/fs"
+	"net/http"
+)
+
+var AssetsEmbedded = true
+
+//go:embed public
+var assets embed.FS
+
+func Assets() (http.FileSystem, error) {
+	fsys, err := fs.Sub(assets, "public")
+
+	if err != nil {
+		return nil, err
+	}
+
+	return http.FS(fsys), nil
+}