Support rendering panel PNGs natively (#224)

* feat: Support rendering PNG natively * Using capture screenshot API of chromedp, we can generate PNG of each panel using current plugin. Thus, we avoid the dependency with grafana-image-renderer * Move all panel related JS to a separate .js file and embed that file with the app. Read the file content at runtime and inject it into chrome tab tasks. * Add duration debug logs for better debugging experience * Add e2e tests to test both native renderer and grafana-image-renderer integrations * test: Use Prom DS for API report tests. This test uses a wider interval simulating a real case where panel loading takes time. This ensures that the plugin works in real life scenarios as well. --------- Signed-off-by: Mahendra Paipuri <[email protected]>
mahendrapaipuri · Jan 6, 2025 · 7c954c1 · 7c954c1
1 parent 76a3d8a
commit 7c954c1
Show file tree

Hide file tree

Showing 23 changed files with 4,057 additions and 245 deletions.
diff --git a/.ci/dashboards/prometheus.json b/.ci/dashboards/prometheus.json
diff --git a/.ci/datasources/sample.yml b/.ci/datasources/sample.yml
@@ -0,0 +1,15 @@
+apiVersion: 1
+
+# A sample Prometheus datasource for testing purposes.
+datasources:
+  - access: proxy
+    isDefault: true
+    jsonData:
+      cacheLevel: Medium
+      incrementalQuerying: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.53.1
+      timeInterval: 10s
+    name: Prometheus
+    type: prometheus
+    url: https://prometheus.demo.do.prometheus.io
diff --git a/.ci/docker-compose.yaml b/.ci/docker-compose.yaml
@@ -8,12 +8,13 @@ services:
       context: ../.config
       args:
         grafana_image: ${GRAFANA_IMAGE:-grafana-oss}
-        grafana_version: ${GRAFANA_VERSION:-11.3.0}
+        grafana_version: ${GRAFANA_VERSION:-11.4.0}
     ports:
       - 3080:${GF_SERVER_HTTP_PORT:-3000}/tcp
     volumes:
       - ../dist:/var/lib/grafana/plugins/mahendrapaipuri-dashboardreporter-app
       - ./dashboards:/etc/grafana/provisioning/dashboards
+      - ./datasources:/etc/grafana/provisioning/datasources
       # Dont set config in provisioning just to ensure that plugin works without any
       # extra config
       - ./config/plain:/etc/grafana/provisioning/plugins
@@ -31,7 +32,7 @@ services:
       # allow anonymous admin so we don't have to set up a password to start testing
       - GF_AUTH_ANONYMOUS_ENABLED=false
       - GF_AUTH_BASIC_ENABLED=true
-      #- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      # - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
       # skip login page
       # - GF_AUTH_DISABLE_LOGIN_FORM=true
       # We need to toggle external service accounts so that Grafana will get
@@ -41,6 +42,8 @@ services:
       # disable alerting because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
       # Grafana image renderer
       - GF_RENDERING_SERVER_URL=http://renderer_plain:8081/render
       - GF_RENDERING_CALLBACK_URL=http://grafana_plain:${GF_SERVER_HTTP_PORT:-3000}/
@@ -49,6 +52,7 @@ services:
       # Set CI mode to remove header in report
       - __REPORTER_APP_CI_MODE=true
       - GF_REPORTER_PLUGIN_REMOTE_CHROME_URL=${GF_REPORTER_PLUGIN_REMOTE_CHROME_URL:-}
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
 
   renderer_plain:
     image: grafana/grafana-image-renderer:latest
@@ -74,12 +78,13 @@ services:
       context: ../.config
       args:
         grafana_image: ${GRAFANA_IMAGE:-grafana-oss}
-        grafana_version: ${GRAFANA_VERSION:-11.3.0}
+        grafana_version: ${GRAFANA_VERSION:-11.4.0}
     ports:
       - 3443:${GF_SERVER_HTTP_PORT:-3000}/tcp
     volumes:
       - ../dist:/var/lib/grafana/plugins/mahendrapaipuri-dashboardreporter-app
       - ./dashboards:/etc/grafana/provisioning/dashboards
+      - ./datasources:/etc/grafana/provisioning/datasources
       - ./config/tls:/etc/grafana/provisioning/plugins
       - ./certs:/etc/grafana/tls
       - ./runtime/tls:/srv
@@ -105,6 +110,8 @@ services:
       # disable alerting because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
        # TLS
       - GF_SERVER_PROTOCOL=https
       - GF_SERVER_CERT_KEY=/etc/grafana/tls/localhost.key
@@ -117,6 +124,7 @@ services:
       # Set CI mode to remove header in report
       - __REPORTER_APP_CI_MODE=true
       - GF_REPORTER_PLUGIN_REMOTE_CHROME_URL=${GF_REPORTER_PLUGIN_REMOTE_CHROME_URL:-}
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
       - GF_REPORTER_PLUGIN_SKIP_TLS_CHECK=true
 
   renderer_tls:

diff --git a/.github/workflows/step_e2e-tests.yml b/.github/workflows/step_e2e-tests.yml
@@ -18,13 +18,15 @@ jobs:
           - grafana-version: 10.3.0
             remote-chrome-url: ''
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: local-chrome
             name: local-chrome-10.3.0-with-features
 
           # Grafana v10 without user cookie and with feature flags
           - grafana-version: 10.4.5
             remote-chrome-url: ''
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: true
             # snapshots-folder: local-chrome
             name: local-chrome-10.4.5-with-features
 
@@ -33,22 +35,33 @@ jobs:
           - grafana-version: 10.4.7
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
             name: remote-chrome-10.4.7-without-features
 
           # Grafana v11 with remote chrome
           - grafana-version: 11.1.0
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
             name: remote-chrome-11.1.0-with-features
 
-          # Latest Grafana with local chrome
-          - grafana-version: 11.3.0
+          # Latest Grafana with local chrome and grafana-image-renderer
+          - grafana-version: 11.4.0
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
-            name: local-chrome-11.3.0-with-features
+            name: local-chrome-11.4.0-with-features
+
+          # Latest Grafana with local chrome and native-renderer
+          - grafana-version: 11.4.0
+            remote-chrome-url: ws://localhost:9222
+            feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: true
+            # snapshots-folder: remote-chrome
+            name: local-chrome-11.4.0-with-features-native-renderer
 
     steps:
       - uses: actions/checkout@v4
@@ -76,6 +89,7 @@ jobs:
         env:
           GRAFANA_VERSION: ${{ matrix.grafana-version }}
           GF_REPORTER_PLUGIN_REMOTE_CHROME_URL: ${{ matrix.remote-chrome-url }}
+          GF_REPORTER_PLUGIN_NATIVE_RENDERER: ${{ matrix.native-rendering }}
           GF_FEATURE_TOGGLES_ENABLE: ${{ matrix.feature-flags }}
         run: |
           # Upload/Download artifacts wont preserve permissions

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -40,20 +40,24 @@ services:
       # the token from a service account to read dashboards
       - GF_FEATURE_TOGGLES_ENABLE=${GF_FEATURE_TOGGLES_ENABLE:-accessControlOnCall,idForwarding,externalServiceAccounts}
       - GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED=${GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED:-true}
-      # disable alerting because it vomits logs
+      # disable alerting and Grafana live because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
       # Grafana image renderer
       - GF_RENDERING_SERVER_URL=http://renderer:8081/render
       - GF_RENDERING_CALLBACK_URL=http://grafana:${GF_SERVER_HTTP_PORT:-3000}/
       - "GF_LOG_FILTERS=rendering:debug plugin.mahendrapaipuri-dashboardreporter-app:debug"
+      # Current plugin config
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
   renderer:
     image: grafana/grafana-image-renderer:latest
     environment:
       # Recommendation of grafana-image-renderer for optimal performance
       # https://grafana.com/docs/grafana/latest/setup-grafana/image-rendering/#configuration
       - RENDERING_MODE=clustered
-      - RENDERING_CLUSTERING_MODE=browser
+      - RENDERING_CLUSTERING_MODE=context
       - RENDERING_CLUSTERING_MAX_CONCURRENCY=5
       - RENDERING_CLUSTERING_TIMEOUT=60
       - IGNORE_HTTPS_ERRORS=true

diff --git a/pkg/plugin/chrome/local.go b/pkg/plugin/chrome/local.go
@@ -141,10 +141,4 @@ func (i *LocalInstance) Close(logger log.Logger) {
 			logger.Error("got error from cancel browser context", "error", err)
 		}
 	}
-
-	if i.allocCtx != nil {
-		if err := chromedp.Cancel(i.allocCtx); err != nil {
-			logger.Error("got error from cancel browser allocator context", "error", err)
-		}
-	}
 }
diff --git a/pkg/plugin/chrome/tab.go b/pkg/plugin/chrome/tab.go
@@ -102,6 +102,11 @@ func (t *Tab) Context() context.Context {
 	return t.ctx
 }
 
+// Target returns tab's target ID.
+func (t *Tab) Target() *chromedp.Target {
+	return chromedp.FromContext(t.Context()).Target
+}
+
 // PrintToPDF returns chroms tasks that print the requested HTML into a PDF and returns the PDF stream handle.
 func (t *Tab) PrintToPDF(options PDFOptions, writer io.Writer) error {
 	err := chromedp.Run(t.ctx, chromedp.Tasks{

diff --git a/pkg/plugin/config/settings.go b/pkg/plugin/config/settings.go
@@ -41,6 +41,7 @@ type Config struct {
 	MaxBrowserWorkers   int    `env:"GF_REPORTER_PLUGIN_MAX_BROWSER_WORKERS, overwrite"    json:"maxBrowserWorkers"`
 	MaxRenderWorkers    int    `env:"GF_REPORTER_PLUGIN_MAX_RENDER_WORKERS, overwrite"     json:"maxRenderWorkers"`
 	RemoteChromeURL     string `env:"GF_REPORTER_PLUGIN_REMOTE_CHROME_URL, overwrite"      json:"remoteChromeUrl"`
+	NativeRendering     bool   `env:"GF_REPORTER_PLUGIN_NATIVE_RENDERER, overwrite"        json:"nativeRenderer"`
 	IncludePanelIDs     []string
 	ExcludePanelIDs     []string
 	IncludePanelDataIDs []string
@@ -141,10 +142,12 @@ func (c *Config) String() string {
 		"Theme: %s; Orientation: %s; Layout: %s; Dashboard Mode: %s; "+
 			"Time Zone: %s; Time Format: %s; Encoded Logo: %s; "+
 			"Max Renderer Workers: %d; Max Browser Workers: %d; Remote Chrome Addr: %s; App URL: %s; "+
-			"TLS Skip verify: %v; Included Panel IDs: %s; Excluded Panel IDs: %s Included Data for Panel IDs: %s",
+			"TLS Skip verify: %v; Included Panel IDs: %s; Excluded Panel IDs: %s Included Data for Panel IDs: %s; "+
+			"Native Renderer: %v; Client Timeout: %d",
 		c.Theme, c.Orientation, c.Layout, c.DashboardMode, c.TimeZone, c.TimeFormat,
 		encodedLogo, c.MaxRenderWorkers, c.MaxBrowserWorkers, c.RemoteChromeURL, appURL,
-		c.SkipTLSCheck, includedPanelIDs, excludedPanelIDs, includeDataPanelIDs,
+		c.SkipTLSCheck, includedPanelIDs, excludedPanelIDs, includeDataPanelIDs, c.NativeRendering,
+		int(c.HTTPClientOptions.Timeouts.Timeout.Seconds()),
 	)
 }
 

diff --git a/pkg/plugin/dashboard/dashboard.go b/pkg/plugin/dashboard/dashboard.go
@@ -2,50 +2,58 @@ package dashboard
 
 import (
 	"context"
+	"embed"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/chrome"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/config"
-	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/worker"
+	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/helpers"
 )
 
-// Regex for parsing X and Y co-ordinates from CSS
-// Scales for converting width and height to Grafana units.
+// Embed the entire directory.
 //
-// This is based on viewportWidth that we used in client.go which
-// is 1952px. Stripping margin 32px we get 1920px / 24 = 80px
-// height scale should be fine with 36px as width and aspect ratio
-// should choose a height appropriately.
-var (
-	scales = map[string]float64{
-		"width":  80,
-		"height": 36,
-	}
-)
+//go:embed js
+var jsFS embed.FS
 
 // New creates a new instance of the Dashboard struct.
 func New(logger log.Logger, conf *config.Config, httpClient *http.Client, chromeInstance chrome.Instance,
-	pools worker.Pools, appURL, appVersion string, model *Model, authHeader http.Header,
-) *Dashboard {
+	appURL, appVersion string, model *Model, authHeader http.Header,
+) (*Dashboard, error) {
+	// Parse app URL
+	u, err := url.Parse(appURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse app URL: %w", errors.Unwrap(err))
+	}
+
+	// Read JS from embedded file
+	js, err := jsFS.ReadFile("js/panels.js")
+	if err != nil {
+		return nil, fmt.Errorf("failed to load JS: %w", err)
+	}
+
 	return &Dashboard{
 		logger,
 		conf,
 		httpClient,
 		chromeInstance,
-		pools,
-		appURL,
+		u,
 		appVersion,
+		string(js),
 		model,
 		authHeader,
-	}
+	}, nil
 }
 
 // GetData fetches dashboard related data.
 func (d *Dashboard) GetData(ctx context.Context) (*Data, error) {
+	defer helpers.TimeTrack(time.Now(), "dashboard data", d.logger)
+
 	// Make panels from loading the dashboard in a browser instance
 	panels, err := d.panels(ctx)
 	if err != nil {