Introducing an ignore bad folders options

.github/workflows/code_scanners.yml

@@ -8,7 +8,7 @@ on:
       - main
 
 env:
-  GO_VERSION: 1.23.0
+  GO_VERSION: 1.23.4
 
 
 permissions:

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23.0"
+          go-version: "1.23.4"
       - name: Verify go version
         run: go version
       - name: Install GoTest

.github/workflows/release-validation.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22.1' 
+          go-version: '1.23.4' 
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22.2' 
+          go-version: '1.23.4' 
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:

.gitignore

@@ -11,3 +11,4 @@ config/token*.yml
 config/templates.yml
 website/node_modules/
 .env
+.DS_Store

cli/backup/dashboard.go

@@ -193,6 +193,7 @@ func newListDashboardsCmd() simplecobra.Commander {
 			}
 			count := 0
 
+			folders := rootCmd.GrafanaSvc().ListFolders(service.NewFolderFilter())
 			for _, link := range boards {
 				urlValue := getDashboardUrl(link)
 				count++
@@ -206,7 +207,7 @@ func newListDashboardsCmd() simplecobra.Commander {
 
 				baseRow := table.Row{link.ID, link.Title, link.Slug, link.FolderTitle}
 				if cfg.GetDashboardSettings().NestedFolders {
-					baseRow = append(baseRow, service.GetNestedFolder(link.FolderTitle, link.FolderUID, rootCmd.GrafanaSvc()))
+					baseRow = append(baseRow, service.GetNestedFolder(link.FolderTitle, link.FolderUID, folders))
 				}
 				baseRow = append(baseRow, table.Row{link.UID, tagVal, urlValue}...)
 				rootCmd.TableObj.AppendRow(baseRow)

cmd/gdg/main.go

@@ -1,17 +1,24 @@
 package main
 
 import (
+	"context"
 	"log"
 	"os"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/esnet/gdg/cli"
 	"github.com/esnet/gdg/cli/support"
 
 	api "github.com/esnet/gdg/internal/service"
 )
 
 var getGrafanaSvc = func() api.GrafanaService {
-	return api.NewApiService()
+	storageEngine, err := api.ConfigureStorage()
+	if err != nil {
+		storageEngine = storage.NewLocalStorage(context.Background())
+	}
+	return api.NewApiService(storageEngine)
 }
 
 func main() {

go.mod

@@ -1,6 +1,6 @@
 module github.com/esnet/gdg
 
-go 1.23.0
+go 1.23.4
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7

internal/config/grafana_config.go

@@ -1,8 +1,9 @@
 package config
 
 type DashboardSettings struct {
-	NestedFolders bool `mapstructure:"nested_folders" yaml:"nested_folders"`
-	IgnoreFilters bool `yaml:"ignore_filters" mapstructure:"ignore_filters" `
+	NestedFolders    bool `mapstructure:"nested_folders" yaml:"nested_folders"`
+	IgnoreFilters    bool `yaml:"ignore_filters" mapstructure:"ignore_filters" `
+	IgnoreBadFolders bool `yaml:"ignore_bad_folders" mapstructure:"ignore_bad_folders"`
 }
 
 // GrafanaConfig model wraps auth and watched list for grafana

internal/service/common_test.go

@@ -1,9 +1,12 @@
 package service
 
 import (
+	"context"
 	"os"
 	"testing"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/pkg/test_tooling/common"
 	"github.com/esnet/gdg/pkg/test_tooling/path"
@@ -23,9 +26,13 @@ func TestRelativePathLogin(t *testing.T) {
 	assert.NoError(t, os.Setenv(envKey, "http://localhost:3000/grafana/"))
 	fixEnvironment(t)
 	config.InitGdgConfig(common.DefaultTestConfig)
-	defer assert.NoError(t, os.Unsetenv(envKey))
+	defer func() {
+		assert.NoError(t, os.Unsetenv(envKey))
+		assert.NoError(t, os.Unsetenv(path.TestEnvKey))
+	}()
 
-	svc := NewApiService("dummy")
+	localEngine := storage.NewLocalStorage(context.Background())
+	svc := NewApiService(localEngine)
 	_, cfg := svc.(*DashNGoImpl).getNewClient()
 	assert.Equal(t, cfg.Host, "localhost:3000")
 	assert.Equal(t, cfg.BasePath, "/grafana/api")

internal/service/dashboards.go

@@ -296,6 +296,14 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 			folderMatch = getNestedFolder(folderMatch, link.FolderUID, folderUid)
 		}
 
+		// validate folder name
+		folderValid := s.checkFolderName(folderMatch)
+		if !folderValid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
+			log.Fatal("Invalid folder name detected, interrupting process.", slog.String("folderTitle", folderMatch))
+		} else if !folderValid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
+			slog.Warn("Invalid folder name detected, Skipping dashboards in folder", slog.String("folderTitle", folderMatch), slog.String("dashboard", link.Title))
+			continue
+		}
 		// accepts all folders
 		if config.Config().GetDefaultGrafanaConfig().GetDashboardSettings().IgnoreFilters {
 			validFolder = true
@@ -312,6 +320,7 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 		if link.FolderID == 0 && string(link.Type) == searchTypeDashboard {
 			link.FolderTitle = DefaultFolderName
 		}
+		// check folder
 
 		if validUid {
 			deduplicatedLinks[link.ID] = boardLinks[ndx]
@@ -371,8 +380,8 @@ func (s *DashNGoImpl) DownloadDashboards(filter filters.Filter) []string {
 
 // GetNestedFolder returns a nested path for a given folder.
 // Public version of GetNestedFolder, do not call from within service, not optimized.
-func GetNestedFolder(folderTitle, folderUID string, svc GrafanaService) string {
-	folderUid := getFolderUIDEntityMap(svc.ListFolders(NewFolderFilter()))
+func GetNestedFolder(folderTitle, folderUID string, folders []*customTypes.FolderDetails) string {
+	folderUid := getFolderUIDEntityMap(folders)
 	return getNestedFolder(folderTitle, folderUID, folderUid)
 }
 

internal/service/folders.go

@@ -205,7 +205,10 @@ func (s *DashNGoImpl) ListFolders(filter filters.Filter) []*types.FolderDetails
 	}
 	for ndx, val := range folderListing {
 		valid := s.checkFolderName(val.Title)
-		if !valid {
+		if !valid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
+			slog.Info("Skipping folder due to invalid character", slog.Any("folderTitle", val.Title))
+			continue
+		} else if !valid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
 			log.Fatalf("Folder has an invalid character and is not supported. Path separators are not allowed. folderName: %s", val.Title)
 		}
 		filterValue := val.Title

internal/service/gdg_api.go

@@ -2,12 +2,15 @@ package service
 
 import (
 	"context"
+	"fmt"
+	"log"
 	"log/slog"
 	"os"
 	"sync"
 
 	"github.com/esnet/gdg/internal/api"
 	"github.com/esnet/gdg/internal/config"
+	"github.com/esnet/gdg/internal/storage"
 	"github.com/spf13/viper"
 )
 
@@ -23,7 +26,7 @@ type DashNGoImpl struct {
 	configRef   *viper.Viper
 	debug       bool
 	apiDebug    bool
-	storage     Storage
+	storage     storage.Storage
 }
 
 func NewDashNGoImpl() *DashNGoImpl {
@@ -51,41 +54,48 @@ func newInstance() *DashNGoImpl {
 		}
 	}
 	obj.Login()
-	configureStorage(obj)
+	storageEngine, err := ConfigureStorage()
+	if err != nil {
+		log.Fatal("Unable to configure a valid storage engine, %w", err)
+	}
+	obj.SetStorage(storageEngine)
 
 	return obj
 }
 
-// Testing Only
-func (s *DashNGoImpl) SetStorage(v Storage) {
+func (s *DashNGoImpl) SetStorage(v storage.Storage) {
 	s.storage = v
 }
 
-func configureStorage(obj *DashNGoImpl) {
+func ConfigureStorage() (storage.Storage, error) {
+	var (
+		storageEngine storage.Storage
+		err           error
+	)
 	// config
 	storageType, appData := config.Config().GetCloudConfiguration(config.Config().GetDefaultGrafanaConfig().Storage)
 
-	var err error
 	ctx := context.Background()
-	ctx = context.WithValue(ctx, StorageContext, appData)
+	ctx = context.WithValue(ctx, storage.Context, appData)
 	switch storageType {
 	case "cloud":
 		{
-			obj.storage, err = NewCloudStorage(ctx)
+			storageEngine, err = storage.NewCloudStorage(ctx)
 			if err != nil {
-				slog.Warn("falling back on Local Storage, Cloud storage configuration error")
-				obj.storage = NewLocalStorage(ctx)
+				return nil, fmt.Errorf("unable to configure CloudStorage Engine:	%w", err)
 			}
 		}
 	default:
-		obj.storage = NewLocalStorage(ctx)
+		storageEngine = storage.NewLocalStorage(ctx)
 	}
+	return storageEngine, nil
 }
 
-func NewApiService(override ...string) GrafanaService {
+func NewApiService(storageEngine storage.Storage) GrafanaService {
 	// Used for Testing purposes
-	if len(override) > 0 {
+	if os.Getenv("TESTING") == "1" {
 		return newInstance()
 	}
+
 	return NewDashNGoImpl()
 }

internal/storage/const.go

@@ -0,0 +1,19 @@
+package storage
+
+type ContextStorage string
+
+const (
+	Context = ContextStorage("storage")
+	// Cloud Specific const
+	CloudType  = "cloud_type"
+	BucketName = "bucket_name"
+	Prefix     = "prefix"
+	Kind       = "kind"
+	Custom     = "custom"
+	AccessId   = "access_id"
+	SecretKey  = "secret_key"
+	Endpoint   = "endpoint"
+	Region     = "region"
+	SSLEnabled = "ssl_enabled"
+	InitBucket = "init_bucket"
+)

internal/service/storage.go → internal/storage/storage.go

@@ -1,15 +1,11 @@
-package service
+package storage
 
 import (
 	_ "gocloud.dev/blob/azureblob"
 	_ "gocloud.dev/blob/gcsblob"
 	_ "gocloud.dev/blob/s3blob"
 )
 
-type ContextStorage string
-
-const StorageContext = ContextStorage("storage")
-
 // TODO: pull all the cloud based interaction into a Plugin System
 type Storage interface {
 	WriteFile(filename string, data []byte) error                // WriteFile returns error or writes byte array to destination

internal/service/storage_cloud.go → internal/storage/storage_cloud.go

@@ -1,4 +1,4 @@
-package service
+package storage
 
 import (
 	"context"
@@ -27,7 +27,7 @@ type Resolver struct {
 	URL *url.URL
 }
 
-func (r *Resolver) ResolveEndpoint(_ context.Context, params s3.EndpointParameters) (transport.Endpoint, error) {
+func (r *Resolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (transport.Endpoint, error) {
 	u := *r.URL
 	u.Path += "/" + *params.Bucket
 	return transport.Endpoint{URI: u}, nil
@@ -42,20 +42,6 @@ type CloudStorage struct {
 	StorageName string
 }
 
-const (
-	CloudType  = "cloud_type"
-	BucketName = "bucket_name"
-	Prefix     = "prefix"
-	Kind       = "kind"
-	Custom     = "custom"
-	AccessId   = "access_id"
-	SecretKey  = "secret_key"
-	Endpoint   = "endpoint"
-	Region     = "region"
-	SSLEnabled = "ssl_enabled"
-	InitBucket = "init_bucket"
-)
-
 var (
 	stringEmpty = func(key string) bool {
 		return key == ""
@@ -142,7 +128,7 @@ func NewCloudStorage(c context.Context) (Storage, error) {
 		errorMsg  string
 	)
 
-	contextVal := c.Value(StorageContext)
+	contextVal := c.Value(Context)
 	if contextVal == nil {
 		return nil, errors.New("cannot configure GCP storage, context missing")
 	}

internal/service/storage_local.go → internal/storage/storage_local.go

@@ -1,4 +1,4 @@
-package service
+package storage
 
 import (
 	"context"

internal/tools/ptr/ptr.go

@@ -1,5 +1,14 @@
 package ptr
 
+// Of returns a pointer to the given value.
+//
+// This is a convenience function to create a pointer from a value.
+//
+// Example:
+//
+//	p := ptr.Of(5)
+//
+// will return a pointer to the value 5.
 func Of[T any](value T) *T {
 	return &value
 }

pkg/test_tooling/common.go

@@ -107,7 +107,9 @@ func CreateSimpleClient(t *testing.T, cfgName *string, container testcontainers.
 	contextName := conf.GetString("context_name")
 	conf.Set(fmt.Sprintf("context.%s.url", contextName), grafanaHost)
 	assert.Equal(t, contextName, "testing")
-	client := service.NewApiService("dummy")
+	storageEngine, err := service.ConfigureStorage()
+	assert.NoError(t, err)
+	client := service.NewApiService(storageEngine)
 	path, _ := os.Getwd()
 	if strings.Contains(path, "test") {
 		err := os.Chdir("..")