Refactoring moving Storage functionality into its own package

.gitignore

@@ -11,3 +11,4 @@ config/token*.yml
 config/templates.yml
 website/node_modules/
 .env
+.DS_Store

cmd/gdg/main.go

@@ -1,17 +1,24 @@
 package main
 
 import (
+	"context"
 	"log"
 	"os"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/esnet/gdg/cli"
 	"github.com/esnet/gdg/cli/support"
 
 	api "github.com/esnet/gdg/internal/service"
 )
 
 var getGrafanaSvc = func() api.GrafanaService {
-	return api.NewApiService()
+	storageEngine, err := api.ConfigureStorage()
+	if err != nil {
+		storageEngine = storage.NewLocalStorage(context.Background())
+	}
+	return api.NewApiService(storageEngine)
 }
 
 func main() {

internal/config/grafana_config.go

@@ -1,9 +1,9 @@
 package config
 
 type DashboardSettings struct {
-	NestedFolders   bool `mapstructure:"nested_folders" yaml:"nested_folders"`
-	IgnoreFilters   bool `yaml:"ignore_filters" mapstructure:"ignore_filters" `
-	IgnoreBadFolder bool `yaml:"ignore_bad_folder" mapstructure:"ignore_bad_folder"`
+	NestedFolders    bool `mapstructure:"nested_folders" yaml:"nested_folders"`
+	IgnoreFilters    bool `yaml:"ignore_filters" mapstructure:"ignore_filters" `
+	IgnoreBadFolders bool `yaml:"ignore_bad_folders" mapstructure:"ignore_bad_folders"`
 }
 
 // GrafanaConfig model wraps auth and watched list for grafana

internal/service/common_test.go

@@ -1,9 +1,12 @@
 package service
 
 import (
+	"context"
 	"os"
 	"testing"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/pkg/test_tooling/common"
 	"github.com/esnet/gdg/pkg/test_tooling/path"
@@ -23,9 +26,13 @@ func TestRelativePathLogin(t *testing.T) {
 	assert.NoError(t, os.Setenv(envKey, "http://localhost:3000/grafana/"))
 	fixEnvironment(t)
 	config.InitGdgConfig(common.DefaultTestConfig)
-	defer assert.NoError(t, os.Unsetenv(envKey))
+	defer func() {
+		assert.NoError(t, os.Unsetenv(envKey))
+		assert.NoError(t, os.Unsetenv(path.TestEnvKey))
+	}()
 
-	svc := NewApiService("dummy")
+	localEngine := storage.NewLocalStorage(context.Background())
+	svc := NewApiService(localEngine)
 	_, cfg := svc.(*DashNGoImpl).getNewClient()
 	assert.Equal(t, cfg.Host, "localhost:3000")
 	assert.Equal(t, cfg.BasePath, "/grafana/api")

internal/service/dashboards.go

@@ -298,9 +298,9 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 
 		// validate folder name
 		folderValid := s.checkFolderName(folderMatch)
-		if !folderValid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolder {
+		if !folderValid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
 			log.Fatal("Invalid folder name detected, interrupting process.", slog.String("folderTitle", folderMatch))
-		} else if !folderValid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolder {
+		} else if !folderValid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
 			slog.Warn("Invalid folder name detected, Skipping dashboards in folder", slog.String("folderTitle", folderMatch), slog.String("dashboard", link.Title))
 			continue
 		}

internal/service/folders.go

@@ -205,10 +205,10 @@ func (s *DashNGoImpl) ListFolders(filter filters.Filter) []*types.FolderDetails
 	}
 	for ndx, val := range folderListing {
 		valid := s.checkFolderName(val.Title)
-		if !valid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolder {
+		if !valid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
 			slog.Info("Skipping folder due to invalid character", slog.Any("folderTitle", val.Title))
 			continue
-		} else if !valid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolder {
+		} else if !valid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders {
 			log.Fatalf("Folder has an invalid character and is not supported. Path separators are not allowed. folderName: %s", val.Title)
 		}
 		filterValue := val.Title

internal/service/gdg_api.go

@@ -2,12 +2,15 @@ package service
 
 import (
 	"context"
+	"fmt"
+	"log"
 	"log/slog"
 	"os"
 	"sync"
 
 	"github.com/esnet/gdg/internal/api"
 	"github.com/esnet/gdg/internal/config"
+	"github.com/esnet/gdg/internal/storage"
 	"github.com/spf13/viper"
 )
 
@@ -23,7 +26,7 @@ type DashNGoImpl struct {
 	configRef   *viper.Viper
 	debug       bool
 	apiDebug    bool
-	storage     Storage
+	storage     storage.Storage
 }
 
 func NewDashNGoImpl() *DashNGoImpl {
@@ -51,41 +54,48 @@ func newInstance() *DashNGoImpl {
 		}
 	}
 	obj.Login()
-	configureStorage(obj)
+	storageEngine, err := ConfigureStorage()
+	if err != nil {
+		log.Fatal("Unable to configure a valid storage engine, %w", err)
+	}
+	obj.SetStorage(storageEngine)
 
 	return obj
 }
 
-// Testing Only
-func (s *DashNGoImpl) SetStorage(v Storage) {
+func (s *DashNGoImpl) SetStorage(v storage.Storage) {
 	s.storage = v
 }
 
-func configureStorage(obj *DashNGoImpl) {
+func ConfigureStorage() (storage.Storage, error) {
+	var (
+		storageEngine storage.Storage
+		err           error
+	)
 	// config
 	storageType, appData := config.Config().GetCloudConfiguration(config.Config().GetDefaultGrafanaConfig().Storage)
 
-	var err error
 	ctx := context.Background()
-	ctx = context.WithValue(ctx, StorageContext, appData)
+	ctx = context.WithValue(ctx, storage.Context, appData)
 	switch storageType {
 	case "cloud":
 		{
-			obj.storage, err = NewCloudStorage(ctx)
+			storageEngine, err = storage.NewCloudStorage(ctx)
 			if err != nil {
-				slog.Warn("falling back on Local Storage, Cloud storage configuration error")
-				obj.storage = NewLocalStorage(ctx)
+				return nil, fmt.Errorf("unable to configure CloudStorage Engine:	%w", err)
 			}
 		}
 	default:
-		obj.storage = NewLocalStorage(ctx)
+		storageEngine = storage.NewLocalStorage(ctx)
 	}
+	return storageEngine, nil
 }
 
-func NewApiService(override ...string) GrafanaService {
+func NewApiService(storageEngine storage.Storage) GrafanaService {
 	// Used for Testing purposes
-	if len(override) > 0 {
+	if os.Getenv("TESTING") == "1" {
 		return newInstance()
 	}
+
 	return NewDashNGoImpl()
 }

internal/storage/const.go

@@ -0,0 +1,19 @@
+package storage
+
+type ContextStorage string
+
+const (
+	Context = ContextStorage("storage")
+	// Cloud Specific const
+	CloudType  = "cloud_type"
+	BucketName = "bucket_name"
+	Prefix     = "prefix"
+	Kind       = "kind"
+	Custom     = "custom"
+	AccessId   = "access_id"
+	SecretKey  = "secret_key"
+	Endpoint   = "endpoint"
+	Region     = "region"
+	SSLEnabled = "ssl_enabled"
+	InitBucket = "init_bucket"
+)

internal/service/storage.go → internal/storage/storage.go

@@ -1,15 +1,11 @@
-package service
+package storage
 
 import (
 	_ "gocloud.dev/blob/azureblob"
 	_ "gocloud.dev/blob/gcsblob"
 	_ "gocloud.dev/blob/s3blob"
 )
 
-type ContextStorage string
-
-const StorageContext = ContextStorage("storage")
-
 // TODO: pull all the cloud based interaction into a Plugin System
 type Storage interface {
 	WriteFile(filename string, data []byte) error                // WriteFile returns error or writes byte array to destination

internal/service/storage_cloud.go → internal/storage/storage_cloud.go

@@ -1,4 +1,4 @@
-package service
+package storage
 
 import (
 	"context"
@@ -27,7 +27,7 @@ type Resolver struct {
 	URL *url.URL
 }
 
-func (r *Resolver) ResolveEndpoint(_ context.Context, params s3.EndpointParameters) (transport.Endpoint, error) {
+func (r *Resolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (transport.Endpoint, error) {
 	u := *r.URL
 	u.Path += "/" + *params.Bucket
 	return transport.Endpoint{URI: u}, nil
@@ -42,20 +42,6 @@ type CloudStorage struct {
 	StorageName string
 }
 
-const (
-	CloudType  = "cloud_type"
-	BucketName = "bucket_name"
-	Prefix     = "prefix"
-	Kind       = "kind"
-	Custom     = "custom"
-	AccessId   = "access_id"
-	SecretKey  = "secret_key"
-	Endpoint   = "endpoint"
-	Region     = "region"
-	SSLEnabled = "ssl_enabled"
-	InitBucket = "init_bucket"
-)
-
 var (
 	stringEmpty = func(key string) bool {
 		return key == ""
@@ -142,7 +128,7 @@ func NewCloudStorage(c context.Context) (Storage, error) {
 		errorMsg  string
 	)
 
-	contextVal := c.Value(StorageContext)
+	contextVal := c.Value(Context)
 	if contextVal == nil {
 		return nil, errors.New("cannot configure GCP storage, context missing")
 	}

internal/service/storage_local.go → internal/storage/storage_local.go

@@ -1,4 +1,4 @@
-package service
+package storage
 
 import (
 	"context"

internal/tools/ptr/ptr.go

@@ -1,5 +1,14 @@
 package ptr
 
+// Of returns a pointer to the given value.
+//
+// This is a convenience function to create a pointer from a value.
+//
+// Example:
+//
+//	p := ptr.Of(5)
+//
+// will return a pointer to the value 5.
 func Of[T any](value T) *T {
 	return &value
 }

pkg/test_tooling/common.go

@@ -107,7 +107,9 @@ func CreateSimpleClient(t *testing.T, cfgName *string, container testcontainers.
 	contextName := conf.GetString("context_name")
 	conf.Set(fmt.Sprintf("context.%s.url", contextName), grafanaHost)
 	assert.Equal(t, contextName, "testing")
-	client := service.NewApiService("dummy")
+	storageEngine, err := service.ConfigureStorage()
+	assert.NoError(t, err)
+	client := service.NewApiService(storageEngine)
 	path, _ := os.Getwd()
 	if strings.Contains(path, "test") {
 		err := os.Chdir("..")

pkg/test_tooling/containers.go

@@ -7,6 +7,8 @@ import (
 	"log/slog"
 	"os"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/gdg/pkg/test_tooling/containers"
@@ -16,7 +18,7 @@ func SetupCloudFunction(params []string) (context.Context, context.CancelFunc, s
 	errorFunc := func(err error) (context.Context, context.CancelFunc, service.GrafanaService, error) {
 		return nil, nil, nil, err
 	}
-	_ = os.Setenv(service.InitBucket, "true")
+	_ = os.Setenv(storage.InitBucket, "true")
 	bucketName := params[1]
 	container, cancel := containers.BootstrapCloudStorage("", "")
 	wwwPort, err := container.PortEndpoint(context.Background(), "9001", "")
@@ -30,37 +32,36 @@ func SetupCloudFunction(params []string) (context.Context, context.CancelFunc, s
 	minioHost := fmt.Sprintf("http://%s", actualPort)
 	slog.Info("Minio container is up and running", slog.Any("hostname", fmt.Sprintf("http://%s", wwwPort)))
 	m := map[string]string{
-		service.InitBucket: "true",
-		service.CloudType:  params[0],
-		service.Prefix:     "dummy",
-		service.AccessId:   "test",
-		service.SecretKey:  "secretsss",
-		service.BucketName: bucketName,
-		service.Kind:       "cloud",
-		service.Custom:     "true",
-		service.Endpoint:   minioHost,
-		service.SSLEnabled: "false",
+		storage.InitBucket: "true",
+		storage.CloudType:  params[0],
+		storage.Prefix:     "dummy",
+		storage.AccessId:   "test",
+		storage.SecretKey:  "secretsss",
+		storage.BucketName: bucketName,
+		storage.Kind:       "cloud",
+		storage.Custom:     "true",
+		storage.Endpoint:   minioHost,
+		storage.SSLEnabled: "false",
 	}
 
 	cfgObj := config.Config().GetGDGConfig()
 	defaultCfg := config.Config().GetDefaultGrafanaConfig()
 	defaultCfg.Storage = "test"
 	cfgObj.StorageEngine["test"] = m
-	apiClient := service.NewApiService("dummy")
 
 	ctx := context.Background()
-	ctx = context.WithValue(ctx, service.StorageContext, m)
+	ctx = context.WithValue(ctx, storage.Context, m)
 	configMap := map[string]string{}
 	for key, value := range m {
 		configMap[key] = fmt.Sprintf("%v", value)
 	}
 
-	s, err := service.NewCloudStorage(ctx)
+	s, err := storage.NewCloudStorage(ctx)
 	if err != nil {
 		log.Fatalf("Could not instantiate cloud storage for type: %s", params[0])
 	}
-	dash := apiClient.(*service.DashNGoImpl)
-	dash.SetStorage(s)
+
+	apiClient := service.NewApiService(s)
 
 	return ctx, cancel, apiClient, nil
 }

pkg/test_tooling/path/path.go

@@ -5,10 +5,10 @@ import (
 	"strings"
 )
 
-const testEnv = "TESTING"
+const TestEnvKey = "TESTING"
 
 func FixTestDir(packageName string, newPath string) error {
-	err := os.Setenv(testEnv, "1")
+	err := os.Setenv(TestEnvKey, "1")
 	if err != nil {
 		return err
 	}