diff --git a/.gitignore b/.gitignore index 6e07c64e..49ba049d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ config/token*.yml config/templates.yml website/node_modules/ .env +.DS_Store diff --git a/cmd/gdg/main.go b/cmd/gdg/main.go index 348972d2..f3366e89 100644 --- a/cmd/gdg/main.go +++ b/cmd/gdg/main.go @@ -1,9 +1,12 @@ package main import ( + "context" "log" "os" + "github.com/esnet/gdg/internal/storage" + "github.com/esnet/gdg/cli" "github.com/esnet/gdg/cli/support" @@ -11,7 +14,11 @@ import ( ) var getGrafanaSvc = func() api.GrafanaService { - return api.NewApiService() + storageEngine, err := api.ConfigureStorage() + if err != nil { + storageEngine = storage.NewLocalStorage(context.Background()) + } + return api.NewApiService(storageEngine) } func main() { diff --git a/internal/config/grafana_config.go b/internal/config/grafana_config.go index 9413316f..d6b954f9 100644 --- a/internal/config/grafana_config.go +++ b/internal/config/grafana_config.go @@ -1,9 +1,9 @@ package config type DashboardSettings struct { - NestedFolders bool `mapstructure:"nested_folders" yaml:"nested_folders"` - IgnoreFilters bool `yaml:"ignore_filters" mapstructure:"ignore_filters" ` - IgnoreBadFolder bool `yaml:"ignore_bad_folder" mapstructure:"ignore_bad_folder"` + NestedFolders bool `mapstructure:"nested_folders" yaml:"nested_folders"` + IgnoreFilters bool `yaml:"ignore_filters" mapstructure:"ignore_filters" ` + IgnoreBadFolders bool `yaml:"ignore_bad_folders" mapstructure:"ignore_bad_folders"` } // GrafanaConfig model wraps auth and watched list for grafana diff --git a/internal/service/common_test.go b/internal/service/common_test.go index df263baf..7fe38ec5 100644 --- a/internal/service/common_test.go +++ b/internal/service/common_test.go @@ -1,9 +1,12 @@ package service import ( + "context" "os" "testing" + "github.com/esnet/gdg/internal/storage" + "github.com/esnet/gdg/internal/config" "github.com/esnet/gdg/pkg/test_tooling/common" "github.com/esnet/gdg/pkg/test_tooling/path" @@ -23,9 +26,13 @@ func TestRelativePathLogin(t *testing.T) { assert.NoError(t, os.Setenv(envKey, "http://localhost:3000/grafana/")) fixEnvironment(t) config.InitGdgConfig(common.DefaultTestConfig) - defer assert.NoError(t, os.Unsetenv(envKey)) + defer func() { + assert.NoError(t, os.Unsetenv(envKey)) + assert.NoError(t, os.Unsetenv(path.TestEnvKey)) + }() - svc := NewApiService("dummy") + localEngine := storage.NewLocalStorage(context.Background()) + svc := NewApiService(localEngine) _, cfg := svc.(*DashNGoImpl).getNewClient() assert.Equal(t, cfg.Host, "localhost:3000") assert.Equal(t, cfg.BasePath, "/grafana/api") diff --git a/internal/service/dashboards.go b/internal/service/dashboards.go index e771868a..092d0190 100644 --- a/internal/service/dashboards.go +++ b/internal/service/dashboards.go @@ -298,9 +298,9 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit { // validate folder name folderValid := s.checkFolderName(folderMatch) - if !folderValid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolder { + if !folderValid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders { log.Fatal("Invalid folder name detected, interrupting process.", slog.String("folderTitle", folderMatch)) - } else if !folderValid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolder { + } else if !folderValid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders { slog.Warn("Invalid folder name detected, Skipping dashboards in folder", slog.String("folderTitle", folderMatch), slog.String("dashboard", link.Title)) continue } diff --git a/internal/service/folders.go b/internal/service/folders.go index 472822f5..fb5aa0dc 100644 --- a/internal/service/folders.go +++ b/internal/service/folders.go @@ -205,10 +205,10 @@ func (s *DashNGoImpl) ListFolders(filter filters.Filter) []*types.FolderDetails } for ndx, val := range folderListing { valid := s.checkFolderName(val.Title) - if !valid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolder { + if !valid && s.grafanaConf.GetDashboardSettings().IgnoreBadFolders { slog.Info("Skipping folder due to invalid character", slog.Any("folderTitle", val.Title)) continue - } else if !valid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolder { + } else if !valid && !s.grafanaConf.GetDashboardSettings().IgnoreBadFolders { log.Fatalf("Folder has an invalid character and is not supported. Path separators are not allowed. folderName: %s", val.Title) } filterValue := val.Title diff --git a/internal/service/gdg_api.go b/internal/service/gdg_api.go index 84ebf4fc..d1d67173 100644 --- a/internal/service/gdg_api.go +++ b/internal/service/gdg_api.go @@ -2,12 +2,15 @@ package service import ( "context" + "fmt" + "log" "log/slog" "os" "sync" "github.com/esnet/gdg/internal/api" "github.com/esnet/gdg/internal/config" + "github.com/esnet/gdg/internal/storage" "github.com/spf13/viper" ) @@ -23,7 +26,7 @@ type DashNGoImpl struct { configRef *viper.Viper debug bool apiDebug bool - storage Storage + storage storage.Storage } func NewDashNGoImpl() *DashNGoImpl { @@ -51,41 +54,48 @@ func newInstance() *DashNGoImpl { } } obj.Login() - configureStorage(obj) + storageEngine, err := ConfigureStorage() + if err != nil { + log.Fatal("Unable to configure a valid storage engine, %w", err) + } + obj.SetStorage(storageEngine) return obj } -// Testing Only -func (s *DashNGoImpl) SetStorage(v Storage) { +func (s *DashNGoImpl) SetStorage(v storage.Storage) { s.storage = v } -func configureStorage(obj *DashNGoImpl) { +func ConfigureStorage() (storage.Storage, error) { + var ( + storageEngine storage.Storage + err error + ) // config storageType, appData := config.Config().GetCloudConfiguration(config.Config().GetDefaultGrafanaConfig().Storage) - var err error ctx := context.Background() - ctx = context.WithValue(ctx, StorageContext, appData) + ctx = context.WithValue(ctx, storage.Context, appData) switch storageType { case "cloud": { - obj.storage, err = NewCloudStorage(ctx) + storageEngine, err = storage.NewCloudStorage(ctx) if err != nil { - slog.Warn("falling back on Local Storage, Cloud storage configuration error") - obj.storage = NewLocalStorage(ctx) + return nil, fmt.Errorf("unable to configure CloudStorage Engine: %w", err) } } default: - obj.storage = NewLocalStorage(ctx) + storageEngine = storage.NewLocalStorage(ctx) } + return storageEngine, nil } -func NewApiService(override ...string) GrafanaService { +func NewApiService(storageEngine storage.Storage) GrafanaService { // Used for Testing purposes - if len(override) > 0 { + if os.Getenv("TESTING") == "1" { return newInstance() } + return NewDashNGoImpl() } diff --git a/internal/storage/const.go b/internal/storage/const.go new file mode 100644 index 00000000..6f2a2cad --- /dev/null +++ b/internal/storage/const.go @@ -0,0 +1,19 @@ +package storage + +type ContextStorage string + +const ( + Context = ContextStorage("storage") + // Cloud Specific const + CloudType = "cloud_type" + BucketName = "bucket_name" + Prefix = "prefix" + Kind = "kind" + Custom = "custom" + AccessId = "access_id" + SecretKey = "secret_key" + Endpoint = "endpoint" + Region = "region" + SSLEnabled = "ssl_enabled" + InitBucket = "init_bucket" +) diff --git a/internal/service/storage.go b/internal/storage/storage.go similarity index 88% rename from internal/service/storage.go rename to internal/storage/storage.go index 652c81b2..5c6674e3 100644 --- a/internal/service/storage.go +++ b/internal/storage/storage.go @@ -1,4 +1,4 @@ -package service +package storage import ( _ "gocloud.dev/blob/azureblob" @@ -6,10 +6,6 @@ import ( _ "gocloud.dev/blob/s3blob" ) -type ContextStorage string - -const StorageContext = ContextStorage("storage") - // TODO: pull all the cloud based interaction into a Plugin System type Storage interface { WriteFile(filename string, data []byte) error // WriteFile returns error or writes byte array to destination diff --git a/internal/service/storage_cloud.go b/internal/storage/storage_cloud.go similarity index 92% rename from internal/service/storage_cloud.go rename to internal/storage/storage_cloud.go index 57c4c442..e1b900a7 100644 --- a/internal/service/storage_cloud.go +++ b/internal/storage/storage_cloud.go @@ -1,4 +1,4 @@ -package service +package storage import ( "context" @@ -27,7 +27,7 @@ type Resolver struct { URL *url.URL } -func (r *Resolver) ResolveEndpoint(_ context.Context, params s3.EndpointParameters) (transport.Endpoint, error) { +func (r *Resolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (transport.Endpoint, error) { u := *r.URL u.Path += "/" + *params.Bucket return transport.Endpoint{URI: u}, nil @@ -42,20 +42,6 @@ type CloudStorage struct { StorageName string } -const ( - CloudType = "cloud_type" - BucketName = "bucket_name" - Prefix = "prefix" - Kind = "kind" - Custom = "custom" - AccessId = "access_id" - SecretKey = "secret_key" - Endpoint = "endpoint" - Region = "region" - SSLEnabled = "ssl_enabled" - InitBucket = "init_bucket" -) - var ( stringEmpty = func(key string) bool { return key == "" @@ -142,7 +128,7 @@ func NewCloudStorage(c context.Context) (Storage, error) { errorMsg string ) - contextVal := c.Value(StorageContext) + contextVal := c.Value(Context) if contextVal == nil { return nil, errors.New("cannot configure GCP storage, context missing") } diff --git a/internal/service/storage_local.go b/internal/storage/storage_local.go similarity index 99% rename from internal/service/storage_local.go rename to internal/storage/storage_local.go index f82015dc..c50c6d79 100644 --- a/internal/service/storage_local.go +++ b/internal/storage/storage_local.go @@ -1,4 +1,4 @@ -package service +package storage import ( "context" diff --git a/internal/tools/ptr/ptr.go b/internal/tools/ptr/ptr.go index c36b2a2d..f9c1124e 100644 --- a/internal/tools/ptr/ptr.go +++ b/internal/tools/ptr/ptr.go @@ -1,5 +1,14 @@ package ptr +// Of returns a pointer to the given value. +// +// This is a convenience function to create a pointer from a value. +// +// Example: +// +// p := ptr.Of(5) +// +// will return a pointer to the value 5. func Of[T any](value T) *T { return &value } diff --git a/pkg/test_tooling/common.go b/pkg/test_tooling/common.go index 101e4531..effd5ef3 100644 --- a/pkg/test_tooling/common.go +++ b/pkg/test_tooling/common.go @@ -107,7 +107,9 @@ func CreateSimpleClient(t *testing.T, cfgName *string, container testcontainers. contextName := conf.GetString("context_name") conf.Set(fmt.Sprintf("context.%s.url", contextName), grafanaHost) assert.Equal(t, contextName, "testing") - client := service.NewApiService("dummy") + storageEngine, err := service.ConfigureStorage() + assert.NoError(t, err) + client := service.NewApiService(storageEngine) path, _ := os.Getwd() if strings.Contains(path, "test") { err := os.Chdir("..") diff --git a/pkg/test_tooling/containers.go b/pkg/test_tooling/containers.go index 8210f14b..a058f2c7 100644 --- a/pkg/test_tooling/containers.go +++ b/pkg/test_tooling/containers.go @@ -7,6 +7,8 @@ import ( "log/slog" "os" + "github.com/esnet/gdg/internal/storage" + "github.com/esnet/gdg/internal/config" "github.com/esnet/gdg/internal/service" "github.com/esnet/gdg/pkg/test_tooling/containers" @@ -16,7 +18,7 @@ func SetupCloudFunction(params []string) (context.Context, context.CancelFunc, s errorFunc := func(err error) (context.Context, context.CancelFunc, service.GrafanaService, error) { return nil, nil, nil, err } - _ = os.Setenv(service.InitBucket, "true") + _ = os.Setenv(storage.InitBucket, "true") bucketName := params[1] container, cancel := containers.BootstrapCloudStorage("", "") wwwPort, err := container.PortEndpoint(context.Background(), "9001", "") @@ -30,37 +32,36 @@ func SetupCloudFunction(params []string) (context.Context, context.CancelFunc, s minioHost := fmt.Sprintf("http://%s", actualPort) slog.Info("Minio container is up and running", slog.Any("hostname", fmt.Sprintf("http://%s", wwwPort))) m := map[string]string{ - service.InitBucket: "true", - service.CloudType: params[0], - service.Prefix: "dummy", - service.AccessId: "test", - service.SecretKey: "secretsss", - service.BucketName: bucketName, - service.Kind: "cloud", - service.Custom: "true", - service.Endpoint: minioHost, - service.SSLEnabled: "false", + storage.InitBucket: "true", + storage.CloudType: params[0], + storage.Prefix: "dummy", + storage.AccessId: "test", + storage.SecretKey: "secretsss", + storage.BucketName: bucketName, + storage.Kind: "cloud", + storage.Custom: "true", + storage.Endpoint: minioHost, + storage.SSLEnabled: "false", } cfgObj := config.Config().GetGDGConfig() defaultCfg := config.Config().GetDefaultGrafanaConfig() defaultCfg.Storage = "test" cfgObj.StorageEngine["test"] = m - apiClient := service.NewApiService("dummy") ctx := context.Background() - ctx = context.WithValue(ctx, service.StorageContext, m) + ctx = context.WithValue(ctx, storage.Context, m) configMap := map[string]string{} for key, value := range m { configMap[key] = fmt.Sprintf("%v", value) } - s, err := service.NewCloudStorage(ctx) + s, err := storage.NewCloudStorage(ctx) if err != nil { log.Fatalf("Could not instantiate cloud storage for type: %s", params[0]) } - dash := apiClient.(*service.DashNGoImpl) - dash.SetStorage(s) + + apiClient := service.NewApiService(s) return ctx, cancel, apiClient, nil } diff --git a/pkg/test_tooling/path/path.go b/pkg/test_tooling/path/path.go index 697fc118..45af35e8 100644 --- a/pkg/test_tooling/path/path.go +++ b/pkg/test_tooling/path/path.go @@ -5,10 +5,10 @@ import ( "strings" ) -const testEnv = "TESTING" +const TestEnvKey = "TESTING" func FixTestDir(packageName string, newPath string) error { - err := os.Setenv(testEnv, "1") + err := os.Setenv(TestEnvKey, "1") if err != nil { return err } diff --git a/test/connections_integration_test.go b/test/connections_integration_test.go index 9fe163d8..0d62c416 100644 --- a/test/connections_integration_test.go +++ b/test/connections_integration_test.go @@ -1,11 +1,14 @@ package test import ( + "context" "log/slog" "os" "strings" "testing" + "github.com/esnet/gdg/internal/storage" + "github.com/esnet/gdg/internal/config" "github.com/esnet/gdg/internal/service" "github.com/esnet/gdg/internal/types" @@ -164,7 +167,8 @@ func TestConnectionFilter(t *testing.T) { testingContext = config.Config().GetGDGConfig().GetContexts()["testing"] _ = testingContext - apiClient := service.NewApiService("dummy") + localEngine := storage.NewLocalStorage(context.Background()) + apiClient := service.NewApiService(localEngine) filtersEntity := service.NewConnectionFilter("") slog.Info("Exporting all connections") diff --git a/test/folder_integration_test.go b/test/folder_integration_test.go index 9f22d5b6..6f010e2e 100644 --- a/test/folder_integration_test.go +++ b/test/folder_integration_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/esnet/gdg/internal/config" + "github.com/gosimple/slug" "github.com/testcontainers/testcontainers-go" @@ -49,6 +51,46 @@ func TestFolderCRUD(t *testing.T) { assert.Equal(t, len(folders), 0) } +func TestFolderCRUDInvalidChar(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + os.Setenv("TESTING", "1") + os.Setenv("GDG__CONTEXTS__TESTING__DASHBOARD_SETTINGS__NESTED_FOLDERS", "true") + os.Setenv("GDG__CONTEXTS__TESTING__DASHBOARD_SETTINGS__IGNORE_BAD_FOLDERS", "true") + + apiClient, _, _, cleanup := test_tooling.InitTest(t, nil, nil) + + defer func() { + os.Unsetenv("GDG__CONTEXTS__TESTING__DASHBOARD_SETTINGS__NESTED_FOLDERS") + os.Unsetenv("GDG__CONTEXTS__TESTING__DASHBOARD_SETTINGS__IGNORE_BAD_FOLDERS") + cleanup() + }() + + config.InitGdgConfig("testing.yml") + mcfg := config.Config() + _ = mcfg + slog.Info("Exporting all folders") + apiClient.UploadFolders(nil) + slog.Info("Listing all Folders") + folders := apiClient.ListFolders(nil) + assert.Equal(t, len(folders), 2) + firstDsItem := folders[0] + assert.Equal(t, firstDsItem.Title, "Ignored") + secondDsItem := folders[1] + assert.Equal(t, secondDsItem.Title, "Other") + // Import Folders + slog.Info("Importing folders") + list := apiClient.DownloadFolders(nil) + assert.Equal(t, len(list), len(folders)) + slog.Info("Deleting Folders") + deleteList := apiClient.DeleteAllFolders(nil) + assert.Equal(t, len(deleteList), len(folders)) + slog.Info("List Folders again") + folders = apiClient.ListFolders(nil) + assert.Equal(t, len(folders), 0) +} + // TODO: write a full CRUD validation of folder permissions func TestFolderPermissions(t *testing.T) { apiClient, _, _, cleanup := test_tooling.InitTest(t, nil, nil)