AUT-10341 Add support for multiple dirs when reading config from fs (#4)

* add support for multiple dirs when reading config from fs

* readme

* fix passing multiple paths as env variable

* readme

* logs

.github/workflows/lint.yaml

@@ -5,7 +5,6 @@ on:
       - master
       - dev
   pull_request:
-
 permissions:
   contents: read
 

README.md

@@ -82,6 +82,12 @@ cac --config examples/e2e/config.yaml pull --workspace cdr_australia-demo-c67evw
 
 Merge configuration from a directory structure and push it into Cloudentity.
 
+#### Push configuration from multiple directories
+
+To push configration from multiple directories, either pass an array to the `storage.dir_path` or use `STORAGE_DIR_PATH` with multiple paths split by a comma.
+
+Configurations are merged in the reverse order, so the first path has the highest priority, and will override everything else.
+
 ```bash
 cac --config examples/e2e/config.yaml push --workspace cdr_australia-demo-c67evw7mj4
 ```

internal/cac/app.go

@@ -11,7 +11,7 @@ import (
 type Application struct {
 	Config  *config.Configuration
 	Client  *client.Client
-	Storage *storage.Storage
+	Storage storage.Storage
 }
 
 func InitApp(configPath string) (app *Application, err error) {
@@ -25,15 +25,17 @@ func InitApp(configPath string) (app *Application, err error) {
 		return app, err
 	}
 
-	slog.Info("config", "c", app.Config.Client)
+	slog.Debug("config", "c", app.Config.Client)
 
 	if app.Client, err = client.InitClient(app.Config.Client); err != nil {
 		return app, err
 	}
 
-	app.Storage = storage.InitStorage(app.Config.Storage)
+	if app.Storage, err = storage.InitMultiStorage(app.Config.Storage); err != nil {
+		return app, err
+	}
 
-	slog.With("app", app).Debug("Initiated application")
+	slog.Info("Initiated application")
 
 	return app, nil
 }

internal/cac/config/config.go

@@ -18,14 +18,14 @@ var (
 	DefaultConfig = Configuration{
 		Client:  client.DefaultConfig,
 		Logging: logging.DefaultLoggingConfig,
-		Storage: storage.DefaultConfig,
+		Storage: storage.DefaultMultiStorageConfig,
 	}
 )
 
 type Configuration struct {
-	Client  client.Configuration  `json:"client"`
-	Logging logging.Configuration `json:"logging"`
-	Storage storage.Configuration `json:"storage"`
+	Client  client.Configuration              `json:"client"`
+	Logging logging.Configuration             `json:"logging"`
+	Storage storage.MultiStorageConfiguration `json:"storage"`
 }
 
 func InitConfig(path string) (_ *Configuration, err error) {
@@ -75,7 +75,7 @@ func InitConfig(path string) (_ *Configuration, err error) {
 func configureDecoder(config *mapstructure.DecoderConfig) {
 	config.TagName = "json"
 	config.WeaklyTypedInput = true
-	config.DecodeHook = mapstructure.ComposeDecodeHookFunc(urlDecoder(), timeDecoder())
+	config.DecodeHook = mapstructure.ComposeDecodeHookFunc(urlDecoder(), timeDecoder(), stringToSlice())
 }
 
 func urlDecoder() mapstructure.DecodeHookFunc {
@@ -125,3 +125,21 @@ func timeDecoder() mapstructure.DecodeHookFunc {
 		}
 	}
 }
+
+func stringToSlice() mapstructure.DecodeHookFunc {
+	return func(
+		f reflect.Kind,
+		t reflect.Kind,
+		data interface{}) (interface{}, error) {
+		if f != reflect.String || t != reflect.Slice {
+			return data, nil
+		}
+
+		raw := data.(string)
+		if raw == "" {
+			return []string{}, nil
+		}
+
+		return strings.Split(raw, ","), nil
+	}
+}

internal/cac/storage/multi.go

@@ -0,0 +1,69 @@
+package storage
+
+import (
+	"github.com/cloudentity/acp-client-go/clients/hub/models"
+	"github.com/imdario/mergo"
+	"github.com/pkg/errors"
+)
+
+type MultiStorageConfiguration struct {
+	DirPath []string `json:"dir_path"`
+}
+
+var DefaultMultiStorageConfig = MultiStorageConfiguration{
+	DirPath: []string{"data"},
+}
+
+func InitMultiStorage(config MultiStorageConfiguration) (*MultiStorage, error) {
+	var storages []Storage
+
+	if len(config.DirPath) == 0 {
+		return nil, errors.New("at least one dir_path is required")
+	}
+
+	for _, config := range config.DirPath {
+		storages = append(storages, InitStorage(Configuration{
+			DirPath: config,
+		}))
+	}
+
+	return &MultiStorage{
+		Storages: storages,
+		Config:   config,
+	}, nil
+}
+
+type MultiStorage struct {
+	Storages []Storage
+	Config   MultiStorageConfiguration
+}
+
+var _ Storage = &MultiStorage{}
+
+// Store for simplicity stores data in first storage only, it is responsibility of the user to move entities to other storages
+func (m *MultiStorage) Store(workspace string, data *models.TreeServer) error {
+	return m.Storages[0].Store(workspace, data)
+}
+
+// Read data from all storages and merge them
+func (m *MultiStorage) Read(workspace string) (models.TreeServer, error) {
+	var (
+		data models.TreeServer
+		err  error
+	)
+
+	for i := len(m.Storages) - 1; i >= 0; i-- {
+		var data2 models.TreeServer
+
+		if data2, err = m.Storages[i].Read(workspace); err != nil {
+			return data, errors.Wrap(err, "failed to read data from storage")
+		}
+
+		if err = mergo.Merge(&data, data2, mergo.WithOverride); err != nil {
+			return data, errors.Wrap(err, "failed to merge data")
+		}
+
+	}
+
+	return data, nil
+}

internal/cac/storage/storage.go

@@ -17,17 +17,24 @@ var DefaultConfig = Configuration{
 	DirPath: "data",
 }
 
-func InitStorage(config Configuration) *Storage {
-	return &Storage{
+func InitStorage(config Configuration) *SingleStorage {
+	return &SingleStorage{
 		Config: config,
 	}
 }
 
-type Storage struct {
+type Storage interface {
+	Store(workspace string, data *models.TreeServer) error
+	Read(workspace string) (models.TreeServer, error)
+}
+
+type SingleStorage struct {
 	Config Configuration
 }
 
-func (s *Storage) Store(workspace string, data *models.TreeServer) error {
+var _ Storage = &SingleStorage{}
+
+func (s *SingleStorage) Store(workspace string, data *models.TreeServer) error {
 	var (
 		workspacePath = s.workspacePath(workspace)
 		err           error
@@ -126,7 +133,7 @@ func (s *Storage) Store(workspace string, data *models.TreeServer) error {
 	return nil
 }
 
-func (s *Storage) Read(workspace string) (models.TreeServer, error) {
+func (s *SingleStorage) Read(workspace string) (models.TreeServer, error) {
 	var (
 		server = models.TreeServer{
 			Clients:              models.TreeClients{},
@@ -235,11 +242,11 @@ func (s *Storage) Read(workspace string) (models.TreeServer, error) {
 	return server, nil
 }
 
-func (s *Storage) workspacePath(workspace string) string {
+func (s *SingleStorage) workspacePath(workspace string) string {
 	return filepath.Join(s.Config.DirPath, "workspaces", workspace)
 }
 
-func (s *Storage) storeServer(workspace string, data *models.TreeServer) error {
+func (s *SingleStorage) storeServer(workspace string, data *models.TreeServer) error {
 	var (
 		path   = filepath.Join(s.workspacePath(workspace), "server")
 		server adminmodels.Server

internal/cac/storage/storage_test.go

@@ -440,34 +440,40 @@ module.exports = async function(context) {
 
 	for _, tc := range tcs {
 		t.Run(tc.desc, func(t *testing.T) {
-			st := storage.InitStorage(storage.Configuration{
-				DirPath: t.TempDir(),
+			st, err := storage.InitMultiStorage(storage.MultiStorageConfiguration{
+				DirPath: []string{t.TempDir(), t.TempDir()},
 			})
 
-			err := st.Store("demo", tc.data)
+			require.NoError(t, err)
+
+			err = st.Store("demo", tc.data)
 			require.NoError(t, err)
 
 			var files []string
-			err = filepath.Walk(st.Config.DirPath, func(path string, info fs.FileInfo, err error) error {
-				if err != nil {
-					return err
-				}
 
-				if !info.IsDir() {
-					if path, err = filepath.Rel(st.Config.DirPath, path); err != nil {
+			for _, dir := range st.Config.DirPath {
+				err = filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
+					if err != nil {
 						return err
 					}
 
-					files = append(files, path)
-				}
-				return nil
-			})
+					if !info.IsDir() {
+						if path, err = filepath.Rel(dir, path); err != nil {
+							return err
+						}
+
+						files = append(files, path)
+					}
+					return nil
+				})
+			}
 
 			require.NoError(t, err)
 			require.ElementsMatch(t, slices.Compact(append(tc.files, "workspaces/demo/server.yaml")), files)
 
 			for _, f := range tc.files {
-				bts, err := os.ReadFile(filepath.Join(st.Config.DirPath, f))
+				// using first dirpath as multi storage stores everything there
+				bts, err := os.ReadFile(filepath.Join(st.Config.DirPath[0], f))
 				require.NoError(t, err)
 
 				if tc.assert != nil {