From fc6f8142495b520ed421b04fa731949e33392cc6 Mon Sep 17 00:00:00 2001 From: Piotrek Janus Date: Mon, 8 Jan 2024 14:30:32 +0100 Subject: [PATCH 1/5] add support for multiple dirs when reading config from fs --- .github/workflows/lint.yaml | 1 - internal/cac/app.go | 6 ++- internal/cac/config/config.go | 8 ++-- internal/cac/storage/multi.go | 69 ++++++++++++++++++++++++++++ internal/cac/storage/storage.go | 21 ++++++--- internal/cac/storage/storage_test.go | 35 ++++++++------ 6 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 internal/cac/storage/multi.go diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6b0096d..3e2de75 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -5,7 +5,6 @@ on: - master - dev pull_request: - permissions: contents: read diff --git a/internal/cac/app.go b/internal/cac/app.go index 61a0fa7..af3c9da 100644 --- a/internal/cac/app.go +++ b/internal/cac/app.go @@ -11,7 +11,7 @@ import ( type Application struct { Config *config.Configuration Client *client.Client - Storage *storage.Storage + Storage storage.Storage } func InitApp(configPath string) (app *Application, err error) { @@ -31,7 +31,9 @@ func InitApp(configPath string) (app *Application, err error) { return app, err } - app.Storage = storage.InitStorage(app.Config.Storage) + if app.Storage, err = storage.InitMultiStorage(app.Config.Storage); err != nil { + return app, err + } slog.With("app", app).Debug("Initiated application") diff --git a/internal/cac/config/config.go b/internal/cac/config/config.go index 29b407d..6a9076d 100644 --- a/internal/cac/config/config.go +++ b/internal/cac/config/config.go @@ -18,14 +18,14 @@ var ( DefaultConfig = Configuration{ Client: client.DefaultConfig, Logging: logging.DefaultLoggingConfig, - Storage: storage.DefaultConfig, + Storage: storage.DefaultMultiStorageConfig, } ) type Configuration struct { - Client client.Configuration `json:"client"` - Logging logging.Configuration `json:"logging"` - Storage storage.Configuration `json:"storage"` + Client client.Configuration `json:"client"` + Logging logging.Configuration `json:"logging"` + Storage storage.MultiStorageConfiguration `json:"storage"` } func InitConfig(path string) (_ *Configuration, err error) { diff --git a/internal/cac/storage/multi.go b/internal/cac/storage/multi.go new file mode 100644 index 0000000..51a3d02 --- /dev/null +++ b/internal/cac/storage/multi.go @@ -0,0 +1,69 @@ +package storage + +import ( + "github.com/cloudentity/acp-client-go/clients/hub/models" + "github.com/imdario/mergo" + "github.com/pkg/errors" +) + +type MultiStorageConfiguration struct { + DirPath []string `json:"dir_path"` +} + +var DefaultMultiStorageConfig = MultiStorageConfiguration{ + DirPath: []string{"data"}, +} + +func InitMultiStorage(config MultiStorageConfiguration) (*MultiStorage, error) { + var storages []Storage + + if len(config.DirPath) == 0 { + return nil, errors.New("at least one dir_path is required") + } + + for _, config := range config.DirPath { + storages = append(storages, InitStorage(Configuration{ + DirPath: config, + })) + } + + return &MultiStorage{ + Storages: storages, + Config: config, + }, nil +} + +type MultiStorage struct { + Storages []Storage + Config MultiStorageConfiguration +} + +var _ Storage = &MultiStorage{} + +// Store for simplicity stores data in first storage only, it is responsibility of the user to move entities to other storages +func (m *MultiStorage) Store(workspace string, data *models.TreeServer) error { + return m.Storages[0].Store(workspace, data) +} + +// Read data from all storages and merge them +func (m *MultiStorage) Read(workspace string) (models.TreeServer, error) { + var ( + data models.TreeServer + err error + ) + + for i := len(m.Storages) - 1; i >= 0; i-- { + var data2 models.TreeServer + + if data2, err = m.Storages[i].Read(workspace); err != nil { + return data, errors.Wrap(err, "failed to read data from storage") + } + + if err = mergo.Merge(&data, data2, mergo.WithOverride); err != nil { + return data, errors.Wrap(err, "failed to merge data") + } + + } + + return data, nil +} diff --git a/internal/cac/storage/storage.go b/internal/cac/storage/storage.go index a9240fc..f298ca7 100644 --- a/internal/cac/storage/storage.go +++ b/internal/cac/storage/storage.go @@ -17,17 +17,24 @@ var DefaultConfig = Configuration{ DirPath: "data", } -func InitStorage(config Configuration) *Storage { - return &Storage{ +func InitStorage(config Configuration) *SingleStorage { + return &SingleStorage{ Config: config, } } -type Storage struct { +type Storage interface { + Store(workspace string, data *models.TreeServer) error + Read(workspace string) (models.TreeServer, error) +} + +type SingleStorage struct { Config Configuration } -func (s *Storage) Store(workspace string, data *models.TreeServer) error { +var _ Storage = &SingleStorage{} + +func (s *SingleStorage) Store(workspace string, data *models.TreeServer) error { var ( workspacePath = s.workspacePath(workspace) err error @@ -126,7 +133,7 @@ func (s *Storage) Store(workspace string, data *models.TreeServer) error { return nil } -func (s *Storage) Read(workspace string) (models.TreeServer, error) { +func (s *SingleStorage) Read(workspace string) (models.TreeServer, error) { var ( server = models.TreeServer{ Clients: models.TreeClients{}, @@ -235,11 +242,11 @@ func (s *Storage) Read(workspace string) (models.TreeServer, error) { return server, nil } -func (s *Storage) workspacePath(workspace string) string { +func (s *SingleStorage) workspacePath(workspace string) string { return filepath.Join(s.Config.DirPath, "workspaces", workspace) } -func (s *Storage) storeServer(workspace string, data *models.TreeServer) error { +func (s *SingleStorage) storeServer(workspace string, data *models.TreeServer) error { var ( path = filepath.Join(s.workspacePath(workspace), "server") server adminmodels.Server diff --git a/internal/cac/storage/storage_test.go b/internal/cac/storage/storage_test.go index cb2c76a..f219a7d 100644 --- a/internal/cac/storage/storage_test.go +++ b/internal/cac/storage/storage_test.go @@ -440,34 +440,41 @@ module.exports = async function(context) { for _, tc := range tcs { t.Run(tc.desc, func(t *testing.T) { - st := storage.InitStorage(storage.Configuration{ - DirPath: t.TempDir(), + st, err := storage.InitMultiStorage(storage.MultiStorageConfiguration{ + DirPath: []string{t.TempDir(), t.TempDir()}, }) - err := st.Store("demo", tc.data) + require.NoError(t, err) + + err = st.Store("demo", tc.data) require.NoError(t, err) var files []string - err = filepath.Walk(st.Config.DirPath, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - if path, err = filepath.Rel(st.Config.DirPath, path); err != nil { + // refactor the code below to loop over dirpath + for _, dir := range st.Config.DirPath { + err = filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if err != nil { return err } - files = append(files, path) - } - return nil - }) + if !info.IsDir() { + if path, err = filepath.Rel(dir, path); err != nil { + return err + } + + files = append(files, path) + } + return nil + }) + } require.NoError(t, err) require.ElementsMatch(t, slices.Compact(append(tc.files, "workspaces/demo/server.yaml")), files) for _, f := range tc.files { - bts, err := os.ReadFile(filepath.Join(st.Config.DirPath, f)) + // using first dirpath as multi storage stores everything there + bts, err := os.ReadFile(filepath.Join(st.Config.DirPath[0], f)) require.NoError(t, err) if tc.assert != nil { From adb84bb4c74f0ccecd98a994401af01ec5e772aa Mon Sep 17 00:00:00 2001 From: Piotrek Janus Date: Mon, 8 Jan 2024 16:46:18 +0100 Subject: [PATCH 2/5] readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 88ca04d..52a264e 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ cac --config examples/e2e/config.yaml pull --workspace cdr_australia-demo-c67evw Merge configuration from a directory structure and push it into Cloudentity. +#### Push configuration from multiple directories + +To push configration from multiple directories, either pass an array to the `storage.dir_path` or use `STORAGE_DIR_PATH` env variable multiple times. + ```bash cac --config examples/e2e/config.yaml push --workspace cdr_australia-demo-c67evw7mj4 ``` From 8413d405b095e351102489bde6edab5f00850c78 Mon Sep 17 00:00:00 2001 From: Piotrek Janus Date: Mon, 8 Jan 2024 22:29:35 +0100 Subject: [PATCH 3/5] fix passing multiple paths as env variable --- README.md | 2 +- internal/cac/config/config.go | 20 +++++++++++++++++++- internal/cac/storage/storage_test.go | 1 - 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 52a264e..92e77c8 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Merge configuration from a directory structure and push it into Cloudentity. #### Push configuration from multiple directories -To push configration from multiple directories, either pass an array to the `storage.dir_path` or use `STORAGE_DIR_PATH` env variable multiple times. +To push configration from multiple directories, either pass an array to the `storage.dir_path` or use `STORAGE_DIR_PATH` with multiple paths split by a comma. ```bash cac --config examples/e2e/config.yaml push --workspace cdr_australia-demo-c67evw7mj4 diff --git a/internal/cac/config/config.go b/internal/cac/config/config.go index 6a9076d..73a6599 100644 --- a/internal/cac/config/config.go +++ b/internal/cac/config/config.go @@ -75,7 +75,7 @@ func InitConfig(path string) (_ *Configuration, err error) { func configureDecoder(config *mapstructure.DecoderConfig) { config.TagName = "json" config.WeaklyTypedInput = true - config.DecodeHook = mapstructure.ComposeDecodeHookFunc(urlDecoder(), timeDecoder()) + config.DecodeHook = mapstructure.ComposeDecodeHookFunc(urlDecoder(), timeDecoder(), stringToSlice()) } func urlDecoder() mapstructure.DecodeHookFunc { @@ -125,3 +125,21 @@ func timeDecoder() mapstructure.DecodeHookFunc { } } } + +func stringToSlice() mapstructure.DecodeHookFunc { + return func( + f reflect.Kind, + t reflect.Kind, + data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.Slice { + return data, nil + } + + raw := data.(string) + if raw == "" { + return []string{}, nil + } + + return strings.Split(raw, ","), nil + } +} diff --git a/internal/cac/storage/storage_test.go b/internal/cac/storage/storage_test.go index f219a7d..118dd75 100644 --- a/internal/cac/storage/storage_test.go +++ b/internal/cac/storage/storage_test.go @@ -451,7 +451,6 @@ module.exports = async function(context) { var files []string - // refactor the code below to loop over dirpath for _, dir := range st.Config.DirPath { err = filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { From e3fa8ae3efa2f9d7c3f7629fee4b3a439f2a1b13 Mon Sep 17 00:00:00 2001 From: Piotrek Janus Date: Mon, 8 Jan 2024 22:49:33 +0100 Subject: [PATCH 4/5] readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 92e77c8..c28e10d 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ Merge configuration from a directory structure and push it into Cloudentity. To push configration from multiple directories, either pass an array to the `storage.dir_path` or use `STORAGE_DIR_PATH` with multiple paths split by a comma. +Configurations are merged in the reverse order, so the first path has the highest priority, and will override everything else. + ```bash cac --config examples/e2e/config.yaml push --workspace cdr_australia-demo-c67evw7mj4 ``` From b3c7beed5e784658dada7d7ebc317dbf8e429834 Mon Sep 17 00:00:00 2001 From: Piotrek Janus Date: Tue, 9 Jan 2024 23:17:59 +0100 Subject: [PATCH 5/5] logs --- internal/cac/app.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cac/app.go b/internal/cac/app.go index af3c9da..50942ce 100644 --- a/internal/cac/app.go +++ b/internal/cac/app.go @@ -25,7 +25,7 @@ func InitApp(configPath string) (app *Application, err error) { return app, err } - slog.Info("config", "c", app.Config.Client) + slog.Debug("config", "c", app.Config.Client) if app.Client, err = client.InitClient(app.Config.Client); err != nil { return app, err @@ -35,7 +35,7 @@ func InitApp(configPath string) (app *Application, err error) { return app, err } - slog.With("app", app).Debug("Initiated application") + slog.Info("Initiated application") return app, nil }