From 7279cc758ea8567262e692262caf815103b86cd4 Mon Sep 17 00:00:00 2001 From: Fritz Durchardt Date: Mon, 17 Jul 2023 09:16:58 +0200 Subject: [PATCH 1/2] feat: vendir sync caching On vendir sync a new vendir.sync.yaml is safed to the "tmp" folder containing the current state of vendir.yaml condensed into hash values. If this file remains unchanged, vendir sync can be omitted. The feature can be enabled on environment and application level with the new application.cache.enabled flag. --- .gitignore | 1 + cmd/root.go | 2 +- internal/myks/application.go | 69 ++--- internal/myks/globe.go | 2 + internal/myks/sync.go | 233 +++++++++++++++ internal/myks/sync_test.go | 281 ++++++++++++++++++ internal/myks/util.go | 74 ++++- internal/myks/util_test.go | 138 +++++++++ testData/sync/lock-file.yaml | 15 + testData/sync/simple.yaml | 5 + testData/sync/sync-file.yaml | 4 + testData/sync/vendir-multiple-contents.yaml | 19 ++ .../sync/vendir-multiple-directories.yaml | 21 ++ .../sync/vendir-simple-different-order.yaml | 11 + testData/sync/vendir-simple.yaml | 11 + testData/sync/vendir-with-subpath.yaml | 11 + testData/ytt/data-file-schema-2.yaml | 8 + testData/ytt/data-file-schema.yaml | 5 + testData/ytt/data-file-values.yaml | 5 + 19 files changed, 875 insertions(+), 40 deletions(-) create mode 100644 internal/myks/sync.go create mode 100644 internal/myks/sync_test.go create mode 100644 internal/myks/util_test.go create mode 100644 testData/sync/lock-file.yaml create mode 100644 testData/sync/simple.yaml create mode 100644 testData/sync/sync-file.yaml create mode 100644 testData/sync/vendir-multiple-contents.yaml create mode 100644 testData/sync/vendir-multiple-directories.yaml create mode 100644 testData/sync/vendir-simple-different-order.yaml create mode 100644 testData/sync/vendir-simple.yaml create mode 100644 testData/sync/vendir-with-subpath.yaml create mode 100644 testData/ytt/data-file-schema-2.yaml create mode 100644 testData/ytt/data-file-schema.yaml create mode 100644 testData/ytt/data-file-values.yaml diff --git a/.gitignore b/.gitignore index 38cacdc1..3982db88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ bin/* dist +.idea diff --git a/cmd/root.go b/cmd/root.go index 356c195b..c7580f56 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,7 +26,7 @@ var rootCmd = &cobra.Command{ Long: "Myks TBD", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // Check positional arguments: - // 1. Comma-separated list of envirmoment search paths or ALL to search everywhere (default: ALL) + // 1. Comma-separated list of environment search paths or ALL to search everywhere (default: ALL) // 2. Comma-separated list of application names or none to process all applications (default: none) targetEnvironments = nil diff --git a/internal/myks/application.go b/internal/myks/application.go index 03dea2f5..1f17d655 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -22,6 +22,7 @@ type Application struct { e *Environment // YTT data files yttDataFiles []string + cached bool } type HelmConfig struct { @@ -48,12 +49,41 @@ func NewApplication(e *Environment, name string, prototypeName string) (*Applica Prototype: prototype, e: e, } + err := app.Init() + if err != nil { + return nil, err + } return app, nil } func (a *Application) Init() error { - // TODO: create application directory if it does not exist + // 1. Collect all ytt data files: + // - environment data files: `envs/**/env-data.ytt.yaml` + // - application prototype data file: `prototypes//app-data.ytt.yaml` + // - application data files: `envs/**/_apps//add-data.ytt.yaml` + + a.collectDataFiles() + + dataYaml, err := renderDataYaml(append(a.e.g.extraYttPaths, a.yttDataFiles...)) + if err != nil { + return err + } + + var applicationData struct { + Application struct { + Cache struct { + Enabled bool + } + } + } + + err = yaml.Unmarshal(dataYaml, &applicationData) + if err != nil { + return err + } + a.cached = applicationData.Application.Cache.Enabled + return nil } @@ -73,12 +103,7 @@ func (a *Application) Sync() error { } func (a *Application) Render() error { - // 1. Collect all ytt data files: - // - environment data files: `envs/**/env-data.ytt.yaml` - // - application prototype data file: `prototypes//app-data.ytt.yaml` - // - application data files: `envs/**/_apps//add-data.ytt.yaml` - a.collectDataFiles() log.Debug().Strs("files", a.yttDataFiles).Msg("Collected ytt data files") // 2. Run built-in rendering steps: @@ -185,38 +210,6 @@ func (a *Application) prepareSync() error { return nil } -func (a *Application) doSync() error { - // TODO: implement selective sync - // TODO: implement secrets-from-env extraction - - // Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`) - vendirConfigFile := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName) - vendirLockFile := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName) - - vendorDir := a.expandPath(a.e.g.VendorDirName) - if _, err := os.Stat(vendorDir); err != nil { - err := os.MkdirAll(vendorDir, 0o750) - if err != nil { - log.Warn().Err(err).Msg("Unable to create vendor directory") - return err - } - } - - log.Info().Str("app", a.Name).Msg("Syncing vendir") - res, err := runCmd("vendir", nil, []string{ - "sync", - "--chdir=" + vendorDir, - "--file=" + vendirConfigFile, - "--lock-file=" + vendirLockFile, - }) - if err != nil { - log.Warn().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") - return err - } - - return nil -} - func (a *Application) expandPath(path string) string { return filepath.Join(a.e.Dir, "_apps", a.Name, path) } diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 341cedae..bc0fd6fb 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -64,6 +64,8 @@ type Globe struct { VendirConfigFileName string `default:"vendir.yaml" yaml:"vendirConfigFileName"` // Rendered vendir lock file name VendirLockFileName string `default:"vendir.lock.yaml" yaml:"vendirLockFileName"` + // Rendered vendir sync file name + VendirSyncFileName string `default:"vendir.sync.yaml" yaml:"vendirSyncFileName"` // Downloaded third-party sources VendorDirName string `default:"vendor" yaml:"vendorDirName"` // Ytt library directory name diff --git a/internal/myks/sync.go b/internal/myks/sync.go new file mode 100644 index 00000000..5fb4b2d2 --- /dev/null +++ b/internal/myks/sync.go @@ -0,0 +1,233 @@ +package myks + +import ( + "errors" + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v3" + "os" + "path/filepath" +) + +type Directory struct { + Path string + ContentHash string `yaml:"contentHash"` +} + +func (a *Application) doSync() error { + // TODO: implement secrets-from-env extraction + + // Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`) + vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName) + vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName) + vendirConfigFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirConfigFileName) + vendirLockFilePath := filepath.Join(a.expandServicePath(""), a.e.g.VendirLockFileName) + vendirSyncPath := a.expandTempPath(a.e.g.VendirSyncFileName) + vendorDir := a.expandPath(a.e.g.VendorDirName) + + vendirDirs, err := readVendirConfig(vendirConfigFilePath) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Error while trying to find directories in vendir config: " + vendirConfigFilePath) + return err + } + + syncFileDirs, err := readSyncFile(vendirSyncPath) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Sync file: " + vendirSyncPath) + return err + } + if len(syncFileDirs) == 0 { + log.Debug().Str("app", a.Name).Msg("Vendir sync file not found. First sync..") + } + + lockFileDirs, err := readLockFile(vendirLockFilePath) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Unable to read Vendir Lock file: " + vendirLockFilePath) + return err + } + + err = createDirectory(vendorDir) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Msg("Unable to create vendor dir: " + vendorDir) + return err + } + + //TODO sync retry + // only sync vendir with directory flag, if the lock file matches the vendir config file and caching is enabled + if a.cached && checkLockFileMatch(vendirDirs, lockFileDirs) { + for _, dir := range vendirDirs { + if checkVersionMatch(dir.Path, dir.ContentHash, syncFileDirs) { + log.Debug().Str("app", a.Name).Msg("Skipping vendir sync for: " + dir.Path) + continue + } + log.Info().Str("app", a.Name).Msg("Syncing vendir for: " + dir.Path) + res, err := runCmd("vendir", nil, []string{ + "sync", + "--chdir=" + vendorDir, + "--directory=" + dir.Path, + "--file=" + vendirConfigFileRelativePath, + "--lock-file=" + vendirLockFileRelativePath, + }) + if err != nil { + log.Warn().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") + return err + } + } + } else { + log.Info().Str("app", a.Name).Msg("Syncing vendir completely for: " + vendirConfigFilePath) + res, err := runCmd("vendir", nil, []string{ + "sync", + "--chdir=" + vendorDir, + "--file=" + vendirConfigFileRelativePath, + "--lock-file=" + vendirLockFileRelativePath, + }) + if err != nil { + log.Error().Err(err).Str("app", a.Name).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg("Unable to sync vendir") + return err + } + } + + err = a.writeSyncFile(vendirDirs) + if err != nil { + log.Error().Str("app", a.Name).Err(err).Msg("Unable to write sync file") + return err + } + + log.Debug().Str("app", a.Name).Msg("Vendir sync file written: " + vendirSyncPath) + log.Info().Str("app", a.Name).Msg("Vendir sync completed!") + + return nil +} + +func (a *Application) writeSyncFile(directories []Directory) error { + + bytes, err := yaml.Marshal(directories) + if err != nil { + return err + } + a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes)) + if err != nil { + return err + } + + return nil +} + +func readVendirConfig(vendirConfigFilePath string) ([]Directory, error) { + config, err := unmarshalYamlToMap(vendirConfigFilePath) + if err != nil { + return nil, err + } + + vendirDirs, err := findDirectories(config) + if err != nil { + return nil, err + } + + return vendirDirs, nil +} + +func readSyncFile(vendirSyncFile string) ([]Directory, error) { + + if _, err := os.Stat(vendirSyncFile); err != nil { + return []Directory{}, nil + } + + syncFile, err := os.ReadFile(vendirSyncFile) + if err != nil { + return nil, err + } + + out := &[]Directory{} + err = yaml.Unmarshal(syncFile, out) + if err != nil { + return nil, err + } + + return *out, nil +} + +func readLockFile(vendirLockFile string) ([]Directory, error) { + + config, err := unmarshalYamlToMap(vendirLockFile) + if err != nil { + return nil, err + } + + if len(config) == 0 { + return []Directory{}, nil + } + + directories, err := findDirectories(config) + if err != nil { + return nil, err + } + + return directories, nil +} + +func findDirectories(config map[string]interface{}) ([]Directory, error) { + // check if directories key exists + if _, ok := config["directories"]; !ok { + return nil, errors.New("No directories found in vendir config") + } + var directories = make(map[string]string) + for _, dir := range config["directories"].([]interface{}) { + dirMap := dir.(map[string]interface{}) + path := dirMap["path"].(string) + // check contents length + if len(dirMap["contents"].([]interface{})) > 1 { + return nil, errors.New("Vendir config contains more than one contents for path: " + path + ". This is not supported") + } + contents := dirMap["contents"].([]interface{})[0].(map[string]interface{}) + subPath := contents["path"].(string) + if subPath != "." { + path += "/" + subPath + } + sortedYaml, err := sortYaml(contents) + if err != nil { + return nil, err + } + directories[path] = sortedYaml + } + return convertDirectoryMapToHashedStruct(directories), nil +} + +func convertDirectoryMapToHashedStruct(directories map[string]string) []Directory { + var syncDirs []Directory + for path, contents := range directories { + syncDirs = append(syncDirs, Directory{ + Path: path, + ContentHash: hash(contents), + }) + } + return syncDirs +} + +func checkVersionMatch(path string, contentHash string, syncDirs []Directory) bool { + for _, dir := range syncDirs { + if dir.Path == path { + if dir.ContentHash == contentHash { + return true + } + } + } + return false +} + +func checkPathMatch(path string, syncDirs []Directory) bool { + for _, dir := range syncDirs { + if dir.Path == path { + return true + } + } + return false +} + +func checkLockFileMatch(vendirDirs []Directory, lockFileDirs []Directory) bool { + for _, dir := range vendirDirs { + if !checkPathMatch(dir.Path, lockFileDirs) { + return false + } + } + return true +} diff --git a/internal/myks/sync_test.go b/internal/myks/sync_test.go new file mode 100644 index 00000000..ed5e91ed --- /dev/null +++ b/internal/myks/sync_test.go @@ -0,0 +1,281 @@ +package myks + +import ( + "os" + "reflect" + "testing" +) + +func TestApplication_writeSyncFile(t *testing.T) { + type fields struct { + Name string + Prototype string + e *Environment + yttDataFiles []string + } + type args struct { + directories []Directory + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + "happy path", + fields{"name", "proto", &Environment{Dir: "/tmp", g: &Globe{VendirSyncFileName: "TestFile"}}, []string{}}, + args{[]Directory{{"path", "hash"}, {"path2", "hash2"}}}, + "- path: path\n contentHash: hash\n- path: path2\n contentHash: hash2\n", + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &Application{ + Name: tt.fields.Name, + Prototype: tt.fields.Prototype, + e: tt.fields.e, + yttDataFiles: tt.fields.yttDataFiles, + } + // write sync file + if err := a.writeSyncFile(tt.args.directories); (err != nil) != tt.wantErr { + t.Errorf("writeSyncFile() error = %v, wantErr %v", err, tt.wantErr) + } + file, err := os.ReadFile(tt.fields.e.Dir + "/_apps/name/" + tt.fields.e.g.VendirSyncFileName) + if err != nil { + t.Errorf("writeSyncFile() error = %v", err) + } + if got := string(file); got != tt.want { + t.Errorf("got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestApplication_readSyncFile(t *testing.T) { + tests := []struct { + name string + filePath string + want []Directory + wantErr bool + }{ + { + "happy path", + "../../testData/sync/sync-file.yaml", + []Directory{{"path", "hash"}, {"path2", "hash2"}}, + false, + }, + { + "no sync file", + "no-existing.yaml", + []Directory{}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // write sync file + var dirs []Directory + var err error + if dirs, err = readSyncFile(tt.filePath); (err != nil) != tt.wantErr { + t.Errorf("writeSyncFile() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(dirs, tt.want) { + t.Errorf("got = %v, want %v", dirs, tt.want) + } + }) + } +} + +func Test_checkVersionMatch(t *testing.T) { + type args struct { + path string + contentHash string + syncDirs []Directory + } + tests := []struct { + name string + args args + want bool + }{ + {"happy path", args{"path1", "hash1", []Directory{{ContentHash: "hash1", Path: "path1"}}}, true}, + {"sad path", args{"path1", "hash1", []Directory{{ContentHash: "no-match", Path: "path1"}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := checkVersionMatch(tt.args.path, tt.args.contentHash, tt.args.syncDirs); got != tt.want { + t.Errorf("checkVersionMatch() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_findDirectories(t *testing.T) { + tests := []struct { + name string + input string + want []Directory + wantErr bool + }{ + { + "happy path", + "../../testData/sync/vendir-simple.yaml", + []Directory{{ContentHash: "5589fa11a8117eefbec30e4190b9649dd282bd747b4acbd6e47201700990870b", Path: "vendor/charts/loki-stack"}}, + false, + }, + { + "yaml order irrelevant for hash", + "../../testData/sync/vendir-simple-different-order.yaml", + []Directory{{ContentHash: "5589fa11a8117eefbec30e4190b9649dd282bd747b4acbd6e47201700990870b", Path: "vendor/charts/loki-stack"}}, + false, + }, + { + "multiple directories", + "../../testData/sync/vendir-multiple-directories.yaml", + []Directory{ + {ContentHash: "84bc14f63b966dcec26278cc66976cdba19a8757f5b06f2be463e8033c8ade9c", Path: "vendor/charts/ingress-nginx"}, + {ContentHash: "4f95153c2130e5967fc97f0977877012b3f1579e6fcd9e66184302252ca83c70", Path: "vendor/ytt/grafana-dashboards"}, + }, + false, + }, + { + "not a vendir file", + "../../testData/sync/simple.yaml", + nil, + true, + }, + { + "multiple contents", + "../../testData/sync/vendir-multiple-contents.yaml", + nil, + true, + }, + { + "with sub path", + "../../testData/sync/vendir-with-subpath.yaml", + []Directory{ + {ContentHash: "5fa245cedee795a9a01fc62f3c56ac809dc8b304f6656897d060b68b8a5f32ef", Path: "vendor/charts/loki-stack"}, + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + yaml, err := unmarshalYamlToMap(tt.input) + if err != nil { + t.Errorf("unmarshalYamlToMap() error = %v", err) + return + } + got, err := findDirectories(yaml) + if (err != nil) != tt.wantErr { + t.Errorf("findDirectories() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("findDirectories() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_readLockFile(t *testing.T) { + type args struct { + vendirLockFile string + } + tests := []struct { + name string + args args + want []Directory + wantErr bool + }{ + {"happy path", args{"../../testData/sync/lock-file.yaml"}, []Directory{{"vendor/charts/loki-stack", "9ebaa03dc8dd419b94a124193f6b597037daa95e208febb0122ca8920667f42a"}, {"vendor/charts/ingress-nginx", "1d535ff265861947e32c890cbcb76d93a9562771dbd7b3367e4d723c1c6d95db"}}, false}, + {"file not exist", args{"file-not-exist.yaml"}, []Directory{}, false}, + {"no lock file", args{"../../testData/sync/simple.yaml"}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := readLockFile(tt.args.vendirLockFile) + if (err != nil) != tt.wantErr { + t.Errorf("readLockFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("readLockFile() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkPathMatch(t *testing.T) { + type args struct { + path string + syncDirs []Directory + } + tests := []struct { + name string + args args + want bool + }{ + {"happy path", args{"path1", []Directory{{Path: "path1"}}}, true}, + {"sad path", args{"non-existing", []Directory{{Path: "path1"}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := checkPathMatch(tt.args.path, tt.args.syncDirs); got != tt.want { + t.Errorf("checkPathMatch() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkLockFileMatch(t *testing.T) { + type args struct { + vendirDirs []Directory + lockFileDirs []Directory + } + tests := []struct { + name string + args args + want bool + }{ + {"happy path", args{[]Directory{{Path: "path1"}}, []Directory{{Path: "path1"}}}, true}, + {"sad path", args{[]Directory{{Path: "path2"}}, []Directory{{Path: "path1"}}}, false}, + {"wrong sort order", args{[]Directory{{Path: "path1"}, {Path: "path2"}}, []Directory{{Path: "path2"}, {Path: "path1"}}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := checkLockFileMatch(tt.args.vendirDirs, tt.args.lockFileDirs); got != tt.want { + t.Errorf("checkLockFileMatch() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_readVendirConfig(t *testing.T) { + type args struct { + vendirConfigFilePath string + } + tests := []struct { + name string + args args + want []Directory + wantErr bool + }{ + {"happy path", args{"../../testData/sync/vendir-simple.yaml"}, []Directory{{"vendor/charts/loki-stack", "5589fa11a8117eefbec30e4190b9649dd282bd747b4acbd6e47201700990870b"}}, false}, + {"file not exist", args{"file-not-exist.yaml"}, nil, true}, + {"no vendir file", args{"../../testData/sync/simple.yaml"}, nil, true}} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := readVendirConfig(tt.args.vendirConfigFilePath) + if (err != nil) != tt.wantErr { + t.Errorf("readVendirConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("readVendirConfig() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/myks/util.go b/internal/myks/util.go index e586ee08..17698ed4 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -2,13 +2,18 @@ package myks import ( "bytes" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" + "gopkg.in/yaml.v3" "io" "io/fs" "os" "os/exec" "path/filepath" "reflect" + "strings" "github.com/rs/zerolog/log" ) @@ -19,7 +24,8 @@ type CmdResult struct { } func runCmd(name string, stdin io.Reader, args []string) (CmdResult, error) { - log.Debug().Str("cmd", name).Interface("args", args).Msg("Running command") + // make this copy-n-pastable + log.Debug().Msg("Running command:\n" + name + " " + strings.Join(args, " ")) cmd := exec.Command(name, args...) if stdin != nil { @@ -163,6 +169,72 @@ func copyFileSystemToPath(source fs.FS, sourcePath string, destinationPath strin return err } +func unmarshalYamlToMap(filePath string) (map[string]interface{}, error) { + + if _, err := os.Stat(filePath); err != nil { + log.Debug().Str("filePath", filePath).Msg("Yaml not found.") + return make(map[string]interface{}), nil + } + + file, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + var config map[string]interface{} + err = yaml.Unmarshal(file, &config) + if err != nil { + return nil, err + } + return config, nil +} + +func sortYaml(yaml map[string]interface{}) (string, error) { + if yaml == nil { + return "", nil + } + var sorted bytes.Buffer + _, err := fmt.Fprint(&sorted, yaml) + if err != nil { + return "", err + } + return string(sorted.Bytes()), nil +} + +// hash string +func hash(s string) string { + hash := sha256.Sum256([]byte(s)) + return hex.EncodeToString(hash[:]) +} + +func renderDataYaml(dataFiles []string) ([]byte, error) { + if len(dataFiles) == 0 { + return nil, errors.New("No data files found") + } + res, err := runYttWithFilesAndStdin(dataFiles, nil, "--data-values-inspect") + if err != nil { + log.Error().Err(err).Str("stderr", res.Stderr).Msg("Unable to render data") + return nil, err + } + if res.Stdout == "" { + return nil, errors.New("Empty output from ytt") + } + + dataYaml := []byte(res.Stdout) + return dataYaml, nil +} + func mergeValuesYaml(valueFilesYaml string) (CmdResult, error) { return runYttWithFilesAndStdin(nil, nil, "--data-values-file="+valueFilesYaml, "--data-values-inspect") } + +func createDirectory(dir string) error { + if _, err := os.Stat(dir); err != nil { + err := os.MkdirAll(dir, 0o750) + if err != nil { + log.Error().Err(err).Msg("Unable to create directory: " + dir) + return err + } + } + return nil +} diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go new file mode 100644 index 00000000..4c1729a9 --- /dev/null +++ b/internal/myks/util_test.go @@ -0,0 +1,138 @@ +package myks + +import ( + "reflect" + "testing" +) + +func Test_hash(t *testing.T) { + tests := []struct { + a string + b string + want string + }{ + {"happy path", "some-string", "a3635c09bda7293ae1f144a240f155cf151451f2420d11ac385d13cce4eb5fa2"}, + } + for _, tt := range tests { + t.Run(tt.a, func(t *testing.T) { + if got := hash(tt.b); got != tt.want { + t.Errorf("hash() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sortYaml(t *testing.T) { + + tests := []struct { + name string + args map[string]interface{} + want string + wantErr bool + }{ + { + "happy path", + map[string]interface{}{"key1": "A", "key2": "B"}, + "map[key1:A key2:B]", + false, + }, + { + "fix sorting", + map[string]interface{}{"key2": "B", "key1": "A"}, + "map[key1:A key2:B]", + false, + }, + { + "empty input", + nil, + "", + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sortYaml(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("sortYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("sortYaml() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_unmarshalYaml(t *testing.T) { + type args struct { + filePath string + } + tests := []struct { + name string + args args + want map[string]interface{} + wantErr bool + }{ + {"happy path", args{"../../testData/sync/simple.yaml"}, map[string]interface{}{"key1": "A", "key2": "B", "arr": []interface{}{"arr1", "arr2"}}, false}, + {"file not exist", args{"non-existing.yaml"}, map[string]interface{}{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := unmarshalYamlToMap(tt.args.filePath) + if (err != nil) != tt.wantErr { + t.Errorf("unmarshalYamlToMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unmarshalYamlToMap() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_renderDataYaml(t *testing.T) { + type args struct { + dataFiles []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"happy path", args{[]string{"../../testData/ytt/data-file-schema.yaml", "../../testData/ytt/data-file-schema-2.yaml", "../../testData/ytt/data-file-values.yaml"}}, "application:\n cache:\n enabled: true\n name: cert-manager\n", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderDataYaml(tt.args.dataFiles) + if (err != nil) != tt.wantErr { + t.Errorf("renderDataYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("renderDataYaml() got = %v, want %v", string(got), tt.want) + } + }) + } +} + +func Test_createDirectory(t *testing.T) { + type args struct { + dir string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"happy path", args{"/tmp/test-dir"}, false}, + {"sad path", args{"/non-existing/test-dir"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := createDirectory(tt.args.dir); (err != nil) != tt.wantErr { + t.Errorf("createDirectory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/testData/sync/lock-file.yaml b/testData/sync/lock-file.yaml new file mode 100644 index 00000000..d4940593 --- /dev/null +++ b/testData/sync/lock-file.yaml @@ -0,0 +1,15 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: + - contents: + - helmChart: + appVersion: v2.6.1 + version: 2.9.10 + path: loki-stack + path: vendor/charts + - contents: + - helmChart: + appVersion: v2.6.1 + version: 1.1.0 + path: . + path: vendor/charts/ingress-nginx +kind: LockConfig diff --git a/testData/sync/simple.yaml b/testData/sync/simple.yaml new file mode 100644 index 00000000..609d31c2 --- /dev/null +++ b/testData/sync/simple.yaml @@ -0,0 +1,5 @@ +key1: A +key2: B +arr: + - arr1 + - arr2 diff --git a/testData/sync/sync-file.yaml b/testData/sync/sync-file.yaml new file mode 100644 index 00000000..be831b8c --- /dev/null +++ b/testData/sync/sync-file.yaml @@ -0,0 +1,4 @@ +- path: path + contentHash: hash +- path: path2 + contentHash: hash2 diff --git a/testData/sync/vendir-multiple-contents.yaml b/testData/sync/vendir-multiple-contents.yaml new file mode 100644 index 00000000..987bb1fa --- /dev/null +++ b/testData/sync/vendir-multiple-contents.yaml @@ -0,0 +1,19 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/charts/ingress-nginx + contents: + - path: ingress-nginx + helmChart: + name: ingress-nginx + version: 4.7.1 + repository: + url: https://kubernetes.github.io/ingress-nginx + - path: dashboards + git: + url: https://github.com/kubernetes/ingress-nginx/ + ref: helm-chart-4.7.1 + newRootPath: deploy/grafana/dashboards + includePaths: + - deploy/grafana/dashboards/nginx.json + - deploy/grafana/dashboards/request-handling-performance.json diff --git a/testData/sync/vendir-multiple-directories.yaml b/testData/sync/vendir-multiple-directories.yaml new file mode 100644 index 00000000..82d1694d --- /dev/null +++ b/testData/sync/vendir-multiple-directories.yaml @@ -0,0 +1,21 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/charts/ingress-nginx + contents: + - path: . + helmChart: + name: ingress-nginx + version: 4.7.1 + repository: + url: https://kubernetes.github.io/ingress-nginx + - path: vendor/ytt/grafana-dashboards + contents: + - path: . + git: + url: https://github.com/kubernetes/ingress-nginx/ + ref: helm-chart-4.7.1 + newRootPath: deploy/grafana/dashboards + includePaths: + - deploy/grafana/dashboards/nginx.json + - deploy/grafana/dashboards/request-handling-performance.json diff --git a/testData/sync/vendir-simple-different-order.yaml b/testData/sync/vendir-simple-different-order.yaml new file mode 100644 index 00000000..4c4077cd --- /dev/null +++ b/testData/sync/vendir-simple-different-order.yaml @@ -0,0 +1,11 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/charts/loki-stack + contents: + - path: . + helmChart: + repository: + url: https://grafana.github.io/helm-charts + version: 2.9.10 + name: loki-stack diff --git a/testData/sync/vendir-simple.yaml b/testData/sync/vendir-simple.yaml new file mode 100644 index 00000000..6265b237 --- /dev/null +++ b/testData/sync/vendir-simple.yaml @@ -0,0 +1,11 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/charts/loki-stack + contents: + - path: . + helmChart: + name: loki-stack + version: 2.9.10 + repository: + url: https://grafana.github.io/helm-charts diff --git a/testData/sync/vendir-with-subpath.yaml b/testData/sync/vendir-with-subpath.yaml new file mode 100644 index 00000000..5df4e3c8 --- /dev/null +++ b/testData/sync/vendir-with-subpath.yaml @@ -0,0 +1,11 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor/charts + contents: + - path: loki-stack + helmChart: + name: loki-stack + version: 2.9.10 + repository: + url: https://grafana.github.io/helm-charts diff --git a/testData/ytt/data-file-schema-2.yaml b/testData/ytt/data-file-schema-2.yaml new file mode 100644 index 00000000..602a2a1b --- /dev/null +++ b/testData/ytt/data-file-schema-2.yaml @@ -0,0 +1,8 @@ +#@data/values-schema + +#@overlay/match-child-defaults missing_ok=True +--- +application: + cache: + enabled: true + name: cert-manager diff --git a/testData/ytt/data-file-schema.yaml b/testData/ytt/data-file-schema.yaml new file mode 100644 index 00000000..a053fc9a --- /dev/null +++ b/testData/ytt/data-file-schema.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +--- +application: + cache: + enabled: false diff --git a/testData/ytt/data-file-values.yaml b/testData/ytt/data-file-values.yaml new file mode 100644 index 00000000..4dbe0bfb --- /dev/null +++ b/testData/ytt/data-file-values.yaml @@ -0,0 +1,5 @@ +#@data/values +--- +application: + cache: + enabled: true From 24ff41ccbb4b002f9f1f434879c0720bd6990924 Mon Sep 17 00:00:00 2001 From: Fritz Durchardt Date: Tue, 18 Jul 2023 10:02:34 +0200 Subject: [PATCH 2/2] feat: Add vendir sync caching On vendir sync a new vendir.sync.yaml is safed to the "tmp" folder containing the current state of vendir.yaml condensed into hash values. If this file remains unchanged, vendir sync can be omitted. The feature can be enabled on environment and application level with the new application.cache.enabled flag. --- internal/myks/sync.go | 4 ++-- internal/myks/util.go | 2 +- internal/myks/util_test.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/myks/sync.go b/internal/myks/sync.go index 5fb4b2d2..ca56d06d 100644 --- a/internal/myks/sync.go +++ b/internal/myks/sync.go @@ -104,7 +104,7 @@ func (a *Application) writeSyncFile(directories []Directory) error { if err != nil { return err } - a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes)) + err = a.writeTempFile(a.e.g.VendirSyncFileName, string(bytes)) if err != nil { return err } @@ -168,7 +168,7 @@ func readLockFile(vendirLockFile string) ([]Directory, error) { func findDirectories(config map[string]interface{}) ([]Directory, error) { // check if directories key exists if _, ok := config["directories"]; !ok { - return nil, errors.New("No directories found in vendir config") + return nil, errors.New("no directories found in vendir config") } var directories = make(map[string]string) for _, dir := range config["directories"].([]interface{}) { diff --git a/internal/myks/util.go b/internal/myks/util.go index 17698ed4..29cc7866 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -198,7 +198,7 @@ func sortYaml(yaml map[string]interface{}) (string, error) { if err != nil { return "", err } - return string(sorted.Bytes()), nil + return sorted.String(), nil } // hash string diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index 4c1729a9..fd321af2 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -1,6 +1,7 @@ package myks import ( + "os" "reflect" "testing" ) @@ -91,6 +92,9 @@ func Test_unmarshalYaml(t *testing.T) { } func Test_renderDataYaml(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("Skipping in pipeline since ytt is not installed") + } type args struct { dataFiles []string }