diff --git a/README.md b/README.md index 7bf1ad8..25c1a6d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ Update the version of components that were updated in the last commit. 1. Open the git repository. 2. Check if the latest commit is not a bump commit. -3. Collect a list of changed files until the previous bump commit is found (matching by author). If the `last` option is +3. Collect a list of changed (new, updated, deleted) files until the previous bump commit is found (matching by author). + If the `last` option is passed, only take changes from the last commit. Prepare a map of resource objects to update. 4. Get the short hash of the last commit. 5. Iterate through the resource map and update the version of each. @@ -27,15 +28,14 @@ Update the version of components that were updated in the last commit. `bump` works closely with `plasmactl compose`. The purpose of `compose` is to build the platform using resources from different places (domain repo, packages outside the repo). The purpose of `bump --sync` is to propagate the versions of changed resources to dependant resources in current build ( -post composition) while preserving history of earlier propagated versions. +post composition) using git history of repositories. ### Overall Propagation Workflow: -- Search and download the artifact to compare the build to. -- Find the list of different files between the build and the artifact. +- Initialize inventory. Which includes calculating variables and resources dependencies. - Identify list of resources which version should be propagated. - Build propagation map. -- Update resources in build dir. +- Update resources in build dir according propagation map. #### Prerequisites: @@ -45,50 +45,23 @@ post composition) while preserving history of earlier propagated versions. 1. **Search and download the artifact to compare the build to:** -- Open the git repository. -- Search for the first bump commit after HEAD and use it's short sha (7 characters) to compute artifact name to - retrieve. -- Check if the artifact was previously downloaded and stored in the `.compose/artifacts` directory. If it exists, use it - as local cache instead of re-downloading it. -- Attempt to download the artifact with the name `repo_name-commit_hash_7_symbols-plasma-src.tar.gz` from the - repository. -- If artifact doesn't exist with that name, recursively look for earlier bump commits. -- Unarchive the artifact into the comparison directory (usually `.compose/comparison-artifact`). - -> You can override the artifact commit with the `override` option, which bypasses the git history search and directly -> attempts to download the artifact. - -2. **Find the list of different files between the build and the artifact:** - -- Compare the files in the build directory to the files in the artifact directory. -- If two files differ, their paths are added to the list of updated items. -- If file doesn't exist in artifact, path is added to updates items. -- If file exists in artifact, but not in the build, path is added to updates items. - -**Excluded Subdirectories and Files:** - -- `.git` -- `.compose` -- `.plasmactl` -- `.gitlab-ci.yml` -- `ansible_collections` -- `scripts/ci/.gitlab-ci.platform.yaml` -- `venv` -- `__pycache__` - -3. **Identify list of resources which version should be propagated:** - - Prepare list of resources names per namespace (domain + packages names), if resource exists in several sources, - identify origin of composed resource, - see [Syncing Resource Versions Across Namespaces](#syncing-resource-versions-across-sources)) and remove duplicates. -- Convert list of changes files to list of resources. if filepath matches [resource criteria](#resource-criteria), - resource object will be created and added to list of changed resources. -- Initialize [timeline](#timeline). -- Iterate through modified resources and populate timeline, - see [Iterating Through Resources](#iterating-through-resources) -- Find in list of modified files `group_vars.yaml` and `vault.yaml` files. -- Iterate through variables files and populate timeline list with changed variables. - see [Iterating Through Variables](#iterating-through-variables) + identify origin of composed resource and remove duplicates, + see [Syncing Resource Versions Across Namespaces](#syncing-resource-versions-across-sources) +- Find all variables which include other variables in their values, store these dependencies in special map. +- Find all usage of variables in resources files (template, configuration files), store usage of each dependency in + special map. + +2.**Identify list of resources which version should be propagated:** + +* Initialize [timeline](#timeline). +* Iterate through all resources (domain + packages) and populate timeline, + * Iterate git history to find latest commit where version for resource was set. + * If resource has non-versioned changes, error will be returned, unless `allow-override` options passed. +* Iterate through all variables in build dir and populate timeline with variables versions. + * Iterate git history to find latest commit where value for variable was set. + * If build variable value is different from committed value, error will be returned, unless `allow-override` options + passed. #### Resource criteria @@ -134,36 +107,7 @@ The timeline is sorted by date to maintain the correct version sequence as if pr Iterating through the timeline (chronologically sorted) and applying dependent resource versions, the latest state is achieved. -#### Iterating Through Resources: - -During this phase, the following rules apply: - -- If a resource not present in build, it’s skipped. The developer should handle dependencies manually and bump related - resources (in this case dependent resource will be considered as `updated` and require propagation). -- If a resource is new, add it to the propagation list. -- If a resource differs between the build and the artifact, compare versions. If the base version (e.g., base, - base-propagation_suffix) differs, add it to the propagation list. Otherwise, copy the resource version from the - artifact to preserve history of earlier propagated versions. -- If entry with the same date and commit existed before, merge entries. -- If build resource version differs from committed resource version - warning will be printed. -- If resource has non-versioned changes, error will be returned. - -#### Iterating Through Variables: - -During this phase, the following rules apply: - -- If variables file is new, find all new variables and commit where they were added, create new timeline entry per - commit. -- If variables file was deleted, find all deleted variables and commit where they were deleted, create new timeline - entry and add to timeline. -- If variables file exists, but one or several variables were changed, search commits where these changes were done, - create new timeline entry per commit, add them to timeline. -- If entry with the same date and commit existed before, merge entries. -- If variable never existed in commit history, error will be returned. -- If build variable value is different from committed value, error will be returned. Same for non-versioned changes. - - -4. **Build propagation map:** +3**Build propagation map:** - Chronologically sort timeline. - Iterate each timeline entry. @@ -171,8 +115,7 @@ During this phase, the following rules apply: entry (propagate). - If resource existed in propagation map, it will be overridden by next timeline entry version. - -5. **Update resources in build dir:** +4**Update resources in build dir:** - Iterate each resource in propagation map. - Build new version for resource, which consists of resource original and propagated versions, final result will look diff --git a/actionSync.go b/actionSync.go index 2eab9ae..7341e7a 100644 --- a/actionSync.go +++ b/actionSync.go @@ -1,22 +1,28 @@ package plasmactlbump import ( + "context" "errors" "fmt" "io" + "log/slog" "os" "path/filepath" + "runtime" + "slices" "sort" "strings" + async "sync" "time" + "github.com/cespare/xxhash/v2" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/launchrctl/compose/compose" "github.com/launchrctl/keyring" "github.com/launchrctl/launchr" + "github.com/pterm/pterm" "github.com/skilld-labs/plasmactl-bump/v2/pkg/repository" "github.com/skilld-labs/plasmactl-bump/v2/pkg/sync" @@ -29,6 +35,7 @@ var ( const ( vaultpassKey = "vaultpass" domainNamespace = "domain" + buildHackAuthor = "override" ) // SyncAction is a type representing a resources version synchronization action. @@ -37,50 +44,41 @@ type SyncAction struct { keyring keyring.Keyring // target dirs. - buildDir string - comparisonDir string - packagesDir string - domainDir string - artifactsDir string - artifactsRepoURL string + buildDir string + packagesDir string + domainDir string // internal. saveKeyring bool + timeline []sync.TimelineItem // options. - dryRun bool - listImpacted bool - vaultPass string - artifactOverride string + dryRun bool + allowOverride bool + vaultPass string + verbosity int } -// Execute the sync action to propagate resources' versions. -func (s *SyncAction) Execute(username, password string) error { - err := s.prepareArtifact(username, password) - if err != nil { - return fmt.Errorf("preparing artifact > %w", err) - } - - modifiedFiles, err := sync.CompareDirs(s.buildDir, s.comparisonDir, sync.InventoryExcluded) - if err != nil { - return err - } +type hashStruct struct { + hash string + hashTime time.Time + author string +} - sort.Strings(modifiedFiles) - launchr.Log().Info(fmt.Sprintf("Build and Artifact diff:")) - for _, file := range modifiedFiles { - launchr.Log().Info(fmt.Sprintf("- %s", file)) - } +// Execute the sync action to propagate resources' versions. +func (s *SyncAction) Execute() error { + launchr.Term().Info().Println("Processing propagation...") - err = s.ensureVaultpassExists() + err := s.ensureVaultpassExists() if err != nil { return err } - err = s.propagate(modifiedFiles) + err = s.propagate() if err != nil { return err } + launchr.Term().Info().Println("Propagation has been finished") if s.saveKeyring { err = s.keyring.Save() @@ -89,45 +87,6 @@ func (s *SyncAction) Execute(username, password string) error { return err } -func (s *SyncAction) prepareArtifact(username, password string) error { - // Get artifact repository credentials or store new. - ci, errGet := s.keyring.GetForURL(s.artifactsRepoURL) - if errGet != nil { - if errors.Is(errGet, keyring.ErrEmptyPass) { - return errGet - } else if !errors.Is(errGet, keyring.ErrNotFound) { - launchr.Log().Debug("keyring error", "error", errGet) - return errMalformedKeyring - } - - ci.URL = s.artifactsRepoURL - ci.Username = username - ci.Password = password - - if ci.Username == "" || ci.Password == "" { - launchr.Term().Printfln("Please add login and password for URL - %s\n", ci.URL) - err := keyring.RequestCredentialsFromTty(&ci) - if err != nil { - return err - } - } - - err := s.keyring.AddItem(ci) - if err != nil { - return err - } - s.saveKeyring = true - } - - artifact, err := sync.NewArtifact(s.artifactsDir, s.artifactsRepoURL, s.artifactOverride, s.comparisonDir) - if err != nil { - return err - } - - err = artifact.Get(ci.Username, ci.Password) - return err -} - func (s *SyncAction) ensureVaultpassExists() error { keyValueItem, errGet := s.getVaultPass(s.vaultPass) if errGet != nil { @@ -170,35 +129,37 @@ func (s *SyncAction) getVaultPass(vaultpass string) (keyring.KeyValueItem, error return keyValueItem, nil } -func (s *SyncAction) propagate(modifiedFiles []string) error { +func (s *SyncAction) propagate() error { + s.timeline = sync.CreateTimeline() + + launchr.Log().Info("Initializing build inventory") inv, err := sync.NewInventory(s.buildDir) if err != nil { return err } - // build timeline and resources to copy. - timeline, history, err := s.buildTimeline(inv, modifiedFiles) + launchr.Log().Info("Calculating variables usage") + err = inv.CalculateVariablesUsage(s.vaultPass) + if err != nil { + return err + } + + err = s.buildTimeline(inv) if err != nil { return fmt.Errorf("building timeline > %w", err) } - if len(timeline) == 0 { + if len(s.timeline) == 0 { launchr.Term().Warning().Println("No resources were found for propagation") return nil } - // sort and iterate timeline, create propagation map. - toPropagate, resourceVersionMap, err := s.buildPropagationMap(inv, timeline) + toPropagate, resourceVersionMap, err := s.buildPropagationMap(inv, s.timeline) if err != nil { return fmt.Errorf("building propagation map > %w", err) } - if s.listImpacted { - s.showImpacted(inv, timeline, toPropagate) - } - - // update resources. - err = s.updateResources(resourceVersionMap, toPropagate, history) + err = s.updateResources(resourceVersionMap, toPropagate) if err != nil { return fmt.Errorf("propagate > %w", err) } @@ -206,335 +167,94 @@ func (s *SyncAction) propagate(modifiedFiles []string) error { return nil } -func (s *SyncAction) findResourceChangeTime(resourceVersion, resourceMetaPath string, repo *git.Repository) (*sync.TimelineResourcesItem, error) { - ref, err := s.ensureResourceIsVersioned(resourceVersion, resourceMetaPath, repo) +func (s *SyncAction) buildTimeline(buildInv *sync.Inventory) error { + launchr.Log().Info("Gathering domain and package resources") + resourcesMap, packagePathMap, err := s.getResourcesMaps(buildInv) if err != nil { - return nil, fmt.Errorf("ensure versioned > %w", err) + return fmt.Errorf("build resource map > %w", err) } - //@TODO use git log -S'value' -- path/to/file instead of full history search? - // or git log -- path/to/file - // go-git log -- filepath looks abysmally slow, to research. - // start from the latest commit and iterate to the past - cIter, err := repo.Log(&git.LogOptions{From: ref.Hash()}) + launchr.Log().Info("Populate timeline with resources") + err = s.populateTimelineResources(resourcesMap, packagePathMap) if err != nil { - return nil, err + return fmt.Errorf("iteraring resources > %w", err) } - var prevHash string - var prevHashTime time.Time - var author string - err = cIter.ForEach(func(c *object.Commit) error { - file, errIt := c.File(resourceMetaPath) - if errIt != nil { - if !errors.Is(errIt, object.ErrFileNotFound) { - return fmt.Errorf("open file %s in commit %s > %w", resourceMetaPath, c.Hash, errIt) - } - if prevHash == "" { - prevHash = c.Hash.String() - prevHashTime = c.Author.When - } - - launchr.Log().Debug("File didn't exist before, take current hash as version", "version", resourceMetaPath) - return storer.ErrStop - } - - metaFile, errIt := s.loadYamlFileFromBytes(file, resourceMetaPath) - if errIt != nil { - return fmt.Errorf("commit %s > %w", c.Hash, errIt) - } - - prevVer := sync.GetMetaVersion(metaFile) - if resourceVersion != prevVer { - return storer.ErrStop - } - - prevHashTime = c.Author.When - prevHash = c.Hash.String() - author = c.Author.Name - return nil - }) - + launchr.Log().Info("Populate timeline with variables") + err = s.populateTimelineVars() if err != nil { - return nil, err - } - - if author != repository.Author { - launchr.Term().Warning().Printfln("Non-bump version selected for %s resource", resourceMetaPath) + return fmt.Errorf("iteraring variables > %w", err) } - tri := sync.NewTimelineResourcesItem(resourceVersion, prevHash, prevHashTime) - - return tri, err + return nil } -func (s *SyncAction) ensureResourceIsVersioned(resourceVersion, resourceMetaPath string, repo *git.Repository) (*plumbing.Reference, error) { - ref, err := repo.Head() - if err != nil { - return nil, err - } +func (s *SyncAction) getResourcesMaps(buildInv *sync.Inventory) (map[string]*sync.OrderedMap[*sync.Resource], map[string]string, error) { + resourcesMap := make(map[string]*sync.OrderedMap[*sync.Resource]) + packagePathMap := make(map[string]string) - headCommit, err := repo.CommitObject(ref.Hash()) - if err != nil { - return nil, err - } - headMeta, err := headCommit.File(resourceMetaPath) + plasmaCompose, err := compose.Lookup(os.DirFS(s.domainDir)) if err != nil { - return nil, fmt.Errorf("meta %s doesn't exist in HEAD commit", resourceMetaPath) + return nil, nil, err } - metaFile, err := s.loadYamlFileFromBytes(headMeta, resourceMetaPath) - if err != nil { - return nil, fmt.Errorf("%w", err) + var priorityOrder []string + for _, dep := range plasmaCompose.Dependencies { + pkg := dep.ToPackage(dep.Name) + packagePathMap[dep.Name] = filepath.Join(s.packagesDir, pkg.GetName(), pkg.GetTarget()) + priorityOrder = append(priorityOrder, dep.Name) } - headVersion := sync.GetMetaVersion(metaFile) - if resourceVersion != headVersion { - return nil, fmt.Errorf("version from %s doesn't match any existing commit", resourceMetaPath) - } + packagePathMap[domainNamespace] = s.domainDir - return ref, nil -} + priorityOrder = append(priorityOrder, domainNamespace) -//func (s *SyncAction) ensureResourceNonVersioned(mrn string, repo *git.Repository) error { -// resourcePath, err := sync.ConvertMRNtoPath(mrn) -// if err != nil { -// return err -// } -// -// buildPath := filepath.Join(s.buildDir, resourcePath) -// resourceFiles, err := sync.GetFiles(buildPath, []string{}) -// if err != nil { -// return err -// } -// -// ref, err := repo.Head() -// if err != nil { -// return err -// } -// -// headCommit, err := repo.CommitObject(ref.Hash()) -// if err != nil { -// return err -// } -// -// for f := range resourceFiles { -// buildHash, err := sync.HashFileByPath(filepath.Join(buildPath, f)) -// if err != nil { -// return err -// } -// -// launchr.Term().Warning().Printfln(filepath.Join(resourcePath, f)) -// headFile, err := headCommit.File(filepath.Join(resourcePath, f)) -// if err != nil { -// return err -// } -// -// reader, err := headFile.Blob.Reader() -// if err != nil { -// return err -// } -// -// headHash, err := sync.HashFileFromReader(reader) -// if err != nil { -// return err -// } -// -// if buildHash != headHash { -// return fmt.Errorf("resource %s has unversioned changes. You need to commit these changes", mrn) -// } -// } -// -// return nil -//} + var wg async.WaitGroup + var mx async.Mutex -func (s *SyncAction) findVariableUpdateTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) { - // @TODO look for several vars during iteration? - ref, err := s.ensureVariableIsVersioned(variable, repo) - if err != nil { - return nil, fmt.Errorf("ensure versioned > %w", err) - } + maxWorkers := min(runtime.NumCPU(), len(packagePathMap)) + workChan := make(chan map[string]string, len(packagePathMap)) + errorChan := make(chan error, 1) - //@TODO use git log -S'value' -- path/to/file instead of full history search? - // or git log -- path/to/file - // go-git log -- filepath looks abysmally slow, to research. - cIter, err := repo.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err - } + for i := 0; i < maxWorkers; i++ { + go func() { + for repo := range workChan { + resources, errRes := getResourcesMapFrom(repo["path"]) + if errRes != nil { + errorChan <- errRes + return + } - var currentHash string - var currentHashTime time.Time - err = cIter.ForEach(func(c *object.Commit) error { - file, errIt := c.File(variable.GetPath()) - if errIt != nil { - if !errors.Is(errIt, object.ErrFileNotFound) { - return fmt.Errorf("open file %s in commit %s > %w", variable.GetPath(), c.Hash, errIt) + mx.Lock() + resourcesMap[repo["package"]] = resources + mx.Unlock() + wg.Done() } - if currentHash == "" { - currentHash = c.Hash.String() - currentHashTime = c.Author.When - } - - return storer.ErrStop - } - - varFile, errIt := s.loadVariablesFileFromBytes(file, variable.GetPath(), variable.IsVault()) - if errIt != nil { - return fmt.Errorf("commit %s > %w", c.Hash, errIt) - } - - prevVar, exists := varFile[variable.GetName()] - if !exists { - // Variable didn't exist before, take current hash as version - return storer.ErrStop - } - - prevVarHash := sync.HashString(fmt.Sprint(prevVar)) - if variable.GetHash() != prevVarHash { - // Variable exists, hashes don't match, stop iterating - return storer.ErrStop - } - - currentHash = c.Hash.String() - currentHashTime = c.Author.When - return nil - }) - - if err != nil { - return nil, err - } - - tvi := sync.NewTimelineVariablesItem(currentHash[:13], currentHash, currentHashTime) - - return tvi, err -} - -func (s *SyncAction) ensureVariableIsVersioned(variable *sync.Variable, repo *git.Repository) (*plumbing.Reference, error) { - ref, err := repo.Head() - if err != nil { - return nil, err - } - - headCommit, err := repo.CommitObject(ref.Hash()) - if err != nil { - return nil, err - } - headVarsFile, err := headCommit.File(variable.GetPath()) - if err != nil { - return nil, fmt.Errorf("file %s doesn't exist in HEAD", variable.GetPath()) - } - - varFile, errIt := s.loadVariablesFileFromBytes(headVarsFile, variable.GetPath(), variable.IsVault()) - if errIt != nil { - return nil, fmt.Errorf("%w", errIt) - } - - headVar, exists := varFile[variable.GetName()] - if !exists { - return nil, fmt.Errorf("variable from %s doesn't exist in HEAD", variable.GetPath()) - } - - headVarHash := sync.HashString(fmt.Sprint(headVar)) - if variable.GetHash() != headVarHash { - return nil, fmt.Errorf("variable from %s is an unversioned change. You need to commit variable change", variable.GetPath()) - } - - return ref, nil -} - -func (s *SyncAction) findVariableDeletionTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) { - // @TODO look for several vars during iteration? - // @TODO ensure variable existed at first place, before starting to search. - - ref, err := repo.Head() - if err != nil { - return nil, err + }() } - cIter, err := repo.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err + for pkg, path := range packagePathMap { + wg.Add(1) + workChan <- map[string]string{"path": path, "package": pkg} } - // Ensure variable existed in first place before propagating. - varExisted := false - var currentHash string - var currentHashTime time.Time - err = cIter.ForEach(func(c *object.Commit) error { - file, errIt := c.File(variable.GetPath()) - if errors.Is(errIt, object.ErrFileNotFound) { - currentHash = c.Hash.String() - currentHashTime = c.Author.When - return nil - } + close(workChan) - varFile, errIt := s.loadVariablesFileFromBytes(file, variable.GetPath(), variable.IsVault()) - if errIt != nil { - return fmt.Errorf("commit %s > %w", c.Hash, errIt) - } + go func() { + wg.Wait() + close(errorChan) + }() - _, exists := varFile[variable.GetName()] - if !exists { - currentHash = c.Hash.String() - currentHashTime = c.Author.When - return nil + for err = range errorChan { + if err != nil { + return nil, nil, err } - - varExisted = true - return storer.ErrStop - }) - - if err != nil { - return nil, err - } - - if !varExisted { - return nil, fmt.Errorf("variable from %s never existed in repository, please ensure your build is correct", variable.GetPath()) - } - - tvi := sync.NewTimelineVariablesItem(currentHash[:13], currentHash, currentHashTime) - - return tvi, err -} - -func (s *SyncAction) getResourcesMaps() (map[string]*sync.OrderedMap[bool], map[string]string, error) { - resourcesMap := make(map[string]*sync.OrderedMap[bool]) - packagePathMap := make(map[string]string) - - buildResources, err := s.getResourcesMapFrom(s.buildDir) - if err != nil { - return nil, nil, err - } - resourcesMap[domainNamespace], err = s.getResourcesMapFrom(s.domainDir) - if err != nil { - return resourcesMap, packagePathMap, err } - plasmaCompose, err := compose.Lookup(os.DirFS(s.domainDir)) + buildResources := buildInv.GetResourcesMap() if err != nil { return nil, nil, err } - - var priorityOrder []string - for _, dep := range plasmaCompose.Dependencies { - pkg := dep.ToPackage(dep.Name) - version := pkg.GetTarget() - - packagePathMap[dep.Name] = filepath.Join(s.packagesDir, pkg.GetName(), version) - priorityOrder = append(priorityOrder, dep.Name) - } - - priorityOrder = append(priorityOrder, domainNamespace) - - for name, packagePath := range packagePathMap { - resources, err := s.getResourcesMapFrom(packagePath) - if err != nil { - return nil, nil, err - } - resourcesMap[name] = resources - } - for _, resourceName := range buildResources.Keys() { conflicts := make(map[string]string) for name, resources := range resourcesMap { @@ -555,12 +275,7 @@ func (s *SyncAction) getResourcesMaps() (map[string]*sync.OrderedMap[bool], map[ var sameVersionNamespaces []string for conflictingNamespace := range conflicts { - var conflictEntity *sync.Resource - if conflictingNamespace == domainNamespace { - conflictEntity = sync.NewResource(resourceName, s.buildDir) - } else { - conflictEntity = sync.NewResource(resourceName, packagePathMap[conflictingNamespace]) - } + conflictEntity := sync.NewResource(resourceName, packagePathMap[conflictingNamespace]) baseVersion, _, err := conflictEntity.GetBaseVersion() if err != nil { @@ -599,341 +314,625 @@ func (s *SyncAction) getResourcesMaps() (map[string]*sync.OrderedMap[bool], map[ return resourcesMap, packagePathMap, nil } -func (s *SyncAction) getResourcesMapFrom(dir string) (*sync.OrderedMap[bool], error) { - inv, err := sync.NewInventory(dir) - if err != nil { - return nil, err - } +func (s *SyncAction) populateTimelineResources(resources map[string]*sync.OrderedMap[*sync.Resource], packagePathMap map[string]string) error { + var wg async.WaitGroup + var mx async.Mutex - rm := inv.GetResourcesMap() - rm.SortKeysAlphabetically() - return rm, nil -} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() -func (s *SyncAction) buildTimeline(buildInv *sync.Inventory, modifiedFiles []string) ([]sync.TimelineItem, *sync.OrderedMap[*sync.Resource], error) { - timeline := sync.CreateTimeline() + errorChan := make(chan error, 1) + maxWorkers := min(runtime.NumCPU(), len(packagePathMap)) + workChan := make(chan map[string]any, len(packagePathMap)) - allDiffResources := buildInv.GetChangedResources(modifiedFiles) - if allDiffResources.Len() > 0 { - allDiffResources.SortKeysAlphabetically() - launchr.Log().Info(fmt.Sprintf("")) - launchr.Log().Info(fmt.Sprintf("Resources diff between build and artifact")) - for _, key := range allDiffResources.Keys() { - r, ok := allDiffResources.Get(key) - if !ok { - continue - } - launchr.Log().Info(fmt.Sprintf("- %s", r.GetName())) - } + multi := pterm.DefaultMultiPrinter - launchr.Term().Info().Printfln("Gathering domain and package resources") - resourcesMap, packagePathMap, err := s.getResourcesMaps() - if err != nil { - return timeline, nil, fmt.Errorf("build resource map > %w", err) + for i := 0; i < maxWorkers; i++ { + go func(workerID int) { + for { + select { + case <-ctx.Done(): + return + case domain, ok := <-workChan: + if !ok { + return + } + + name := domain["name"].(string) + path := domain["path"].(string) + pb := domain["pb"].(*pterm.ProgressbarPrinter) + + if err := s.findResourcesChangeTime(resources[name], path, &mx, pb); err != nil { + select { + case errorChan <- fmt.Errorf("worker %d error processing %s: %w", workerID, name, err): + cancel() + default: + } + return + } + wg.Done() + } + } + }(i) + } + + for name, path := range packagePathMap { + if resources[name].Len() == 0 { + // Skipping packages with 0 composed resources. + continue } - // Find new or updated resources in diff. - launchr.Log().Info(fmt.Sprintf("Checking domain resources")) - timeline, err = s.populateTimelineResources(allDiffResources, resourcesMap[domainNamespace], timeline, s.domainDir) - if err != nil { - return nil, nil, fmt.Errorf("iteraring domain resources > %w", err) + wg.Add(1) + + var p *pterm.ProgressbarPrinter + var err error + if s.verbosity < 1 { + p, err = pterm.DefaultProgressbar.WithTotal(resources[name].Len()).WithWriter(multi.NewWriter()).Start(fmt.Sprintf("Collecting resources from %s", name)) + if err != nil { + return err + } } - // Iterate each package, find new or updated resources in diff. - launchr.Log().Info(fmt.Sprintf("Checking packages resources")) - for name, packagePath := range packagePathMap { - timeline, err = s.populateTimelineResources(allDiffResources, resourcesMap[name], timeline, packagePath) + workChan <- map[string]any{"name": name, "path": path, "pb": p} + } + close(workChan) + go func() { + if s.verbosity < 1 { + _, err := multi.Start() if err != nil { - return nil, nil, fmt.Errorf("iteraring package %s resources > %w", name, err) + errorChan <- fmt.Errorf("error starting multi progress bar: %w", err) } } + + wg.Wait() + close(errorChan) + }() + + for err := range errorChan { + if err != nil { + return err + } } - launchr.Term().Info().Printfln("Checking variables change") - timeline, err := s.populateTimelineVars(buildInv, modifiedFiles, timeline, s.domainDir) + // Sleep to re-render progress bar. Needed to achieve latest state. + if s.verbosity < 1 { + time.Sleep(multi.UpdateDelay) + multi.Stop() //nolint + } + + return nil +} + +func (s *SyncAction) collectCommits(r *git.Repository) (map[string]bool, error) { + // @todo get commits per files meta/vars to iterate only commits where files were changed. + result := make(map[string]bool) + ref, err := r.Head() if err != nil { - return nil, nil, fmt.Errorf("iteraring variables > %w", err) + return result, err } - return timeline, allDiffResources, nil + // start from the latest commit and iterate to the past + cIter, err := r.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return result, err + } + + _ = cIter.ForEach(func(c *object.Commit) error { + hash := c.Hash.String() + hash = hash[:13] + result[hash] = true + return nil + }) + + return result, nil } -func (s *SyncAction) populateTimelineResources(allUpdatedResources *sync.OrderedMap[*sync.Resource], namespaceResources *sync.OrderedMap[bool], timeline []sync.TimelineItem, gitPath string) ([]sync.TimelineItem, error) { +func (s *SyncAction) findResourcesChangeTime(namespaceResources *sync.OrderedMap[*sync.Resource], gitPath string, mx *async.Mutex, p *pterm.ProgressbarPrinter) error { repo, err := git.PlainOpen(gitPath) if err != nil { - return nil, fmt.Errorf("%s - %w", gitPath, err) + return fmt.Errorf("%s - %w", gitPath, err) } - for _, resourceName := range namespaceResources.Keys() { - buildResource, ok := allUpdatedResources.Get(resourceName) - if !ok { - continue + commitsMap, err := s.collectCommits(repo) + if err != nil { + return err + } + + ref, err := repo.Head() + if err != nil { + return err + } + + // start from the latest commit and iterate to the past + cIter, err := repo.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return err + } + + hashesMap := make(map[string]*hashStruct) + toIterate := namespaceResources.ToDict() + currentVersions := map[string]string{} + + for k, resource := range toIterate { + if _, ok := hashesMap[k]; !ok { + hashesMap[k] = &hashStruct{} } - // Return in case when Inventory.GetChangedResources() stops checking IsValidResource(). - //if !domainResource.IsValidResource() { - // allUpdatedResources.Unset(domainResource.GetName()) - // continue - //} + hashesMap[k].hash = buildHackAuthor + hashesMap[k].hashTime = time.Now() + hashesMap[k].author = buildHackAuthor - buildBaseVersion, buildFullVersion, err := buildResource.GetBaseVersion() + buildResource := sync.NewResource(resource.GetName(), s.buildDir) + version, err := buildResource.GetVersion() if err != nil { - return timeline, err + return err + } + + currentVersions[k] = version + } + + remainingDebug := len(toIterate) + err = cIter.ForEach(func(c *object.Commit) error { + if len(toIterate) == 0 { + return storer.ErrStop + } + + if len(toIterate) != remainingDebug { + remainingDebug = len(toIterate) + launchr.Log().Debug(fmt.Sprintf("Remaining unidentified resources %d", remainingDebug), slog.String("source", gitPath)) } - // If resource doesn't exist in artifact, consider it as new and add to timeline. - artifactResource := sync.NewResource(buildResource.GetName(), s.comparisonDir) - if !artifactResource.IsValidResource() { - ti, errTi := s.findResourceChangeTime(buildBaseVersion, buildResource.BuildMetaPath(), repo) - if errTi != nil { - return timeline, fmt.Errorf("find resource `%s` timeline (new) > %w", buildResource.GetName(), errTi) + for k, resource := range toIterate { + if _, ok := hashesMap[k]; !ok { + hashesMap[k] = &hashStruct{} } - ti.AddResource(buildResource) - timeline = sync.AddToTimeline(timeline, ti) + resourceMetaPath := resource.BuildMetaPath() + resourceVersion, ok := currentVersions[k] + if !ok { + resourceVersion, err = resource.GetVersion() + if err != nil { + return err + } + currentVersions[k] = resourceVersion + } - allUpdatedResources.Unset(buildResource.GetName()) + file, errIt := c.File(resourceMetaPath) + if errIt != nil { + if !errors.Is(errIt, object.ErrFileNotFound) { + return fmt.Errorf("open file %s in commit %s > %w", resourceMetaPath, c.Hash, errIt) + } - launchr.Log().Info(fmt.Sprintf("- %s - new resource from %s", buildResource.GetName(), gitPath)) - continue - } + if hashesMap[k].hash == "" { + hashesMap[k].hash = c.Hash.String() + hashesMap[k].hashTime = c.Author.When + hashesMap[k].author = c.Author.Name + } - artifactBaseVersion, artifactFullVersion, err := artifactResource.GetBaseVersion() - if err != nil { - return timeline, err - } + // File didn't exist before, take current hash as version, + delete(toIterate, k) - // If domain and artifact resource have different versions, consider it as an update. Push update into timeline. - if buildBaseVersion != artifactBaseVersion { - ti, errTi := s.findResourceChangeTime(buildBaseVersion, buildResource.BuildMetaPath(), repo) - if errTi != nil { - return timeline, fmt.Errorf("find resource `%s` timeline (update) > %w", buildResource.GetName(), errTi) + if p != nil { + p.Increment() + } + + continue } - ti.AddResource(buildResource) - timeline = sync.AddToTimeline(timeline, ti) - allUpdatedResources.Unset(buildResource.GetName()) + metaFile, errIt := s.loadYamlFileFromBytes(file, resourceMetaPath) + if errIt != nil { + return fmt.Errorf("commit %s > %w", c.Hash, errIt) + } - launchr.Log().Info(fmt.Sprintf("- %s - updated resource from %s", buildResource.GetName(), gitPath)) - continue + prevVer := sync.GetMetaVersion(metaFile) + if resourceVersion != prevVer { + delete(toIterate, k) + if p != nil { + p.Increment() + } + continue + } + + hashesMap[k].hash = c.Hash.String() + hashesMap[k].hashTime = c.Author.When + hashesMap[k].author = c.Author.Name } - if buildFullVersion == artifactFullVersion { - //errNotVersioned := s.ensureResourceNonVersioned(resourceName, repo) - //if errNotVersioned != nil { - // launchr.Term().Printfln("ensureResourceNonVersioned") - // return nil, errNotVersioned - //} + return nil + }) + + if err != nil { + return err + } + + // Ensure progress bar showing correct progress. + if p != nil && p.Total != p.Current { + p.Add(p.Total - p.Current) + } + + mx.Lock() + defer mx.Unlock() + + for n, hm := range hashesMap { + r, _ := namespaceResources.Get(n) + resourceVersion := currentVersions[n] + + launchr.Log().Debug("add resource to timeline", + slog.String("mrn", r.GetName()), + slog.String("commit", hm.hash), + slog.String("version", resourceVersion), + slog.Time("date", hm.hashTime), + ) + + if hm.author == buildHackAuthor { + msg := fmt.Sprintf("Version of `%s` doesn't match HEAD commit", n) + if !s.allowOverride { + return errors.New(msg) + } - launchr.Log().Info(fmt.Sprintf("- skipping %s (identical build and artifact version)", resourceName)) - allUpdatedResources.Unset(buildResource.GetName()) + launchr.Log().Warn(msg) + } else if hm.author != repository.Author { + launchr.Log().Warn(fmt.Sprintf("Latest commit of %s is not a bump commit", r.GetName())) } + + if _, ok := commitsMap[resourceVersion]; !ok { + launchr.Log().Warn(fmt.Sprintf("Latest version of `%s` doesn't match any existing commit", r.GetName())) + } + + tri := sync.NewTimelineResourcesItem(resourceVersion, hm.hash, hm.hashTime) + tri.AddResource(r) + + s.timeline = sync.AddToTimeline(s.timeline, tri) } - return timeline, nil + return err } -func (s *SyncAction) populateTimelineVars(buildInv *sync.Inventory, modifiedFiles []string, timeline []sync.TimelineItem, gitPath string) ([]sync.TimelineItem, error) { - updatedVariables, deletedVariables, err := buildInv.GetChangedVariables(modifiedFiles, s.comparisonDir, s.vaultPass) +func (s *SyncAction) populateTimelineVars() error { + filesCrawler := sync.NewFilesCrawler(s.domainDir) + groupedFiles, err := filesCrawler.FindVarsFiles("") if err != nil { - return timeline, err + return err + } + + var varsFiles []string + for _, paths := range groupedFiles { + varsFiles = append(varsFiles, paths...) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg async.WaitGroup + var mx async.Mutex + + maxWorkers := min(runtime.NumCPU(), len(varsFiles)) + workChan := make(chan string, len(varsFiles)) + errorChan := make(chan error, 1) + + var p *pterm.ProgressbarPrinter + if s.verbosity < 1 { + p, _ = pterm.DefaultProgressbar.WithTotal(len(varsFiles)).WithTitle("Processing variables files").Start() + } + + for i := 0; i < maxWorkers; i++ { + go func(workerID int) { + for { + select { + case <-ctx.Done(): + return + case varsFile, ok := <-workChan: + if !ok { + return + } + if err = s.findVariableUpdateTime(varsFile, s.domainDir, &mx, p); err != nil { + select { + case errorChan <- fmt.Errorf("worker %d error processing %s: %w", workerID, varsFile, err): + cancel() + default: + } + return + } + wg.Done() + } + } + }(i) } - if updatedVariables.Len() == 0 && deletedVariables.Len() == 0 { - launchr.Log().Info(fmt.Sprintf("- no variables were updated or deleted\n")) - return timeline, nil + for _, f := range varsFiles { + wg.Add(1) + workChan <- f + } + close(workChan) + + go func() { + wg.Wait() + close(errorChan) + }() + + for err = range errorChan { + if err != nil { + return err + } } + return nil +} + +func (s *SyncAction) findVariableUpdateTime(varsFile string, gitPath string, mx *async.Mutex, p *pterm.ProgressbarPrinter) error { repo, err := git.PlainOpen(gitPath) if err != nil { - return nil, err + return fmt.Errorf("%s - %w", gitPath, err) } - for _, varName := range updatedVariables.Keys() { - variable, _ := updatedVariables.Get(varName) - ti, errTi := s.findVariableUpdateTime(variable, repo) - if errTi != nil { - launchr.Term().Error().Printfln("find variable `%s` timeline (update) > %s", variable.GetName(), errTi.Error()) - launchr.Log().Warn(fmt.Sprintf("skipping %s", variable.GetName())) - continue - } + ref, err := repo.Head() + if err != nil { + return err + } - ti.AddVariable(variable) - timeline = sync.AddToTimeline(timeline, ti) + var varsYaml map[string]any + hashesMap := make(map[string]*hashStruct) + variablesMap := sync.NewOrderedMap[*sync.Variable]() + isVault := sync.IsVaultFile(varsFile) - launchr.Log().Info(fmt.Sprintf("- %s - new or updated variable from %s", variable.GetName(), variable.GetPath())) + varsYaml, err = sync.LoadVariablesFile(filepath.Join(s.buildDir, varsFile), s.vaultPass, isVault) + if err != nil { + return err } - for _, varName := range deletedVariables.Keys() { - variable, _ := deletedVariables.Get(varName) - ti, errTi := s.findVariableDeletionTime(variable, repo) - if errTi != nil { - launchr.Term().Error().Printfln("find variable `%s` timeline (delete) > %s", variable.GetName(), errTi.Error()) - launchr.Log().Warn(fmt.Sprintf("skipping %s", variable.GetName())) - continue - } + for k, value := range varsYaml { + v := sync.NewVariable(varsFile, k, HashString(fmt.Sprint(value)), isVault) + variablesMap.Set(k, v) - ti.AddVariable(variable) - timeline = sync.AddToTimeline(timeline, ti) + if _, ok := hashesMap[k]; !ok { + hashesMap[k] = &hashStruct{} + } - launchr.Log().Info(fmt.Sprintf("- %s - deleted variable from %s", variable.GetName(), variable.GetPath())) + hashesMap[k].hash = fmt.Sprint(v.GetHash()) + hashesMap[k].hashTime = time.Now() + hashesMap[k].author = buildHackAuthor } - return timeline, nil -} + toIterate := variablesMap.ToDict() -func (s *SyncAction) buildPropagationMap(buildInv *sync.Inventory, timeline []sync.TimelineItem) (*sync.OrderedMap[*sync.Resource], map[string]string, error) { - resourceVersionMap := make(map[string]string) - toPropagate := sync.NewOrderedMap[*sync.Resource]() + cIter, err := repo.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return err + } - sync.SortTimeline(timeline) - launchr.Term().Info().Printfln("Iterating timeline") - for _, item := range timeline { - launchr.Log().Debug("timeline item", "version", item.GetVersion(), "date", item.GetDate(), "commit", item.GetCommit()) - switch i := item.(type) { - case *sync.TimelineResourcesItem: - resources := i.GetResources() - resources.SortKeysAlphabetically() - for _, key := range resources.Keys() { - r, ok := resources.Get(key) - if !ok { - return nil, nil, fmt.Errorf("unknown key %s detected during timeline iteration", key) - } + remainingDebug := len(toIterate) + err = cIter.ForEach(func(c *object.Commit) error { + if len(toIterate) == 0 { + return storer.ErrStop + } - launchr.Log().Info(fmt.Sprintf("Collecting %s dependencies:", r.GetName())) - err := s.propagateResourceDeps(r, i.GetVersion(), toPropagate, buildInv.GetRequiredByMap(), resourceVersionMap) - if err != nil { - return toPropagate, resourceVersionMap, err - } - } + if len(toIterate) != remainingDebug { + remainingDebug = len(toIterate) + launchr.Log().Debug(fmt.Sprintf("Remaining unidentified variables, %s - %d", varsFile, remainingDebug)) + } - for _, key := range resources.Keys() { - r, _ := resources.Get(key) - _, ok := toPropagate.Get(r.GetName()) - if ok { - // Ensure new version removes previous propagation for that resource. - toPropagate.Unset(r.GetName()) - delete(resourceVersionMap, r.GetName()) - } + file, errIt := c.File(varsFile) + if errIt != nil { + if !errors.Is(errIt, object.ErrFileNotFound) { + return fmt.Errorf("open file %s in commit %s > %w", varsFile, c.Hash, errIt) } - case *sync.TimelineVariablesItem: - variables := i.GetVariables() - variables.SortKeysAlphabetically() - resources, _, err := buildInv.SearchVariablesAffectedResources(variables.ToList()) - if err != nil { - return toPropagate, resourceVersionMap, fmt.Errorf("search variable affected resources > %w", err) - } + return storer.ErrStop + } - launchr.Log().Info(fmt.Sprintf("Variables:")) - for _, variable := range variables.Keys() { - launchr.Log().Info(fmt.Sprintf("- %s", variable)) + varFile, errIt := s.loadVariablesFileFromBytes(file, varsFile, isVault) + if errIt != nil { + if strings.Contains(errIt.Error(), "did not find expected key") || strings.Contains(errIt.Error(), "could not find expected") { + launchr.Log().Warn("Bad YAML structured detected", + slog.String("file", varsFile), + slog.String("commit", c.Hash.String()), + slog.String("error", errIt.Error()), + ) + + return nil } - launchr.Log().Info(fmt.Sprintf("Collecting dependencies:")) - version := i.GetVersion() + if strings.Contains(errIt.Error(), "invalid password for vault") { + launchr.Log().Warn("Invalid password for vault", + slog.String("file", varsFile), + slog.String("commit", c.Hash.String()), + ) - resources.SortKeysAlphabetically() - for _, key := range resources.Keys() { - r, ok := resources.Get(key) - if !ok { - continue - } - - s.propagateDepsRecursively(r, version, toPropagate, buildInv.GetRequiredByMap(), resourceVersionMap) + return storer.ErrStop } - } - } - return toPropagate, resourceVersionMap, nil -} + if strings.Contains(errIt.Error(), "invalid secret format") { + launchr.Log().Warn("invalid secret format for vault", + slog.String("file", varsFile), + slog.String("commit", c.Hash.String()), + ) + return nil + } -func (s *SyncAction) copyHistory(history *sync.OrderedMap[*sync.Resource]) error { - launchr.Term().Info().Printfln("Copying history from artifact") - for _, key := range history.Keys() { - r, ok := history.Get(key) - if !ok { - continue + return fmt.Errorf("commit %s > %w", c.Hash, errIt) } - // set version from artifact to build dir - artifactResource := sync.NewResource(r.GetName(), s.comparisonDir) - if artifactResource.IsValidResource() { - artifactVersion, err := artifactResource.GetVersion() - if err != nil { - return err + for k, hh := range toIterate { + prevVar, exists := varFile[k] + if !exists { + // Variable didn't exist before, take current hash as version + delete(toIterate, k) + continue } - launchr.Log().Info(fmt.Sprintf("- copy %s - %s", r.GetName(), artifactVersion)) - if s.dryRun { + prevVarHash := HashString(fmt.Sprint(prevVar)) + if hh.GetHash() != prevVarHash { + // Variable exists, hashes don't match, stop iterating + delete(toIterate, k) continue } - err = r.UpdateVersion(artifactVersion) - if err != nil { - return err + hashesMap[k].hash = c.Hash.String() + hashesMap[k].hashTime = c.Author.When + hashesMap[k].author = c.Author.Name + } + + return nil + }) + + if err != nil { + return err + } + + mx.Lock() + defer mx.Unlock() + + for n, hm := range hashesMap { + v, _ := variablesMap.Get(n) + version := hm.hash[:13] + launchr.Log().Debug("add variable to timeline", + slog.String("variable", v.GetName()), + slog.String("version", version), + slog.Time("date", hm.hashTime), + slog.String("path", v.GetPath()), + ) + + if hm.author == buildHackAuthor { + msg := fmt.Sprintf("Value of `%s` doesn't match HEAD commit", n) + if !s.allowOverride { + if p != nil { + p.Stop() //nolint + } + + return errors.New(msg) } + + launchr.Log().Warn(msg) } + + tri := sync.NewTimelineVariablesItem(version, hm.hash, hm.hashTime) + tri.AddVariable(v) + + s.timeline = sync.AddToTimeline(s.timeline, tri) } - return nil + if p != nil { + p.Increment() + } + + return err } -func (s *SyncAction) showImpacted(inv *sync.Inventory, timeline []sync.TimelineItem, propagated *sync.OrderedMap[*sync.Resource]) { - result := sync.NewOrderedMap[bool]() - timelineResources := sync.NewOrderedMap[bool]() +func (s *SyncAction) buildPropagationMap(buildInv *sync.Inventory, timeline []sync.TimelineItem) (*sync.OrderedMap[*sync.Resource], map[string]string, error) { + resourceVersionMap := make(map[string]string) + toPropagate := sync.NewOrderedMap[*sync.Resource]() + resourcesMap := buildInv.GetResourcesMap() + sync.SortTimeline(timeline) + launchr.Log().Info("Iterating timeline") for _, item := range timeline { switch i := item.(type) { case *sync.TimelineResourcesItem: resources := i.GetResources() - for _, mrn := range resources.Keys() { - timelineResources.Set(mrn, true) + resources.SortKeysAlphabetically() + + dependenciesLog := sync.NewOrderedMap[bool]() + + for _, key := range resources.Keys() { + r, ok := resources.Get(key) + if !ok { + return nil, nil, fmt.Errorf("unknown key %s detected during timeline iteration", key) + } + + dependentResources := buildInv.GetRequiredByResources(r.GetName(), -1) + for dep := range dependentResources { + depResource, okR := resourcesMap.Get(dep) + if !okR { + continue + } + + toPropagate.Set(dep, depResource) + resourceVersionMap[dep] = i.GetVersion() + + if _, okD := resources.Get(dep); !okD { + dependenciesLog.Set(dep, true) + } + } } - } - } -iterateTimelineResources: - for _, mrn := range timelineResources.Keys() { - deps := inv.GetRequiredByResources(mrn, -1) - for d := range deps { - if _, ok := propagated.Get(d); ok { - continue iterateTimelineResources + for _, key := range resources.Keys() { + // Ensure new version removes previous propagation for that resource. + toPropagate.Unset(key) + delete(resourceVersionMap, key) } - if _, ok := timelineResources.Get(d); ok { - continue iterateTimelineResources + launchr.Log().Debug("timeline item (resources)", + slog.String("version", item.GetVersion()), + slog.Time("date", item.GetDate()), + slog.String("resources", fmt.Sprintf("%v", resources.Keys())), + slog.String("dependencies", fmt.Sprintf("%v", dependenciesLog.Keys())), + ) + + case *sync.TimelineVariablesItem: + variables := i.GetVariables() + variables.SortKeysAlphabetically() + + var resources []string + for _, v := range variables.Keys() { + variable, _ := variables.Get(v) + vr, err := buildInv.GetVariableResources(variable.GetName(), variable.GetPlatform()) + if err != nil { + return nil, nil, err + } + resources = append(resources, vr...) } - } - result.Set(mrn, true) - } + slices.Sort(resources) + resources = slices.Compact(resources) + + dependenciesLog := sync.NewOrderedMap[bool]() -iteratePropagate: - for _, mrn := range propagated.Keys() { - deps := inv.GetRequiredByResources(mrn, -1) - for d := range deps { - if _, ok := propagated.Get(d); ok { - continue iteratePropagate + for _, r := range resources { + // First set version for main resource. + mainResource, okM := resourcesMap.Get(r) + if !okM { + launchr.Log().Warn(fmt.Sprintf("skipping not valid resource %s (direct vars dependency)", r)) + continue + } + toPropagate.Set(r, mainResource) + resourceVersionMap[r] = i.GetVersion() + + dependenciesLog.Set(r, true) + + // Set versions for dependent resources. + dependentResources := buildInv.GetRequiredByResources(r, -1) + for dep := range dependentResources { + depResource, okR := resourcesMap.Get(dep) + if !okR { + launchr.Log().Warn(fmt.Sprintf("skipping not valid resource %s (dependency of %s)", dep, r)) + continue + } + + toPropagate.Set(dep, depResource) + resourceVersionMap[dep] = i.GetVersion() + + dependenciesLog.Set(dep, true) + } } - } - result.Set(mrn, true) + launchr.Log().Debug("timeline item (variables)", + slog.String("version", item.GetVersion()), + slog.Time("date", item.GetDate()), + slog.String("variables", fmt.Sprintf("%v", variables.Keys())), + slog.String("resources", fmt.Sprintf("%v", dependenciesLog.Keys())), + ) + } } - result.SortKeysAlphabetically() - launchr.Term().Info().Printf("List of impacted resources:\n") - for _, k := range result.Keys() { - launchr.Term().Printf("- %s\n", k) - } + return toPropagate, resourceVersionMap, nil } -func (s *SyncAction) updateResources(resourceVersionMap map[string]string, toPropagate, history *sync.OrderedMap[*sync.Resource]) error { +func (s *SyncAction) updateResources(resourceVersionMap map[string]string, toPropagate *sync.OrderedMap[*sync.Resource]) error { var sortList []string updateMap := make(map[string]map[string]string) stopPropagation := false - launchr.Log().Info(fmt.Sprintf("Filtering identical versions:")) + launchr.Log().Info("Sorting resources before update") for _, key := range toPropagate.Keys() { r, _ := toPropagate.Get(key) baseVersion, currentVersion, errVersion := r.GetBaseVersion() @@ -942,15 +941,15 @@ func (s *SyncAction) updateResources(resourceVersionMap map[string]string, toPro } if currentVersion == "" { - launchr.Log().Warn(fmt.Sprintf("resource %s has no version", r.GetName())) + launchr.Term().Warning().Printfln("resource %s has no version", r.GetName()) stopPropagation = true } - newVersion := s.composeVersion(currentVersion, resourceVersionMap[r.GetName()]) + newVersion := composeVersion(currentVersion, resourceVersionMap[r.GetName()]) if baseVersion == resourceVersionMap[r.GetName()] { launchr.Log().Debug("skip identical", "baseVersion", baseVersion, "currentVersion", currentVersion, "propagateVersion", resourceVersionMap[r.GetName()], "newVersion", newVersion) - launchr.Log().Warn(fmt.Sprintf("- skip %s (identical versions)", r.GetName())) + launchr.Term().Warning().Printfln("- skip %s (identical versions)", r.GetName()) continue } @@ -967,27 +966,33 @@ func (s *SyncAction) updateResources(resourceVersionMap map[string]string, toPro return errors.New("empty version has been detected, please check log") } - // copy history from artifact. - err := s.copyHistory(history) - if err != nil { - return fmt.Errorf("history copy > %w", err) - } - if len(updateMap) == 0 { launchr.Term().Printfln("No version to propagate") return nil } sort.Strings(sortList) - launchr.Term().Info().Printfln("Propagating versions:") + launchr.Log().Info("Propagating versions") + + var p *pterm.ProgressbarPrinter + if s.verbosity < 1 { + p, _ = pterm.DefaultProgressbar.WithTotal(len(sortList)).WithTitle("Updating resources").Start() + } for _, key := range sortList { + if p != nil { + p.Increment() + } + val := updateMap[key] - r, _ := toPropagate.Get(key) + r, ok := toPropagate.Get(key) currentVersion := val["current"] newVersion := val["new"] + if !ok { + return fmt.Errorf("unidentified resource found during update %s", key) + } - launchr.Term().Printfln("- %s from %s to %s", r.GetName(), currentVersion, newVersion) + launchr.Log().Info(fmt.Sprintf("%s from %s to %s", r.GetName(), currentVersion, newVersion)) if s.dryRun { continue } @@ -1001,65 +1006,6 @@ func (s *SyncAction) updateResources(resourceVersionMap map[string]string, toPro return nil } -func (s *SyncAction) propagateResourceDeps(resource *sync.Resource, version string, toPropagate *sync.OrderedMap[*sync.Resource], resourcesGraph map[string]*sync.OrderedMap[bool], resourceVersionMap map[string]string) error { - if itemsMap, ok := resourcesGraph[resource.GetName()]; ok { - itemsMap.SortKeysAlphabetically() - for _, resourceName := range itemsMap.Keys() { - depResource, resourceExists := toPropagate.Get(resourceName) - if !resourceExists { - depResource = sync.NewResource(resourceName, s.buildDir) - if !depResource.IsValidResource() { - continue - } - } - - s.propagateDepsRecursively(depResource, version, toPropagate, resourcesGraph, resourceVersionMap) - } - } - - return nil -} - -func (s *SyncAction) propagateDepsRecursively(resource *sync.Resource, version string, toPropagate *sync.OrderedMap[*sync.Resource], resourcesGraph map[string]*sync.OrderedMap[bool], resourceVersionMap map[string]string) { - if _, ok := toPropagate.Get(resource.GetName()); !ok { - toPropagate.Set(resource.GetName(), resource) - } - resourceVersionMap[resource.GetName()] = version - launchr.Log().Info(fmt.Sprintf("- %s", resource.GetName())) - - if itemsMap, ok := resourcesGraph[resource.GetName()]; ok { - itemsMap.SortKeysAlphabetically() - for _, resourceName := range itemsMap.Keys() { - depResource, resourceExists := toPropagate.Get(resourceName) - if !resourceExists { - depResource = sync.NewResource(resourceName, s.buildDir) - if !depResource.IsValidResource() { - continue - } - } - s.propagateDepsRecursively(depResource, version, toPropagate, resourcesGraph, resourceVersionMap) - } - } -} - -func (s *SyncAction) composeVersion(oldVersion string, newVersion string) string { - var version string - if len(strings.Split(newVersion, "-")) > 1 { - version = newVersion - } else { - split := strings.Split(oldVersion, "-") - if len(split) == 1 { - version = fmt.Sprintf("%s-%s", oldVersion, newVersion) - } else if len(split) > 1 { - version = fmt.Sprintf("%s-%s", split[0], newVersion) - } else { - version = newVersion - } - } - - return version -} - func (s *SyncAction) loadYamlFileFromBytes(file *object.File, path string) (map[string]any, error) { reader, errIt := file.Blob.Reader() if errIt != nil { @@ -1097,3 +1043,150 @@ func (s *SyncAction) loadVariablesFileFromBytes(file *object.File, path string, return varFile, nil } + +func composeVersion(oldVersion string, newVersion string) string { + var version string + if len(strings.Split(newVersion, "-")) > 1 { + version = newVersion + } else { + split := strings.Split(oldVersion, "-") + if len(split) == 1 { + version = fmt.Sprintf("%s-%s", oldVersion, newVersion) + } else if len(split) > 1 { + version = fmt.Sprintf("%s-%s", split[0], newVersion) + } else { + version = newVersion + } + } + + return version +} + +func getResourcesMapFrom(dir string) (*sync.OrderedMap[*sync.Resource], error) { + inv, err := sync.NewInventory(dir) + if err != nil { + return nil, err + } + + rm := inv.GetResourcesMap() + rm.SortKeysAlphabetically() + return rm, nil +} + +// HashString is wrapper for hashing string. +func HashString(item string) uint64 { + return xxhash.Sum64String(item) +} + +//func (s *SyncAction) ensureResourceIsVersioned(resourceVersion, resourceMetaPath string, repo *git.Repository) (*plumbing.Reference, error) { +// ref, err := repo.Head() +// if err != nil { +// return nil, err +// } +// +// headCommit, err := repo.CommitObject(ref.Hash()) +// if err != nil { +// return nil, err +// } +// headMeta, err := headCommit.File(resourceMetaPath) +// if err != nil { +// return nil, fmt.Errorf("meta %s doesn't exist in HEAD commit", resourceMetaPath) +// } +// +// metaFile, err := s.loadYamlFileFromBytes(headMeta, resourceMetaPath) +// if err != nil { +// return nil, fmt.Errorf("%w", err) +// } +// +// headVersion := sync.GetMetaVersion(metaFile) +// if resourceVersion != headVersion { +// return nil, fmt.Errorf("version from %s doesn't match any existing commit", resourceMetaPath) +// } +// +// return ref, nil +//} + +//func (s *SyncAction) ensureResourceNonVersioned(mrn string, repo *git.Repository) error { +// resourcePath, err := sync.ConvertMRNtoPath(mrn) +// if err != nil { +// return err +// } +// +// buildPath := filepath.Join(s.buildDir, resourcePath) +// resourceFiles, err := sync.GetFiles(buildPath, []string{}) +// if err != nil { +// return err +// } +// +// ref, err := repo.Head() +// if err != nil { +// return err +// } +// +// headCommit, err := repo.CommitObject(ref.Hash()) +// if err != nil { +// return err +// } +// +// for f := range resourceFiles { +// buildHash, err := sync.HashFileByPath(filepath.Join(buildPath, f)) +// if err != nil { +// return err +// } +// +// launchr.Term().Warning().Printfln(filepath.Join(resourcePath, f)) +// headFile, err := headCommit.File(filepath.Join(resourcePath, f)) +// if err != nil { +// return err +// } +// +// reader, err := headFile.Blob.Reader() +// if err != nil { +// return err +// } +// +// headHash, err := sync.HashFileFromReader(reader) +// if err != nil { +// return err +// } +// +// if buildHash != headHash { +// return fmt.Errorf("resource %s has unversioned changes. You need to commit these changes", mrn) +// } +// } +// +// return nil +//} + +//func (s *SyncAction) ensureVariableIsVersioned(variable *sync.Variable, repo *git.Repository) (*plumbing.Reference, error) { +// ref, err := repo.Head() +// if err != nil { +// return nil, err +// } +// +// headCommit, err := repo.CommitObject(ref.Hash()) +// if err != nil { +// return nil, err +// } +// headVarsFile, err := headCommit.File(variable.GetPath()) +// if err != nil { +// return nil, fmt.Errorf("file %s doesn't exist in HEAD", variable.GetPath()) +// } +// +// varFile, errIt := s.loadVariablesFileFromBytes(headVarsFile, variable.GetPath(), variable.IsVault()) +// if errIt != nil { +// return nil, fmt.Errorf("%w", errIt) +// } +// +// headVar, exists := varFile[variable.GetName()] +// if !exists { +// return nil, fmt.Errorf("variable from %s doesn't exist in HEAD", variable.GetPath()) +// } +// +// headVarHash := sync.HashString(fmt.Sprint(headVar)) +// if variable.GetHash() != headVarHash { +// return nil, fmt.Errorf("variable from %s is an unversioned change. You need to commit variable change", variable.GetPath()) +// } +// +// return ref, nil +//} diff --git a/go.mod b/go.mod index b0fe819..daae877 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/launchrctl/compose v0.11.0 github.com/launchrctl/keyring v0.2.5 github.com/launchrctl/launchr v0.16.4 + github.com/pterm/pterm v0.12.79 github.com/sosedoff/ansible-vault-go v0.2.0 github.com/spf13/cobra v1.8.1 github.com/stevenle/topsort v0.2.0 @@ -82,7 +83,6 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pterm/pterm v0.12.79 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect diff --git a/pkg/sync/artifact.go b/pkg/sync/artifact.go deleted file mode 100644 index 0d68a9e..0000000 --- a/pkg/sync/artifact.go +++ /dev/null @@ -1,311 +0,0 @@ -// Package sync contains tools to provide bump propagation. -package sync - -import ( - "archive/tar" - "compress/gzip" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/launchrctl/launchr" - - "github.com/skilld-labs/plasmactl-bump/v2/pkg/repository" -) - -const ( - // ArtifactTruncateLength contains bump commit hash truncate length. - ArtifactTruncateLength = 7 - dirPermissions = 0755 -) - -var ( - // ErrArtifactNotFound not artifact error. - ErrArtifactNotFound = errors.New("artifact was not found") -) - -// Artifact represents a storage for artifacts. -type Artifact struct { - bumper *repository.Bumper - artifactsDir string - artifactsRepositoryURL string - override string - comparisonDir string - retryLimit int -} - -// NewArtifact returns new instance of artifact to get. -func NewArtifact(artifactsDir, artifactsRepoURL, override, comparisonDir string) (*Artifact, error) { - b, err := repository.NewBumper() - if err != nil { - return nil, err - } - - return &Artifact{ - bumper: b, - override: override, - artifactsDir: artifactsDir, - artifactsRepositoryURL: artifactsRepoURL, - comparisonDir: comparisonDir, - retryLimit: 50, - }, nil -} - -// Get prepares the artifact for comparison by downloading and extracting it into the specified directory. -func (a *Artifact) Get(username, password string) error { - repoName, err := a.bumper.GetRepoName() - if err != nil { - return err - } - - launchr.Term().Info().Printfln("Repository name: %s", repoName) - var archivePath string - if a.override != "" { - comparisonRef := a.override - launchr.Term().Info().Printfln("OVERRIDDEN_COMPARISON_REF has been set: %s", a.override) - artifactFile, artifactPath := a.buildArtifactPaths(repoName, comparisonRef) - err = a.downloadArtifact(username, password, artifactFile, artifactPath, repoName) - if err != nil { - return err - } - - archivePath = artifactPath - } else { - hash, errHash := a.bumper.GetGit().ResolveRevision(plumbing.Revision(plumbing.HEAD)) - if errHash != nil { - return errHash - } - - from := hash - retryCount := 0 - for retryCount < a.retryLimit { - comparisonHash, errHash := a.bumper.GetComparisonCommit(*from, repository.BumpMessage) - if errHash != nil { - return errHash - } - - commit := []rune(comparisonHash.String()) - comparisonRef := string(commit[:ArtifactTruncateLength]) - - launchr.Term().Info().Printfln("Bump commit identified: %s", comparisonRef) - artifactFile, artifactPath := a.buildArtifactPaths(repoName, comparisonRef) - errDownload := a.downloadArtifact(username, password, artifactFile, artifactPath, repoName) - if errDownload != nil { - if errors.Is(errDownload, ErrArtifactNotFound) { - retryCount++ - from = comparisonHash - continue - } - - return errDownload - } - - archivePath = artifactPath - break - } - - if archivePath == "" { - return ErrArtifactNotFound - } - } - - launchr.Term().Printf("Processing...\n") - err = a.prepareComparisonDir(a.comparisonDir) - if err != nil { - return err - } - _, err = a.unarchiveTar(archivePath, a.comparisonDir) - if err != nil { - return err - } - - return nil -} - -func (a *Artifact) buildArtifactPaths(repoName, comparisonRef string) (string, string) { - artifactFile := fmt.Sprintf("%s-%s-plasma-src.tar.gz", repoName, comparisonRef) - artifactPath := filepath.Join(a.artifactsDir, artifactFile) - - return artifactFile, artifactPath -} - -func (a *Artifact) prepareComparisonDir(path string) error { - err := os.MkdirAll(path, dirPermissions) - if err != nil { - return err - } - err = os.RemoveAll(path) - if err != nil { - return err - } - return nil -} - -func (a *Artifact) downloadArtifact(username, password, artifactFile, artifactPath, repo string) error { - launchr.Term().Printfln("Attempting to get %s from local storage", artifactFile) - _, errExists := os.Stat(artifactPath) - if errExists == nil { - launchr.Term().Printf("Local artifact found\n") - return nil - } - - launchr.Term().Printfln("Local artifact %s not found", artifactFile) - url := fmt.Sprintf("%s/repository/%s-artifacts/%s", a.artifactsRepositoryURL, repo, artifactFile) - launchr.Term().Printfln("Attempting to download artifact: %s", url) - - err := os.MkdirAll(a.artifactsDir, dirPermissions) - if err != nil { - return err - } - - out, err := os.Create(filepath.Clean(artifactPath)) - if err != nil { - return err - } - - defer func() { - if err = out.Close(); err != nil { - launchr.Term().Error().Printf("error closing stream\n") - } - }() - - client := &http.Client{} - req, err := http.NewRequest(http.MethodGet, url, nil) - - req.SetBasicAuth(username, password) - - resp, err := client.Do(req) - if err != nil { - return err - } - - defer func() { - if err = resp.Body.Close(); err != nil { - launchr.Term().Error().Printf("error closing stream\n") - } - }() - - statusCode := resp.StatusCode - if statusCode != http.StatusOK { - errRemove := os.Remove(artifactPath) - if errRemove != nil { - launchr.Log().Debug("Error during removing invalid artifact: msg", "msg", errRemove.Error()) - } - - if statusCode == http.StatusUnauthorized { - return errors.New("invalid credentials") - } - if statusCode == http.StatusNotFound { - return ErrArtifactNotFound - } - - return fmt.Errorf("unexpected status code: %d while trying to get %s", statusCode, url) - } - - _, err = io.Copy(out, resp.Body) - - return err -} - -func (a *Artifact) unarchiveTar(fpath, tpath string) (string, error) { - var rootDir string - rgxPathRoot := regexp.MustCompile(`^[^/]*`) - - r, err := os.Open(filepath.Clean(fpath)) - if err != nil { - return rootDir, err - } - - gzr, err := gzip.NewReader(r) - if err != nil { - return rootDir, err - } - defer gzr.Close() - - tr := tar.NewReader(gzr) - - for { - header, readErr := tr.Next() - - switch { - - // if no more files are found return - case readErr == io.EOF: - if rootDir != "" { - rootDir = rgxPathRoot.FindString(rootDir) - } - - return rootDir, nil - - // return any other error - case readErr != nil: - return rootDir, readErr - - // if the header is nil, just skip it (not sure how this happens) - case header == nil: - continue - } - - // the target location where the dir/file should be created - target, readErr := a.sanitizeArchivePath(tpath, header.Name) - if readErr != nil { - return rootDir, errors.New("invalid filepath") - } - - if !strings.HasPrefix(target, filepath.Clean(tpath)) { - return rootDir, errors.New("invalid filepath") - } - - // check the file type - switch header.Typeflag { - - // if it's a dir, and it doesn't exist create it - case tar.TypeDir: - rootDir = header.Name - if _, errStat := os.Stat(target); errStat != nil { - if errMk := os.MkdirAll(target, 0750); errMk != nil { - return rootDir, err - } - } - - // if it's a file create it - case tar.TypeReg: - f, errOpen := os.OpenFile(filepath.Clean(target), os.O_CREATE|os.O_RDWR, header.FileInfo().Mode()) - if errOpen != nil { - return rootDir, errOpen - } - - for { - _, errCopy := io.CopyN(f, tr, 1024) - if errCopy != nil { - if errCopy != io.EOF { - return rootDir, errCopy - } - break - } - } - - // manually close here after each file operation; deferring would cause each file close - // to wait until all operations have completed. - err = f.Close() - if err != nil { - return rootDir, err - } - } - } -} - -func (a *Artifact) sanitizeArchivePath(d, t string) (v string, err error) { - v = filepath.Join(d, t) - if strings.HasPrefix(v, filepath.Clean(d)) { - return v, nil - } - - return "", fmt.Errorf("%s: %s", "content filepath is tainted", t) -} diff --git a/pkg/sync/diff.go b/pkg/sync/diff.go deleted file mode 100644 index a656cfc..0000000 --- a/pkg/sync/diff.go +++ /dev/null @@ -1,137 +0,0 @@ -package sync - -import ( - "io" - "os" - "path/filepath" - "strings" - - "github.com/cespare/xxhash/v2" -) - -// CompareDirs takes two directory paths as inputs and returns a slice of different files between them. -func CompareDirs(dirA, dirB string, excludeSubDirs []string) ([]string, error) { - filesInDirA, err := GetFiles(dirA, excludeSubDirs) - if err != nil { - return nil, err - } - filesInDirB, err := GetFiles(dirB, excludeSubDirs) - if err != nil { - return nil, err - } - - var updated []string - - for f := range filesInDirA { - _, found := filesInDirB[f] - if !found { - updated = append(updated, f) - continue - } - fileA := filepath.Join(dirA, f) - fileB := filepath.Join(dirB, f) - - areEqual, err := fileEqual(fileA, fileB) - if err != nil { - return nil, err - } - if !areEqual { - updated = append(updated, f) - } - } - - updated = append(updated, mapKeyDiff(filesInDirB, filesInDirA)...) - - return updated, nil -} - -func mapKeyDiff(m1, m2 map[string]bool) []string { - var diff []string - for key := range m1 { - if !m2[key] { - diff = append(diff, key) - } - } - return diff -} - -// GetFiles returns list of files as map. -func GetFiles(path string, excludeSubDirs []string) (map[string]bool, error) { - path = ensureTrailingSlash(path) - files := make(map[string]bool) - err := filepath.Walk(path, func(fpath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - relPath := strings.TrimPrefix(fpath, path) - // skip symlinks - if info.Mode()&os.ModeSymlink != 0 || info.IsDir() { - return nil - } - - // exclude subdirectories - for _, d := range excludeSubDirs { - if strings.Contains(relPath, d) { - return nil - } - } - - files[relPath] = true - return nil - }) - - return files, err -} - -func ensureTrailingSlash(s string) string { - if !strings.HasSuffix(s, "/") { - s += "/" - } - return s -} - -func fileEqual(fileA, fileB string) (bool, error) { - hashA, err := HashFileByPath(fileA) - if err != nil { - return false, err - } - hashB, err := HashFileByPath(fileB) - if err != nil { - return false, err - } - - return hashA == hashB, nil -} - -// HashFileByPath opens file and hash content. -func HashFileByPath(path string) (uint64, error) { - file, err := os.Open(filepath.Clean(path)) - if err != nil { - return 0, err - } - defer file.Close() - - hash := xxhash.New() - _, err = io.Copy(hash, file) - if err != nil { - return 0, err - } - - return hash.Sum64(), nil -} - -// HashFileFromReader uses reader to copy and hash file. -func HashFileFromReader(reader io.ReadCloser) (uint64, error) { - hash := xxhash.New() - _, err := io.Copy(hash, reader) - if err != nil { - return 0, err - } - - return hash.Sum64(), nil -} - -// HashString is wrapper for hashing string. -func HashString(item string) uint64 { - return xxhash.Sum64String(item) -} diff --git a/pkg/sync/filesCrawler.go b/pkg/sync/filesCrawler.go new file mode 100644 index 0000000..7655cdc --- /dev/null +++ b/pkg/sync/filesCrawler.go @@ -0,0 +1,131 @@ +package sync + +import ( + "os" + "path/filepath" + "strings" +) + +const roles = "roles" + +// FilesCrawler is a type that represents a crawler for resources in a given directory. +type FilesCrawler struct { + rootDir string +} + +// NewFilesCrawler creates a new instance of FilesCrawler with initialized taskSources and templateSources maps. +func NewFilesCrawler(directory string) *FilesCrawler { + return &FilesCrawler{ + rootDir: directory, + } +} + +// FindVarsFiles return list of variables files in platform. +// If platform is empty, search across all. +func (cr *FilesCrawler) FindVarsFiles(platform string) (map[string][]string, error) { + allowedKinds := map[string]bool{} + for _, k := range Kinds { + allowedKinds[k] = true + } + + partsCount := 3 + platformPart := 0 + rolePart := 0 + kindPart := 1 + + files := make(map[string][]string) + dir := filepath.Join(cr.rootDir, platform) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath := strings.TrimPrefix(path, cr.rootDir+"/") + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + + if strings.Contains(path, "scripts") { + return filepath.SkipDir + } + + if info.IsDir() { + return nil + } + + parts := strings.Split(relPath, "/") + if len(parts) >= partsCount && (platform == "" || parts[platformPart] == platform) && + (rolePart == 0 || parts[rolePart] == roles) { + + if parts[kindPart] == "group_vars" { + filename := filepath.Base(path) + if filename == "vars.yaml" || filename == vaultFile { + files[parts[platformPart]] = append(files[parts[platformPart]], relPath) + } + } + } + + return nil + }) + + return files, err +} + +// FindResourcesFiles return list of resources files in platform. +// If platform is empty, search across all. +func (cr *FilesCrawler) FindResourcesFiles(platform string) (map[string][]string, error) { + allowedKinds := map[string]bool{} + for _, k := range Kinds { + allowedKinds[k] = true + } + + partsCount := 4 + platformPart := 0 + rolePart := 2 + kindPart := 4 + + files := make(map[string][]string) + dir := filepath.Join(cr.rootDir, platform) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath := strings.TrimPrefix(path, cr.rootDir+"/") + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + + if strings.Contains(path, "scripts") { + return filepath.SkipDir + } + + if info.IsDir() { + return nil + } + + parts := strings.Split(relPath, "/") + if len(parts) >= partsCount && (platform == "" || parts[platformPart] == platform) && + (rolePart == 0 || parts[rolePart] == roles) { + + if ok := allowedKinds[parts[1]]; !ok { + return nil + } + + if parts[kindPart] == "templates" { + ext := filepath.Ext(path) + if ext != ".j2" { + return nil + } + files[parts[platformPart]] = append(files[parts[platformPart]], relPath) + + } else if parts[kindPart] == "tasks" && filepath.Base(path) == "configuration.yaml" { + files[parts[platformPart]] = append(files[parts[platformPart]], relPath) + } + } + + return nil + }) + + return files, err +} diff --git a/pkg/sync/inventory.go b/pkg/sync/inventory.go index 63f743a..9939bbe 100644 --- a/pkg/sync/inventory.go +++ b/pkg/sync/inventory.go @@ -1,16 +1,12 @@ +// Package sync contains tools to provide bump propagation. package sync import ( - "bufio" - "errors" "fmt" "os" "path/filepath" - "regexp" "strings" - "github.com/launchrctl/launchr" - vault "github.com/sosedoff/ansible-vault-go" "github.com/stevenle/topsort" "gopkg.in/yaml.v3" ) @@ -18,8 +14,8 @@ import ( const ( invalidPasswordErrText = "invalid password" - rootPlatform = "platform" - variablePattern = "(?:\\s|{|\\|)(%s)(?:\\s|}|\\|)" + rootPlatform = "platform" + vaultFile = "vault.yaml" ) // InventoryExcluded is list of excluded files and folders from inventory. @@ -47,42 +43,6 @@ var Kinds = map[string]string{ "entity": "entities", } -// Variable represents a variable used in the application. -// It contains information about the variable filepath, platform, name, hash, -// and whether it is from a vault or not. -type Variable struct { - filepath string - platform string - name string - hash uint64 - isVault bool -} - -// GetPath returns path to variable file. -func (v *Variable) GetPath() string { - return v.filepath -} - -// GetPlatform returns variable platform. -func (v *Variable) GetPlatform() string { - return v.platform -} - -// GetName returns variable name. -func (v *Variable) GetName() string { - return v.name -} - -// GetHash returns variable [Variable.hash] -func (v *Variable) GetHash() uint64 { - return v.hash -} - -// IsVault tells if variable from vault. -func (v *Variable) IsVault() bool { - return v.isVault -} - type resourceDependencies struct { Name string `yaml:"name"` IncludeRole struct { @@ -93,14 +53,18 @@ type resourceDependencies struct { // Inventory represents the inventory used in the application to search and collect resources and variable resources. type Inventory struct { // services - ResourcesCrawler *ResourcesCrawler + fc *FilesCrawler //internal - resourcesMap *OrderedMap[bool] + resourcesMap *OrderedMap[*Resource] requiredBy map[string]*OrderedMap[bool] dependsOn map[string]*OrderedMap[bool] topOrder []string + variablesCalculated bool + variableVariablesDependencyMap map[string]map[string]*VariableDependency + variableResourcesDependencyMap map[string]map[string][]string + // options sourceDir string } @@ -110,11 +74,13 @@ type Inventory struct { // the initialized Inventory or any error that occurred during initialization. func NewInventory(sourceDir string) (*Inventory, error) { inv := &Inventory{ - sourceDir: sourceDir, - ResourcesCrawler: NewResourcesCrawler(sourceDir), - resourcesMap: NewOrderedMap[bool](), - requiredBy: make(map[string]*OrderedMap[bool]), - dependsOn: make(map[string]*OrderedMap[bool]), + sourceDir: sourceDir, + fc: NewFilesCrawler(sourceDir), + resourcesMap: NewOrderedMap[*Resource](), + requiredBy: make(map[string]*OrderedMap[bool]), + dependsOn: make(map[string]*OrderedMap[bool]), + variableVariablesDependencyMap: make(map[string]map[string]*VariableDependency), + variableResourcesDependencyMap: make(map[string]map[string][]string), } err := inv.Init() @@ -133,26 +99,6 @@ func (i *Inventory) Init() error { return err } -// GetResourcesMap returns map of all resources found in source dir. -func (i *Inventory) GetResourcesMap() *OrderedMap[bool] { - return i.resourcesMap -} - -// GetResourcesOrder returns the order of resources in the inventory. -func (i *Inventory) GetResourcesOrder() []string { - return i.topOrder -} - -// GetRequiredByMap returns the required by map, which represents the `required by` dependencies between resources in the Inventory. -func (i *Inventory) GetRequiredByMap() map[string]*OrderedMap[bool] { - return i.requiredBy -} - -// GetDependsOnMap returns the map, which represents the 'depends on' dependencies between resources in the Inventory. -func (i *Inventory) GetDependsOnMap() map[string]*OrderedMap[bool] { - return i.dependsOn -} - func (i *Inventory) buildResourcesGraph() error { err := filepath.Walk(i.sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -183,7 +129,7 @@ func (i *Inventory) buildResourcesGraph() error { } resourceName := resource.GetName() - i.resourcesMap.Set(resourceName, true) + i.resourcesMap.Set(resourceName, resource) case "dependencies.yaml": resource := BuildResourceFromPath(relPath, i.sourceDir) if resource == nil { @@ -194,7 +140,9 @@ func (i *Inventory) buildResourcesGraph() error { } resourceName := resource.GetName() - i.resourcesMap.Set(resourceName, true) + if _, ok := i.resourcesMap.Get(resourceName); !ok { + i.resourcesMap.Set(resourceName, resource) + } data, errRead := os.ReadFile(filepath.Clean(path)) if errRead != nil { @@ -269,230 +217,24 @@ func (i *Inventory) buildResourcesGraph() error { return nil } -// GetChangedResources returns an OrderedResourceMap containing the resources that have been modified, based on the provided list of modified files. -// It iterates over the modified files, builds a resource from each file path, and adds it to the result map if it is not already present. -func (i *Inventory) GetChangedResources(modifiedFiles []string) *OrderedMap[*Resource] { - resources := NewOrderedMap[*Resource]() - for _, path := range modifiedFiles { - resource := BuildResourceFromPath(path, i.sourceDir) - if resource == nil { - continue - } - if _, ok := resources.Get(resource.GetName()); ok { - continue - } - resources.Set(resource.GetName(), resource) - } - - return resources +// GetResourcesMap returns map of all resources found in source dir. +func (i *Inventory) GetResourcesMap() *OrderedMap[*Resource] { + return i.resourcesMap } -// GetChangedVariables fetches variables file from list, compare them with comparison dir files and fetches changed vars. -func (i *Inventory) GetChangedVariables(modifiedFiles []string, comparisonDir, vaultpass string) (*OrderedMap[*Variable], *OrderedMap[*Variable], error) { - changedVariables := NewOrderedMap[*Variable]() - deletedVariables := NewOrderedMap[*Variable]() - for _, path := range modifiedFiles { - platform, kind, role, errPath := ProcessResourcePath(path) - if errPath != nil { - continue - } - - if platform == "" || kind == "" || role == "" { - continue - } - - if kind != "group_vars" { - continue - } - - sourcePath := filepath.Join(i.sourceDir, path) - artifactPath := filepath.Join(comparisonDir, path) - isVault := isVaultFile(path) - - _, err1 := os.Stat(sourcePath) - sourceFileExists := !os.IsNotExist(err1) - _, err2 := os.Stat(artifactPath) - artifactFileExists := !os.IsNotExist(err2) - - if sourceFileExists && artifactFileExists { - sourceData, err := LoadVariablesFile(sourcePath, vaultpass, isVault) - if err != nil { - return changedVariables, deletedVariables, err - } - - artifactData, err := LoadVariablesFile(artifactPath, vaultpass, isVault) - if err != nil { - return changedVariables, deletedVariables, err - } - - // Check for updated vars in existing files - for k, sv := range sourceData { - if av, ok := artifactData[k]; ok { - sourceValue := fmt.Sprint(sv) - artifactValue := fmt.Sprint(av) - - sourceHash := HashString(sourceValue) - artifactHash := HashString(artifactValue) - - if sourceHash != artifactHash { - changedVar := &Variable{ - filepath: path, - name: k, - hash: sourceHash, - platform: platform, - isVault: isVault, - } - changedVariables.Set(changedVar.GetName(), changedVar) - } - } else { - // Handle new variable. - sourceValue := fmt.Sprint(sv) - newVar := &Variable{ - filepath: path, - name: k, - hash: HashString(sourceValue), - platform: platform, - isVault: isVault, - } - - changedVariables.Set(newVar.GetName(), newVar) - } - } - - // Check for deleted vars in existing files - for k, av := range artifactData { - if _, ok := sourceData[k]; !ok { - artifactValue := fmt.Sprint(av) - deletedVar := &Variable{ - filepath: path, - name: k, - hash: HashString(artifactValue), - platform: platform, - isVault: isVault, - } - - deletedVariables.Set(deletedVar.GetName(), deletedVar) - } - } - - } else if sourceFileExists && !artifactFileExists { - // New vars file, add all variable to propagate. - sourceData, err := LoadVariablesFile(sourcePath, vaultpass, isVault) - if err != nil { - return changedVariables, deletedVariables, err - } - for k, sv := range sourceData { - sourceValue := fmt.Sprint(sv) - newVar := &Variable{ - filepath: path, - name: k, - hash: HashString(sourceValue), - platform: platform, - isVault: isVault, - } - - changedVariables.Set(newVar.GetName(), newVar) - } - - } else if !sourceFileExists && artifactFileExists { - // Vars file was deleted, find all removed variables. - artifactData, err := LoadVariablesFile(artifactPath, vaultpass, isVault) - if err != nil { - return changedVariables, deletedVariables, err - } - - for k, av := range artifactData { - artifactValue := fmt.Sprint(av) - deletedVar := &Variable{ - filepath: path, - name: k, - hash: HashString(artifactValue), - platform: platform, - isVault: isVault, - } - - deletedVariables.Set(deletedVar.GetName(), deletedVar) - } - } - } - - changedVariables.SortKeysAlphabetically() - deletedVariables.SortKeysAlphabetically() - return changedVariables, deletedVariables, nil +// GetResourcesOrder returns the order of resources in the inventory. +func (i *Inventory) GetResourcesOrder() []string { + return i.topOrder } -// SearchVariablesAffectedResources crawls inventory to find if variables were used in [Inventory.SourceDir] resources. -func (i *Inventory) SearchVariablesAffectedResources(variables []*Variable) (*OrderedMap[*Resource], map[string]map[string]bool, error) { - resources := NewOrderedMap[*Resource]() - resourceVariablesMap := make(map[string]map[string]bool) - - splitByPlatform := make(map[string]map[string]*Variable) - dependentVariables := make(map[string]map[string]bool) - - for _, v := range variables { - if _, ok := splitByPlatform[v.platform]; !ok { - splitByPlatform[v.platform] = make(map[string]*Variable) - } - splitByPlatform[v.platform][v.name] = v - descendents := make(map[string]*Variable) - err := i.crawlVariableUsage(v, descendents) - if err != nil { - return resources, resourceVariablesMap, err - } - - for _, d := range descendents { - if _, ok := splitByPlatform[d.platform]; !ok { - splitByPlatform[d.platform] = make(map[string]*Variable) - } - - splitByPlatform[d.platform][d.name] = d - - if _, ok := dependentVariables[d.name]; !ok { - dependentVariables[d.name] = make(map[string]bool) - } - - dependentVariables[d.name][v.name] = true - } - } - - i.pushRequiredVariables(dependentVariables) - - assetsMap := make(map[string]map[string]bool) - for platform, platformVars := range splitByPlatform { - err := i.ResourcesCrawler.SearchVariableResources(platform, platformVars, assetsMap) - if err != nil { - return resources, resourceVariablesMap, err - } - } - - for path, vars := range assetsMap { - resource := BuildResourceFromPath(path, i.sourceDir) - if resource == nil { - continue - } - if val, ok := resources.Get(resource.GetName()); !ok { - launchr.Log().Debug("Processing resource", "resource", resource.GetName()) - resources.Set(resource.GetName(), resource) - } else { - resource = val - } - - if _, ok := resourceVariablesMap[resource.GetName()]; !ok { - resourceVariablesMap[resource.GetName()] = make(map[string]bool) - } - - for variable := range vars { - if _, ok := dependentVariables[variable]; !ok { - resourceVariablesMap[resource.GetName()][variable] = true - } else { - for name := range dependentVariables[variable] { - resourceVariablesMap[resource.GetName()][name] = true - } - } - } - } +// GetRequiredByMap returns the required by map, which represents the `required by` dependencies between resources in the Inventory. +func (i *Inventory) GetRequiredByMap() map[string]*OrderedMap[bool] { + return i.requiredBy +} - return resources, resourceVariablesMap, nil +// GetDependsOnMap returns the map, which represents the 'depends on' dependencies between resources in the Inventory. +func (i *Inventory) GetDependsOnMap() map[string]*OrderedMap[bool] { + return i.dependsOn } // GetRequiredByResources returns list of resources which depend on argument resource (directly or not). @@ -530,441 +272,20 @@ func (i *Inventory) lookupDependenciesRecursively(resourceName string, resources } } -func (i *Inventory) pushRequiredVariables(requiredMap map[string]map[string]bool) { - for k, v := range requiredMap { - for name := range v { - if _, ok := requiredMap[name]; ok { - delete(v, name) - for nameParent := range requiredMap[name] { - requiredMap[k][nameParent] = true - } - i.pushRequiredVariables(requiredMap) - } - } - } -} - -func (i *Inventory) crawlVariableUsage(variable *Variable, dependents map[string]*Variable) error { - //@TODO crawl several variables - var files []string - var err error - - if variable.isVault || variable.platform != rootPlatform { - files, err = i.ResourcesCrawler.FindGroupVarsFiles(variable.platform) - if err != nil { - return err - } - } else if variable.platform == rootPlatform { - files, err = i.ResourcesCrawler.FindGroupVarsFiles("") - if err != nil { - return err - } - } - - //if i.variablesTree == nil { - // i.variablesTree = make(map[string]map[string]bool) - //} - if len(files) > 0 { - dependencies, err := i.ResourcesCrawler.SearchVariablesInGroupFiles(variable.name, files) - if err != nil { - return err - } - - for _, dep := range dependencies { - if _, ok := dependents[dep.name]; ok { - continue - } - dependents[dep.name] = dep - - //if _, ok := i.variablesTree[variable.name]; !ok { - // i.variablesTree[variable.name] = make(map[string]bool) - //} - //i.variablesTree[variable.name][dep.name] = true - - errUsage := i.crawlVariableUsage(dep, dependents) - if errUsage != nil { - return errUsage - } - } - } - - return nil -} - -// ResourcesCrawler is a type that represents a crawler for resources in a given directory. -type ResourcesCrawler struct { - taskSources map[string][]string - templateSources map[string][]string - defaultsSources map[string][]string - rootDir string -} - -// NewResourcesCrawler creates a new instance of ResourcesCrawler with initialized taskSources and templateSources maps. -func NewResourcesCrawler(directory string) *ResourcesCrawler { - return &ResourcesCrawler{ - taskSources: make(map[string][]string), - templateSources: make(map[string][]string), - defaultsSources: make(map[string][]string), - rootDir: directory, - } -} - -func (cr *ResourcesCrawler) findFilesByPattern(filenamePattern, kind, platform string, partsCount, platformPart, rolePart, kindPart int) ([]string, error) { - var files []string - dir := filepath.Join(cr.rootDir, platform) - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath := strings.TrimPrefix(path, cr.rootDir+"/") - if info.Mode()&os.ModeSymlink != 0 { - return nil - } - - if info.IsDir() || strings.Contains(path, "scripts") { - return nil - } - - match, err := filepath.Match(filenamePattern, filepath.Base(path)) - if err != nil { - return err - } - if !match { - return nil - } - - parts := strings.Split(relPath, "/") - if len(parts) >= partsCount && (platform == "" || parts[platformPart] == platform) && - (rolePart == 0 || parts[rolePart] == "roles") && parts[kindPart] == kind { - files = append(files, relPath) - } - - return nil - }) - - return files, err -} - -// FindTaskFiles returns a slice of file paths that match certain criteria for tasks on a specific platform. -func (cr *ResourcesCrawler) FindTaskFiles(platform string) ([]string, error) { - return cr.findFilesByPattern( - "*.yaml", - "tasks", - platform, - 4, - 0, - 2, - 4, - ) -} - -// FindDefaultFiles returns a slice of file paths that match certain criteria for defaults on a specific platform. -func (cr *ResourcesCrawler) FindDefaultFiles(platform string) ([]string, error) { - return cr.findFilesByPattern( - "*.yaml", - "defaults", - platform, - 4, - 0, - 2, - 4, - ) -} - -// FindTemplateFiles returns a slice of file paths that match certain criteria for templates on a specific platform. -func (cr *ResourcesCrawler) FindTemplateFiles(platform string) ([]string, error) { - return cr.findFilesByPattern( - "*.j2", - "templates", - platform, - 4, - 0, - 2, - 4, - ) -} - -// FindGroupVarsFiles returns a slice of file paths that match certain criteria for group vars on a specific platform. -func (cr *ResourcesCrawler) FindGroupVarsFiles(platform string) ([]string, error) { - return cr.findFilesByPattern( - "vars.yaml", - "group_vars", - platform, - 3, - 0, - 0, - 1, - ) -} - -// SearchVariablesInGroupFiles searches for variables in a group of files that match the specified name. -// It takes a name string and a slice of files. -// It returns a slice of Variable pointers that contain information about each found variable. -// The function iterates over each file in the files slice, read it and if the string representation -// of the value contains the specified name, it creates a Variable object and appends it to the variables slice. -// Finally, it returns the variables slice. -// Example usage: -// -// crawler := &ResourcesCrawler{} -// files := crawler.FindGroupVarsFiles("platform") -// variables := crawler.SearchVariablesInGroupFiles("variable", files) -func (cr *ResourcesCrawler) SearchVariablesInGroupFiles(name string, files []string) ([]*Variable, error) { - var variables []*Variable - regexName := regexp.QuoteMeta(name) - searchRegexp, err := regexp.Compile(fmt.Sprintf(variablePattern, regexName)) - if err != nil { - return variables, err - } - +// GetChangedResources returns an OrderedResourceMap containing the resources that have been modified, based on the provided list of modified files. +// It iterates over the modified files, builds a resource from each file path, and adds it to the result map if it is not already present. +func (i *Inventory) GetChangedResources(files []string) *OrderedMap[*Resource] { + resources := NewOrderedMap[*Resource]() for _, path := range files { - platform, _, _, errPath := ProcessResourcePath(path) - if errPath != nil { - return variables, errPath - } - - sourcePath := filepath.Clean(filepath.Join(cr.rootDir, path)) - sourceVariables, errRead := os.ReadFile(sourcePath) - if errRead != nil { - return variables, errRead - } - - var sourceData map[string]any - errMarshal := yaml.Unmarshal(sourceVariables, &sourceData) - if errMarshal != nil { - if !strings.Contains(errMarshal.Error(), "already defined at line") { - return variables, errMarshal - } - - sourceData, errMarshal = UnmarshallFixDuplicates(sourceVariables) - if err != nil { - return variables, errMarshal - } - } - - for k, v := range sourceData { - sourceValue := fmt.Sprint(v) - if k == name { - continue - } - - if searchRegexp.MatchString(sourceValue) { - variables = append(variables, &Variable{ - filepath: path, - name: k, - hash: HashString(sourceValue), - isVault: isVaultFile(path), - platform: platform, - }) - } - } - } - - return variables, nil -} - -// SearchVariableResources searches for variable resources on a specific platform using the provided names and resources map. -// If the platform is rootPlatform, an empty string is used as the search platform. -// It retrieves task and template files for the platform if they are not already stored in the taskSources and templateSources maps of the ResourcesCrawler. -// It scans each file and checks if the variables in the names map are present in the resources map for that file. If a variable is missing, it adds it to the toIterate slice. -// It then scans each line of the file and checks if it contains any of the variables in the toIterate slice. If a variable is found, it adds it to the foundList map. -// After scanning all the files, it updates the resources map with the variables found in each file. -func (cr *ResourcesCrawler) SearchVariableResources(platform string, variables map[string]*Variable, resources map[string]map[string]bool) error { - var err error - searchPlatform := platform - if searchPlatform == rootPlatform { - searchPlatform = "" - } - - if _, ok := cr.taskSources[platform]; !ok { - cr.taskSources[platform], err = cr.FindTaskFiles(searchPlatform) - if err != nil { - return err - } - } - - if _, ok := cr.templateSources[platform]; !ok { - cr.templateSources[platform], err = cr.FindTemplateFiles(searchPlatform) - if err != nil { - return err - } - } - - if _, ok := cr.defaultsSources[platform]; !ok { - cr.defaultsSources[platform], err = cr.FindDefaultFiles(searchPlatform) - if err != nil { - return err - } - } - - files := append(cr.taskSources[platform], cr.templateSources[platform]...) - files = append(files, cr.defaultsSources[platform]...) - - regExpressions := make(map[string]*regexp.Regexp) - - for name := range variables { - regexName := regexp.QuoteMeta(name) - searchRegexp, err := regexp.Compile(fmt.Sprintf(variablePattern, regexName)) - if err != nil { - return err - } - regExpressions[name] = searchRegexp - } - - for _, file := range files { - sourcePath := filepath.Clean(filepath.Join(cr.rootDir, file)) - f, err := os.Open(sourcePath) - if err != nil { - launchr.Log().Debug("error opening file for variables usage", "file", file, "msg", err) - return err - } - - s := bufio.NewScanner(f) - - var toIterate []string - for name := range variables { - if _, ok := resources[file][name]; !ok { - toIterate = append(toIterate, name) - } - } - - if len(toIterate) == 0 { + resource := BuildResourceFromPath(path, i.sourceDir) + if resource == nil { continue } - - foundList := make(map[string]bool) - for s.Scan() { - for _, name := range toIterate { - if _, ok := foundList[name]; ok { - continue - } - - regExpressions[name].MatchString(s.Text()) - if regExpressions[name].MatchString(s.Text()) { - foundList[name] = true - } - - if len(foundList) == len(toIterate) { - break - } - } - } - errClose := f.Close() - if errClose != nil { - return errClose - } - - if err = s.Err(); err != nil { - launchr.Log().Debug("Error reading file: msg", "file", file, "msg", err) + if _, ok := resources.Get(resource.GetName()); ok { continue } - - for n := range foundList { - if resources[file] == nil { - resources[file] = make(map[string]bool) - } - resources[file][n] = true - } - } - - return nil -} - -func isVaultFile(path string) bool { - return filepath.Base(path) == "vault.yaml" -} - -// LoadVariablesFile loads vars yaml file from path. -func LoadVariablesFile(path, vaultPassword string, isVault bool) (map[string]any, error) { - var data map[string]any - var rawData []byte - var err error - - cleanPath := filepath.Clean(path) - if isVault { - sourceVault, errDecrypt := vault.DecryptFile(cleanPath, vaultPassword) - if errDecrypt != nil { - if errors.Is(errDecrypt, vault.ErrEmptyPassword) { - return data, fmt.Errorf("error decrypting vault %s, password is blank", cleanPath) - } else if errors.Is(errDecrypt, vault.ErrInvalidFormat) { - return data, fmt.Errorf("error decrypting vault %s, invalid secret format", cleanPath) - } else if errDecrypt.Error() == invalidPasswordErrText { - return data, fmt.Errorf("invalid password for vault '%s'", cleanPath) - } - - return data, errDecrypt - } - rawData = []byte(sourceVault) - } else { - rawData, err = os.ReadFile(cleanPath) - if err != nil { - return data, err - } - } - - err = yaml.Unmarshal(rawData, &data) - if err != nil { - if !strings.Contains(err.Error(), "already defined at line") { - return data, err - } - - launchr.Log().Warn("duplicate found, parsing YAML file manually") - data, err = UnmarshallFixDuplicates(rawData) - if err != nil { - return data, err - } - } - - return data, err -} - -// LoadVariablesFileFromBytes loads vars yaml file from bytes input. -func LoadVariablesFileFromBytes(input []byte, vaultPassword string, isVault bool) (map[string]any, error) { - var data map[string]any - var rawData []byte - var err error - - if isVault { - sourceVault, errDecrypt := vault.Decrypt(string(input), vaultPassword) - if errDecrypt != nil { - if errors.Is(errDecrypt, vault.ErrEmptyPassword) { - return data, fmt.Errorf("error decrypting vaults, password is blank") - } else if errors.Is(errDecrypt, vault.ErrInvalidFormat) { - return data, fmt.Errorf("error decrypting vault, invalid secret format") - } else if errDecrypt.Error() == invalidPasswordErrText { - return data, fmt.Errorf("invalid password for vault") - } - - return data, errDecrypt - } - rawData = []byte(sourceVault) - } else { - rawData = input - } - - err = yaml.Unmarshal(rawData, &data) - if err != nil { - if !strings.Contains(err.Error(), "already defined at line") { - return data, err - } - - launchr.Log().Warn("duplicate found, parsing YAML file manually") - data, err = UnmarshallFixDuplicates(rawData) - if err != nil { - return data, err - } + resources.Set(resource.GetName(), resource) } - return data, err -} - -// LoadYamlFileFromBytes loads yaml file from bytes input. -func LoadYamlFileFromBytes(input []byte) (map[string]any, error) { - var data map[string]any - var rawData []byte - var err error - - rawData = input - err = yaml.Unmarshal(rawData, &data) - return data, err + return resources } diff --git a/pkg/sync/resource.go b/pkg/sync/inventory.resource.go similarity index 98% rename from pkg/sync/resource.go rename to pkg/sync/inventory.resource.go index 2c5b701..8b08b0b 100644 --- a/pkg/sync/resource.go +++ b/pkg/sync/inventory.resource.go @@ -91,7 +91,7 @@ func (r *Resource) GetVersion() (string, error) { version := GetMetaVersion(meta) if version == "" { - launchr.Log().Warn("Empty meta file, return empty string as version") + launchr.Log().Warn(fmt.Sprintf("Empty meta file %s version, return empty string as version", metaFile)) } return version, nil diff --git a/pkg/sync/inventory.variable.go b/pkg/sync/inventory.variable.go new file mode 100644 index 0000000..189fd4e --- /dev/null +++ b/pkg/sync/inventory.variable.go @@ -0,0 +1,647 @@ +package sync + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +// IsVaultFile is helper to determine if file is vault file. +func IsVaultFile(path string) bool { + return filepath.Base(path) == vaultFile +} + +// Variable represents a variable used in the application. +type Variable struct { + filepath string + name string + platform string + hash uint64 + isVault bool +} + +// VariableDependency stores variable name, platform and reference to dependent vars. +type VariableDependency struct { + Name string + Platform string + Dependent map[string]map[string]*VariableDependency +} + +// NewVariableDependency creates new instance of [VariableDependency] +func NewVariableDependency(name, platform string) *VariableDependency { + return &VariableDependency{ + Name: name, + Platform: platform, + Dependent: make(map[string]map[string]*VariableDependency), + } +} + +// SetDependent adds new dependent variable to variable. +func (v *VariableDependency) SetDependent(d *VariableDependency) { + if v.Dependent[d.Name] == nil { + v.Dependent[d.Name] = make(map[string]*VariableDependency) + } + + v.Dependent[d.Name][d.Platform] = d +} + +// GatherDependentKeys recursively gathers dependent keys and stores them in the result map. +func GatherDependentKeys(vd *VariableDependency, result map[string]map[string]bool) { + if vd == nil { + return + } + + for name, deps := range vd.Dependent { + if _, exists := result[name]; !exists { + result[name] = make(map[string]bool) + } + + for platform, dependency := range deps { + result[name][platform] = true + + GatherDependentKeys(dependency, result) + } + } +} + +// NewVariable returns instance of [Variable] struct. +func NewVariable(filepath, name string, hash uint64, isVault bool) *Variable { + // @todo + parts := strings.Split(filepath, "/") + return &Variable{filepath, name, parts[0], hash, isVault} +} + +// GetPath returns path to variable file. +func (v *Variable) GetPath() string { + return v.filepath +} + +// GetName returns variable name. +func (v *Variable) GetName() string { + return v.name +} + +// GetPlatform returns variable platform. +func (v *Variable) GetPlatform() string { + return v.platform +} + +// GetHash returns variable [Variable.hash] +func (v *Variable) GetHash() uint64 { + return v.hash +} + +// IsVault tells if variable from vault. +func (v *Variable) IsVault() bool { + return v.isVault +} + +// GetVariableResources returns list of resources which depends on variable. +func (i *Inventory) GetVariableResources(variableName, variablePlatform string) ([]string, error) { + var result []string + + if !i.variablesCalculated { + panic("use inventory.CalculateVariablesUsage first") + } + + variablesList := make(map[string]map[string]bool) + i.getVariableVariables(variableName, variablePlatform, variablesList) + + if variablesList[variableName] == nil { + variablesList[variableName] = make(map[string]bool) + } + variablesList[variableName][variablePlatform] = true + + for v, m := range variablesList { + if _, ok := i.variableResourcesDependencyMap[v]; !ok { + continue + } + + for p := range m { + items, ok := i.variableResourcesDependencyMap[v][p] + if !ok { + continue + } + + result = append(result, items...) + } + } + + return result, nil +} + +// GetVariableResources returns list of variables which depends on variable. +func (i *Inventory) getVariableVariables(variableName, variablePlatform string, result map[string]map[string]bool) { + if p, ok := i.variableVariablesDependencyMap[variableName]; ok { + if v, okP := p[variablePlatform]; okP { + GatherDependentKeys(v, result) + } + } +} + +// CalculateVariablesUsage precalculates all variables dependencies across platform. +func (i *Inventory) CalculateVariablesUsage(vaultpass string) error { + // Find all variables files + // Find all variables + // Determine from variables values list of potential variables which may use other variables + // Build variables usage map using previous list + // - all variables from the same playbook can be used only on that playbook + // - variables from platform playbook can be used in others + keys, vars, err := i.buildVarsGroups(vaultpass) + if err != nil { + return err + } + + variableVariablesDependencyMap := i.buildVariableDependencies(keys, vars) + + // Find all resources related template (templates/*.j2) and config (tasks/configuration.yaml) files. split by playbook + // Iterate all files to find {{ and/or }}, get these lines + // iterate potential lines with vars usage and check each variable in it. + + variableResourcesDependencyMap, err := i.buildVariableResourcesDependencies(keys, false) + if err != nil { + return err + } + + i.variableVariablesDependencyMap = variableVariablesDependencyMap + i.variableResourcesDependencyMap = variableResourcesDependencyMap + + i.variablesCalculated = true + + return nil +} + +func (i *Inventory) buildVarsGroups(vaultPass string) (map[string]map[string]bool, map[string]map[string]string, error) { + groups, err := i.fc.FindVarsFiles("") + if err != nil { + return nil, nil, err + } + + // Output maps + groupKeys := make(map[string]map[string]bool) // group -> keys + groupVars := make(map[string]map[string]string) // group -> key -> string that contains {{ or }} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + var mx sync.Mutex + + maxWorkers := min(runtime.NumCPU(), len(groups)) + groupChan := make(chan string, len(groups)) + errorChan := make(chan error, 1) + + for w := 0; w < maxWorkers; w++ { + go func(workerID int) { + for { + select { + case <-ctx.Done(): + return + case group, ok := <-groupChan: + if !ok { + return + } + if err = i.processGroup(ctx, vaultPass, group, groups[group], groupKeys, groupVars, &mx); err != nil { + select { + case errorChan <- fmt.Errorf("worker %d error processing %s: %w", workerID, group, err): + cancel() + default: + } + return + } + wg.Done() + } + } + }(w) + } + + for group := range groups { + wg.Add(1) + groupChan <- group + } + close(groupChan) + + go func() { + wg.Wait() + close(errorChan) + }() + + for err = range errorChan { + if err != nil { + return nil, nil, err + } + } + + return groupKeys, groupVars, nil +} + +func (i *Inventory) buildVariableDependencies(groupKeys map[string]map[string]bool, groupVars map[string]map[string]string) map[string]map[string]*VariableDependency { + // Map to store the result: key -> map of keys that use it + reverseDependencyMap := make(map[string]map[string]*VariableDependency) + + var mx sync.Mutex + var wg sync.WaitGroup + for group, vars := range groupVars { + wg.Add(1) + go func(group string, vars map[string]string) { + defer wg.Done() + processGroupDependencies(group, vars, groupKeys, reverseDependencyMap, &mx) + }(group, vars) + } + + wg.Wait() + + return reverseDependencyMap +} + +// Helper function to process each group's dependencies +func processGroupDependencies(group string, vars map[string]string, groupKeys map[string]map[string]bool, reverseDependencyMap map[string]map[string]*VariableDependency, mx *sync.Mutex) { + currentGroupKeys := groupKeys[group] + platformKeys := groupKeys[rootPlatform] + + for varKey, varValue := range vars { + deps := findDependencies(group, varValue, currentGroupKeys, platformKeys) + if len(deps) == 0 { + continue + } + + mx.Lock() + + if reverseDependencyMap[varKey] == nil { + reverseDependencyMap[varKey] = make(map[string]*VariableDependency) + } + + v, okV := reverseDependencyMap[varKey][group] + if !okV { + v = NewVariableDependency(varKey, group) + reverseDependencyMap[varKey][group] = v + } + + for dep, depGr := range deps { + if reverseDependencyMap[dep] == nil { + reverseDependencyMap[dep] = make(map[string]*VariableDependency) + } + + depV, okD := reverseDependencyMap[dep][depGr] + if !okD { + depV = NewVariableDependency(dep, depGr) + reverseDependencyMap[dep][depGr] = depV + } + depV.SetDependent(v) + } + mx.Unlock() + } +} + +// Helper function to find dependencies in a string +func findDependencies(group, value string, currentGroupKeys, platformKeys map[string]bool) map[string]string { + dependencies := make(map[string]string) + + // Check current group keys + for key := range currentGroupKeys { + if strings.Contains(value, " "+key+" ") { + if _, ok := dependencies[key]; !ok { + dependencies[key] = group + } + } + } + + if group == rootPlatform { + return dependencies + } + + // Check platform keys + for key := range platformKeys { + if strings.Contains(value, " "+key+" ") { + if _, ok := dependencies[key]; !ok { + dependencies[key] = rootPlatform + } + } + } + + return dependencies +} + +// Helper function to convert map[string]bool to []string +//func getMapKeys(m map[string]bool) []string { +// keys := make([]string, 0, len(m)) +// for key := range m { +// keys = append(keys, key) +// } +// return keys +//} + +func (i *Inventory) processGroup(ctx context.Context, vaultPass, group string, files []string, groupKeys map[string]map[string]bool, groupVars map[string]map[string]string, mx *sync.Mutex) error { + mx.Lock() + if _, exists := groupKeys[group]; !exists { + groupKeys[group] = make(map[string]bool) + } + if _, exists := groupVars[group]; !exists { + groupVars[group] = make(map[string]string) + } + mx.Unlock() + + const fileWorkers = 2 + var wg sync.WaitGroup + fileChan := make(chan string, len(files)) + errChan := make(chan error, 1) + + for w := 0; w < fileWorkers; w++ { + go func() { + for { + select { + case <-ctx.Done(): + return + case file, ok := <-fileChan: + if !ok { + return + } + if err := i.processFile(file, group, vaultPass, groupKeys, groupVars, mx); err != nil { + select { + case errChan <- err: + default: + } + } + wg.Done() + } + } + }() + } + + for _, file := range files { + wg.Add(1) + fileChan <- file + } + close(fileChan) + + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + if err != nil { + return err + } + } + + return nil +} + +func (i *Inventory) processFile(file, group, vaultPass string, groupKeys map[string]map[string]bool, groupVars map[string]map[string]string, mx *sync.Mutex) error { + data, err := LoadVariablesFile(filepath.Join(i.sourceDir, file), vaultPass, IsVaultFile(file)) + if err != nil { + return err + } + + i.extractKeysAndVars(data, group, groupKeys, groupVars, "", 0, mx) + return nil +} + +func (i *Inventory) extractKeysAndVars(data interface{}, group string, groupKeys map[string]map[string]bool, groupVars map[string]map[string]string, currentKey string, level int, mx *sync.Mutex) { + switch v := data.(type) { + case map[string]interface{}: + for key, value := range v { + if level == 0 { + fullKey := key + if currentKey != "" { + fullKey = currentKey + //fullKey = currentKey + "." + key // Nested keys + } + + mx.Lock() + if _, exists := groupKeys[group]; !exists { + groupKeys[group] = make(map[string]bool) + } + groupKeys[group][key] = true + mx.Unlock() + + // Recurse for nested structures + i.extractKeysAndVars(value, group, groupKeys, groupVars, fullKey, level+1, mx) + } else { + mapStr := fmt.Sprintf("%v", v) + mx.Lock() + if strings.Contains(mapStr, "{{") || strings.Contains(mapStr, "}}") { + if _, exists := groupVars[group]; !exists { + groupVars[group] = make(map[string]string) + } + groupVars[group][currentKey] = mapStr + } + mx.Unlock() + } + + } + case []interface{}: + listStr := fmt.Sprintf("%v", v) + mx.Lock() + if strings.Contains(listStr, "{{") || strings.Contains(listStr, "}}") { + if _, exists := groupVars[group]; !exists { + groupVars[group] = make(map[string]string) + } + groupVars[group][currentKey] = listStr + } + mx.Unlock() + + // Recurse into list items + //for _, item := range v { + // if test { + // panic(currentKey) + // } + // i.extractKeysAndVars(item, group, groupKeys, groupVars, currentKey, mx) + //} + case string: + if strings.Contains(v, "{{") || strings.Contains(v, "}}") { + mx.Lock() + if _, exists := groupVars[group]; !exists { + groupVars[group] = make(map[string]string) + } + groupVars[group][currentKey] = v + mx.Unlock() + } + } +} + +func (i *Inventory) buildVariableResourcesDependencies(groupKeys map[string]map[string]bool, filesOnly bool) (map[string]map[string][]string, error) { + groupFiles, err := i.fc.FindResourcesFiles("") + if err != nil { + return nil, err + } + + reverseDependencyMap := make(map[string]map[string][]string) + + errChan := make(chan error, 1) + var wg sync.WaitGroup + var mx sync.Mutex + for group, files := range groupFiles { + wg.Add(1) + go func(group string, files []string) { + defer wg.Done() + if err = i.processGroupFiles(group, files, groupKeys, reverseDependencyMap, &mx); err != nil { + errChan <- fmt.Errorf("group %s: %w", group, err) + } + }(group, files) + } + + go func() { + wg.Wait() + close(errChan) + }() + + for err = range errChan { + if err != nil { + return nil, err + } + } + + if filesOnly { + return reverseDependencyMap, nil + } + + varToResourcesDependencyMap := make(map[string]map[string][]string) + for v, pl := range reverseDependencyMap { + for p, files := range pl { + var res []string + for _, path := range files { + platform, kind, role, err := ProcessResourcePath(path) + if err != nil || (platform == "" || kind == "" || role == "") || !IsUpdatableKind(kind) { + continue + } + + resourceName := PrepareMachineResourceName(platform, kind, role) + res = append(res, resourceName) + } + if varToResourcesDependencyMap[v] == nil { + varToResourcesDependencyMap[v] = make(map[string][]string) + } + + varToResourcesDependencyMap[v][p] = res + } + } + + return varToResourcesDependencyMap, nil +} + +func (i *Inventory) processGroupFiles(group string, files []string, groupKeys map[string]map[string]bool, reverseDependencyMap map[string]map[string][]string, mx *sync.Mutex) error { + // Get keys for the current group and the platform group + currentGroupKeys := groupKeys[group] + platformKeys := groupKeys[rootPlatform] + + var keysToCheck map[string]bool + if group == rootPlatform { + keysToCheck = platformKeys + } else { + keysToCheck = combineKeys(currentGroupKeys, platformKeys) + } + + // Extract relevant lines from all files for the current group + linesWithVariablesByFile := make(map[string][]string) + for _, filePath := range files { + lines, err := extractLinesWithVariables(filepath.Join(i.sourceDir, filePath)) + if err != nil { + return fmt.Errorf("failed to process file %s: %w", filePath, err) + } + linesWithVariablesByFile[filePath] = lines + } + + // Iterate over each key to find its usage in the relevant lines + for key := range keysToCheck { + keyGroup := group + if group != rootPlatform { + // in case if key doesn't exist in target group, but exists in platform + // assign all group related resources to platform + okP := platformKeys[key] + okG := currentGroupKeys[key] + if okP && !okG { + keyGroup = rootPlatform + } + } + + for filePath, lines := range linesWithVariablesByFile { + // Check if the file path prefix has been processed for the key + + filePrefix := getPathPrefix(filePath, 4) + mx.Lock() + isProcessed := isProcessedFile(keyGroup, filePrefix, reverseDependencyMap[key]) + mx.Unlock() + if isProcessed { + continue + } + + // Check if any of the lines contain the key + for _, line := range lines { + if strings.Contains(line, key) { + mx.Lock() + if _, ok := reverseDependencyMap[key]; !ok { + reverseDependencyMap[key] = make(map[string][]string) + } + + reverseDependencyMap[key][keyGroup] = append(reverseDependencyMap[key][keyGroup], filePath) + mx.Unlock() + break + } + } + } + } + return nil +} + +func combineKeys(current, platform map[string]bool) map[string]bool { + combined := make(map[string]bool) + for k := range current { + combined[k] = true + } + for k := range platform { + combined[k] = true + } + return combined +} + +func isProcessedFile(key, filePrefix string, reverseDependencyMap map[string][]string) bool { + if filePaths, exists := reverseDependencyMap[key]; exists { + for _, path := range filePaths { + if getPathPrefix(path, 4) == filePrefix { + return true + } + } + } + return false +} + +func getPathPrefix(filePath string, parts int) string { + pathParts := strings.Split(filePath, string(filepath.Separator)) + if len(pathParts) > parts { + pathParts = pathParts[:parts] + } + return strings.Join(pathParts, string(filepath.Separator)) +} + +func extractLinesWithVariables(filePath string) ([]string, error) { + file, err := os.Open(filepath.Clean(filePath)) + if err != nil { + return nil, fmt.Errorf("error opening file %s: %w", filePath, err) + } + + defer file.Close() + + var linesWithVariables []string + scanner := bufio.NewScanner(file) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 && !strings.HasPrefix(line, "#") && strings.Contains(line, "{{") || strings.Contains(line, "}}") { + linesWithVariables = append(linesWithVariables, line) + } + } + if err = scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading file %s: %w", filePath, err) + } + + return linesWithVariables, nil +} diff --git a/pkg/sync/timeline.go b/pkg/sync/timeline.go index 26804f7..aee0dce 100644 --- a/pkg/sync/timeline.go +++ b/pkg/sync/timeline.go @@ -195,7 +195,50 @@ func AddToTimeline(list []TimelineItem, item TimelineItem) []TimelineItem { // SortTimeline sorts timeline items in slice. func SortTimeline(list []TimelineItem) { sort.Slice(list, func(i, j int) bool { - return list[i].GetDate().Before(list[j].GetDate()) + dateI := list[i].GetDate() + dateJ := list[j].GetDate() + + // First, compare by date + if !dateI.Equal(dateJ) { + return dateI.Before(dateJ) + } + + // If dates are the same, prioritize by type + switch list[i].(type) { + case *TimelineVariablesItem: + switch list[j].(type) { + case *TimelineVariablesItem: + // Both are Variables, maintain current order + return false + case *TimelineResourcesItem: + // Variables come before Resources + return true + default: + // Variables come before unknown types + return true + } + case *TimelineResourcesItem: + switch list[j].(type) { + case *TimelineVariablesItem: + // Resources come after Variables + return false + case *TimelineResourcesItem: + // Both are Resources, maintain current order + return false + default: + // Resources come before unknown types + return true + } + default: + switch list[j].(type) { + case *TimelineVariablesItem, *TimelineResourcesItem: + // Unknown types come after Variables and Resources + return false + default: + // Maintain current order for unknown types + return false + } + } }) } diff --git a/pkg/sync/yaml.go b/pkg/sync/yaml.go index a665fc8..cbfbc6a 100644 --- a/pkg/sync/yaml.go +++ b/pkg/sync/yaml.go @@ -2,12 +2,112 @@ package sync import ( "bytes" + "errors" "fmt" + "os" + "path/filepath" + "strings" "github.com/launchrctl/launchr" + vault "github.com/sosedoff/ansible-vault-go" "gopkg.in/yaml.v3" ) +// LoadVariablesFile loads vars yaml file from path. +func LoadVariablesFile(path, vaultPassword string, isVault bool) (map[string]any, error) { + var data map[string]any + var rawData []byte + var err error + + cleanPath := filepath.Clean(path) + if isVault { + sourceVault, errDecrypt := vault.DecryptFile(cleanPath, vaultPassword) + if errDecrypt != nil { + if errors.Is(errDecrypt, vault.ErrEmptyPassword) { + return data, fmt.Errorf("error decrypting vault %s, password is blank", cleanPath) + } else if errors.Is(errDecrypt, vault.ErrInvalidFormat) { + return data, fmt.Errorf("error decrypting vault %s, invalid secret format", cleanPath) + } else if errDecrypt.Error() == invalidPasswordErrText { + return data, fmt.Errorf("invalid password for vault '%s'", cleanPath) + } + + return data, errDecrypt + } + rawData = []byte(sourceVault) + } else { + rawData, err = os.ReadFile(cleanPath) + if err != nil { + return data, err + } + } + + err = yaml.Unmarshal(rawData, &data) + if err != nil { + if !strings.Contains(err.Error(), "already defined at line") { + return data, err + } + + launchr.Log().Debug("duplicate found, parsing YAML file manually") + data, err = UnmarshallFixDuplicates(rawData) + if err != nil { + return data, err + } + } + + return data, err +} + +// LoadVariablesFileFromBytes loads vars yaml file from bytes input. +func LoadVariablesFileFromBytes(input []byte, vaultPassword string, isVault bool) (map[string]any, error) { + var data map[string]any + var rawData []byte + var err error + + if isVault { + sourceVault, errDecrypt := vault.Decrypt(string(input), vaultPassword) + if errDecrypt != nil { + if errors.Is(errDecrypt, vault.ErrEmptyPassword) { + return data, fmt.Errorf("error decrypting vaults, password is blank") + } else if errors.Is(errDecrypt, vault.ErrInvalidFormat) { + return data, fmt.Errorf("error decrypting vault, invalid secret format") + } else if errDecrypt.Error() == invalidPasswordErrText { + return data, fmt.Errorf("invalid password for vault") + } + + return data, errDecrypt + } + rawData = []byte(sourceVault) + } else { + rawData = input + } + + err = yaml.Unmarshal(rawData, &data) + if err != nil { + if !strings.Contains(err.Error(), "already defined at line") { + return data, err + } + + launchr.Log().Debug("duplicate found, parsing YAML file manually") + data, err = UnmarshallFixDuplicates(rawData) + if err != nil { + return data, err + } + } + + return data, err +} + +// LoadYamlFileFromBytes loads yaml file from bytes input. +func LoadYamlFileFromBytes(input []byte) (map[string]any, error) { + var data map[string]any + var rawData []byte + var err error + + rawData = input + err = yaml.Unmarshal(rawData, &data) + return data, err +} + // UnmarshallFixDuplicates handles duplicated values in yaml instead throwing error. func UnmarshallFixDuplicates(data []byte) (map[string]any, error) { reader := bytes.NewReader(data) diff --git a/plugin.go b/plugin.go index 37efd9f..cf5eab7 100644 --- a/plugin.go +++ b/plugin.go @@ -5,8 +5,6 @@ import ( "github.com/launchrctl/keyring" "github.com/launchrctl/launchr" "github.com/spf13/cobra" - - "github.com/skilld-labs/plasmactl-bump/v2/pkg/sync" ) func init() { @@ -37,10 +35,7 @@ func (p *Plugin) OnAppInit(app launchr.App) error { func (p *Plugin) CobraAddCommands(rootCmd *cobra.Command) error { var doSync bool var dryRun bool - var listImpacted bool - var override string - var username string - var password string + var allowOverride bool var vaultpass string var last bool @@ -51,6 +46,11 @@ func (p *Plugin) CobraAddCommands(rootCmd *cobra.Command) error { // Don't show usage help on a runtime error. cmd.SilenceUsage = true + verboseCount, err := rootCmd.Flags().GetCount("verbose") + if err != nil { + return err + } + if !doSync { bumpAction := BumpAction{last: last, dryRun: dryRun} return bumpAction.Execute() @@ -59,42 +59,31 @@ func (p *Plugin) CobraAddCommands(rootCmd *cobra.Command) error { syncAction := SyncAction{ keyring: p.k, - domainDir: ".", - buildDir: ".compose/build", - comparisonDir: ".compose/comparison-artifact", - packagesDir: ".compose/packages", - artifactsDir: ".compose/artifacts", - artifactsRepoURL: "https://repositories.skilld.cloud", - - dryRun: dryRun, - listImpacted: listImpacted, - vaultPass: vaultpass, - artifactOverride: truncateOverride(override), + domainDir: ".", + buildDir: ".compose/build", + packagesDir: ".compose/packages", + + dryRun: dryRun, + allowOverride: allowOverride, + vaultPass: vaultpass, + verbosity: verboseCount, + } + + err = syncAction.Execute() + if err != nil { + return err } - return syncAction.Execute(username, password) + return nil }, } bumpCmd.Flags().BoolVarP(&doSync, "sync", "s", false, "Propagate versions of updated components to their dependencies") bumpCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Simulate bump or sync without updating anything") - bumpCmd.Flags().BoolVar(&listImpacted, "list-impacted", false, "Print list of impacted resources") - bumpCmd.Flags().StringVar(&override, "override", "", "Override comparison artifact name (commit)") - bumpCmd.Flags().StringVar(&username, "username", "", "Username for artifact repository") - bumpCmd.Flags().StringVar(&password, "password", "", "Password for artifact repository") + bumpCmd.Flags().BoolVar(&allowOverride, "allow-override", false, "Allow override committed version by current build value") bumpCmd.Flags().StringVar(&vaultpass, "vault-pass", "", "Password for Ansible Vault") bumpCmd.Flags().BoolVarP(&last, "last", "l", false, "Bump resources modified in last commit only") rootCmd.AddCommand(bumpCmd) return nil } - -func truncateOverride(override string) string { - truncateLength := sync.ArtifactTruncateLength - - if len(override) > truncateLength { - launchr.Term().Info().Printfln("Truncated override value to %d chars: %s", truncateLength, override) - return override[:truncateLength] - } - return override -}