From d3b9111729706abe100a02767a53140a117e6688 Mon Sep 17 00:00:00 2001 From: Igor Ignatyev <106526746+iignatevich@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:19:00 +0300 Subject: [PATCH] handle yaml parse error (duplicated key) (#52) --- actionSync.go | 4 ++ pkg/sync/inventory.go | 34 +++++++++++++- pkg/sync/yaml.go | 102 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 pkg/sync/yaml.go diff --git a/actionSync.go b/actionSync.go index 653f45c..132ec53 100644 --- a/actionSync.go +++ b/actionSync.go @@ -372,6 +372,8 @@ func (s *SyncAction) ensureResourceIsVersioned(resourceVersion, resourceMetaPath //} func (s *SyncAction) findVariableUpdateTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) { + //launchr.Log().Debug("find variable update: var, path", "var", variable.GetName(), "path", variable.GetPath()) + // @TODO look for several vars during iteration? ref, err := s.ensureVariableIsVersioned(variable, repo) if err != nil { @@ -479,6 +481,8 @@ func (s *SyncAction) ensureVariableIsVersioned(variable *sync.Variable, repo *gi } func (s *SyncAction) findVariableDeletionTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) { + //launchr.Log().Debug("find variable delete: var, path", "var", variable.GetName(), "path", variable.GetPath()) + // @TODO look for several vars during iteration? // @TODO ensure variable existed at first place, before starting to search. diff --git a/pkg/sync/inventory.go b/pkg/sync/inventory.go index 8e77cf3..647a0de 100644 --- a/pkg/sync/inventory.go +++ b/pkg/sync/inventory.go @@ -506,6 +506,7 @@ func (i *Inventory) pushRequiredVariables(requiredMap map[string]map[string]bool } func (i *Inventory) crawlVariableUsage(variable *Variable, dependents map[string]*Variable) error { + //@TODO crawl several variables var files []string var err error @@ -692,7 +693,14 @@ func (cr *ResourcesCrawler) SearchVariablesInGroupFiles(name string, files []str var sourceData map[string]any errMarshal := yaml.Unmarshal(sourceVariables, &sourceData) if errMarshal != nil { - return variables, errMarshal + if !strings.Contains(errMarshal.Error(), "already defined at line") { + return variables, errMarshal + } + + sourceData, errMarshal = UnmarshallFixDuplicates(sourceVariables) + if err != nil { + return variables, errMarshal + } } for k, v := range sourceData { @@ -856,6 +864,18 @@ func LoadVariablesFile(path, vaultPassword string, isVault bool) (map[string]any } err = yaml.Unmarshal(rawData, &data) + if err != nil { + if !strings.Contains(err.Error(), "already defined at line") { + return data, err + } + + launchr.Log().Warn("duplicate found, parsing YAML file manually") + data, err = UnmarshallFixDuplicates(rawData) + if err != nil { + return data, err + } + } + return data, err } @@ -884,6 +904,18 @@ func LoadVariablesFileFromBytes(input []byte, vaultPassword string, isVault bool } err = yaml.Unmarshal(rawData, &data) + if err != nil { + if !strings.Contains(err.Error(), "already defined at line") { + return data, err + } + + launchr.Log().Warn("duplicate found, parsing YAML file manually") + data, err = UnmarshallFixDuplicates(rawData) + if err != nil { + return data, err + } + } + return data, err } diff --git a/pkg/sync/yaml.go b/pkg/sync/yaml.go new file mode 100644 index 0000000..a665fc8 --- /dev/null +++ b/pkg/sync/yaml.go @@ -0,0 +1,102 @@ +package sync + +import ( + "bytes" + "fmt" + + "github.com/launchrctl/launchr" + "gopkg.in/yaml.v3" +) + +// UnmarshallFixDuplicates handles duplicated values in yaml instead throwing error. +func UnmarshallFixDuplicates(data []byte) (map[string]any, error) { + reader := bytes.NewReader(data) + decoder := yaml.NewDecoder(reader) + + result := make(map[string]any) + for { + var rootNode yaml.Node + err := decoder.Decode(&rootNode) + if err != nil { + if err.Error() == "EOF" { + break + } + return nil, fmt.Errorf("error parsing YAML document: %w", err) + } + + // Parse each document and merge + res, err := recursiveParse(&rootNode) + if err != nil { + return nil, err + } + doc := res.(map[string]any) + for k, v := range doc { + if _, found := result[k]; found { + //launchr.Log().Debug("overriding duplicated key in YAML", "key", k) + result[k] = v + } else { + result[k] = v + } + } + } + + return result, nil +} + +// Recursive parsing function to handle YAML data with duplicate keys. +func recursiveParse(node *yaml.Node) (any, error) { + switch node.Kind { + case yaml.DocumentNode: + if len(node.Content) > 0 { + return recursiveParse(node.Content[0]) + } + return nil, nil + + case yaml.AliasNode: + return recursiveParse(node.Alias) + + case yaml.ScalarNode: + var value any + if err := node.Decode(&value); err != nil { + launchr.Term().Warning().Printfln("Failed to decode scalar at line %d, column %d: %v", node.Line, node.Column, err) + return node.Value, nil + } + return value, nil + + case yaml.SequenceNode: + var result []any + for _, n := range node.Content { + value, err := recursiveParse(n) + if err != nil { + return nil, err + } + result = append(result, value) + } + + return result, nil + + case yaml.MappingNode: + // Handle mappings and detect duplicates + result := make(map[string]any) + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + key := keyNode.Value + value, err := recursiveParse(valueNode) + if err != nil { + return result, err + } + + // Manage duplicate keys + if _, found := result[key]; found { + //launchr.Log().Debug("overriding duplicated key in YAML", "key", key) + result[key] = value + } else { + result[key] = value + } + } + return result, nil + default: + return nil, fmt.Errorf("unhandled YAML node kind at line %d, column %d: %v", node.Line, node.Column, node.Kind) + } +}