handle yaml parse error (duplicated key)

actionSync.go

@@ -372,6 +372,8 @@ func (s *SyncAction) ensureResourceIsVersioned(resourceVersion, resourceMetaPath
 //}
 
 func (s *SyncAction) findVariableUpdateTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) {
+	//launchr.Log().Debug("find variable update: var, path", "var", variable.GetName(), "path", variable.GetPath())
+
 	// @TODO look for several vars during iteration?
 	ref, err := s.ensureVariableIsVersioned(variable, repo)
 	if err != nil {
@@ -479,6 +481,8 @@ func (s *SyncAction) ensureVariableIsVersioned(variable *sync.Variable, repo *gi
 }
 
 func (s *SyncAction) findVariableDeletionTime(variable *sync.Variable, repo *git.Repository) (*sync.TimelineVariablesItem, error) {
+	//launchr.Log().Debug("find variable delete: var, path", "var", variable.GetName(), "path", variable.GetPath())
+
 	// @TODO look for several vars during iteration?
 	// @TODO ensure variable existed at first place, before starting to search.
 

pkg/sync/inventory.go

@@ -506,6 +506,7 @@ func (i *Inventory) pushRequiredVariables(requiredMap map[string]map[string]bool
 }
 
 func (i *Inventory) crawlVariableUsage(variable *Variable, dependents map[string]*Variable) error {
+	//@TODO crawl several variables
 	var files []string
 	var err error
 
@@ -692,7 +693,14 @@ func (cr *ResourcesCrawler) SearchVariablesInGroupFiles(name string, files []str
 		var sourceData map[string]any
 		errMarshal := yaml.Unmarshal(sourceVariables, &sourceData)
 		if errMarshal != nil {
-			return variables, errMarshal
+			if !strings.Contains(errMarshal.Error(), "already defined at line") {
+				return variables, errMarshal
+			}
+
+			sourceData, errMarshal = UnmarshallFixDuplicates(sourceVariables)
+			if err != nil {
+				return variables, errMarshal
+			}
 		}
 
 		for k, v := range sourceData {
@@ -856,6 +864,18 @@ func LoadVariablesFile(path, vaultPassword string, isVault bool) (map[string]any
 	}
 
 	err = yaml.Unmarshal(rawData, &data)
+	if err != nil {
+		if !strings.Contains(err.Error(), "already defined at line") {
+			return data, err
+		}
+
+		launchr.Log().Warn("duplicate found, parsing YAML file manually")
+		data, err = UnmarshallFixDuplicates(rawData)
+		if err != nil {
+			return data, err
+		}
+	}
+
 	return data, err
 }
 
@@ -884,6 +904,18 @@ func LoadVariablesFileFromBytes(input []byte, vaultPassword string, isVault bool
 	}
 
 	err = yaml.Unmarshal(rawData, &data)
+	if err != nil {
+		if !strings.Contains(err.Error(), "already defined at line") {
+			return data, err
+		}
+
+		launchr.Log().Warn("duplicate found, parsing YAML file manually")
+		data, err = UnmarshallFixDuplicates(rawData)
+		if err != nil {
+			return data, err
+		}
+	}
+
 	return data, err
 }
 

pkg/sync/yaml.go

@@ -0,0 +1,102 @@
+package sync
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/launchrctl/launchr"
+	"gopkg.in/yaml.v3"
+)
+
+// UnmarshallFixDuplicates handles duplicated values in yaml instead throwing error.
+func UnmarshallFixDuplicates(data []byte) (map[string]any, error) {
+	reader := bytes.NewReader(data)
+	decoder := yaml.NewDecoder(reader)
+
+	result := make(map[string]any)
+	for {
+		var rootNode yaml.Node
+		err := decoder.Decode(&rootNode)
+		if err != nil {
+			if err.Error() == "EOF" {
+				break
+			}
+			return nil, fmt.Errorf("error parsing YAML document: %w", err)
+		}
+
+		// Parse each document and merge
+		res, err := recursiveParse(&rootNode)
+		if err != nil {
+			return nil, err
+		}
+		doc := res.(map[string]any)
+		for k, v := range doc {
+			if _, found := result[k]; found {
+				//launchr.Log().Debug("overriding duplicated key in YAML", "key", k)
+				result[k] = v
+			} else {
+				result[k] = v
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// Recursive parsing function to handle YAML data with duplicate keys.
+func recursiveParse(node *yaml.Node) (any, error) {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) > 0 {
+			return recursiveParse(node.Content[0])
+		}
+		return nil, nil
+
+	case yaml.AliasNode:
+		return recursiveParse(node.Alias)
+
+	case yaml.ScalarNode:
+		var value any
+		if err := node.Decode(&value); err != nil {
+			launchr.Term().Warning().Printfln("Failed to decode scalar at line %d, column %d: %v", node.Line, node.Column, err)
+			return node.Value, nil
+		}
+		return value, nil
+
+	case yaml.SequenceNode:
+		var result []any
+		for _, n := range node.Content {
+			value, err := recursiveParse(n)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, value)
+		}
+
+		return result, nil
+
+	case yaml.MappingNode:
+		// Handle mappings and detect duplicates
+		result := make(map[string]any)
+		for i := 0; i < len(node.Content); i += 2 {
+			keyNode := node.Content[i]
+			valueNode := node.Content[i+1]
+			key := keyNode.Value
+			value, err := recursiveParse(valueNode)
+			if err != nil {
+				return result, err
+			}
+
+			// Manage duplicate keys
+			if _, found := result[key]; found {
+				//launchr.Log().Debug("overriding duplicated key in YAML", "key", key)
+				result[key] = value
+			} else {
+				result[key] = value
+			}
+		}
+		return result, nil
+	default:
+		return nil, fmt.Errorf("unhandled YAML node kind at line %d, column %d: %v", node.Line, node.Column, node.Kind)
+	}
+}