feat: support multiple archives with "priority" field (canonical#160)

This commit adds support for fetching packages from multiple archives. It introduces a new field archives.<archive-name>.priority which takes in a signed integer and specifies the priority of a certain archive. A package is fetched from the archive with highest priority and negative priorities are ignored unless explicitly pinned in a package.

The concept of default archive in chisel.yaml is now deprecated. However, the field can still be present and it will be parsed and interpreted in a compatible way for the time being.

DEPRECATED: "archives.*.default" field in chisel.yaml.

cmd/chisel/cmd_find_test.go

@@ -33,8 +33,6 @@ func makeSamplePackage(pkg string, slices []string) *setup.Package {
 }
 
 var sampleRelease = &setup.Release{
-	DefaultArchive: "ubuntu",
-
 	Archives: map[string]*setup.Archive{
 		"ubuntu": {
 			Name:       "ubuntu",

cmd/chisel/cmd_info_test.go

@@ -25,7 +25,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -37,7 +36,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg2"},
 	stdout: `
 		package: mypkg2
-		archive: ubuntu
 		slices:
 			myslice:
 				contents:
@@ -49,7 +47,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice2", "mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -65,7 +62,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg2", "mypkg1_myslice2"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -76,7 +72,6 @@ var infoTests = []infoTest{{
 					- mypkg2_myslice
 		---
 		package: mypkg2
-		archive: ubuntu
 		slices:
 			myslice:
 				contents:
@@ -88,7 +83,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -104,7 +98,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg1_myslice1", "mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -121,7 +114,6 @@ var infoTests = []infoTest{{
 	query:   []string{"foo", "mypkg1_myslice1", "bar_foo"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:

internal/setup/setup.go

@@ -21,10 +21,9 @@ import (
 // Release is a collection of package slices targeting a particular
 // distribution version.
 type Release struct {
-	Path           string
-	Packages       map[string]*Package
-	Archives       map[string]*Archive
-	DefaultArchive string
+	Path     string
+	Packages map[string]*Package
+	Archives map[string]*Archive
 }
 
 // Archive is the location from which binary packages are obtained.
@@ -33,6 +32,7 @@ type Archive struct {
 	Version    string
 	Suites     []string
 	Components []string
+	Priority   int
 	PubKeys    []*packet.PublicKey
 }
 
@@ -229,6 +229,28 @@ func (r *Release) validate() error {
 		return err
 	}
 
+	// Check for archive priority conflicts.
+	priorities := make(map[int]*Archive)
+	for _, archive := range r.Archives {
+		if old, ok := priorities[archive.Priority]; ok {
+			if old.Name > archive.Name {
+				archive, old = old, archive
+			}
+			return fmt.Errorf("chisel.yaml: archives %q and %q have the same priority value of %d", old.Name, archive.Name, archive.Priority)
+		}
+		priorities[archive.Priority] = archive
+	}
+
+	// Check that archives pinned in packages are defined.
+	for _, pkg := range r.Packages {
+		if pkg.Archive == "" {
+			continue
+		}
+		if _, ok := r.Archives[pkg.Archive]; !ok {
+			return fmt.Errorf("%s: package refers to undefined archive %q", pkg.Path, pkg.Archive)
+		}
+	}
+
 	return nil
 }
 
@@ -355,9 +377,6 @@ func readSlices(release *Release, baseDir, dirName string) error {
 		if err != nil {
 			return err
 		}
-		if pkg.Archive == "" {
-			pkg.Archive = release.DefaultArchive
-		}
 
 		release.Packages[pkg.Name] = pkg
 	}
@@ -370,10 +389,16 @@ type yamlRelease struct {
 	PubKeys  map[string]yamlPubKey  `yaml:"public-keys"`
 }
 
+const (
+	MaxArchivePriority = 1000
+	MinArchivePriority = -1000
+)
+
 type yamlArchive struct {
 	Version    string   `yaml:"version"`
 	Suites     []string `yaml:"suites"`
 	Components []string `yaml:"components"`
+	Priority   *int     `yaml:"priority"`
 	Default    bool     `yaml:"default"`
 	PubKeys    []string `yaml:"public-keys"`
 }
@@ -513,6 +538,11 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) {
 		pubKeys[keyName] = key
 	}
 
+	// For compatibility if there is a default archive set and priorities are
+	// not being used, we will revert back to the default archive behaviour.
+	hasPriority := false
+	var defaultArchive string
+	var archiveNoPriority string
 	for archiveName, details := range yamlVar.Archives {
 		if details.Version == "" {
 			return nil, fmt.Errorf("%s: archive %q missing version field", fileName, archiveName)
@@ -523,13 +553,14 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) {
 		if len(details.Components) == 0 {
 			return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName)
 		}
-		if len(yamlVar.Archives) == 1 {
-			details.Default = true
-		} else if details.Default && release.DefaultArchive != "" {
-			return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, release.DefaultArchive, archiveName)
+		if details.Default && defaultArchive != "" {
+			if archiveName < defaultArchive {
+				archiveName, defaultArchive = defaultArchive, archiveName
+			}
+			return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, defaultArchive, archiveName)
 		}
 		if details.Default {
-			release.DefaultArchive = archiveName
+			defaultArchive = archiveName
 		}
 		if len(details.PubKeys) == 0 {
 			return nil, fmt.Errorf("%s: archive %q missing public-keys field", fileName, archiveName)
@@ -542,14 +573,47 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) {
 			}
 			archiveKeys = append(archiveKeys, key)
 		}
+		priority := 0
+		if details.Priority != nil {
+			hasPriority = true
+			priority = *details.Priority
+			if priority > MaxArchivePriority || priority < MinArchivePriority || priority == 0 {
+				return nil, fmt.Errorf("%s: archive %q has invalid priority value of %d", fileName, archiveName, priority)
+			}
+		} else {
+			if archiveNoPriority == "" || archiveName < archiveNoPriority {
+				// Make it deterministic.
+				archiveNoPriority = archiveName
+			}
+		}
 		release.Archives[archiveName] = &Archive{
 			Name:       archiveName,
 			Version:    details.Version,
 			Suites:     details.Suites,
 			Components: details.Components,
+			Priority:   priority,
 			PubKeys:    archiveKeys,
 		}
 	}
+	if (hasPriority && archiveNoPriority != "") ||
+		(!hasPriority && defaultArchive == "" && len(yamlVar.Archives) > 1) {
+		return nil, fmt.Errorf("%s: archive %q is missing the priority setting", fileName, archiveNoPriority)
+	}
+	if defaultArchive != "" && !hasPriority {
+		// For compatibility with the default archive behaviour we will set
+		// negative priorities to all but the default one, which means all
+		// others will be ignored unless pinned.
+		var archiveNames []string
+		for archiveName := range yamlVar.Archives {
+			archiveNames = append(archiveNames, archiveName)
+		}
+		// Make it deterministic.
+		slices.Sort(archiveNames)
+		for i, archiveName := range archiveNames {
+			release.Archives[archiveName].Priority = -i - 1
+		}
+		release.Archives[defaultArchive].Priority = 1
+	}
 
 	return release, err
 }