Skip to content

Commit

Permalink
feat: support multiple archives with "priority"
Browse files Browse the repository at this point in the history 
This commit adds support for fetching packages from multiple archives.
It introduces a new field ``archives.<archive-name>.priority`` which
takes in a signed integer and specifies the priority of a certain
archive. This value is particularly useful when there are multiple
archives. A package is fetched from the archive with highest priority
regardless of the "default" archive, unless the slice definition file of
that package specifies a particular archive using the "archive" field.

Example chisel.yaml:

    format: chisel-v1
    archives:
	foo:
	    version: 22.04
	    components: [main, universe]
	    priority: 20
	    public-keys: [..]
	bar:
	    version: 22.04
	    components: [main]
	    default: true
	    priority: 10
	    public-keys: [..]

In the above example, if a package exists in both of the archives, the
package will be fetched from archive "foo" since it has higher priority.
Note that, archive "bar" is the default. However, if in a particular
package such as below, the "archive" field is used to specify a
particular archive, that particular archive will be fetched from.

    package: test-package
    archive: bar
    slices:
    	...

The above package will be fetched from archive "bar", regardless of
priority.

Reference: Specification RK018.
  • Loading branch information
@rebornplusplus
rebornplusplus committed Jul 9, 2024
1 parent 504ad63 commit a3075cd
Show file tree
Hide file tree
Showing 4 changed files with 405 additions and 104 deletions.
18 changes: 15 additions & 3 deletions internal/setup/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Archive struct {
Version string
Suites []string
Components []string
Priority int32
PubKeys []*packet.PublicKey
}

Expand Down Expand Up @@ -189,6 +190,18 @@ func (r *Release) validate() error {
paths[newPath] = new
}

// Check for archive priority conflicts.
for _, archive1 := range r.Archives {
for _, archive2 := range r.Archives {
if archive1 != archive2 && archive1.Priority == archive2.Priority {
if archive1.Name > archive2.Name {
archive1, archive2 = archive2, archive1
}
return fmt.Errorf("archives %q and %q have the same priority value of %v", archive1.Name, archive2.Name, archive1.Priority)
}
}
}

return nil
}

Expand Down Expand Up @@ -315,9 +328,6 @@ func readSlices(release *Release, baseDir, dirName string) error {
if err != nil {
return err
}
if pkg.Archive == "" {
pkg.Archive = release.DefaultArchive
}

release.Packages[pkg.Name] = pkg
}
Expand All @@ -337,6 +347,7 @@ type yamlArchive struct {
Suites []string `yaml:"suites"`
Components []string `yaml:"components"`
Default bool `yaml:"default"`
Priority int32 `yaml:"priority"`
PubKeys []string `yaml:"public-keys"`
// V1PubKeys is used for compatibility with format "chisel-v1".
V1PubKeys []string `yaml:"v1-public-keys"`
Expand Down Expand Up @@ -497,6 +508,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) {
Version: details.Version,
Suites: details.Suites,
Components: details.Components,
Priority: details.Priority,
PubKeys: archiveKeys,
}
}
Expand Down
108 changes: 66 additions & 42 deletions internal/setup/setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,9 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{},
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{},
},
},
},
Expand Down Expand Up @@ -129,9 +128,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice1": {
Package: "mypkg",
Expand Down Expand Up @@ -191,9 +189,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice1": {
Package: "mypkg",
Expand Down Expand Up @@ -452,9 +449,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice1": {
Package: "mypkg",
Expand Down Expand Up @@ -667,9 +663,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice": {
Package: "mypkg",
Expand Down Expand Up @@ -707,9 +702,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice": {
Package: "mypkg",
Expand Down Expand Up @@ -748,9 +742,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice": {
Package: "mypkg",
Expand All @@ -765,7 +758,7 @@ var setupTests = []setupTest{{
},
},
}, {
summary: "Multiple archives",
summary: "Multiple archives with priorities",
input: map[string]string{
"chisel.yaml": `
format: chisel-v1
Expand All @@ -775,11 +768,13 @@ var setupTests = []setupTest{{
components: [main, universe]
suites: [jammy]
default: true
priority: 20
v1-public-keys: [test-key]
bar:
version: 22.04
components: [universe]
suites: [jammy-updates]
priority: -10
v1-public-keys: [test-key]
v1-public-keys:
test-key:
Expand All @@ -788,6 +783,7 @@ var setupTests = []setupTest{{
`,
"slices/mydir/mypkg.yaml": `
package: mypkg
archive: foo
`,
},
release: &setup.Release{
Expand All @@ -799,13 +795,15 @@ var setupTests = []setupTest{{
Version: "22.04",
Suites: []string{"jammy"},
Components: []string{"main", "universe"},
Priority: 20,
PubKeys: []*packet.PublicKey{testKey.PubKey},
},
"bar": {
Name: "bar",
Version: "22.04",
Suites: []string{"jammy-updates"},
Components: []string{"universe"},
Priority: -10,
PubKeys: []*packet.PublicKey{testKey.PubKey},
},
},
Expand All @@ -818,6 +816,34 @@ var setupTests = []setupTest{{
},
},
},
}, {
summary: "Two archives cannot have same priority",
input: map[string]string{
"chisel.yaml": `
format: chisel-v1
archives:
foo:
version: 22.04
components: [main, universe]
suites: [jammy]
priority: 20
v1-public-keys: [test-key]
bar:
version: 22.04
components: [universe]
suites: [jammy-updates]
priority: 20
v1-public-keys: [test-key]
v1-public-keys:
test-key:
id: ` + testKey.ID + `
armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + `
`,
"slices/mydir/mypkg.yaml": `
package: mypkg
`,
},
relerror: `archives "bar" and "foo" have the same priority value of 20`,
}, {
summary: "Extra fields in YAML are ignored (necessary for forward compatibility)",
input: map[string]string{
Expand Down Expand Up @@ -861,9 +887,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"myslice": {
Package: "mypkg",
Expand All @@ -888,11 +913,13 @@ var setupTests = []setupTest{{
suites: [jammy]
v1-public-keys: [extra-key]
default: true
priority: 20
bar:
version: 22.04
components: [universe]
suites: [jammy-updates]
v1-public-keys: [test-key, extra-key]
priority: 10
v1-public-keys:
extra-key:
id: ` + extraTestKey.ID + `
Expand All @@ -914,22 +941,23 @@ var setupTests = []setupTest{{
Version: "22.04",
Suites: []string{"jammy"},
Components: []string{"main", "universe"},
Priority: 20,
PubKeys: []*packet.PublicKey{extraTestKey.PubKey},
},
"bar": {
Name: "bar",
Version: "22.04",
Suites: []string{"jammy-updates"},
Components: []string{"universe"},
Priority: 10,
PubKeys: []*packet.PublicKey{testKey.PubKey, extraTestKey.PubKey},
},
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "foo",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{},
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{},
},
},
},
Expand Down Expand Up @@ -1040,9 +1068,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"jq": {
Archive: "ubuntu",
Name: "jq",
Path: "slices/mydir/jq.yaml",
Name: "jq",
Path: "slices/mydir/jq.yaml",
Slices: map[string]*setup.Slice{
"bins": {
Package: "jq",
Expand Down Expand Up @@ -1093,9 +1120,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"slice1": {
Package: "mypkg",
Expand Down Expand Up @@ -1163,9 +1189,8 @@ var setupTests = []setupTest{{
},
Packages: map[string]*setup.Package{
"mypkg": {
Archive: "ubuntu",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Name: "mypkg",
Path: "slices/mydir/mypkg.yaml",
Slices: map[string]*setup.Slice{
"slice1": {
Package: "mypkg",
Expand All @@ -1186,9 +1211,8 @@ var setupTests = []setupTest{{
},
},
"myotherpkg": {
Archive: "ubuntu",
Name: "myotherpkg",
Path: "slices/mydir/myotherpkg.yaml",
Name: "myotherpkg",
Path: "slices/mydir/myotherpkg.yaml",
Slices: map[string]*setup.Slice{
"slice1": {
Package: "myotherpkg",
Expand Down
Loading

0 comments on commit a3075cd

Please sign in to comment.