From 3b110122fbbe520bf293d402c689b97acf5e22cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 18 Apr 2023 09:47:13 +0200 Subject: [PATCH] Add support for Pro FIPS archives Support Pro archives by a pro property in chisel.yaml. When the pro property is specified, it has to be either "fips" or "fips-updates". The repository URL is inferred from the value. Also, credentials for these URLs are searched. If credentials are not found, the corresponding pro archive is silently disabled. Otherwise, all requests to the archive are sent with the credentials found. --- cmd/chisel/cmd_cut.go | 12 ++++- internal/archive/archive.go | 86 +++++++++++++++++++++++++++----- internal/archive/archive_test.go | 53 ++++++++++++++++++++ internal/setup/setup.go | 3 ++ internal/setup/setup_test.go | 44 ++++++++++++++++ internal/slicer/slicer_test.go | 1 + 6 files changed, 184 insertions(+), 15 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 7a20c8d9..a8ebccf2 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -92,11 +92,19 @@ func (cmd *cmdCut) Execute(args []string) error { Components: archiveInfo.Components, CacheDir: cache.DefaultDir("chisel"), Priority: archiveInfo.Priority, + Pro: archiveInfo.Pro, }) if err != nil { - return err + if err != archive.ErrCredentialsNotFound { + return err + } + } else { + archives[archiveName] = openArchive } - archives[archiveName] = openArchive + } + + if len(archives) == 0 { + return fmt.Errorf("no valid archives (%d skipped)", len(release.Archives)) } return slicer.Run(&slicer.RunOptions{ diff --git a/internal/archive/archive.go b/internal/archive/archive.go index bd5f23c6..f8c99870 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "sort" "strings" "time" @@ -35,6 +36,7 @@ type Options struct { Components []string CacheDir string Priority int32 + Pro string } func Open(options *Options) (Archive, error) { @@ -73,6 +75,8 @@ type ubuntuArchive struct { options Options indexes []*ubuntuIndex cache *cache.Cache + baseURL string + auth string } type ubuntuIndex struct { @@ -83,7 +87,7 @@ type ubuntuIndex struct { component string release control.Section packages control.File - cache *cache.Cache + archive *ubuntuArchive } func (a *ubuntuArchive) Options() *Options { @@ -148,6 +152,52 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +const ubuntuProURL = "https://esm.ubuntu.com/" + +// keep it sorted +var validPro = []string{ + "fips", + "fips-updates", +} + +func initProArchive(pro string, archive *ubuntuArchive) error { + if i := sort.SearchStrings(validPro, pro); !(i < len(validPro) && validPro[i] == pro) { + strvals := strings.Join(validPro, ", ") + return fmt.Errorf("invalid pro type, supported types: %s", strvals) + } + + baseURL := ubuntuProURL + pro + "/ubuntu/" + creds, err := findCredentials(baseURL) + if err != nil { + return err + } + + // Check that credentials are valid. + // It appears that only pool/ URLs are protected. + req, err := http.NewRequest("HEAD", baseURL+"pool/", nil) + if err != nil { + return fmt.Errorf("cannot create HTTP request: %w", err) + } + req.SetBasicAuth(creds.Username, creds.Password) + + resp, err := httpDo(req) + if err != nil { + return fmt.Errorf("cannot talk to the archive: %w", err) + } + resp.Body.Close() + switch resp.StatusCode { + case 200: // ok + case 401: + return fmt.Errorf("cannot authenticate to the archive") + default: + return fmt.Errorf("error from the archive: %v", resp.Status) + } + + archive.baseURL = baseURL + archive.auth = req.Header.Get("Authorization") + + return nil +} func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { @@ -167,6 +217,18 @@ func openUbuntu(options *Options) (Archive, error) { }, } + if options.Pro != "" { + if err := initProArchive(options.Pro, archive); err != nil { + return nil, err + } + } else { + if options.Arch == "amd64" || options.Arch == "i386" { + archive.baseURL = ubuntuURL + } else { + archive.baseURL = ubuntuPortsURL + } + } + for _, suite := range options.Suites { var release control.Section for _, component := range options.Components { @@ -177,7 +239,7 @@ func openUbuntu(options *Options) (Archive, error) { suite: suite, component: component, release: release, - cache: archive.cache, + archive: archive, } if release == nil { err := index.fetchRelease() @@ -265,29 +327,27 @@ func (index *ubuntuIndex) checkComponents(components []string) error { } func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.ReadCloser, error) { - reader, err := index.cache.Open(digest) + reader, err := index.archive.cache.Open(digest) if err == nil { return reader, nil } else if err != cache.MissErr { return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } - var url string if strings.HasPrefix(suffix, "pool/") { - url = baseURL + suffix + url = index.archive.baseURL + suffix } else { - url = baseURL + "dists/" + index.suite + "/" + suffix + url = index.archive.baseURL + "dists/" + index.suite + "/" + suffix } req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if index.archive.auth != "" { + req.Header.Set("Authorization", index.archive.auth) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) @@ -302,7 +362,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) @@ -318,7 +378,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea body = reader } - writer := index.cache.Create(digest) + writer := index.archive.cache.Create(digest) defer writer.Close() _, err = io.Copy(writer, body) @@ -329,5 +389,5 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, fmt.Errorf("cannot fetch from archive: %v", err) } - return index.cache.Open(writer.Digest()) + return index.archive.cache.Open(writer.Digest()) } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b18fc60c..c941a352 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -365,6 +365,59 @@ func (s *httpSuite) TestPackageInfo(c *C) { c.Assert(info99, IsNil) } +func (s *httpSuite) TestFetchProPackage(c *C) { + var err error + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + s.base = "https://esm.ubuntu.com/fips/ubuntu/" + s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + + invalidOptions := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + Pro: "invalid", + } + + _, err = archive.Open(&invalidOptions) + c.Assert(err, ErrorMatches, "invalid pro type, supported types: fips, fips-updates") + + fipsOptions := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + Pro: "fips", + } + + _, err = archive.Open(&fipsOptions) + c.Assert(err, Equals, archive.ErrCredentialsNotFound) + + credsFile := filepath.Join(credsDir, "90ubuntu-advantage") + credsData := "machine https://esm.ubuntu.com/fips/ubuntu/ login user password pw\n" + err = os.WriteFile(credsFile, []byte(credsData), 0600) + c.Assert(err, IsNil) + + archive, err := archive.Open(&fipsOptions) + c.Assert(err, IsNil) + + pkg, err := archive.Fetch("mypkg1") + c.Assert(err, IsNil) + c.Assert(read(pkg), Equals, "mypkg1 1.1 data") + + pkg, err = archive.Fetch("mypkg4") + c.Assert(err, IsNil) + c.Assert(read(pkg), Equals, "mypkg4 1.4 data") +} + func read(r io.Reader) string { data, err := io.ReadAll(r) if err != nil { diff --git a/internal/setup/setup.go b/internal/setup/setup.go index e7c616e9..6fa7fadb 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -30,6 +30,7 @@ type Archive struct { Suites []string Components []string Priority int32 + Pro string } // Package holds a collection of slices that represent parts of themselves. @@ -323,6 +324,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Priority int32 `yaml:"priority"` + Pro string `yaml:"pro"` } type yamlPackage struct { @@ -430,6 +432,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Suites: details.Suites, Components: details.Components, Priority: details.Priority, + Pro: details.Pro, } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 0a954db2..eb409d2d 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -790,6 +790,50 @@ var setupTests = []setupTest{{ `, }, relerror: "(?s).*\\bcannot unmarshal !!int `2147483648` into int32\\b.*", +}, { + summary: "Pro property", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + ubuntu: + version: 22.04 + components: [main, universe] + suites: [jammy, jammy-updates, jammy-security] + ubuntu-fips: + pro: fips + version: 22.04 + components: [main] + suites: [jammy] + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy", "jammy-updates", "jammy-security"}, + Components: []string{"main", "universe"}, + }, + "ubuntu-fips": { + Name: "ubuntu-fips", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + Pro: "fips", + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }} const defaultChiselYaml = ` diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index a7122901..0049bac3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -948,6 +948,7 @@ func (s *S) TestRun(c *C) { Suites: setupArchive.Suites, Components: setupArchive.Components, Priority: setupArchive.Priority, + Pro: setupArchive.Pro, Arch: test.arch, }, pkgs: archivePkgs,