Add support for Pro FIPS archives

Support Pro archives by a pro property in chisel.yaml. When the pro
property is specified, it has to be either "fips" or "fips-updates". The
repository URL is inferred from the value. Also, credentials for these
URLs are searched. If credentials are not found, the corresponding pro
archive is silently disabled. Otherwise, all requests to the archive are
sent with the credentials found.

cmd/chisel/cmd_cut.go

@@ -92,11 +92,19 @@ func (cmd *cmdCut) Execute(args []string) error {
 			Components: archiveInfo.Components,
 			CacheDir:   cache.DefaultDir("chisel"),
 			Priority:   archiveInfo.Priority,
+			Pro:        archiveInfo.Pro,
 		})
 		if err != nil {
-			return err
+			if err != archive.ErrCredentialsNotFound {
+				return err
+			}
+		} else {
+			archives[archiveName] = openArchive
 		}
-		archives[archiveName] = openArchive
+	}
+
+	if len(archives) == 0 {
+		return fmt.Errorf("no valid archives (%d skipped)", len(release.Archives))
 	}
 
 	return slicer.Run(&slicer.RunOptions{

internal/archive/archive.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 
@@ -35,6 +36,7 @@ type Options struct {
 	Components []string
 	CacheDir   string
 	Priority   int32
+	Pro        string
 }
 
 func Open(options *Options) (Archive, error) {
@@ -73,6 +75,8 @@ type ubuntuArchive struct {
 	options Options
 	indexes []*ubuntuIndex
 	cache   *cache.Cache
+	baseURL string
+	auth    string
 }
 
 type ubuntuIndex struct {
@@ -83,7 +87,7 @@ type ubuntuIndex struct {
 	component string
 	release   control.Section
 	packages  control.File
-	cache     *cache.Cache
+	archive   *ubuntuArchive
 }
 
 func (a *ubuntuArchive) Options() *Options {
@@ -148,6 +152,52 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, error) {
 
 const ubuntuURL = "http://archive.ubuntu.com/ubuntu/"
 const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/"
+const ubuntuProURL = "https://esm.ubuntu.com/"
+
+// keep it sorted
+var validPro = []string{
+	"fips",
+	"fips-updates",
+}
+
+func initProArchive(pro string, archive *ubuntuArchive) error {
+	if i := sort.SearchStrings(validPro, pro); !(i < len(validPro) && validPro[i] == pro) {
+		strvals := strings.Join(validPro, ", ")
+		return fmt.Errorf("invalid pro type, supported types: %s", strvals)
+	}
+
+	baseURL := ubuntuProURL + pro + "/ubuntu/"
+	creds, err := findCredentials(baseURL)
+	if err != nil {
+		return err
+	}
+
+	// Check that credentials are valid.
+	// It appears that only pool/ URLs are protected.
+	req, err := http.NewRequest("HEAD", baseURL+"pool/", nil)
+	if err != nil {
+		return fmt.Errorf("cannot create HTTP request: %w", err)
+	}
+	req.SetBasicAuth(creds.Username, creds.Password)
+
+	resp, err := httpDo(req)
+	if err != nil {
+		return fmt.Errorf("cannot talk to the archive: %w", err)
+	}
+	resp.Body.Close()
+	switch resp.StatusCode {
+	case 200: // ok
+	case 401:
+		return fmt.Errorf("cannot authenticate to the archive")
+	default:
+		return fmt.Errorf("error from the archive: %v", resp.Status)
+	}
+
+	archive.baseURL = baseURL
+	archive.auth = req.Header.Get("Authorization")
+
+	return nil
+}
 
 func openUbuntu(options *Options) (Archive, error) {
 	if len(options.Components) == 0 {
@@ -167,6 +217,18 @@ func openUbuntu(options *Options) (Archive, error) {
 		},
 	}
 
+	if options.Pro != "" {
+		if err := initProArchive(options.Pro, archive); err != nil {
+			return nil, err
+		}
+	} else {
+		if options.Arch == "amd64" || options.Arch == "i386" {
+			archive.baseURL = ubuntuURL
+		} else {
+			archive.baseURL = ubuntuPortsURL
+		}
+	}
+
 	for _, suite := range options.Suites {
 		var release control.Section
 		for _, component := range options.Components {
@@ -177,7 +239,7 @@ func openUbuntu(options *Options) (Archive, error) {
 				suite:     suite,
 				component: component,
 				release:   release,
-				cache:     archive.cache,
+				archive:   archive,
 			}
 			if release == nil {
 				err := index.fetchRelease()
@@ -265,29 +327,27 @@ func (index *ubuntuIndex) checkComponents(components []string) error {
 }
 
 func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.ReadCloser, error) {
-	reader, err := index.cache.Open(digest)
+	reader, err := index.archive.cache.Open(digest)
 	if err == nil {
 		return reader, nil
 	} else if err != cache.MissErr {
 		return nil, err
 	}
 
-	baseURL := ubuntuURL
-	if index.arch != "amd64" && index.arch != "i386" {
-		baseURL = ubuntuPortsURL
-	}
-
 	var url string
 	if strings.HasPrefix(suffix, "pool/") {
-		url = baseURL + suffix
+		url = index.archive.baseURL + suffix
 	} else {
-		url = baseURL + "dists/" + index.suite + "/" + suffix
+		url = index.archive.baseURL + "dists/" + index.suite + "/" + suffix
 	}
 
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP request: %v", err)
 	}
+	if index.archive.auth != "" {
+		req.Header.Set("Authorization", index.archive.auth)
+	}
 	var resp *http.Response
 	if flags&fetchBulk != 0 {
 		resp, err = bulkDo(req)
@@ -302,7 +362,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 	switch resp.StatusCode {
 	case 200:
 		// ok
-	case 401, 404:
+	case 404:
 		return nil, fmt.Errorf("cannot find archive data")
 	default:
 		return nil, fmt.Errorf("error from archive: %v", resp.Status)
@@ -318,7 +378,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 		body = reader
 	}
 
-	writer := index.cache.Create(digest)
+	writer := index.archive.cache.Create(digest)
 	defer writer.Close()
 
 	_, err = io.Copy(writer, body)
@@ -329,5 +389,5 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 		return nil, fmt.Errorf("cannot fetch from archive: %v", err)
 	}
 
-	return index.cache.Open(writer.Digest())
+	return index.archive.cache.Open(writer.Digest())
 }

internal/archive/archive_test.go

@@ -365,6 +365,59 @@ func (s *httpSuite) TestPackageInfo(c *C) {
 	c.Assert(info99, IsNil)
 }
 
+func (s *httpSuite) TestFetchProPackage(c *C) {
+	var err error
+
+	credsDir := c.MkDir()
+	restore := fakeEnv("CHISEL_AUTH_DIR", credsDir)
+	defer restore()
+
+	s.base = "https://esm.ubuntu.com/fips/ubuntu/"
+	s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"})
+
+	invalidOptions := archive.Options{
+		Label:      "ubuntu",
+		Version:    "22.04",
+		Arch:       "amd64",
+		Suites:     []string{"jammy"},
+		Components: []string{"main", "universe"},
+		CacheDir:   c.MkDir(),
+		Pro:        "invalid",
+	}
+
+	_, err = archive.Open(&invalidOptions)
+	c.Assert(err, ErrorMatches, "invalid pro type, supported types: fips, fips-updates")
+
+	fipsOptions := archive.Options{
+		Label:      "ubuntu",
+		Version:    "22.04",
+		Arch:       "amd64",
+		Suites:     []string{"jammy"},
+		Components: []string{"main", "universe"},
+		CacheDir:   c.MkDir(),
+		Pro:        "fips",
+	}
+
+	_, err = archive.Open(&fipsOptions)
+	c.Assert(err, Equals, archive.ErrCredentialsNotFound)
+
+	credsFile := filepath.Join(credsDir, "90ubuntu-advantage")
+	credsData := "machine https://esm.ubuntu.com/fips/ubuntu/ login user password pw\n"
+	err = os.WriteFile(credsFile, []byte(credsData), 0600)
+	c.Assert(err, IsNil)
+
+	archive, err := archive.Open(&fipsOptions)
+	c.Assert(err, IsNil)
+
+	pkg, err := archive.Fetch("mypkg1")
+	c.Assert(err, IsNil)
+	c.Assert(read(pkg), Equals, "mypkg1 1.1 data")
+
+	pkg, err = archive.Fetch("mypkg4")
+	c.Assert(err, IsNil)
+	c.Assert(read(pkg), Equals, "mypkg4 1.4 data")
+}
+
 func read(r io.Reader) string {
 	data, err := io.ReadAll(r)
 	if err != nil {

internal/setup/setup.go

@@ -30,6 +30,7 @@ type Archive struct {
 	Suites     []string
 	Components []string
 	Priority   int32
+	Pro        string
 }
 
 // Package holds a collection of slices that represent parts of themselves.
@@ -323,6 +324,7 @@ type yamlArchive struct {
 	Suites     []string `yaml:"suites"`
 	Components []string `yaml:"components"`
 	Priority   int32    `yaml:"priority"`
+	Pro        string   `yaml:"pro"`
 }
 
 type yamlPackage struct {
@@ -430,6 +432,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) {
 			Suites:     details.Suites,
 			Components: details.Components,
 			Priority:   details.Priority,
+			Pro:        details.Pro,
 		}
 	}
 

internal/setup/setup_test.go

@@ -790,6 +790,50 @@ var setupTests = []setupTest{{
 		`,
 	},
 	relerror: "(?s).*\\bcannot unmarshal !!int `2147483648` into int32\\b.*",
+}, {
+	summary: "Pro property",
+	input: map[string]string{
+		"chisel.yaml": `
+			format: chisel-v1
+			archives:
+				ubuntu:
+					version: 22.04
+					components: [main, universe]
+					suites: [jammy, jammy-updates, jammy-security]
+				ubuntu-fips:
+					pro: fips
+					version: 22.04
+					components: [main]
+					suites: [jammy]
+		`,
+		"slices/mydir/mypkg.yaml": `
+			package: mypkg
+		`,
+	},
+	release: &setup.Release{
+		Archives: map[string]*setup.Archive{
+			"ubuntu": {
+				Name:       "ubuntu",
+				Version:    "22.04",
+				Suites:     []string{"jammy", "jammy-updates", "jammy-security"},
+				Components: []string{"main", "universe"},
+			},
+			"ubuntu-fips": {
+				Name:       "ubuntu-fips",
+				Version:    "22.04",
+				Suites:     []string{"jammy"},
+				Components: []string{"main"},
+				Pro:        "fips",
+			},
+		},
+		Packages: map[string]*setup.Package{
+			"mypkg": {
+				Name:   "mypkg",
+				Path:   "slices/mydir/mypkg.yaml",
+				Slices: map[string]*setup.Slice{},
+			},
+		},
+	},
 }}
 
 const defaultChiselYaml = `

internal/slicer/slicer_test.go

@@ -948,6 +948,7 @@ func (s *S) TestRun(c *C) {
 					Suites:     setupArchive.Suites,
 					Components: setupArchive.Components,
 					Priority:   setupArchive.Priority,
+					Pro:        setupArchive.Pro,
 					Arch:       test.arch,
 				},
 				pkgs: archivePkgs,