feat: support "Pro" archives

In chisel.yaml, archive definitions can now use the "pro" value to
specify Ubuntu Pro archives. The `archives.<archive>.pro` value
currently accepts the following values: "fips", "fips-updates", "apps"
and "infra". Any other values are ignored.

By default, Chisel will look for credentials in the
`/etc/apt/auth.conf.d/` directory, unless the environment variable
`CHISEL_AUTH_DIR` is set. In which case, it will look for
configuration files in that directory. The configuration files may only
have the ".conf" extensions or no extensions, the format is described
in https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html.

.github/workflows/spread.yml

@@ -27,6 +27,8 @@ jobs:
           go-version: '>=1.17.0'
 
       - name: Build and run spread
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
         run: |
           (cd _spread/cmd/spread && go build)
           _spread/cmd/spread/spread -v focal jammy mantic noble

.github/workflows/tests.yaml

@@ -43,3 +43,46 @@ jobs:
         with:
           name: chisel-test-coverage.html
           path: ./*.html
+
+  real-archive-tests:
+    # Do not change to newer releases as "fips" may not be available there.
+    runs-on: ubuntu-20.04
+    name: Real Archive Tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run real archive tests
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
+        run: |
+          set -ex
+
+          detach() {
+            sudo pro detach --assume-yes || true
+            sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage
+          }
+          trap detach EXIT
+
+          # Attach pro token and enable services
+          sudo pro attach ${PRO_TOKEN} --no-auto-enable
+
+          # Cannot enable fips and fips-updates at the same time.
+          # Hack: enable fips, copy the credentials and then after enabling
+          # other services, add the credentials back.
+          sudo pro enable fips --assume-yes
+          sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+          # This will disable the fips service.
+          sudo pro enable fips-updates esm-apps esm-infra --assume-yes
+          # Add the fips credentials back.
+          sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage'
+          sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+
+          # Make apt credentials accessible to USER.
+          sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage
+
+          # Run tests on Pro and non-Pro real archives.
+          go test ./internal/archive/ -v --real-archive --real-pro-archive

README.md

@@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs*
 folder, according to the slice definitions available in the
 ["ubuntu-22.04" chisel-releases branch](<https://github.com/canonical/chisel-releases/tree/ubuntu-22.04>).
 
+## Chisel support for Pro archives
+
+Chisel can also fetch and install packages from Ubuntu Pro archives. For this,
+the archive has to be defined with the `archives.<archive>.pro` field in
+chisel.yaml and its credentials have to be made available to Chisel.
+
+
+```yaml
+# chisel.yaml
+format: v1
+archives:
+  <archive-name>:
+    pro: <value>
+    ...
+...
+```
+
+Chisel currently supports the following Pro archives:
+
+| `pro` value | Archive URL | Related Ubuntu Pro service |
+| - | - | - |
+| fips         | https://esm.ubuntu.com/fips/ubuntu         | fips         |
+| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates |
+| apps         | https://esm.ubuntu.com/apps/ubuntu         | esm-apps     |
+| infra        | https://esm.ubuntu.com/infra/ubuntu        | esm-infra    |
+
+Authentication to Pro archives requires that the host is Pro or it is equipped
+with the Pro credentials. By default, Chisel will support using credentials
+from the `/etc/apt/auth.conf.d/` directory, but this location can be configured
+using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have
+read permission for the necessary credentials files.
+
+The format of the files is documented in the
+[apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html)
+man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage`
+file from a host with the `fips-updates` and `infra` archives enabled:
+
+```
+machine esm.ubuntu.com/infra/ubuntu/ login bearer password <infra-token>
+machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password <fips-updates-token>
+```
+
 ## Reference
 
 ### Chisel releases

cmd/chisel/cmd_cut.go

@@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error {
 			Arch:       cmd.Arch,
 			Suites:     archiveInfo.Suites,
 			Components: archiveInfo.Components,
+			Pro:        archiveInfo.Pro,
 			CacheDir:   cache.DefaultDir("chisel"),
 			PubKeys:    archiveInfo.PubKeys,
 		})
 		if err != nil {
+			if err == archive.ErrCredentialsNotFound {
+				logf("Ignoring archive %q (credentials not found)...", archiveName)
+				continue
+			}
 			return err
 		}
 		archives[archiveName] = openArchive

cmd/chisel/main.go

@@ -327,6 +327,7 @@ func run() error {
 	deb.SetLogger(log.Default())
 	setup.SetLogger(log.Default())
 	slicer.SetLogger(log.Default())
+	SetLogger(log.Default())
 
 	parser := Parser()
 	xtra, err := parser.Parse()

internal/archive/archive.go

@@ -14,6 +14,7 @@ import (
 	"github.com/canonical/chisel/internal/control"
 	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/pgputil"
+	"github.com/canonical/chisel/internal/setup"
 )
 
 type Archive interface {
@@ -36,6 +37,7 @@ type Options struct {
 	Arch       string
 	Suites     []string
 	Components []string
+	Pro        string
 	CacheDir   string
 	PubKeys    []*packet.PublicKey
 }
@@ -77,6 +79,8 @@ type ubuntuArchive struct {
 	indexes []*ubuntuIndex
 	cache   *cache.Cache
 	pubKeys []*packet.PublicKey
+	baseURL string
+	creds   *credentials
 }
 
 type ubuntuIndex struct {
@@ -147,6 +151,39 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) {
 const ubuntuURL = "http://archive.ubuntu.com/ubuntu/"
 const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/"
 
+var proArchiveInfo = map[string]struct {
+	BaseURL, Label string
+}{
+	setup.ProFIPS: {
+		BaseURL: "https://esm.ubuntu.com/fips/ubuntu/",
+		Label:   "UbuntuFIPS",
+	},
+	setup.ProFIPSUpdates: {
+		BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/",
+		Label:   "UbuntuFIPSUpdates",
+	},
+	setup.ProApps: {
+		BaseURL: "https://esm.ubuntu.com/apps/ubuntu/",
+		Label:   "UbuntuESMApps",
+	},
+	setup.ProInfra: {
+		BaseURL: "https://esm.ubuntu.com/infra/ubuntu/",
+		Label:   "UbuntuESM",
+	},
+}
+
+// archiveURL returns the archive base URL depending on the "pro" value and
+// selected architecture "arch".
+func archiveURL(pro, arch string) string {
+	if pro != "" {
+		return proArchiveInfo[pro].BaseURL
+	}
+	if arch == "amd64" || arch == "i386" {
+		return ubuntuURL
+	}
+	return ubuntuPortsURL
+}
+
 func openUbuntu(options *Options) (Archive, error) {
 	if len(options.Components) == 0 {
 		return nil, fmt.Errorf("archive options missing components")
@@ -157,13 +194,30 @@ func openUbuntu(options *Options) (Archive, error) {
 	if len(options.Version) == 0 {
 		return nil, fmt.Errorf("archive options missing version")
 	}
+	if options.Pro != "" {
+		if _, ok := proArchiveInfo[options.Pro]; !ok {
+			return nil, fmt.Errorf("invalid pro value: %q", options.Pro)
+		}
+	}
+
+	baseURL := archiveURL(options.Pro, options.Arch)
+	var creds *credentials
+	if options.Pro != "" {
+		var err error
+		creds, err = findCredentials(baseURL)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	archive := &ubuntuArchive{
 		options: *options,
 		cache: &cache.Cache{
 			Dir: options.CacheDir,
 		},
 		pubKeys: options.PubKeys,
+		baseURL: baseURL,
+		creds:   creds,
 	}
 
 	for _, suite := range options.Suites {
@@ -184,6 +238,13 @@ func openUbuntu(options *Options) (Archive, error) {
 					return nil, err
 				}
 				release = index.release
+				if !index.supportsArch(options.Arch) {
+					// Release does not support the specified architecture, do
+					// not add any of its indexes.
+					logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...",
+						index.proSuffixedLabel(), index.version, index.suite, options.Arch)
+					break
+				}
 				err = index.checkComponents(options.Components)
 				if err != nil {
 					return nil, err
@@ -201,7 +262,7 @@ func openUbuntu(options *Options) (Archive, error) {
 }
 
 func (index *ubuntuIndex) fetchRelease() error {
-	logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite)
+	logf("Fetching %s %s %s suite details...", index.proSuffixedLabel(), index.version, index.suite)
 	reader, err := index.fetch("InRelease", "", fetchDefault)
 	if err != nil {
 		return err
@@ -235,12 +296,14 @@ func (index *ubuntuIndex) fetchRelease() error {
 	if err != nil {
 		return fmt.Errorf("cannot parse InRelease file: %v", err)
 	}
-	section := ctrl.Section("Ubuntu")
+	// Parse the appropriate section for the type of archive.
+	label := "Ubuntu"
+	if index.archive.options.Pro != "" {
+		label = proArchiveInfo[index.archive.options.Pro].Label
+	}
+	section := ctrl.Section(label)
 	if section == nil {
-		section = ctrl.Section("UbuntuProFIPS")
-		if section == nil {
-			return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section")
-		}
+		return fmt.Errorf("corrupted archive InRelease file: no %s section", label)
 	}
 	logf("Release date: %s", section.Get("Date"))
 
@@ -256,7 +319,7 @@ func (index *ubuntuIndex) fetchIndex() error {
 		return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component)
 	}
 
-	logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component)
+	logf("Fetching index for %s %s %s %s component...", index.proSuffixedLabel(), index.version, index.suite, index.component)
 	reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk)
 	if err != nil {
 		return err
@@ -270,6 +333,17 @@ func (index *ubuntuIndex) fetchIndex() error {
 	return nil
 }
 
+// supportsArch returns true if the Architectures field in the index release
+// contains "arch". Per the Debian wiki [1], index release files should list the
+// supported architectures in the "Architectures" field.
+// The "ubuntuURL" archive only supports the amd64 and i386 architectures
+// whereas the "ubuntuPortsURL" one supports the rest. But each of them
+// (faultly) specifies all those architectures in their InRelease files.
+// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures
+func (index *ubuntuIndex) supportsArch(arch string) bool {
+	return strings.Contains(index.release.Get("Architectures"), arch)
+}
+
 func (index *ubuntuIndex) checkComponents(components []string) error {
 	releaseComponents := strings.Fields(index.release.Get("Components"))
 	for _, c1 := range components {
@@ -295,10 +369,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 		return nil, err
 	}
 
-	baseURL := ubuntuURL
-	if index.arch != "amd64" && index.arch != "i386" {
-		baseURL = ubuntuPortsURL
-	}
+	baseURL, creds := index.archive.baseURL, index.archive.creds
 
 	var url string
 	if strings.HasPrefix(suffix, "pool/") {
@@ -311,6 +382,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP request: %v", err)
 	}
+	if creds != nil && !creds.Empty() {
+		req.SetBasicAuth(creds.Username, creds.Password)
+	}
 	var resp *http.Response
 	if flags&fetchBulk != 0 {
 		resp, err = bulkDo(req)
@@ -325,7 +399,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 	switch resp.StatusCode {
 	case 200:
 		// ok
-	case 401, 404:
+	case 401:
+		return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label)
+	case 404:
 		return nil, fmt.Errorf("cannot find archive data")
 	default:
 		return nil, fmt.Errorf("error from archive: %v", resp.Status)
@@ -363,3 +439,13 @@ func sectionPackageInfo(section control.Section) *PackageInfo {
 		SHA256:  section.Get("SHA256"),
 	}
 }
+
+// proSuffixedLabel adds "<pro value> (pro)" suffix to the label and returns it
+// if the archive is specified with pro value. Otherwise, it returns the
+// original label.
+func (index *ubuntuIndex) proSuffixedLabel() string {
+	if index.archive.options.Pro == "" {
+		return index.label
+	}
+	return index.label + " " + index.archive.options.Pro + " (pro)"
+}