pkg/cache: memory efficient cache building

Signed-off-by: Joe Lanford <[email protected]>

pkg/cache/cache.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -208,37 +209,70 @@ func (c *cache) Build(ctx context.Context, fbcFsys fs.FS) error {
 		return fmt.Errorf("init cache: %v", err)
 	}
 
-	fbc, err := declcfg.LoadFS(ctx, fbcFsys)
+	tmpFile, err := os.CreateTemp("", "opm-cache-build-*.json")
 	if err != nil {
 		return err
 	}
-	fbcModel, err := declcfg.ConvertToModel(*fbc)
-	if err != nil {
+	byPackageReaders := make(map[string][]io.Reader, 0)
+	defer func() {
+		tmpFile.Close()
+		os.Remove(tmpFile.Name())
+	}()
+	offset := int64(0)
+	if err := declcfg.WalkMetasFS(fbcFsys, func(path string, meta *declcfg.Meta, err error) error {
+		if err != nil {
+			return err
+		}
+		if _, err := tmpFile.Write(meta.Blob); err != nil {
+			return err
+		}
+		packageName := meta.Package
+		if meta.Schema == declcfg.SchemaPackage {
+			packageName = meta.Name
+		}
+		sr := io.NewSectionReader(tmpFile, offset, int64(len(meta.Blob)))
+		byPackageReaders[packageName] = append(byPackageReaders[packageName], sr)
+		offset += int64(len(meta.Blob))
+		return nil
+	}); err != nil {
 		return err
 	}
 
-	pkgs, err := packagesFromModel(fbcModel)
-	if err != nil {
-		return err
+	pkgs := packageIndex{}
+	for pkgName, readers := range byPackageReaders {
+		mr := io.MultiReader(readers...)
+		pkgFbc, err := declcfg.LoadReader(mr)
+		if err != nil {
+			return err
+		}
+		pkgModel, err := declcfg.ConvertToModel(*pkgFbc)
+		if err != nil {
+			return err
+		}
+		pkgIndex, err := packagesFromModel(pkgModel)
+		if err != nil {
+			return err
+		}
+		for _, p := range pkgModel {
+			for _, ch := range p.Channels {
+				for _, b := range ch.Bundles {
+					apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
+					if err != nil {
+						return err
+					}
+					if err := c.backend.PutBundle(ctx, bundleKey{p.Name, ch.Name, b.Name}, apiBundle); err != nil {
+						return fmt.Errorf("store bundle %q: %v", b.Name, err)
+					}
+				}
+			}
+		}
+		pkgs[pkgName] = pkgIndex[pkgName]
 	}
 
 	if err := c.backend.PutPackageIndex(ctx, pkgs); err != nil {
 		return fmt.Errorf("store package index: %v", err)
 	}
 
-	for _, p := range fbcModel {
-		for _, ch := range p.Channels {
-			for _, b := range ch.Bundles {
-				apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
-				if err != nil {
-					return err
-				}
-				if err := c.backend.PutBundle(ctx, bundleKey{p.Name, ch.Name, b.Name}, apiBundle); err != nil {
-					return fmt.Errorf("store bundle %q: %v", b.Name, err)
-				}
-			}
-		}
-	}
 	digest, err := c.backend.ComputeDigest(ctx, fbcFsys)
 	if err != nil {
 		return fmt.Errorf("compute digest: %v", err)