From b7e18d1fb233db0399c731ae20e700406c98da69 Mon Sep 17 00:00:00 2001 From: Arjun Sreedharan Date: Thu, 8 Jun 2023 21:29:04 +0000 Subject: [PATCH] inspector: support flattened buildpacks With platforms introducing the ability to create flattened buildpackages[1], the idea of 1 layer = 1 buildpack is not always true anymore - a single layer can contain multiple buildpacks. This change adds to the inspector the ability to read buildpack metadata off flattened buildpacks. This fixes "jam summarize" when creating summary/release notes for flattened buildpacks. Previously it picked up the first buildpack.toml it could find in the layer, and assumed it to represent the buildpackage. [1]: https://github.com/buildpacks/pack/pull/1691 --- internal/buildpack_inspector.go | 74 ++++++++----- internal/buildpack_inspector_test.go | 156 ++++++++++++++++++++++++--- 2 files changed, 189 insertions(+), 41 deletions(-) diff --git a/internal/buildpack_inspector.go b/internal/buildpack_inspector.go index 8c52f92..a4003bf 100644 --- a/internal/buildpack_inspector.go +++ b/internal/buildpack_inspector.go @@ -2,6 +2,7 @@ package internal import ( "archive/tar" + "bytes" "compress/gzip" "encoding/json" "fmt" @@ -31,7 +32,7 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro } defer file.Close() - indexJSON, err := fetchArchivedFile(tar.NewReader(file), "index.json") + indicesJSON, err := fetchFromArchive(tar.NewReader(file), "index.json", true) if err != nil { return nil, err } @@ -42,7 +43,8 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro } `json:"manifests"` } - err = json.NewDecoder(indexJSON).Decode(&index) + // There can only be 1 image index + err = json.NewDecoder(indicesJSON[0]).Decode(&index) if err != nil { return nil, err } @@ -52,7 +54,7 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro return nil, err } - manifest, err := fetchArchivedFile(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(index.Manifests[0].Digest, "sha256:"))) + manifests, err := fetchFromArchive(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(index.Manifests[0].Digest, "sha256:")), true) if err != nil { return nil, err } @@ -65,7 +67,8 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro } `json:"layers"` } - err = json.NewDecoder(manifest).Decode(&m) + // We only support single manifest images + err = json.NewDecoder(manifests[0]).Decode(&m) if err != nil { return nil, err } @@ -77,35 +80,40 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro return nil, err } - buildpack, err := fetchArchivedFile(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(layer.Digest, "sha256:"))) + layerBlobs, err := fetchFromArchive(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(layer.Digest, "sha256:")), true) if err != nil { return nil, err } - buildpackGR, err := gzip.NewReader(buildpack) + layerGR, err := gzip.NewReader(layerBlobs[0]) if err != nil { - return nil, fmt.Errorf("failed to read buildpack gzip: %w", err) + return nil, fmt.Errorf("failed to read layer blob: %w", err) } - defer buildpackGR.Close() + defer layerGR.Close() - buildpackTOML, err := fetchArchivedFile(tar.NewReader(buildpackGR), "buildpack.toml") + // Generally, each layer corresponds to a buildpack. + // But certain buildpacks are "flattened" and contain multiple buildpacks + // in the same layer. + buildpackTOMLs, err := fetchFromArchive(tar.NewReader(layerGR), "buildpack.toml", false) if err != nil { return nil, err } - var config cargo.Config - err = cargo.DecodeConfig(buildpackTOML, &config) - if err != nil { - return nil, err - } - - metadata := BuildpackMetadata{ - Config: config, - } - if len(config.Order) > 0 { - metadata.SHA256 = buildpackageDigest + for _, buildpackTOML := range buildpackTOMLs { + var config cargo.Config + err = cargo.DecodeConfig(buildpackTOML, &config) + if err != nil { + return nil, err + } + + metadata := BuildpackMetadata{ + Config: config, + } + if len(config.Order) > 0 { + metadata.SHA256 = buildpackageDigest + } + metadataCollection = append(metadataCollection, metadata) } - metadataCollection = append(metadataCollection, metadata) } if len(metadataCollection) == 1 { @@ -115,7 +123,14 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro return metadataCollection, nil } -func fetchArchivedFile(tr *tar.Reader, filename string) (io.Reader, error) { +// This function takes a boolean to stop search after the first match because +// tar.Reader is a streaming reader, and once you move to the next entry via a +// Next() call, the previous file reader becomes invalid. This forces us to +// copy the file contents to memory if we want to fetch multiple matches, and +// we only want to do so for small text files, and not large files like layer +// blobs. +func fetchFromArchive(tr *tar.Reader, filename string, stopAtFirstMatch bool) ([]io.Reader, error) { + var readers []io.Reader for { hdr, err := tr.Next() if err == io.EOF { @@ -126,9 +141,20 @@ func fetchArchivedFile(tr *tar.Reader, filename string) (io.Reader, error) { } if strings.HasSuffix(hdr.Name, filename) { - return tr, nil + if stopAtFirstMatch { + return []io.Reader{tr}, nil + } + buff := bytes.NewBuffer(nil) + _, err = io.CopyN(buff, tr, hdr.Size) + if err != nil { + return nil, fmt.Errorf("failed to copy file %s: %w", hdr.Name, err) + } + readers = append(readers, buff) } } - return nil, fmt.Errorf("failed to fetch archived file %s", filename) + if len(readers) < 1 { + return nil, fmt.Errorf("failed to fetch archived file %s", filename) + } + return readers, nil } diff --git a/internal/buildpack_inspector_test.go b/internal/buildpack_inspector_test.go index 00edbd6..a5ab0ae 100644 --- a/internal/buildpack_inspector_test.go +++ b/internal/buildpack_inspector_test.go @@ -19,8 +19,9 @@ func testBuildpackInspector(t *testing.T, context spec.G, it spec.S) { var ( Expect = NewWithT(t).Expect - buildpackage string - inspector internal.BuildpackInspector + inspector internal.BuildpackInspector + buildpackage string + contentBp1, contentBp2, contentMetaBp []byte ) it.Before(func() { @@ -33,7 +34,7 @@ func testBuildpackInspector(t *testing.T, context spec.G, it spec.S) { firstBuildpackGW := gzip.NewWriter(firstBuildpack) firstBuildpackTW := tar.NewWriter(firstBuildpackGW) - content := []byte(`[buildpack] + contentBp1 = []byte(`[buildpack] id = "some-buildpack" version = "1.2.3" @@ -61,11 +62,11 @@ other-dependency = "2.3.x" err = firstBuildpackTW.WriteHeader(&tar.Header{ Name: "./buildpack.toml", Mode: 0644, - Size: int64(len(content)), + Size: int64(len(contentBp1)), }) Expect(err).NotTo(HaveOccurred()) - _, err = firstBuildpackTW.Write(content) + _, err = firstBuildpackTW.Write(contentBp1) Expect(err).NotTo(HaveOccurred()) Expect(firstBuildpackTW.Close()).To(Succeed()) @@ -85,7 +86,7 @@ other-dependency = "2.3.x" secondBuildpackGW := gzip.NewWriter(secondBuildpack) secondBuildpackTW := tar.NewWriter(secondBuildpackGW) - content = []byte(`[buildpack] + contentBp2 = []byte(`[buildpack] id = "other-buildpack" version = "2.3.4" @@ -113,11 +114,11 @@ second-dependency = "5.6.x" err = secondBuildpackTW.WriteHeader(&tar.Header{ Name: "./buildpack.toml", Mode: 0644, - Size: int64(len(content)), + Size: int64(len(contentBp2)), }) Expect(err).NotTo(HaveOccurred()) - _, err = secondBuildpackTW.Write(content) + _, err = secondBuildpackTW.Write(contentBp2) Expect(err).NotTo(HaveOccurred()) Expect(secondBuildpackTW.Close()).To(Succeed()) @@ -137,7 +138,7 @@ second-dependency = "5.6.x" thirdBuildpackGW := gzip.NewWriter(thirdBuildpack) thirdBuildpackTW := tar.NewWriter(thirdBuildpackGW) - content = []byte(`[buildpack] + contentMetaBp = []byte(`[buildpack] id = "meta-buildpack" version = "3.4.5" @@ -155,11 +156,11 @@ version = "2.3.4" err = thirdBuildpackTW.WriteHeader(&tar.Header{ Name: "./buildpack.toml", Mode: 0644, - Size: int64(len(content)), + Size: int64(len(contentMetaBp)), }) Expect(err).NotTo(HaveOccurred()) - _, err = thirdBuildpackTW.Write(content) + _, err = thirdBuildpackTW.Write(contentMetaBp) Expect(err).NotTo(HaveOccurred()) Expect(thirdBuildpackTW.Close()).To(Succeed()) @@ -226,10 +227,13 @@ version = "2.3.4" }) context("Dependencies", func() { - it("returns a list of dependencies", func() { - configs, err := inspector.Dependencies(buildpackage) - Expect(err).NotTo(HaveOccurred()) - Expect(configs).To(Equal([]internal.BuildpackMetadata{ + var ( + expectedMetadata []internal.BuildpackMetadata + buildpackageFlat string + ) + + it.Before(func() { + expectedMetadata = []internal.BuildpackMetadata{ { Config: cargo.Config{ Buildpack: cargo.ConfigBuildpack{ @@ -317,7 +321,125 @@ version = "2.3.4" }, SHA256: "sha256:manifest-sha", }, - })) + } + }) + + context("Unflattened buildpack", func() { + it("returns a list of dependencies", func() { + configs, err := inspector.Dependencies(buildpackage) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(Equal(expectedMetadata)) + }) + }) + + context("Flattened buildpack", func() { + it.Before(func() { + /* Flattened buildpackage, but with the same buildpack metadata as the unflattened */ + fileFlat, err := os.CreateTemp("", "buildpackage-flattened") + Expect(err).NotTo(HaveOccurred()) + + twf := tar.NewWriter(fileFlat) + + flatLayer := bytes.NewBuffer(nil) + flatLayerGW := gzip.NewWriter(flatLayer) + flatLayerTW := tar.NewWriter(flatLayerGW) + + err = flatLayerTW.WriteHeader(&tar.Header{ + Name: "./buildpack-one/buildpack.toml", + Mode: 0644, + Size: int64(len(contentBp1)), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = flatLayerTW.Write(contentBp1) + Expect(err).NotTo(HaveOccurred()) + + err = flatLayerTW.WriteHeader(&tar.Header{ + Name: "./buildpack-two/buildpack.toml", + Mode: 0644, + Size: int64(len(contentBp2)), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = flatLayerTW.Write(contentBp2) + Expect(err).NotTo(HaveOccurred()) + + err = flatLayerTW.WriteHeader(&tar.Header{ + Name: "./buildpack-three-meta/buildpack.toml", + Mode: 0644, + Size: int64(len(contentMetaBp)), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = flatLayerTW.Write(contentMetaBp) + Expect(err).NotTo(HaveOccurred()) + + Expect(flatLayerTW.Close()).To(Succeed()) + Expect(flatLayerGW.Close()).To(Succeed()) + + err = twf.WriteHeader(&tar.Header{ + Name: "blobs/sha256/all-buildpacks-flattened-layer-sha", + Mode: 0644, + Size: int64(flatLayer.Len()), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = twf.Write(flatLayer.Bytes()) + Expect(err).NotTo(HaveOccurred()) + + manifestFlat := bytes.NewBuffer(nil) + err = json.NewEncoder(manifestFlat).Encode(map[string]interface{}{ + "layers": []map[string]interface{}{ + {"digest": "sha256:all-buildpacks-flattened-layer-sha"}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + err = twf.WriteHeader(&tar.Header{ + Name: "blobs/sha256/manifest-sha", + Mode: 0644, + Size: int64(manifestFlat.Len()), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = twf.Write(manifestFlat.Bytes()) + Expect(err).NotTo(HaveOccurred()) + + indexFlat := bytes.NewBuffer(nil) + err = json.NewEncoder(indexFlat).Encode(map[string]interface{}{ + "manifests": []map[string]interface{}{ + {"digest": "sha256:manifest-sha"}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + err = twf.WriteHeader(&tar.Header{ + Name: "index.json", + Mode: 0644, + Size: int64(indexFlat.Len()), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = twf.Write(indexFlat.Bytes()) + Expect(err).NotTo(HaveOccurred()) + + buildpackageFlat = fileFlat.Name() + + Expect(twf.Close()).To(Succeed()) + Expect(fileFlat.Close()).To(Succeed()) + + inspector = internal.NewBuildpackInspector() + }) + + it.After(func() { + Expect(os.Remove(buildpackageFlat)).To(Succeed()) + }) + + it("returns a list of dependencies", func() { + configs, err := inspector.Dependencies(buildpackageFlat) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(Equal(expectedMetadata)) + }) }) context("failure cases", func() { @@ -559,7 +681,7 @@ version = "2.3.4" it("returns an error", func() { _, err := inspector.Dependencies(buildpackage) - Expect(err).To(MatchError("failed to read buildpack gzip: unexpected EOF")) + Expect(err).To(MatchError("failed to read layer blob: unexpected EOF")) }) })