Skip to content

Commit

Permalink
inspector: support flattened buildpacks
Browse files Browse the repository at this point in the history 
With platforms introducing the ability to create flattened
buildpackages[1], the idea of 1 layer = 1 buildpack is not always true
anymore - a single layer can contain multiple buildpacks.
This change adds to the inspector the ability to read buildpack
metadata off flattened buildpacks.

This fixes "jam summarize" when creating summary/release notes
for flattened buildpacks. Previously it picked up the first
buildpack.toml it could find in the layer, and assumed it to represent
the buildpackage.

[1]: buildpacks/pack#1691
  • Loading branch information
@arjun024 @sophiewigmore
arjun024 authored and sophiewigmore committed Jun 9, 2023
1 parent d4052cb commit b7e18d1
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 41 deletions.
74 changes: 50 additions & 24 deletions internal/buildpack_inspector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package internal

import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -31,7 +32,7 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
}
defer file.Close()

indexJSON, err := fetchArchivedFile(tar.NewReader(file), "index.json")
indicesJSON, err := fetchFromArchive(tar.NewReader(file), "index.json", true)
if err != nil {
return nil, err
}
Expand All @@ -42,7 +43,8 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
} `json:"manifests"`
}

err = json.NewDecoder(indexJSON).Decode(&index)
// There can only be 1 image index
err = json.NewDecoder(indicesJSON[0]).Decode(&index)
if err != nil {
return nil, err
}
Expand All @@ -52,7 +54,7 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
return nil, err
}

manifest, err := fetchArchivedFile(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(index.Manifests[0].Digest, "sha256:")))
manifests, err := fetchFromArchive(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(index.Manifests[0].Digest, "sha256:")), true)
if err != nil {
return nil, err
}
Expand All @@ -65,7 +67,8 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
} `json:"layers"`
}

err = json.NewDecoder(manifest).Decode(&m)
// We only support single manifest images
err = json.NewDecoder(manifests[0]).Decode(&m)
if err != nil {
return nil, err
}
Expand All @@ -77,35 +80,40 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
return nil, err
}

buildpack, err := fetchArchivedFile(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(layer.Digest, "sha256:")))
layerBlobs, err := fetchFromArchive(tar.NewReader(file), filepath.Join("blobs", "sha256", strings.TrimPrefix(layer.Digest, "sha256:")), true)
if err != nil {
return nil, err
}

buildpackGR, err := gzip.NewReader(buildpack)
layerGR, err := gzip.NewReader(layerBlobs[0])
if err != nil {
return nil, fmt.Errorf("failed to read buildpack gzip: %w", err)
return nil, fmt.Errorf("failed to read layer blob: %w", err)
}
defer buildpackGR.Close()
defer layerGR.Close()

buildpackTOML, err := fetchArchivedFile(tar.NewReader(buildpackGR), "buildpack.toml")
// Generally, each layer corresponds to a buildpack.
// But certain buildpacks are "flattened" and contain multiple buildpacks
// in the same layer.
buildpackTOMLs, err := fetchFromArchive(tar.NewReader(layerGR), "buildpack.toml", false)
if err != nil {
return nil, err
}

var config cargo.Config
err = cargo.DecodeConfig(buildpackTOML, &config)
if err != nil {
return nil, err
}

metadata := BuildpackMetadata{
Config: config,
}
if len(config.Order) > 0 {
metadata.SHA256 = buildpackageDigest
for _, buildpackTOML := range buildpackTOMLs {
var config cargo.Config
err = cargo.DecodeConfig(buildpackTOML, &config)
if err != nil {
return nil, err
}

metadata := BuildpackMetadata{
Config: config,
}
if len(config.Order) > 0 {
metadata.SHA256 = buildpackageDigest
}
metadataCollection = append(metadataCollection, metadata)
}
metadataCollection = append(metadataCollection, metadata)
}

if len(metadataCollection) == 1 {
Expand All @@ -115,7 +123,14 @@ func (i BuildpackInspector) Dependencies(path string) ([]BuildpackMetadata, erro
return metadataCollection, nil
}

func fetchArchivedFile(tr *tar.Reader, filename string) (io.Reader, error) {
// This function takes a boolean to stop search after the first match because
// tar.Reader is a streaming reader, and once you move to the next entry via a
// Next() call, the previous file reader becomes invalid. This forces us to
// copy the file contents to memory if we want to fetch multiple matches, and
// we only want to do so for small text files, and not large files like layer
// blobs.
func fetchFromArchive(tr *tar.Reader, filename string, stopAtFirstMatch bool) ([]io.Reader, error) {
var readers []io.Reader
for {
hdr, err := tr.Next()
if err == io.EOF {
Expand All @@ -126,9 +141,20 @@ func fetchArchivedFile(tr *tar.Reader, filename string) (io.Reader, error) {
}

if strings.HasSuffix(hdr.Name, filename) {
return tr, nil
if stopAtFirstMatch {
return []io.Reader{tr}, nil
}
buff := bytes.NewBuffer(nil)
_, err = io.CopyN(buff, tr, hdr.Size)
if err != nil {
return nil, fmt.Errorf("failed to copy file %s: %w", hdr.Name, err)
}
readers = append(readers, buff)
}
}

return nil, fmt.Errorf("failed to fetch archived file %s", filename)
if len(readers) < 1 {
return nil, fmt.Errorf("failed to fetch archived file %s", filename)
}
return readers, nil
}
156 changes: 139 additions & 17 deletions internal/buildpack_inspector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ func testBuildpackInspector(t *testing.T, context spec.G, it spec.S) {
var (
Expect = NewWithT(t).Expect

buildpackage string
inspector internal.BuildpackInspector
inspector internal.BuildpackInspector
buildpackage string
contentBp1, contentBp2, contentMetaBp []byte
)

it.Before(func() {
Expand All @@ -33,7 +34,7 @@ func testBuildpackInspector(t *testing.T, context spec.G, it spec.S) {
firstBuildpackGW := gzip.NewWriter(firstBuildpack)
firstBuildpackTW := tar.NewWriter(firstBuildpackGW)

content := []byte(`[buildpack]
contentBp1 = []byte(`[buildpack]
id = "some-buildpack"
version = "1.2.3"
Expand Down Expand Up @@ -61,11 +62,11 @@ other-dependency = "2.3.x"
err = firstBuildpackTW.WriteHeader(&tar.Header{
Name: "./buildpack.toml",
Mode: 0644,
Size: int64(len(content)),
Size: int64(len(contentBp1)),
})
Expect(err).NotTo(HaveOccurred())

_, err = firstBuildpackTW.Write(content)
_, err = firstBuildpackTW.Write(contentBp1)
Expect(err).NotTo(HaveOccurred())

Expect(firstBuildpackTW.Close()).To(Succeed())
Expand All @@ -85,7 +86,7 @@ other-dependency = "2.3.x"
secondBuildpackGW := gzip.NewWriter(secondBuildpack)
secondBuildpackTW := tar.NewWriter(secondBuildpackGW)

content = []byte(`[buildpack]
contentBp2 = []byte(`[buildpack]
id = "other-buildpack"
version = "2.3.4"
Expand Down Expand Up @@ -113,11 +114,11 @@ second-dependency = "5.6.x"
err = secondBuildpackTW.WriteHeader(&tar.Header{
Name: "./buildpack.toml",
Mode: 0644,
Size: int64(len(content)),
Size: int64(len(contentBp2)),
})
Expect(err).NotTo(HaveOccurred())

_, err = secondBuildpackTW.Write(content)
_, err = secondBuildpackTW.Write(contentBp2)
Expect(err).NotTo(HaveOccurred())

Expect(secondBuildpackTW.Close()).To(Succeed())
Expand All @@ -137,7 +138,7 @@ second-dependency = "5.6.x"
thirdBuildpackGW := gzip.NewWriter(thirdBuildpack)
thirdBuildpackTW := tar.NewWriter(thirdBuildpackGW)

content = []byte(`[buildpack]
contentMetaBp = []byte(`[buildpack]
id = "meta-buildpack"
version = "3.4.5"
Expand All @@ -155,11 +156,11 @@ version = "2.3.4"
err = thirdBuildpackTW.WriteHeader(&tar.Header{
Name: "./buildpack.toml",
Mode: 0644,
Size: int64(len(content)),
Size: int64(len(contentMetaBp)),
})
Expect(err).NotTo(HaveOccurred())

_, err = thirdBuildpackTW.Write(content)
_, err = thirdBuildpackTW.Write(contentMetaBp)
Expect(err).NotTo(HaveOccurred())

Expect(thirdBuildpackTW.Close()).To(Succeed())
Expand Down Expand Up @@ -226,10 +227,13 @@ version = "2.3.4"
})

context("Dependencies", func() {
it("returns a list of dependencies", func() {
configs, err := inspector.Dependencies(buildpackage)
Expect(err).NotTo(HaveOccurred())
Expect(configs).To(Equal([]internal.BuildpackMetadata{
var (
expectedMetadata []internal.BuildpackMetadata
buildpackageFlat string
)

it.Before(func() {
expectedMetadata = []internal.BuildpackMetadata{
{
Config: cargo.Config{
Buildpack: cargo.ConfigBuildpack{
Expand Down Expand Up @@ -317,7 +321,125 @@ version = "2.3.4"
},
SHA256: "sha256:manifest-sha",
},
}))
}
})

context("Unflattened buildpack", func() {
it("returns a list of dependencies", func() {
configs, err := inspector.Dependencies(buildpackage)
Expect(err).NotTo(HaveOccurred())
Expect(configs).To(Equal(expectedMetadata))
})
})

context("Flattened buildpack", func() {
it.Before(func() {
/* Flattened buildpackage, but with the same buildpack metadata as the unflattened */
fileFlat, err := os.CreateTemp("", "buildpackage-flattened")
Expect(err).NotTo(HaveOccurred())

twf := tar.NewWriter(fileFlat)

flatLayer := bytes.NewBuffer(nil)
flatLayerGW := gzip.NewWriter(flatLayer)
flatLayerTW := tar.NewWriter(flatLayerGW)

err = flatLayerTW.WriteHeader(&tar.Header{
Name: "./buildpack-one/buildpack.toml",
Mode: 0644,
Size: int64(len(contentBp1)),
})
Expect(err).NotTo(HaveOccurred())

_, err = flatLayerTW.Write(contentBp1)
Expect(err).NotTo(HaveOccurred())

err = flatLayerTW.WriteHeader(&tar.Header{
Name: "./buildpack-two/buildpack.toml",
Mode: 0644,
Size: int64(len(contentBp2)),
})
Expect(err).NotTo(HaveOccurred())

_, err = flatLayerTW.Write(contentBp2)
Expect(err).NotTo(HaveOccurred())

err = flatLayerTW.WriteHeader(&tar.Header{
Name: "./buildpack-three-meta/buildpack.toml",
Mode: 0644,
Size: int64(len(contentMetaBp)),
})
Expect(err).NotTo(HaveOccurred())

_, err = flatLayerTW.Write(contentMetaBp)
Expect(err).NotTo(HaveOccurred())

Expect(flatLayerTW.Close()).To(Succeed())
Expect(flatLayerGW.Close()).To(Succeed())

err = twf.WriteHeader(&tar.Header{
Name: "blobs/sha256/all-buildpacks-flattened-layer-sha",
Mode: 0644,
Size: int64(flatLayer.Len()),
})
Expect(err).NotTo(HaveOccurred())

_, err = twf.Write(flatLayer.Bytes())
Expect(err).NotTo(HaveOccurred())

manifestFlat := bytes.NewBuffer(nil)
err = json.NewEncoder(manifestFlat).Encode(map[string]interface{}{
"layers": []map[string]interface{}{
{"digest": "sha256:all-buildpacks-flattened-layer-sha"},
},
})
Expect(err).NotTo(HaveOccurred())

err = twf.WriteHeader(&tar.Header{
Name: "blobs/sha256/manifest-sha",
Mode: 0644,
Size: int64(manifestFlat.Len()),
})
Expect(err).NotTo(HaveOccurred())

_, err = twf.Write(manifestFlat.Bytes())
Expect(err).NotTo(HaveOccurred())

indexFlat := bytes.NewBuffer(nil)
err = json.NewEncoder(indexFlat).Encode(map[string]interface{}{
"manifests": []map[string]interface{}{
{"digest": "sha256:manifest-sha"},
},
})
Expect(err).NotTo(HaveOccurred())

err = twf.WriteHeader(&tar.Header{
Name: "index.json",
Mode: 0644,
Size: int64(indexFlat.Len()),
})
Expect(err).NotTo(HaveOccurred())

_, err = twf.Write(indexFlat.Bytes())
Expect(err).NotTo(HaveOccurred())

buildpackageFlat = fileFlat.Name()

Expect(twf.Close()).To(Succeed())
Expect(fileFlat.Close()).To(Succeed())

inspector = internal.NewBuildpackInspector()
})

it.After(func() {
Expect(os.Remove(buildpackageFlat)).To(Succeed())
})

it("returns a list of dependencies", func() {
configs, err := inspector.Dependencies(buildpackageFlat)
Expect(err).NotTo(HaveOccurred())
Expect(configs).To(Equal(expectedMetadata))
})
})

context("failure cases", func() {
Expand Down Expand Up @@ -559,7 +681,7 @@ version = "2.3.4"

it("returns an error", func() {
_, err := inspector.Dependencies(buildpackage)
Expect(err).To(MatchError("failed to read buildpack gzip: unexpected EOF"))
Expect(err).To(MatchError("failed to read layer blob: unexpected EOF"))
})
})

Expand Down

0 comments on commit b7e18d1

Please sign in to comment.