Skip to content

Commit

Permalink
fix: use go-containerregistry to extract package.yaml (#52)
Browse files Browse the repository at this point in the history 
* fix: use go-containerregistry to extract package.yaml

Signed-off-by: Maximilian Braun (SAP) <[email protected]>

* cleanup

Signed-off-by: Maximilian Braun (SAP) <[email protected]>

* cleanup

Signed-off-by: Maximilian Braun (SAP) <[email protected]>

---------

Signed-off-by: Maximilian Braun (SAP) <[email protected]>
  • Loading branch information
@maximilianbraun
maximilianbraun authored Mar 19, 2024
1 parent 66179b4 commit aca814f
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 151 deletions.
7 changes: 7 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,17 @@ require (
)

require (
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/cli v24.0.6+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v24.0.6+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand Down Expand Up @@ -52,13 +57,15 @@ require (
github.com/vbatts/tar-split v0.11.5 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
17 changes: 17 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
Expand All @@ -20,6 +24,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
Expand All @@ -28,6 +34,10 @@ github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/
github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8=
github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
Expand Down Expand Up @@ -118,11 +128,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
Expand Down Expand Up @@ -189,6 +201,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -225,6 +239,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -249,6 +264,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -299,6 +315,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.27.4 h1:0pCo/AN9hONazBKlNUdhQymmnfLRbSZjd5H5H3f0bSs=
Expand Down
173 changes: 22 additions & 151 deletions internal/xpkg/xpkg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,30 @@ package xpkg
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"strings"

"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/daemon"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/pkg/errors"
)

const (
errFmtNoPackageFileFound = "couldn't find package.yaml file after checking %d files in the archive"
)

const packageFile = "package.yaml"
const layerTar = "layer.tar"

var (

// required for testing with fake data
extractContainerImage = extractPackageYamlFromImage
)

// tarFile represents a single file inside a tar. Closing it closes the tar itself.
type tarFile struct {
io.Reader
io.Closer
}

// FetchPackageContent returns the content of the package.yaml file from the givecn crossplane package
func FetchPackageContent(crossplanePackage string) (string, error) {
tmpDir, err := os.MkdirTemp("", "xpkg-*")
Expand Down Expand Up @@ -78,35 +73,17 @@ func SavePackage(crossplanePackage string, targetFile string) error {
func extractPackageYamlFromImage(imageName, tempDirPath string) error {
// this func has dependencies only on Google container registry and docker moby

// Save the Docker image to a tar file
var imageBuff bytes.Buffer
saveCmd := exec.Command("docker", "save", imageName)
saveCmd.Stdout = &imageBuff
if err := saveCmd.Run(); err != nil {
return fmt.Errorf("error saving image: %w", err)
}

// require imageREader for multiple passes over io.ReadCloser/io.Reader
// ps: no need to close as it is a wrapper around []byte
imageReader := bytes.NewReader(imageBuff.Bytes())

// retrieve manifest from the xpgk, we need this to get the layers digest and path
layerPath, errPackageLayer := findPackageYamlInImage(ioOpener(io.NopCloser(imageReader)))
if errPackageLayer != nil {
return errPackageLayer
reference, err := name.ParseReference(imageName)
if err != nil {
return fmt.Errorf("error parsing image name %w", err)
}

// reset to start
_, err := imageReader.Seek(0, io.SeekStart)
// Save the Docker image to a tar file
image, err := daemon.Image(reference)
if err != nil {
return fmt.Errorf("error seeking to start of imageReader: %w", err)
return err
}

// e.g. layerPath := "6a19324dac365085b6cf6d286dc0afd4cba84f98ef896f512ecf58d5b9e1566c/layer.tar"
layerRC, errF := extractFileFromTar(ioOpener(io.NopCloser(imageReader)), layerPath)
if errF != nil {
return errF
}
tarc := mutate.Extract(image)

localFile, errCreate := os.Create(filepath.Join(tempDirPath, filepath.Base(packageFile))) //nolint:gosec, we dictate path, not user provided
if errCreate != nil {
Expand All @@ -117,127 +94,21 @@ func extractPackageYamlFromImage(imageName, tempDirPath string) error {
}(localFile)

// search for the desired file within the tarball, only on the current layer
tr := tar.NewReader(layerRC)
t := tar.NewReader(tarc)
var read int
for {
header, errNext := tr.Next()
if errNext == io.EOF {
break
}
if errNext != nil {
return fmt.Errorf("failed to read tar header from layer: %w", errNext)
h, err := t.Next()
if err != nil {
return errors.Wrapf(err, errFmtNoPackageFileFound, read)
}
if header.Name == packageFile {
_, errCopy := io.Copy(localFile, tr) //nolint:gosec
if h.Name == packageFile {
_, errCopy := io.Copy(localFile, t) //nolint:gosec
if errCopy != nil {
return fmt.Errorf("failed to copy content to local %s file: %w", packageFile, errCopy)
}
// if successfully errCopy is nil
return nil
}
}

return errors.New(fmt.Sprintf("no %s file found in xpkg image", packageFile))
}

// extractFileFromTar
func extractFileFromTar(opener tarball.Opener, filePath string) (io.ReadCloser, error) {
f, err := opener()
if err != nil {
return nil, err
}
needClose := true
defer func() {
if needClose {
errClose := f.Close()
if errClose != nil {
return
}
}
}()

tf := tar.NewReader(f)
for {
hdr, err := tf.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, err
}
if hdr.Name == filePath {
if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink {
currentDir := filepath.Dir(filePath)
return extractFileFromTar(opener, path.Join(currentDir, path.Clean(hdr.Linkname)))
}
needClose = false
return tarFile{
Reader: tf,
Closer: f,
}, nil
}
}
return nil, fmt.Errorf("file %s not found in tar", filePath)
}

// findPackageYamlInImage finds the layer containing the package.yaml file.
func findPackageYamlInImage(opener tarball.Opener) (string, error) {
f, err := opener()
if err != nil {
return "", err
}
needClose := true
defer func() {
if needClose {
errClose := f.Close()
if errClose != nil {
return
}
}
}()

tarReader := tar.NewReader(f)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return "", err
}

// Check if this is a layer
if header.Typeflag == tar.TypeReg && strings.Contains(header.Name, layerTar) {
if layerContainsPackageYaml(tarReader) {
return header.Name, nil
}
}
}
return "", fmt.Errorf("%s not found in any layer", packageFile)
}

// layerContainsPackageYaml checks if the given layer contains the package.yaml file.
func layerContainsPackageYaml(layerReader io.Reader) bool {

tarLayerReader := tar.NewReader(layerReader)
for {
header, err := tarLayerReader.Next()
if err == io.EOF {
break
}
if err != nil {
return false
}

if header.Typeflag == tar.TypeReg && header.Name == packageFile {
return true
}
}
return false
}

// ioOpener is a func for opening a tar file
func ioOpener(closer io.ReadCloser) tarball.Opener {
return func() (io.ReadCloser, error) {
return closer, nil
read++
}
}

0 comments on commit aca814f

Please sign in to comment.