Skip to content

Commit

Permalink
test: sif image_simple / image_symlinks adaptations
Browse files Browse the repository at this point in the history 
Modify the TestSimpleImage and TestImageSymlinks code to incorporate
testing of a singularity sif image source.

This requires signficant adaptations as singularity squashes
containers down to a single layer.

Singularity is expected to be available, and is is now installed in
the ci-bootstrap Makefile target (from a GitHub release).

Signed-off-by: David Trudgian <[email protected]>
  • Loading branch information
@dtrudg
dtrudg committed Jun 24, 2022
1 parent 83a74e2 commit 24c442d
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 26 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ help:
.PHONY: ci-bootstrap
ci-bootstrap: bootstrap
sudo apt install -y bc
sudo apt install -y https://github.com/sylabs/singularity/releases/download/v3.10.0/singularity-ce_3.10.0-focal_amd64.deb

$(RESULTSDIR):
mkdir -p $(RESULTSDIR)
Expand Down
4 changes: 3 additions & 1 deletion pkg/image/sif/image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/sylabs/sif/v2/pkg/sif"
)

const SingularityMediaType = "application/vnd.sylabs.sif.layer.v1.sif"

// fileSectionReader implements an io.ReadCloser that reads from r and closes c.
type fileSectionReader struct {
*io.SectionReader
Expand Down Expand Up @@ -135,7 +137,7 @@ func (im *sifImage) RawConfigFile() ([]byte, error) {

// MediaType of this image's manifest.
func (im *sifImage) MediaType() (types.MediaType, error) {
return "application/vnd.sylabs.sif.layer.v1.sif", nil
return SingularityMediaType, nil
}

// LayerByDiffID is a variation on the v1.Image method, which returns an UncompressedLayer instead.
Expand Down
63 changes: 63 additions & 0 deletions pkg/imagetest/image_fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ func PrepareFixtureImage(t testing.TB, source, name string) string {
skopeoCopyDockerArchiveToPath(t, dockerArchivePath, fmt.Sprintf("oci:%s", ociDirPath))
}
location = ociDirPath
case image.SingularitySource:
location = GetFixtureImageSIFPath(t, name)
default:
t.Fatalf("could not determine source: %+v", source)
}
Expand Down Expand Up @@ -251,3 +253,64 @@ func saveImage(t testing.TB, image, path string) error {
cmd.Stdin = os.Stdin
return cmd.Run()
}

func GetFixtureImageSIFPath(t testing.TB, name string) string {
imageName, imageVersion := getFixtureImageInfo(t, name)
sifFileName := fmt.Sprintf("%s-%s.sif", imageName, imageVersion)
return getFixtureImageSIFPath(t, name, CacheDir, sifFileName)
}

func getFixtureImageSIFPath(t testing.TB, fixtureName, sifStoreDir, sifFileName string) string {
imageName, imageVersion := getFixtureImageInfo(t, fixtureName)
fullImageName := fmt.Sprintf("%s:%s", imageName, imageVersion)
sifPath := path.Join(sifStoreDir, sifFileName)

// create the cache dir if it does not already exist...
if !fileOrDirExists(t, CacheDir) {
err := os.Mkdir(CacheDir, 0o755)
if err != nil {
t.Fatalf("could not create sif cache dir (%s): %+v", CacheDir, err)
}
}

// if the image sif does not exist, make it
if !fileOrDirExists(t, sifPath) {
if !isImageInDocker(fullImageName) {
contextPath := path.Join(testutils.TestFixturesDir, fixtureName)
buildDockerImage(t, contextPath, imageName, imageVersion)
}
err := buildSIFFromDocker(t, fullImageName, sifPath)
if err != nil {
t.Fatal("could not save fixture image:", err)
}
}

return sifPath
}

func buildSIFFromDocker(t testing.TB, image, path string) error {
singularity, err := exec.LookPath("singularity")
if err != nil {
t.Skipf("singularity not found: %v", err)
}

outfile, err := os.Create(path)
if err != nil {
t.Fatal("unable to create file for SIF image:", err)
}
defer func() {
err := outfile.Close()
if err != nil {
t.Fatalf("unable to close file path=%q : %+v", path, err)
}
}()

cmdArgs := []string{"build", "--disable-cache", "--force", path, "docker-daemon:" + image}
cmd := exec.Command(singularity, cmdArgs...)
cmd.Env = os.Environ()

cmd.Stdout = outfile
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
return cmd.Run()
}
86 changes: 61 additions & 25 deletions test/integration/fixture_image_simple_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,49 +15,95 @@ import (
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/image/sif"
"github.com/anchore/stereoscope/pkg/imagetest"
v1Types "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/scylladb/go-set"
"github.com/stretchr/testify/require"
)

// Common layer metadata for OCI / Docker / Podman. MediaType will be filled in during test.
var simpleImageLayers = []image.LayerMetadata{
{
Index: 0,
Size: 22,
},
{
Index: 1,
Size: 16,
},
{
Index: 2,
Size: 27,
},
}

// Singularity images are squashed to a single layer.
var simpleImageSingularityLayer = []image.LayerMetadata{
{
Index: 0,
// Disable size check. Size can vary - image build embeds variable length timestamps etc.
Size: -1,
},
}

var simpleImageTestCases = []testCase{
{
name: "FromTarball",
source: "docker-archive",
imageMediaType: v1Types.DockerManifestSchema2,
layerMediaType: v1Types.DockerLayer,
layers: simpleImageLayers,
tagCount: 1,
size: 65,
},
{
name: "FromDocker",
source: "docker",
imageMediaType: v1Types.DockerManifestSchema2,
layerMediaType: v1Types.DockerLayer,
layers: simpleImageLayers,
// name:hash
// name:latest
tagCount: 2,
size: 65,
},
{
name: "FromPodman",
source: "podman",
imageMediaType: v1Types.DockerManifestSchema2,
layerMediaType: v1Types.DockerLayer,
layers: simpleImageLayers,
tagCount: 2,
size: 65,
},
{
name: "FromOciTarball",
source: "oci-archive",
imageMediaType: v1Types.OCIManifestSchema1,
layerMediaType: v1Types.OCILayer,
layers: simpleImageLayers,
tagCount: 0,
size: 65,
},
{
name: "FromOciDirectory",
source: "oci-dir",
imageMediaType: v1Types.OCIManifestSchema1,
layerMediaType: v1Types.OCILayer,
layers: simpleImageLayers,
tagCount: 0,
size: 65,
},
{
name: "FromSingularity",
source: "singularity",
imageMediaType: sif.SingularityMediaType,
layerMediaType: image.SingularitySquashFSLayer,
layers: simpleImageSingularityLayer,
tagCount: 0,
// Disable size check. Size can vary - image build embeds timestamps etc.
size: -1,
},
}

Expand All @@ -66,7 +112,9 @@ type testCase struct {
source string
imageMediaType v1Types.MediaType
layerMediaType v1Types.MediaType
layers []image.LayerMetadata
tagCount int
size int
}

func TestSimpleImage(t *testing.T) {
Expand All @@ -81,8 +129,11 @@ func TestSimpleImage(t *testing.T) {
i := imagetest.GetFixtureImage(t, c.source, "image-simple")

assertImageSimpleMetadata(t, i, c)
assertImageSimpleTrees(t, i)
assertImageSimpleSquashedTrees(t, i)
// Singularity images are a single layer. Don't verify content per layer.
if c.source != "singularity" {
assertImageSimpleTrees(t, i)
assertImageSimpleSquashedTrees(t, i)
}
assertImageSimpleContents(t, i)
})
}
Expand Down Expand Up @@ -148,9 +199,6 @@ func BenchmarkSimpleImage_FetchSquashedContents(b *testing.B) {
func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues testCase) {
t.Helper()
t.Log("Asserting metadata...")
if i.Metadata.Size != 65 {
t.Errorf("unexpected image size: %d", i.Metadata.Size)
}
if i.Metadata.MediaType != expectedValues.imageMediaType {
t.Errorf("unexpected image media type: %+v", i.Metadata.MediaType)
}
Expand All @@ -162,36 +210,24 @@ func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues test
}
}

expected := []image.LayerMetadata{
{
Index: 0,
Size: 22,
MediaType: expectedValues.layerMediaType,
},
{
Index: 1,
Size: 16,
MediaType: expectedValues.layerMediaType,
},
{
Index: 2,
Size: 27,
MediaType: expectedValues.layerMediaType,
},
if expectedValues.size >= 0 && i.Metadata.Size != int64(expectedValues.size) {
t.Errorf("unexpected image size: %d", i.Metadata.Size)
}

if len(expected) != len(i.Layers) {
if len(expectedValues.layers) != len(i.Layers) {
t.Fatal("unexpected number of layers:", len(i.Layers))
}

for idx, l := range i.Layers {
if expected[idx].Size != l.Metadata.Size {
expected := expectedValues.layers[idx]
expected.MediaType = expectedValues.layerMediaType
if expected.Size >= 0 && expected.Size != l.Metadata.Size {
t.Errorf("mismatched layer 'Size' (layer %d): %+v", idx, l.Metadata.Size)
}
if expected[idx].MediaType != l.Metadata.MediaType {
if expected.MediaType != l.Metadata.MediaType {
t.Errorf("mismatched layer 'MediaType' (layer %d): %+v", idx, l.Metadata.MediaType)
}
if expected[idx].Index != l.Metadata.Index {
if expected.Index != l.Metadata.Index {
t.Errorf("mismatched layer 'Index' (layer %d): %+v", idx, l.Metadata.Index)
}
}
Expand Down
67 changes: 67 additions & 0 deletions test/integration/fixture_image_symlinks_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ func TestImageSymlinks(t *testing.T) {
name: "FromOciDirectory",
source: "oci-dir",
},
{
name: "FromSingularity",
source: "singularity",
},
}

expectedSet := set.NewIntSet()
Expand All @@ -61,6 +65,12 @@ func TestImageSymlinks(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
i := imagetest.GetFixtureImage(t, c.source, "image-symlinks")

if c.source == "singularity" {
assertSquashedSymlinkLinkResolution(t, i)
return
}

assertImageSymlinkLinkResolution(t, i)
})
}
Expand Down Expand Up @@ -236,3 +246,60 @@ func assertImageSymlinkLinkResolution(t *testing.T, i *image.Image) {
})
}
}

// Check symlinks in image after it has been squashed to a single layer (Singularity)
func assertSquashedSymlinkLinkResolution(t *testing.T, i *image.Image) {
tests := []linkFetchConfig{
// # link with previous data
// LAYER 1 > ADD file-1.txt .
// LAYER 2 > RUN ln -s ./file-1.txt link-1
{
linkLayer: 0,
linkPath: "/link-1",
resolveLayer: 0,
expectedPath: "/file-1.txt",
perspectiveLayer: 0,
contents: "file 1!",
},

// # link with current data
// LAYER 5 > RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
{
linkLayer: 0,
linkPath: "/link-within",
resolveLayer: 0,
expectedPath: "/file-3.txt",
perspectiveLayer: 0,
// since echo was used a newline character will be present
contents: "file 3\n",
},

// # dead link (link-indirect > [non-existant file])
// LAYER 8 > RUN unlink link-2
{
linkLayer: 0,
linkPath: "/link-indirect",
resolveLayer: 0,
expectedPath: "/link-indirect",
perspectiveLayer: 0,
linkOptions: []filetree.LinkResolutionOption{filetree.DoNotFollowDeadBasenameLinks},
},
}

for _, cfg := range tests {
name := fmt.Sprintf("[%d:%s]-->[%d:%s]@%d", cfg.linkLayer, cfg.linkPath, cfg.resolveLayer, cfg.expectedPath, cfg.perspectiveLayer)
t.Run(name, func(t *testing.T) {
expectedResolve, actualResolve := fetchRefs(t, i, cfg)
assertMatch(t, i, cfg, expectedResolve, actualResolve)

if cfg.contents == "" {
return
}

actualContents := fetchContents(t, i, cfg)
if actualContents != cfg.contents {
t.Errorf("mismatched contents: '%+v'!='%+v'", cfg.contents, actualContents)
}
})
}
}

0 comments on commit 24c442d

Please sign in to comment.