From 24c442dc47b68a7486585f67d62ef4077148a49a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 24 Jun 2022 11:38:03 -0500 Subject: [PATCH] test: sif image_simple / image_symlinks adaptations Modify the TestSimpleImage and TestImageSymlinks code to incorporate testing of a singularity sif image source. This requires signficant adaptations as singularity squashes containers down to a single layer. Singularity is expected to be available, and is is now installed in the ci-bootstrap Makefile target (from a GitHub release). Signed-off-by: David Trudgian --- Makefile | 1 + pkg/image/sif/image.go | 4 +- pkg/imagetest/image_fixtures.go | 63 ++++++++++++++ test/integration/fixture_image_simple_test.go | 86 +++++++++++++------ .../fixture_image_symlinks_test.go | 67 +++++++++++++++ 5 files changed, 195 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 86505912..682d7841 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ help: .PHONY: ci-bootstrap ci-bootstrap: bootstrap sudo apt install -y bc + sudo apt install -y https://github.com/sylabs/singularity/releases/download/v3.10.0/singularity-ce_3.10.0-focal_amd64.deb $(RESULTSDIR): mkdir -p $(RESULTSDIR) diff --git a/pkg/image/sif/image.go b/pkg/image/sif/image.go index 62d952a4..bcbf9d9a 100644 --- a/pkg/image/sif/image.go +++ b/pkg/image/sif/image.go @@ -14,6 +14,8 @@ import ( "github.com/sylabs/sif/v2/pkg/sif" ) +const SingularityMediaType = "application/vnd.sylabs.sif.layer.v1.sif" + // fileSectionReader implements an io.ReadCloser that reads from r and closes c. type fileSectionReader struct { *io.SectionReader @@ -135,7 +137,7 @@ func (im *sifImage) RawConfigFile() ([]byte, error) { // MediaType of this image's manifest. func (im *sifImage) MediaType() (types.MediaType, error) { - return "application/vnd.sylabs.sif.layer.v1.sif", nil + return SingularityMediaType, nil } // LayerByDiffID is a variation on the v1.Image method, which returns an UncompressedLayer instead. diff --git a/pkg/imagetest/image_fixtures.go b/pkg/imagetest/image_fixtures.go index 4b1e501a..56527b1a 100644 --- a/pkg/imagetest/image_fixtures.go +++ b/pkg/imagetest/image_fixtures.go @@ -48,6 +48,8 @@ func PrepareFixtureImage(t testing.TB, source, name string) string { skopeoCopyDockerArchiveToPath(t, dockerArchivePath, fmt.Sprintf("oci:%s", ociDirPath)) } location = ociDirPath + case image.SingularitySource: + location = GetFixtureImageSIFPath(t, name) default: t.Fatalf("could not determine source: %+v", source) } @@ -251,3 +253,64 @@ func saveImage(t testing.TB, image, path string) error { cmd.Stdin = os.Stdin return cmd.Run() } + +func GetFixtureImageSIFPath(t testing.TB, name string) string { + imageName, imageVersion := getFixtureImageInfo(t, name) + sifFileName := fmt.Sprintf("%s-%s.sif", imageName, imageVersion) + return getFixtureImageSIFPath(t, name, CacheDir, sifFileName) +} + +func getFixtureImageSIFPath(t testing.TB, fixtureName, sifStoreDir, sifFileName string) string { + imageName, imageVersion := getFixtureImageInfo(t, fixtureName) + fullImageName := fmt.Sprintf("%s:%s", imageName, imageVersion) + sifPath := path.Join(sifStoreDir, sifFileName) + + // create the cache dir if it does not already exist... + if !fileOrDirExists(t, CacheDir) { + err := os.Mkdir(CacheDir, 0o755) + if err != nil { + t.Fatalf("could not create sif cache dir (%s): %+v", CacheDir, err) + } + } + + // if the image sif does not exist, make it + if !fileOrDirExists(t, sifPath) { + if !isImageInDocker(fullImageName) { + contextPath := path.Join(testutils.TestFixturesDir, fixtureName) + buildDockerImage(t, contextPath, imageName, imageVersion) + } + err := buildSIFFromDocker(t, fullImageName, sifPath) + if err != nil { + t.Fatal("could not save fixture image:", err) + } + } + + return sifPath +} + +func buildSIFFromDocker(t testing.TB, image, path string) error { + singularity, err := exec.LookPath("singularity") + if err != nil { + t.Skipf("singularity not found: %v", err) + } + + outfile, err := os.Create(path) + if err != nil { + t.Fatal("unable to create file for SIF image:", err) + } + defer func() { + err := outfile.Close() + if err != nil { + t.Fatalf("unable to close file path=%q : %+v", path, err) + } + }() + + cmdArgs := []string{"build", "--disable-cache", "--force", path, "docker-daemon:" + image} + cmd := exec.Command(singularity, cmdArgs...) + cmd.Env = os.Environ() + + cmd.Stdout = outfile + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + return cmd.Run() +} diff --git a/test/integration/fixture_image_simple_test.go b/test/integration/fixture_image_simple_test.go index 22eca625..a0f61308 100644 --- a/test/integration/fixture_image_simple_test.go +++ b/test/integration/fixture_image_simple_test.go @@ -15,49 +15,95 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/image/sif" "github.com/anchore/stereoscope/pkg/imagetest" v1Types "github.com/google/go-containerregistry/pkg/v1/types" "github.com/scylladb/go-set" "github.com/stretchr/testify/require" ) +// Common layer metadata for OCI / Docker / Podman. MediaType will be filled in during test. +var simpleImageLayers = []image.LayerMetadata{ + { + Index: 0, + Size: 22, + }, + { + Index: 1, + Size: 16, + }, + { + Index: 2, + Size: 27, + }, +} + +// Singularity images are squashed to a single layer. +var simpleImageSingularityLayer = []image.LayerMetadata{ + { + Index: 0, + // Disable size check. Size can vary - image build embeds variable length timestamps etc. + Size: -1, + }, +} + var simpleImageTestCases = []testCase{ { name: "FromTarball", source: "docker-archive", imageMediaType: v1Types.DockerManifestSchema2, layerMediaType: v1Types.DockerLayer, + layers: simpleImageLayers, tagCount: 1, + size: 65, }, { name: "FromDocker", source: "docker", imageMediaType: v1Types.DockerManifestSchema2, layerMediaType: v1Types.DockerLayer, + layers: simpleImageLayers, // name:hash // name:latest tagCount: 2, + size: 65, }, { name: "FromPodman", source: "podman", imageMediaType: v1Types.DockerManifestSchema2, layerMediaType: v1Types.DockerLayer, + layers: simpleImageLayers, tagCount: 2, + size: 65, }, { name: "FromOciTarball", source: "oci-archive", imageMediaType: v1Types.OCIManifestSchema1, layerMediaType: v1Types.OCILayer, + layers: simpleImageLayers, tagCount: 0, + size: 65, }, { name: "FromOciDirectory", source: "oci-dir", imageMediaType: v1Types.OCIManifestSchema1, layerMediaType: v1Types.OCILayer, + layers: simpleImageLayers, + tagCount: 0, + size: 65, + }, + { + name: "FromSingularity", + source: "singularity", + imageMediaType: sif.SingularityMediaType, + layerMediaType: image.SingularitySquashFSLayer, + layers: simpleImageSingularityLayer, tagCount: 0, + // Disable size check. Size can vary - image build embeds timestamps etc. + size: -1, }, } @@ -66,7 +112,9 @@ type testCase struct { source string imageMediaType v1Types.MediaType layerMediaType v1Types.MediaType + layers []image.LayerMetadata tagCount int + size int } func TestSimpleImage(t *testing.T) { @@ -81,8 +129,11 @@ func TestSimpleImage(t *testing.T) { i := imagetest.GetFixtureImage(t, c.source, "image-simple") assertImageSimpleMetadata(t, i, c) - assertImageSimpleTrees(t, i) - assertImageSimpleSquashedTrees(t, i) + // Singularity images are a single layer. Don't verify content per layer. + if c.source != "singularity" { + assertImageSimpleTrees(t, i) + assertImageSimpleSquashedTrees(t, i) + } assertImageSimpleContents(t, i) }) } @@ -148,9 +199,6 @@ func BenchmarkSimpleImage_FetchSquashedContents(b *testing.B) { func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues testCase) { t.Helper() t.Log("Asserting metadata...") - if i.Metadata.Size != 65 { - t.Errorf("unexpected image size: %d", i.Metadata.Size) - } if i.Metadata.MediaType != expectedValues.imageMediaType { t.Errorf("unexpected image media type: %+v", i.Metadata.MediaType) } @@ -162,36 +210,24 @@ func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues test } } - expected := []image.LayerMetadata{ - { - Index: 0, - Size: 22, - MediaType: expectedValues.layerMediaType, - }, - { - Index: 1, - Size: 16, - MediaType: expectedValues.layerMediaType, - }, - { - Index: 2, - Size: 27, - MediaType: expectedValues.layerMediaType, - }, + if expectedValues.size >= 0 && i.Metadata.Size != int64(expectedValues.size) { + t.Errorf("unexpected image size: %d", i.Metadata.Size) } - if len(expected) != len(i.Layers) { + if len(expectedValues.layers) != len(i.Layers) { t.Fatal("unexpected number of layers:", len(i.Layers)) } for idx, l := range i.Layers { - if expected[idx].Size != l.Metadata.Size { + expected := expectedValues.layers[idx] + expected.MediaType = expectedValues.layerMediaType + if expected.Size >= 0 && expected.Size != l.Metadata.Size { t.Errorf("mismatched layer 'Size' (layer %d): %+v", idx, l.Metadata.Size) } - if expected[idx].MediaType != l.Metadata.MediaType { + if expected.MediaType != l.Metadata.MediaType { t.Errorf("mismatched layer 'MediaType' (layer %d): %+v", idx, l.Metadata.MediaType) } - if expected[idx].Index != l.Metadata.Index { + if expected.Index != l.Metadata.Index { t.Errorf("mismatched layer 'Index' (layer %d): %+v", idx, l.Metadata.Index) } } diff --git a/test/integration/fixture_image_symlinks_test.go b/test/integration/fixture_image_symlinks_test.go index 62389864..7e05344a 100644 --- a/test/integration/fixture_image_symlinks_test.go +++ b/test/integration/fixture_image_symlinks_test.go @@ -50,6 +50,10 @@ func TestImageSymlinks(t *testing.T) { name: "FromOciDirectory", source: "oci-dir", }, + { + name: "FromSingularity", + source: "singularity", + }, } expectedSet := set.NewIntSet() @@ -61,6 +65,12 @@ func TestImageSymlinks(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { i := imagetest.GetFixtureImage(t, c.source, "image-symlinks") + + if c.source == "singularity" { + assertSquashedSymlinkLinkResolution(t, i) + return + } + assertImageSymlinkLinkResolution(t, i) }) } @@ -236,3 +246,60 @@ func assertImageSymlinkLinkResolution(t *testing.T, i *image.Image) { }) } } + +// Check symlinks in image after it has been squashed to a single layer (Singularity) +func assertSquashedSymlinkLinkResolution(t *testing.T, i *image.Image) { + tests := []linkFetchConfig{ + // # link with previous data + // LAYER 1 > ADD file-1.txt . + // LAYER 2 > RUN ln -s ./file-1.txt link-1 + { + linkLayer: 0, + linkPath: "/link-1", + resolveLayer: 0, + expectedPath: "/file-1.txt", + perspectiveLayer: 0, + contents: "file 1!", + }, + + // # link with current data + // LAYER 5 > RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within + { + linkLayer: 0, + linkPath: "/link-within", + resolveLayer: 0, + expectedPath: "/file-3.txt", + perspectiveLayer: 0, + // since echo was used a newline character will be present + contents: "file 3\n", + }, + + // # dead link (link-indirect > [non-existant file]) + // LAYER 8 > RUN unlink link-2 + { + linkLayer: 0, + linkPath: "/link-indirect", + resolveLayer: 0, + expectedPath: "/link-indirect", + perspectiveLayer: 0, + linkOptions: []filetree.LinkResolutionOption{filetree.DoNotFollowDeadBasenameLinks}, + }, + } + + for _, cfg := range tests { + name := fmt.Sprintf("[%d:%s]-->[%d:%s]@%d", cfg.linkLayer, cfg.linkPath, cfg.resolveLayer, cfg.expectedPath, cfg.perspectiveLayer) + t.Run(name, func(t *testing.T) { + expectedResolve, actualResolve := fetchRefs(t, i, cfg) + assertMatch(t, i, cfg, expectedResolve, actualResolve) + + if cfg.contents == "" { + return + } + + actualContents := fetchContents(t, i, cfg) + if actualContents != cfg.contents { + t.Errorf("mismatched contents: '%+v'!='%+v'", cfg.contents, actualContents) + } + }) + } +}