From cb8b3eae27bfdc402d17a604bf74370838c46fb4 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 10 Feb 2022 02:38:06 +0100 Subject: [PATCH] image exporter: return image descriptor in response Signed-off-by: CrazyMax --- client/client_test.go | 40 ++++++++++ cmd/buildctl/build.go | 18 ++++- cmd/buildctl/build_test.go | 14 +++- cmd/buildctl/buildctl_test.go | 97 +++++++++++++++++++++++ exporter/containerimage/export.go | 10 +++ exporter/containerimage/exptypes/types.go | 1 + exporter/oci/export.go | 9 +++ 7 files changed, 187 insertions(+), 2 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 0a79ddd7127fb..fdd8042338793 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -7,6 +7,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/x509" + "encoding/base64" "encoding/json" "encoding/pem" "fmt" @@ -143,6 +144,7 @@ func TestIntegration(t *testing.T) { testMergeOpCacheMax, testRmSymlink, testMoveParentDir, + testExporterImageDescriptor, ) tests = append(tests, diffOpTestCases()...) integration.Run(t, tests, mirrors) @@ -4860,6 +4862,44 @@ func testRelativeMountpoint(t *testing.T, sb integration.Sandbox) { require.Equal(t, dt, []byte(id)) } +func testExporterImageDescriptor(t *testing.T, sb integration.Sandbox) { + skipDockerd(t, sb) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + st := llb.Image("busybox:latest") + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + res, err := c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterOCI, + Attrs: map[string]string{}, + Output: func(m map[string]string) (io.WriteCloser, error) { + return nil, nil + }, + }, + }, + }, nil) + require.NoError(t, err) + + require.Contains(t, res.ExporterResponse, exptypes.ExporterImageDescriptorKey) + dt, err := base64.StdEncoding.DecodeString(res.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.NoError(t, err) + + var desc *ocispecs.Descriptor + err = json.Unmarshal(dt, &desc) + require.NoError(t, err) + + require.NotEmpty(t, desc.MediaType) + require.NotEmpty(t, desc.Digest.String()) + require.True(t, strings.HasPrefix(desc.Digest.String(), "sha256:")) + require.Equal(t, res.ExporterResponse[exptypes.ExporterImageDigestKey], desc.Digest.String()) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-client") if err != nil { diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index c7684f3c016b8..51dddb4a83af8 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "encoding/json" "io" "os" @@ -315,7 +316,22 @@ func buildAction(clicontext *cli.Context) error { } func writeMetadataFile(filename string, exporterResponse map[string]string) error { - b, err := json.Marshal(exporterResponse) + var err error + out := make(map[string]interface{}) + for k, v := range exporterResponse { + dt, err := base64.StdEncoding.DecodeString(v) + if err != nil { + out[k] = v + continue + } + var raw map[string]interface{} + if err = json.Unmarshal(dt, &raw); err != nil || len(raw) == 0 { + out[k] = v + continue + } + out[k] = json.RawMessage(dt) + } + b, err := json.Marshal(out) if err != nil { return err } diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go index 7df12f1241feb..bde880c9095c9 100644 --- a/cmd/buildctl/build_test.go +++ b/cmd/buildctl/build_test.go @@ -19,6 +19,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/util/testutil/integration" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" ) @@ -142,15 +143,26 @@ func testBuildMetadataFile(t *testing.T, sb integration.Sandbox) { metadataBytes, err := ioutil.ReadFile(metadataFile) require.NoError(t, err) - var metadata map[string]string + var metadata map[string]interface{} err = json.Unmarshal(metadataBytes, &metadata) require.NoError(t, err) + require.Contains(t, metadata, "image.name") require.Equal(t, imageName, metadata["image.name"]) + require.Contains(t, metadata, exptypes.ExporterImageDigestKey) digest := metadata[exptypes.ExporterImageDigestKey] require.NotEmpty(t, digest) + require.Contains(t, metadata, exptypes.ExporterImageDescriptorKey) + var desc *ocispecs.Descriptor + dtdesc, err := json.Marshal(metadata[exptypes.ExporterImageDescriptorKey]) + require.NoError(t, err) + err = json.Unmarshal(dtdesc, &desc) + require.NoError(t, err) + require.NotEmpty(t, desc.MediaType) + require.NotEmpty(t, desc.Digest.String()) + cdAddress := sb.ContainerdAddress() if cdAddress == "" { t.Log("no containerd worker, skipping digest verification") diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go index e3c7697bbb135..66fef818f919d 100644 --- a/cmd/buildctl/buildctl_test.go +++ b/cmd/buildctl/buildctl_test.go @@ -1,6 +1,10 @@ package main import ( + "encoding/json" + "io/ioutil" + "os" + "path" "testing" "github.com/moby/buildkit/util/testutil/integration" @@ -31,3 +35,96 @@ func testUsage(t *testing.T, sb integration.Sandbox) { require.NoError(t, sb.Cmd("--help").Run()) } + +func TestWriteMetadataFile(t *testing.T) { + tmpdir, err := os.MkdirTemp("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + cases := []struct { + name string + exporterResponse map[string]string + excpected map[string]interface{} + }{ + { + name: "common", + exporterResponse: map[string]string{ + "containerimage.config.digest": "sha256:2937f66a9722f7f4a2df583de2f8cb97fc9196059a410e7f00072fc918930e66", + "containerimage.descriptor": "eyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQub2NpLmltYWdlLm1hbmlmZXN0LnYxK2pzb24iLCJkaWdlc3QiOiJzaGEyNTY6MTlmZmVhYjZmOGJjOTI5M2FjMmMzZmRmOTRlYmUyODM5NjI1NGM5OTNhZWEwYjVhNTQyY2ZiMDJlMDg4M2ZhMyIsInNpemUiOjUwNiwiYW5ub3RhdGlvbnMiOnsib3JnLm9wZW5jb250YWluZXJzLmltYWdlLmNyZWF0ZWQiOiIyMDIyLTAyLTA4VDE5OjIxOjAzWiJ9fQ==", // {"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3","size":506,"annotations":{"org.opencontainers.image.created":"2022-02-08T19:21:03Z"}} + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + excpected: map[string]interface{}{ + "containerimage.config.digest": "sha256:2937f66a9722f7f4a2df583de2f8cb97fc9196059a410e7f00072fc918930e66", + "containerimage.descriptor": map[string]interface{}{ + "annotations": map[string]interface{}{ + "org.opencontainers.image.created": "2022-02-08T19:21:03Z", + }, + "digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": float64(506), + }, + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + }, + { + name: "b64json", + exporterResponse: map[string]string{ + "key": "MTI=", // 12 + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + excpected: map[string]interface{}{ + "key": "MTI=", + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + }, + { + name: "emptyjson", + exporterResponse: map[string]string{ + "key": "e30=", // {} + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + excpected: map[string]interface{}{ + "key": "e30=", + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + }, + { + name: "invalidjson", + exporterResponse: map[string]string{ + "key": "W10=", // [] + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + excpected: map[string]interface{}{ + "key": "W10=", + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + }, + { + name: "nullobject", + exporterResponse: map[string]string{ + "key": "eyJmb28iOm51bGwsImJhciI6ImJheiJ9", // {"foo":null,"bar":"baz"} + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + excpected: map[string]interface{}{ + "key": map[string]interface{}{ + "foo": nil, + "bar": "baz", + }, + "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3", + }, + }, + } + + for _, tt := range cases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + fname := path.Join(tmpdir, "metadata_"+tt.name) + require.NoError(t, writeMetadataFile(fname, tt.exporterResponse)) + current, err := ioutil.ReadFile(fname) + require.NoError(t, err) + var raw map[string]interface{} + require.NoError(t, json.Unmarshal(current, &raw)) + require.Equal(t, tt.excpected, raw) + }) + } +} diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 64c4acb3070ec..00b5bd19ac6b6 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -2,6 +2,8 @@ package containerimage import ( "context" + "encoding/base64" + "encoding/json" "fmt" "strconv" "strings" @@ -346,7 +348,15 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { resp[exptypes.ExporterImageConfigDigestKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) } + + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + return resp, nil } diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index 11ff4cec598c1..75219b87ddb52 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -11,6 +11,7 @@ const ( ExporterImageDigestKey = "containerimage.digest" ExporterImageConfigKey = "containerimage.config" ExporterImageConfigDigestKey = "containerimage.config.digest" + ExporterImageDescriptorKey = "containerimage.descriptor" ExporterInlineCache = "containerimage.inlinecache" ExporterBuildInfo = "containerimage.buildinfo" ExporterPlatformsKey = "refs.platforms" diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 8b110f1e36f3d..3f22610f9516d 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -2,6 +2,8 @@ package oci import ( "context" + "encoding/base64" + "encoding/json" "strconv" "strings" "time" @@ -208,12 +210,19 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, desc.Annotations[ocispecs.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339) resp := make(map[string]string) + resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { resp[exptypes.ExporterImageConfigDigestKey] = v delete(desc.Annotations, exptypes.ExporterConfigDigestKey) } + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + if n, ok := src.Metadata["image.name"]; e.name == "*" && ok { e.name = string(n) }