Skip to content

Commit

Permalink
dockerfile2llb: emit source image config
Browse files Browse the repository at this point in the history 
The source image config will be used later for avoiding applying
`SOURCE_DATE_EPOCH` to the source image layers (issue 4614).

The exporter stores this as the `ExporterImageSourceConfigKey` metadata.

NOTE: For a multi-stage Dockerfile like below, the source image refers to
`busybox`, not to `foo`:
```dockerfile
FROM busybox AS foo
FROM foo AS bar
```

Signed-off-by: Akihiro Suda <[email protected]>
  • Loading branch information
@AkihiroSuda
AkihiroSuda committed Feb 23, 2024
1 parent 85e9df3 commit 2586152
Show file tree
Hide file tree
Showing 7 changed files with 76 additions and 25 deletions.
10 changes: 9 additions & 1 deletion examples/dockerfile2llb/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
type buildOpt struct {
target string
partialImageConfigFile string
sourceImageConfigFile string
}

func main() {
Expand All @@ -31,6 +32,7 @@ func xmain() error {
var opt buildOpt
flag.StringVar(&opt.target, "target", "", "target stage")
flag.StringVar(&opt.partialImageConfigFile, "partial-image-config-file", "", "Output partial image config as a JSON file")
flag.StringVar(&opt.sourceImageConfigFile, "source-image-config-file", "", "Output source image config as a JSON file")
flag.Parse()

df, err := io.ReadAll(os.Stdin)
Expand All @@ -40,7 +42,7 @@ func xmain() error {

caps := pb.Caps.CapSet(pb.Caps.All())

state, img, _, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{
state, img, srcImg, _, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{
MetaResolver: imagemetaresolver.Default(),
LLBCaps: &caps,
Config: dockerui.Config{
Expand All @@ -63,6 +65,12 @@ func xmain() error {
return err
}
}
if opt.sourceImageConfigFile != "" {
if err := writeJSON(opt.sourceImageConfigFile, srcImg); err != nil {
return err
}
}

return nil
}

Expand Down
2 changes: 2 additions & 0 deletions exporter/containerimage/exptypes/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ const (
ExporterImageConfigKey = "containerimage.config"
ExporterImageConfigDigestKey = "containerimage.config.digest"
ExporterImageDescriptorKey = "containerimage.descriptor"
ExporterImageSourceConfigKey = "containerimage.source.config"
ExporterPlatformsKey = "refs.platforms"
)

// KnownRefMetadataKeys are the subset of exporter keys that can be suffixed by
// a platform to become platform specific
var KnownRefMetadataKeys = []string{
ExporterImageConfigKey,
ExporterImageSourceConfigKey,
}

type Platforms struct {
Expand Down
14 changes: 7 additions & 7 deletions frontend/dockerfile/builder/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,34 +115,34 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {

scanTargets := sync.Map{}

rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (client.Reference, *dockerspec.DockerOCIImage, error) {
rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (client.Reference, *dockerspec.DockerOCIImage, *dockerspec.DockerOCIImage, error) {
opt := convertOpt
opt.TargetPlatform = platform
if idx != 0 {
opt.Warn = nil
}

st, img, scanTarget, err := dockerfile2llb.Dockerfile2LLB(ctx, src.Data, opt)
st, img, srcImg, scanTarget, err := dockerfile2llb.Dockerfile2LLB(ctx, src.Data, opt)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}

def, err := st.Marshal(ctx)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to marshal LLB definition")
return nil, nil, nil, errors.Wrapf(err, "failed to marshal LLB definition")
}

r, err := c.Solve(ctx, client.SolveRequest{
Definition: def.ToPB(),
CacheImports: bc.CacheImports,
})
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}

ref, err := r.SingleRef()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}

p := platforms.DefaultSpec()
Expand All @@ -151,7 +151,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
}
scanTargets.Store(platforms.Format(platforms.Normalize(p)), scanTarget)

return ref, img, nil
return ref, img, srcImg, nil
})
if err != nil {
return nil, err
Expand Down
11 changes: 7 additions & 4 deletions frontend/dockerfile/dockerfile2llb/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,13 @@ type SBOMTargets struct {
IgnoreCache bool
}

func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *dockerspec.DockerOCIImage, *SBOMTargets, error) {
func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (st *llb.State, img, srcImg *dockerspec.DockerOCIImage, sbom *SBOMTargets, err error) {
ds, err := toDispatchState(ctx, dt, opt)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}

sbom := SBOMTargets{
sbom = &SBOMTargets{
Core: ds.state,
Extras: map[string]llb.State{},
}
Expand All @@ -97,7 +97,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
}

return &ds.state, &ds.image, &sbom, nil
return &ds.state, &ds.image, ds.srcImg, sbom, nil
}

func Dockefile2Outline(ctx context.Context, dt []byte, opt ConvertOpt) (*outline.Outline, error) {
Expand Down Expand Up @@ -445,6 +445,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
if err := json.Unmarshal(dt, &img); err != nil {
return errors.Wrap(err, "failed to parse image config")
}
d.srcImg = cloneX(&img) // immutable
img.Created = nil
// if there is no explicit target platform, try to match based on image config
if d.platform == nil && platformOpt.implicitTarget {
Expand Down Expand Up @@ -507,6 +508,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
d.state = d.base.state
d.platform = d.base.platform
d.image = clone(d.base.image)
d.srcImg = cloneX(d.base.srcImg)
// Utilize the same path index as our base image so we propagate
// the paths we use back to the base image.
d.paths = d.base.paths
Expand Down Expand Up @@ -834,6 +836,7 @@ type dispatchState struct {
platform *ocispecs.Platform
stage instructions.Stage
base *dispatchState
srcImg *dockerspec.DockerOCIImage // immutable, unlike image
noinit bool
deps map[*dispatchState]instructions.Command
buildArgs []instructions.KeyValuePairOptional
Expand Down
38 changes: 27 additions & 11 deletions frontend/dockerfile/dockerfile2llb/convert_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/moby/buildkit/frontend/dockerui"
"github.com/moby/buildkit/util/appcontext"
digest "github.com/opencontainers/go-digest"
"github.com/stretchr/testify/assert"
)

Expand All @@ -33,7 +34,7 @@ ENV FOO bar
COPY f1 f2 /sub/
RUN ls -l
`
_, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.NoError(t, err)

df = `FROM scratch AS foo
Expand All @@ -42,7 +43,7 @@ FROM foo
COPY --from=foo f1 /
COPY --from=0 f2 /
`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.NoError(t, err)

df = `FROM scratch AS foo
Expand All @@ -51,14 +52,14 @@ FROM foo
COPY --from=foo f1 /
COPY --from=0 f2 /
`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{
Config: dockerui.Config{
Target: "Foo",
},
})
assert.NoError(t, err)

_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{
Config: dockerui.Config{
Target: "nosuch",
},
Expand All @@ -68,21 +69,21 @@ COPY --from=0 f2 /
df = `FROM scratch
ADD http://github.com/moby/buildkit/blob/master/README.md /
`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.NoError(t, err)

df = `FROM scratch
COPY http://github.com/moby/buildkit/blob/master/README.md /
`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.EqualError(t, err, "source can't be a URL for COPY")

df = `FROM "" AS foo`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.Error(t, err)

df = `FROM ${BLANK} AS foo`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.Error(t, err)
}

Expand All @@ -93,7 +94,7 @@ ENV FOO bar
COPY f1 f2 /sub/
RUN ls -l
`
state, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
state, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.NoError(t, err)

_, err = state.Marshal(context.TODO())
Expand Down Expand Up @@ -194,7 +195,7 @@ func TestDockerfileCircularDependencies(t *testing.T) {
df := `FROM busybox AS stage0
COPY --from=stage0 f1 /sub/
`
_, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.EqualError(t, err, "circular dependency detected on stage: stage0")

// multiple stages with circular dependency
Expand All @@ -205,6 +206,21 @@ COPY --from=stage0 f2 /sub/
FROM busybox AS stage2
COPY --from=stage1 f2 /sub/
`
_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
_, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.EqualError(t, err, "circular dependency detected on stage: stage0")
}

func TestSourceImageConfig(t *testing.T) {
df := `FROM --platform=linux/amd64 busybox:1.36.1@sha256:6d9ac9237a84afe1516540f40a0fafdc86859b2141954b4d643af7066d598b74 AS foo
RUN echo foo
# the source image of bar is busybox, not foo
FROM foo AS bar
RUN echo bar
`
_, _, srcImg, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
assert.NoError(t, err)
t.Logf("srcImg=%+v", srcImg)
assert.Equal(t, []digest.Digest{"sha256:2e112031b4b923a873c8b3d685d48037e4d5ccd967b658743d93a6e56c3064b9"}, srcImg.RootFS.DiffIDs)
assert.Equal(t, "2024-01-17 21:49:12 +0000 UTC", srcImg.Created.String())
}
8 changes: 8 additions & 0 deletions frontend/dockerfile/dockerfile2llb/image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,14 @@ func clone(src dockerspec.DockerOCIImage) dockerspec.DockerOCIImage {
return img
}

func cloneX(src *dockerspec.DockerOCIImage) *dockerspec.DockerOCIImage {
if src == nil {
return nil
}
img := clone(*src)
return &img
}

func emptyImage(platform ocispecs.Platform) dockerspec.DockerOCIImage {
img := dockerspec.DockerOCIImage{}
img.Architecture = platform.Architecture
Expand Down
18 changes: 16 additions & 2 deletions frontend/dockerui/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"golang.org/x/sync/errgroup"
)

type BuildFunc func(ctx context.Context, platform *ocispecs.Platform, idx int) (client.Reference, *dockerspec.DockerOCIImage, error)
type BuildFunc func(ctx context.Context, platform *ocispecs.Platform, idx int) (r client.Reference, img, srcImg *dockerspec.DockerOCIImage, err error)

func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, error) {
res := client.NewResult()
Expand All @@ -36,7 +36,7 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
for i, tp := range targets {
i, tp := i, tp
eg.Go(func() error {
ref, img, err := fn(ctx, tp, i)
ref, img, srcImg, err := fn(ctx, tp, i)
if err != nil {
return err
}
Expand All @@ -46,6 +46,14 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
return errors.Wrapf(err, "failed to marshal image config")
}

var srcConfig []byte
if srcImg != nil {
srcConfig, err = json.Marshal(srcImg)
if err != nil {
return errors.Wrapf(err, "failed to marshal source image config")
}
}

p := platforms.DefaultSpec()
if tp != nil {
p = *tp
Expand All @@ -67,9 +75,15 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
if bc.MultiPlatformRequested {
res.AddRef(k, ref)
res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, k), config)
if len(srcConfig) > 0 {
res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageSourceConfigKey, k), srcConfig)
}
} else {
res.SetRef(ref)
res.AddMeta(exptypes.ExporterImageConfigKey, config)
if len(srcConfig) > 0 {
res.AddMeta(exptypes.ExporterImageSourceConfigKey, srcConfig)
}
}
expPlatforms.Platforms[i] = exptypes.Platform{
ID: k,
Expand Down

0 comments on commit 2586152

Please sign in to comment.