diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go
index 4236650ad5fd..b6365f9460d5 100644
--- a/source/containerimage/pull.go
+++ b/source/containerimage/pull.go
@@ -173,6 +173,10 @@ func mainManifestKey(ctx context.Context, desc specs.Descriptor, platform specs.
 func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cacheKey string, cacheOpts solver.CacheOpts, cacheDone bool, err error) {
 	p.Puller.Resolver = resolver.DefaultPool.GetResolver(p.RegistryHosts, p.Ref, "pull", p.SessionManager, g).WithImageStore(p.ImageStore, p.id.ResolveMode)
 
+	// progressFactory needs the outer context, the context in `p.g.Do` will
+	// be canceled before the progress output is complete
+	progressFactory := progress.FromContext(ctx)
+
 	_, err = p.g.Do(ctx, "", func(ctx context.Context) (_ interface{}, err error) {
 		if p.cacheKeyErr != nil || p.cacheKeyDone == true {
 			return nil, p.cacheKeyErr
@@ -201,7 +205,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach
 
 		if len(p.manifest.Descriptors) > 0 {
 			progressController := &controller.Controller{
-				WriterFactory: progress.FromContext(ctx),
+				WriterFactory: progressFactory,
 			}
 			if p.vtx != nil {
 				progressController.Digest = p.vtx.Digest()