From effb37a122ed70c31b01c6a19184b695a728211b Mon Sep 17 00:00:00 2001 From: taohong Date: Tue, 1 Aug 2023 17:24:15 +0800 Subject: [PATCH] feat: expand Nydusify to support encrypted Nydus images * convert: generate encrypted Ndyus images from OCI images. * check: check encrypted Nydus images. * mount: mount encrypted Nydus images. * build: build Nydus image with encrypted data blobs from a source directory. Signed-off-by: taohong --- contrib/nydusify/cmd/nydusify.go | 56 ++++++++++++++++----- contrib/nydusify/go.mod | 2 +- contrib/nydusify/pkg/build/builder.go | 5 ++ contrib/nydusify/pkg/checker/checker.go | 3 +- contrib/nydusify/pkg/checker/output.go | 10 +++- contrib/nydusify/pkg/converter/config.go | 3 ++ contrib/nydusify/pkg/converter/converter.go | 21 ++++---- contrib/nydusify/pkg/packer/packer.go | 11 ++-- contrib/nydusify/pkg/parser/parser.go | 21 +++++++- contrib/nydusify/pkg/utils/constant.go | 1 + contrib/nydusify/pkg/utils/utils.go | 35 +++++++++++++ contrib/nydusify/pkg/viewer/viewer.go | 13 ++++- 12 files changed, 149 insertions(+), 32 deletions(-) diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 8a56b786076..050f3bcb6a6 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -424,6 +424,13 @@ func main() { Usage: "File path to save the metrics collected during conversion in JSON format, for example: './output.json'", EnvVars: []string{"OUTPUT_JSON"}, }, + &cli.StringSliceFlag{ + Name: "encrypt-recipients", + Value: nil, + Usage: "Recipients to encrypt the nydus bootstrap layer, like " + + "jwe:, provider:, pgp:, pkcs7:", + EnvVars: []string{"ENCRYPT_RECIPIENTS"}, + }, }, Action: func(c *cli.Context) error { setupLogLevel(c) @@ -506,14 +513,15 @@ func main() { ChunkDictRef: chunkDictRef, ChunkDictInsecure: c.Bool("chunk-dict-insecure"), - PrefetchPatterns: prefetchPatterns, - MergePlatform: c.Bool("merge-platform"), - Docker2OCI: docker2OCI, - FsVersion: fsVersion, - FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"), - Compressor: c.String("compressor"), - ChunkSize: c.String("chunk-size"), - BatchSize: c.String("batch-size"), + PrefetchPatterns: prefetchPatterns, + MergePlatform: c.Bool("merge-platform"), + Docker2OCI: docker2OCI, + FsVersion: fsVersion, + FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"), + Compressor: c.String("compressor"), + ChunkSize: c.String("chunk-size"), + BatchSize: c.String("batch-size"), + EncryptRecipients: c.StringSlice("encrypt-recipients"), OCIRef: c.Bool("oci-ref"), WithReferrer: c.Bool("with-referrer"), @@ -605,6 +613,12 @@ func main() { Usage: "Path to the nydusd binary, default to search in PATH", EnvVars: []string{"NYDUSD"}, }, + &cli.StringSliceFlag{ + Name: "decrypt-keys", + Value: nil, + Usage: "Keys to decrypt nydus bootstrap layer.", + EnvVars: []string{"DECRYPT_KEYS"}, + }, }, Action: func(c *cli.Context) error { setupLogLevel(c) @@ -631,6 +645,7 @@ func main() { BackendType: backendType, BackendConfig: backendConfig, ExpectedArch: arch, + DecryptKeys: c.StringSlice("decrypt-keys"), }) if err != nil { return err @@ -702,6 +717,12 @@ func main() { Usage: "The nydusd binary path, if unset, search in PATH environment", EnvVars: []string{"NYDUSD"}, }, + &cli.StringSliceFlag{ + Name: "decrypt-keys", + Value: nil, + Usage: "Keys to decrypt nydus bootstrap layer.", + EnvVars: []string{"DECRYPT_KEYS"}, + }, }, Action: func(c *cli.Context) error { setupLogLevel(c) @@ -746,6 +767,7 @@ func main() { BackendType: backendType, BackendConfig: backendConfig, ExpectedArch: arch, + DecryptKeys: c.StringSlice("decrypt-keys"), }) if err != nil { return err @@ -858,6 +880,14 @@ func main() { Usage: "Path to the nydus-image binary, default to search in PATH", EnvVars: []string{"NYDUS_IMAGE"}, }, + + &cli.StringSliceFlag{ + Name: "encrypt-recipients", + Value: nil, + Usage: "Recipients to encrypt the nydus bootstrap layer, like " + + "jwe:, provider:, pgp:, pkcs7:", + EnvVars: []string{"ENCRYPT_RECIPIENTS"}, + }, }, Before: func(ctx *cli.Context) error { sourcePath := ctx.String("source-dir") @@ -895,10 +925,11 @@ func main() { } if p, err = packer.New(packer.Opt{ - LogLevel: logrus.GetLevel(), - NydusImagePath: c.String("nydus-image"), - OutputDir: c.String("output-dir"), - BackendConfig: backendConfig, + LogLevel: logrus.GetLevel(), + NydusImagePath: c.String("nydus-image"), + OutputDir: c.String("output-dir"), + BackendConfig: backendConfig, + EncryptRecipients: c.StringSlice("encrypt-recipients"), }); err != nil { return err } @@ -915,6 +946,7 @@ func main() { Parent: c.String("parent-bootstrap"), TryCompact: c.Bool("compact"), CompactConfigPath: c.String("compact-config-file"), + Encrypt: len(c.StringSlice("encrypt-recipients")) != 0, }); err != nil { return err } diff --git a/contrib/nydusify/go.mod b/contrib/nydusify/go.mod index f82afcd999a..e1f9e74ef37 100644 --- a/contrib/nydusify/go.mod +++ b/contrib/nydusify/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6 github.com/containerd/containerd v1.7.2 + github.com/containers/ocicrypt v1.1.7 github.com/docker/cli v23.0.3+incompatible github.com/docker/distribution v2.8.2+incompatible github.com/goharbor/acceleration-service v0.2.6 @@ -58,7 +59,6 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containerd/ttrpc v1.2.2 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect - github.com/containers/ocicrypt v1.1.7 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/contrib/nydusify/pkg/build/builder.go b/contrib/nydusify/pkg/build/builder.go index 22f1a71296b..9a595790c14 100644 --- a/contrib/nydusify/pkg/build/builder.go +++ b/contrib/nydusify/pkg/build/builder.go @@ -29,6 +29,7 @@ type BuilderOption struct { Compressor string ChunkSize string FsVersion string + Encrypt bool } type CompactOption struct { @@ -139,6 +140,10 @@ func (builder *Builder) Run(option BuilderOption) error { args = append(args, "--chunk-size", option.ChunkSize) } + if option.Encrypt { + args = append(args, "--encrypt") + } + args = append(args, option.RootfsPath) return builder.run(args, option.PrefetchPatterns) diff --git a/contrib/nydusify/pkg/checker/checker.go b/contrib/nydusify/pkg/checker/checker.go index aa72cbc638b..1de3d769950 100644 --- a/contrib/nydusify/pkg/checker/checker.go +++ b/contrib/nydusify/pkg/checker/checker.go @@ -34,6 +34,7 @@ type Opt struct { BackendType string BackendConfig string ExpectedArch string + DecryptKeys []string } // Checker validates Nydus image manifest, bootstrap and mounts filesystem @@ -119,7 +120,7 @@ func (checker *Checker) check(ctx context.Context) error { return errors.Wrap(err, "create work directory") } - if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil { + if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir, checker.Opt); err != nil { return errors.Wrap(err, "output image information") } diff --git a/contrib/nydusify/pkg/checker/output.go b/contrib/nydusify/pkg/checker/output.go index f1446297fc2..cb22e7e3805 100644 --- a/contrib/nydusify/pkg/checker/output.go +++ b/contrib/nydusify/pkg/checker/output.go @@ -28,7 +28,7 @@ func prettyDump(obj interface{}, name string) error { // Output outputs OCI and Nydus image manifest, index, config to JSON file. // Prefer to use source image to output OCI image information. func (checker *Checker) Output( - ctx context.Context, sourceParsed, targetParsed *parser.Parsed, outputPath string, + ctx context.Context, sourceParsed, targetParsed *parser.Parsed, outputPath string, opt Opt, ) error { logrus.Infof("Dumping OCI and Nydus manifests to %s", outputPath) @@ -87,6 +87,14 @@ func (checker *Checker) Output( } defer bootstrapReader.Close() + if len(opt.DecryptKeys) != 0 && utils.IsEncryptedNydusImage(&targetParsed.NydusImage.Manifest) { + logrus.Infof("Decrypting Nydus bootstrap layer") + bootstrapReader, err = checker.targetParser.DecryptNydusBootstrap(ctx, bootstrapReader, targetParsed.NydusImage, opt.DecryptKeys) + if err != nil { + return errors.Wrap(err, "decrypt Nydus bootstrap layer") + } + } + if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, target); err != nil { return errors.Wrap(err, "unpack Nydus bootstrap layer") } diff --git a/contrib/nydusify/pkg/converter/config.go b/contrib/nydusify/pkg/converter/config.go index c1f58f19e67..7b943ec201f 100644 --- a/contrib/nydusify/pkg/converter/config.go +++ b/contrib/nydusify/pkg/converter/config.go @@ -6,6 +6,7 @@ package converter import ( "strconv" + "strings" ) func getConfig(opt Opt) map[string]string { @@ -36,5 +37,7 @@ func getConfig(opt Opt) map[string]string { cfg["cache_version"] = opt.CacheVersion cfg["cache_max_records"] = strconv.FormatUint(uint64(opt.CacheMaxRecords), 10) + cfg["encrypt_recipients"] = strings.Join(opt.EncryptRecipients, ",") + return cfg } diff --git a/contrib/nydusify/pkg/converter/converter.go b/contrib/nydusify/pkg/converter/converter.go index 49a5a36bd89..48bf5594f5b 100644 --- a/contrib/nydusify/pkg/converter/converter.go +++ b/contrib/nydusify/pkg/converter/converter.go @@ -36,16 +36,17 @@ type Opt struct { BackendConfig string BackendForcePush bool - MergePlatform bool - Docker2OCI bool - FsVersion string - FsAlignChunk bool - Compressor string - ChunkSize string - BatchSize string - PrefetchPatterns string - OCIRef bool - WithReferrer bool + MergePlatform bool + Docker2OCI bool + FsVersion string + FsAlignChunk bool + Compressor string + ChunkSize string + BatchSize string + PrefetchPatterns string + OCIRef bool + WithReferrer bool + EncryptRecipients []string AllPlatforms bool Platforms string diff --git a/contrib/nydusify/pkg/packer/packer.go b/contrib/nydusify/pkg/packer/packer.go index 6d2bf7d53f1..487a0d47242 100644 --- a/contrib/nydusify/pkg/packer/packer.go +++ b/contrib/nydusify/pkg/packer/packer.go @@ -28,10 +28,11 @@ var ( ) type Opt struct { - LogLevel logrus.Level - NydusImagePath string - OutputDir string - BackendConfig BackendConfig + LogLevel logrus.Level + NydusImagePath string + OutputDir string + BackendConfig BackendConfig + EncryptRecipients []string } type Builder interface { @@ -63,6 +64,7 @@ type PackRequest struct { Parent string TryCompact bool CompactConfigPath string + Encrypt bool } type PackResult struct { @@ -253,6 +255,7 @@ func (p *Packer) Pack(_ context.Context, req PackRequest) (PackResult, error) { Compressor: req.Compressor, ChunkSize: req.ChunkSize, FsVersion: req.FsVersion, + Encrypt: req.Encrypt, }); err != nil { return PackResult{}, errors.Wrapf(err, "failed to build image from directory %s", req.SourceDir) } diff --git a/contrib/nydusify/pkg/parser/parser.go b/contrib/nydusify/pkg/parser/parser.go index d937fbdf45d..c8da9005dd3 100644 --- a/contrib/nydusify/pkg/parser/parser.go +++ b/contrib/nydusify/pkg/parser/parser.go @@ -9,8 +9,11 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "strings" + "github.com/containers/ocicrypt" + enchelpers "github.com/containers/ocicrypt/helpers" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" @@ -66,7 +69,8 @@ func FindNydusBootstrapDesc(manifest *ocispec.Manifest) *ocispec.Descriptor { if len(layers) != 0 { desc := &layers[len(layers)-1] if (desc.MediaType == ocispec.MediaTypeImageLayerGzip || - desc.MediaType == images.MediaTypeDockerSchema2LayerGzip) && + desc.MediaType == images.MediaTypeDockerSchema2LayerGzip || + desc.MediaType == images.MediaTypeImageLayerGzipEncrypted) && desc.Annotations[utils.LayerAnnotationNydusBootstrap] == "true" { return desc } @@ -174,6 +178,21 @@ func (parser *Parser) PullNydusBootstrap(ctx context.Context, image *Image) (io. return reader, nil } +// Decrypt Nydus bootstrap layer if decryption key is provided. +func (parser *Parser) DecryptNydusBootstrap(ctx context.Context, reader io.Reader, image *Image, decryptKeys []string) (io.ReadCloser, error) { + bootstrapDesc := FindNydusBootstrapDesc(&image.Manifest) + dcc, err := enchelpers.CreateCryptoConfig([]string{}, decryptKeys) + if err != nil { + return nil, errors.Wrap(err, "Create crypto config failed") + } + + resultReader, _, err := ocicrypt.DecryptLayer(dcc.DecryptConfig, reader, *bootstrapDesc, false) + if err != nil { + return nil, errors.Wrap(err, "Decrypt Nydus bootstrap layer") + } + return ioutil.NopCloser(resultReader), nil +} + func (parser *Parser) matchImagePlatform(desc *ocispec.Descriptor) bool { if parser.interestedArch == desc.Platform.Architecture && desc.Platform.OS == "linux" { return true diff --git a/contrib/nydusify/pkg/utils/constant.go b/contrib/nydusify/pkg/utils/constant.go index 0c9694c0339..0b4f60766c1 100644 --- a/contrib/nydusify/pkg/utils/constant.go +++ b/contrib/nydusify/pkg/utils/constant.go @@ -17,6 +17,7 @@ const ( LayerAnnotationNydusBootstrap = "containerd.io/snapshot/nydus-bootstrap" LayerAnnotationNydusFsVersion = "containerd.io/snapshot/nydus-fs-version" LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid" + LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob" LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids" diff --git a/contrib/nydusify/pkg/utils/utils.go b/contrib/nydusify/pkg/utils/utils.go index e720bfb287e..d5f5fa7dc73 100644 --- a/contrib/nydusify/pkg/utils/utils.go +++ b/contrib/nydusify/pkg/utils/utils.go @@ -15,6 +15,7 @@ import ( "time" "github.com/containerd/containerd/archive/compression" + "github.com/containerd/containerd/images" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -224,3 +225,37 @@ func HashFile(path string) ([]byte, error) { return hasher.Sum(nil), nil } + +// IsEncryptedNydusImage checks if the nydus image is encrypted. +func IsEncryptedNydusImage(manifest *ocispec.Manifest) bool { + layers := manifest.Layers + if len(layers) != 0 { + desc := layers[len(layers)-1] + if IsEncryptedNydusBootstrap(desc) { + return true + } + } + return false +} + +// IsEncryptedNydusBlob returns true when the specified descriptor is nydus encrypted blob. +func IsEncryptedNydusBlob(desc ocispec.Descriptor) bool { + if desc.Annotations == nil { + return false + } + + _, hasAnno := desc.Annotations[LayerAnnotationNydusEncryptedBlob] + return hasAnno +} + +// IsEncryptedNydusBootstrap returns true when the specified descriptor is nydus encrypted bootstrap. +func IsEncryptedNydusBootstrap(desc ocispec.Descriptor) bool { + if desc.Annotations == nil { + return false + } + + _, hasAnno := desc.Annotations[LayerAnnotationNydusBootstrap] + encrypted := desc.MediaType == images.MediaTypeImageLayerEncrypted || + desc.MediaType == images.MediaTypeImageLayerGzipEncrypted + return encrypted && hasAnno +} diff --git a/contrib/nydusify/pkg/viewer/viewer.go b/contrib/nydusify/pkg/viewer/viewer.go index fcd9a82a0ec..7d57a4362d7 100644 --- a/contrib/nydusify/pkg/viewer/viewer.go +++ b/contrib/nydusify/pkg/viewer/viewer.go @@ -37,6 +37,7 @@ type Opt struct { BackendConfig string ExpectedArch string FsVersion string + DecryptKeys []string } // fsViewer provides complete view of file system in nydus image @@ -84,7 +85,7 @@ func New(opt Opt) (*FsViewer, error) { } // Pull Bootstrap, includes nydus_manifest.json and nydus_config.json -func (fsViewer *FsViewer) PullBootstrap(ctx context.Context, targetParsed *parser.Parsed) error { +func (fsViewer *FsViewer) PullBootstrap(ctx context.Context, targetParsed *parser.Parsed, opt Opt) error { if err := os.RemoveAll(fsViewer.WorkDir); err != nil { return errors.Wrap(err, "failed to clean up working directory") } @@ -115,6 +116,14 @@ func (fsViewer *FsViewer) PullBootstrap(ctx context.Context, targetParsed *parse } defer bootstrapReader.Close() + if len(opt.DecryptKeys) != 0 && utils.IsEncryptedNydusImage(&targetParsed.NydusImage.Manifest) { + logrus.Infof("Decrypting Nydus bootstrap layer") + bootstrapReader, err = fsViewer.Parser.DecryptNydusBootstrap(ctx, bootstrapReader, targetParsed.NydusImage, opt.DecryptKeys) + if err != nil { + return errors.Wrap(err, "decrypt Nydus bootstrap layer") + } + } + if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, target); err != nil { return errors.Wrap(err, "failed to unpack Nydus bootstrap layer") } @@ -168,7 +177,7 @@ func (fsViewer *FsViewer) view(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to parse image reference") } - err = fsViewer.PullBootstrap(ctx, targetParsed) + err = fsViewer.PullBootstrap(ctx, targetParsed, fsViewer.Opt) if err != nil { return errors.Wrap(err, "failed to pull Nydus image bootstrap") }