diff --git a/src/cloud-api-adaptor/cmd/process-user-data/main.go b/src/cloud-api-adaptor/cmd/process-user-data/main.go index 2ade87908..05fab1ee1 100644 --- a/src/cloud-api-adaptor/cmd/process-user-data/main.go +++ b/src/cloud-api-adaptor/cmd/process-user-data/main.go @@ -7,10 +7,6 @@ import ( "os" cmdUtil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/aa" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/agent" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh" - daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/userdata" "github.com/spf13/cobra" ) @@ -19,9 +15,6 @@ const ( programName = "process-user-data" providerAzure = "azure" providerAws = "aws" - - defaultAuthJsonPath = "/run/peerpod/auth.json" - defaultAgentConfigPath = "/run/peerpod/agent-config.toml" ) var versionFlag bool @@ -38,20 +31,14 @@ var rootCmd = &cobra.Command{ } func init() { - var aaConfigPath, agentConfigPath, cdhConfigPath, daemonConfigPath string var fetchTimeout int - rootCmd.PersistentFlags().BoolVarP(&versionFlag, "version", "v", false, "Print the version") - rootCmd.PersistentFlags().StringVarP(&daemonConfigPath, "daemon-config-path", "d", daemon.DefaultConfigPath, "Path to a daemon config file") - rootCmd.PersistentFlags().StringVarP(&aaConfigPath, "aa-config-path", "a", aa.DefaultAaConfigPath, "Path to a AA config file") - rootCmd.PersistentFlags().StringVarP(&agentConfigPath, "agent-config-path", "k", agent.ConfigFilePath, "Path to a kata agent config file") - rootCmd.PersistentFlags().StringVarP(&cdhConfigPath, "cdh-config-path", "c", cdh.ConfigFilePath, "Path to a CDH config file") var provisionFilesCmd = &cobra.Command{ Use: "provision-files", Short: "Provision required files based on user data", RunE: func(_ *cobra.Command, _ []string) error { - cfg := userdata.NewConfig(aaConfigPath, agentConfigPath, defaultAuthJsonPath, daemonConfigPath, cdhConfigPath, fetchTimeout) + cfg := userdata.NewConfig(fetchTimeout) return userdata.ProvisionFiles(cfg) }, SilenceUsage: true, // Silence usage on error diff --git a/src/cloud-api-adaptor/docs/initdata.md b/src/cloud-api-adaptor/docs/initdata.md new file mode 100644 index 000000000..fad88016b --- /dev/null +++ b/src/cloud-api-adaptor/docs/initdata.md @@ -0,0 +1,111 @@ +# Initdata + +The document describes the implementation of the [initdata](https://github.com/confidential-containers/trustee/blob/main/kbs/docs/initdata.md) spec in PeerPods. + +## Initdata example + +[attestation-agent](https://github.com/confidential-containers/guest-components/tree/main/attestation-agent) config file `aa.toml`, [confidential-data-hub](https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub) config file `cdh.toml` and a lightweight policy file `polciy.rego` can be passed into PeerPod via initdata. + +Example: +```toml +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = 'http://127.0.0.1:8080' + +[token_configs.kbs] +url = 'http://127.0.0.1:8080' +''' + +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = 'cc_kbc' +url = 'http://1.2.3.4:8080' +''' + +"policy.rego" = ''' +package agent_policy + +import future.keywords.in +import future.keywords.every + +import input + +# Default values, returned by OPA when rules cannot be evaluated to true. +default CopyFileRequest := false +default CreateContainerRequest := false +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := false +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default OnlineCPUMemRequest := true +default PullImageRequest := true +default ReadStreamRequest := false +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StatsContainerRequest := true +default TtyWinResizeRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := false +''' +``` + +## Annotation in Pod yaml +Generate base64 encoded string based on above example and pass it into PeerPod via annotation `io.katacontainers.config.runtime.cc_init_data`: +```yaml +apiVersion: v1 +kind: Pod +metadata: + labels: + run: busybox + name: busybox + annotations: + io.katacontainers.config.runtime.cc_init_data: YWxnb3JpdGhtID0gInNoYTM4NCIKdmVyc2lvbiA9ICIwLjEuMCIKCltkYXRhXQoiYWEudG9tbCIgPSAnJycKW3Rva2VuX2NvbmZpZ3NdClt0b2tlbl9jb25maWdzLmNvY29fYXNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCgpbdG9rZW5fY29uZmlncy5rYnNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCicnJwoKImNkaC50b21sIiAgPSAnJycKc29ja2V0ID0gJ3VuaXg6Ly8vcnVuL2NvbmZpZGVudGlhbC1jb250YWluZXJzL2NkaC5zb2NrJwpjcmVkZW50aWFscyA9IFtdCgpba2JjXQpuYW1lID0gJ2NjX2tiYycKdXJsID0gJ2h0dHA6Ly8xLjIuMy40OjgwODAnCicnJwoKInBvbGljeS5yZWdvIiA9ICcnJwpwYWNrYWdlIGFnZW50X3BvbGljeQoKaW1wb3J0IGZ1dHVyZS5rZXl3b3Jkcy5pbgppbXBvcnQgZnV0dXJlLmtleXdvcmRzLmV2ZXJ5CgppbXBvcnQgaW5wdXQKCiMgRGVmYXVsdCB2YWx1ZXMsIHJldHVybmVkIGJ5IE9QQSB3aGVuIHJ1bGVzIGNhbm5vdCBiZSBldmFsdWF0ZWQgdG8gdHJ1ZS4KZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBDcmVhdGVDb250YWluZXJSZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgQ3JlYXRlU2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IERlc3Ryb3lTYW5kYm94UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgRXhlY1Byb2Nlc3NSZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBPbmxpbmVDUFVNZW1SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVHR5V2luUmVzaXplUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlRXBoZW1lcmFsTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlSW50ZXJmYWNlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlUm91dGVzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV2FpdFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXcml0ZVN0cmVhbVJlcXVlc3QgOj0gZmFsc2UKJycn +spec: + containers: + - image: quay.io/prometheus/busybox + name: busybox + resources: {} + dnsPolicy: ClusterFirst + restartPolicy: Never + runtimeClassName: kata-remote +``` + +## Structure in `write_files` +cloud-api-adaptor will read the annotation and write it to [write_files](../../cloud-providers/util/cloudinit/cloudconfig.go). Note: files unrelated to initdata (like network tunnel configuration in `/run/peerpod/daemon.json`) are also part of the `write_files` directive. +```yaml +write_files: +- path: /run/peerpod/agent-config.toml + content: +- path: /run/peerpod/daemon.json + content: +- path: /run/peerpod/auth.json + content: +- path: /run/peerpod/initdata + content: +``` + +## Provision initdata files. +`/run/peerpod/aa.toml`, `/run/peerpod/cdh.toml` and `/run/peerpod/policy.rego` will be provisioned from `/run/peerpod/initdata` via [process-user-data](../cmd/process-user-data/main.go). + +It also calculates the digest `/run/peerpod/initdata.digest` based on the `algorithm` in `/run/peerpod/initdata` and its contents. + +`/run/peerpod/initdata.digest` could be used by the TEE drivers. + +The digest can be calculated manually and set to attestation service policy before hand if needed. To calculate the digest, use a tool (for example some online sha tools) to calculate the hash value based on the initdata annotation string. The calculated sha384 is: `14980c75860de9adcba2e0e494fc612f0f4fe3d86f5dc8e238a3255acfdf43bf82b9ccfc21da95d639ff0c98cc15e05e` for above sample. + +## TODO +A large policy bodies that cannot be provisioned via IMDS user-data, the limitation depends on providers IMDS limitation. We need add checking and limitations according to test result future. diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 5a3ab9669..9ae921f2f 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -26,12 +26,12 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/userdata" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit" - - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" ) const ( @@ -286,7 +286,25 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r }, } - if s.aaKBCParams != "" { + if authJSON != nil { + if len(authJSON) > cloudinit.DefaultAuthfileLimit { + logger.Printf("Credentials file is too large to be included in cloud-config") + } else { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ + Path: AuthFilePath, + Content: string(authJSON), + }) + } + } + + initdataStr := util.GetInitdataFromAnnotation(req.Annotations) + logger.Printf("initdata: %s", initdataStr) + if initdataStr != "" { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ + Path: userdata.InitdataPath, + Content: initdataStr, + }) + } else if s.aaKBCParams != "" { // Keep AA_KBC_PARAMS support as it is used by e2e test, KBS is dynamic k8s service in e2e test logger.Printf("aaKBCParams: %s, support cc_kbc::*", s.aaKBCParams) toml, err := cdh.CreateConfigFile(s.aaKBCParams) if err != nil { @@ -307,17 +325,6 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r }) } - if authJSON != nil { - if len(authJSON) > cloudinit.DefaultAuthfileLimit { - logger.Printf("Credentials file is too large to be included in cloud-config") - } else { - cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ - Path: AuthFilePath, - Content: string(authJSON), - }) - } - } - sandbox := &sandbox{ id: sid, podName: pod, diff --git a/src/cloud-api-adaptor/pkg/cdh/config.go b/src/cloud-api-adaptor/pkg/cdh/config.go index 1dd92c166..0cc405a9c 100644 --- a/src/cloud-api-adaptor/pkg/cdh/config.go +++ b/src/cloud-api-adaptor/pkg/cdh/config.go @@ -8,7 +8,7 @@ import ( ) const ( - ConfigFilePath = "/run/confidential-containers/cdh.toml" + ConfigFilePath = "/run/peerpod/cdh.toml" Socket = "unix:///run/confidential-containers/cdh.sock" ) diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index 5c3b6caf5..00dd7ef4d 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -2,45 +2,52 @@ package userdata import ( "context" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" "fmt" "log" "os" "path/filepath" + "strings" "time" "github.com/avast/retry-go/v4" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/aws" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/azure" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/docker" + toml "github.com/pelletier/go-toml/v2" "gopkg.in/yaml.v2" ) -var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) +const ( + ConfigParent = "/run/peerpod" + DigestPath = "/run/peerpod/initdata.digest" + InitdataPath = "/run/peerpod/initdata" +) -type paths struct { - aaConfig string - agentConfig string - authJson string - cdhConfig string - daemonConfig string -} +var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) +var WriteFilesList = []string{"agent-config.toml", "daemon.json", "auth.json", "initdata"} +var InitdDataFilesList = []string{"aa.toml", "cdh.toml", "policy.rego"} type Config struct { - fetchTimeout int - paths paths + fetchTimeout int + digestPath string + initdataPath string + parentPath string + writeFiles []string + initdataFiles []string } -func NewConfig(aaConfigPath, agentConfig, authJsonPath, daemonConfigPath, cdhConfig string, fetchTimeout int) *Config { - ps := paths{ - aaConfig: aaConfigPath, - agentConfig: agentConfig, - authJson: authJsonPath, - cdhConfig: cdhConfig, - daemonConfig: daemonConfigPath, - } +func NewConfig(fetchTimeout int) *Config { return &Config{ - fetchTimeout: fetchTimeout, - paths: ps, + fetchTimeout: fetchTimeout, + parentPath: ConfigParent, + initdataPath: InitdataPath, + digestPath: DigestPath, + writeFiles: WriteFilesList, + initdataFiles: InitdDataFilesList, } } @@ -53,6 +60,12 @@ type CloudConfig struct { WriteFiles []WriteFile `yaml:"write_files"` } +type InitData struct { + Algorithm string `toml:"algorithm"` + Version string `toml:"version"` + Data map[string]string `toml:"data,omitempty"` +} + type UserDataProvider interface { GetUserData(ctx context.Context) ([]byte, error) GetRetryDelay() time.Duration @@ -149,60 +162,102 @@ func parseUserData(userData []byte) (*CloudConfig, error) { return &cc, nil } -func findConfigEntry(path string, cc *CloudConfig) []byte { - for _, wf := range cc.WriteFiles { - if wf.Path != path { - continue - } - return []byte(wf.Content) +func writeFile(path string, bytes []byte) error { + // Ensure the parent directory exists + err := os.MkdirAll(filepath.Dir(path), 0755) + if err != nil { + return fmt.Errorf("failed to create directory: %w", err) } + + err = os.WriteFile(path, bytes, 0644) + if err != nil { + return fmt.Errorf("failed to write file %s: %w", path, err) + } + logger.Printf("Wrote %s\n", path) return nil } -type entry struct { - path string - optional bool +func isAllowed(path string, filesList []string) bool { + for _, listedFile := range filesList { + if listedFile == path { + return true + } + if strings.HasSuffix(path, listedFile) { + return true + } + } + return false } -func (f *entry) writeFile(cc *CloudConfig) error { - bytes := findConfigEntry(f.path, cc) - if bytes == nil { - if !f.optional { - return fmt.Errorf("failed to find %s entry in cloud config", f.path) +func processCloudConfig(cfg *Config, cc *CloudConfig) error { + for _, wf := range cc.WriteFiles { + path := wf.Path + bytes := []byte(wf.Content) + if isAllowed(path, cfg.writeFiles) { + if bytes != nil { + if err := writeFile(path, bytes); err != nil { + return fmt.Errorf("failed to write config file %s: %w", path, err) + } + } + } else { + logger.Printf("File: %s is not allowed in WriteFiles.\n", path) } - return nil } - // Ensure the parent directory exists - err := os.MkdirAll(filepath.Dir(f.path), 0755) - if err != nil { - return fmt.Errorf("failed to create directory: %w", err) + return nil +} + +func extractInitdataAndHash(cfg *Config) error { + if _, err := os.Stat(cfg.initdataPath); err != nil { + return fmt.Errorf("Error stat initdata file: %w", err) } - err = os.WriteFile(f.path, bytes, 0644) + dataBytes, err := os.ReadFile(cfg.initdataPath) if err != nil { - return fmt.Errorf("failed to write file: %w", err) + return fmt.Errorf("Error read initdata file: %w", err) } - logger.Printf("Wrote %s\n", f.path) - return nil -} -func processCloudConfig(cfg *Config, cc *CloudConfig) error { - entries := []entry{ - {path: cfg.paths.agentConfig, optional: false}, - {path: cfg.paths.daemonConfig, optional: false}, - {path: cfg.paths.aaConfig, optional: true}, - {path: cfg.paths.cdhConfig, optional: true}, - {path: cfg.paths.authJson, optional: true}, + decodedBytes, err := base64.StdEncoding.DecodeString(string(dataBytes)) + if err != nil { + return fmt.Errorf("Error base64 decode initdata: %w", err) + } + initdata := InitData{} + err = toml.Unmarshal(decodedBytes, &initdata) + if err != nil { + return fmt.Errorf("Error unmarshalling initdata: %w", err) } - for _, e := range entries { - err := e.writeFile(cc) - if err != nil { - return err + for key, value := range initdata.Data { + if isAllowed(key, cfg.initdataFiles) { + err := writeFile(filepath.Join(cfg.parentPath, key), []byte(value)) + if err != nil { + return fmt.Errorf("Error write a file in initdata: %w", err) + } + } else { + logger.Printf("File: %s is not allowed in initdata.\n", key) } } + checksumStr := "" + switch initdata.Algorithm { + case "sha256": + hash := sha256.Sum256(dataBytes) + checksumStr = hex.EncodeToString(hash[:]) + case "sha384": + hash := sha512.Sum384(dataBytes) + checksumStr = hex.EncodeToString(hash[:]) + case "sha512": + hash := sha512.Sum512(dataBytes) + checksumStr = hex.EncodeToString(hash[:]) + default: + return fmt.Errorf("Error creating initdata hash, the Algorithm %s not supported", initdata.Algorithm) + } + + err = writeFile(cfg.digestPath, []byte(checksumStr)) // the hash in digestPath will also be used by attester + if err != nil { + return fmt.Errorf("failed to write file %s: %w", cfg.digestPath, err) + } + return nil } @@ -212,18 +267,25 @@ func ProvisionFiles(cfg *Config) error { ctx, cancel := context.WithTimeout(bg, duration) defer cancel() - provider, err := newProvider(ctx) - if err != nil { - return fmt.Errorf("failed to create UserData provider: %w", err) - } + // some providers provision config files via process-user-data + // some providers rely on cloud-init provision config files + // all providers need extract files from initdata and calculate the hash value for attesters usage + provider, _ := newProvider(ctx) + if provider != nil { + cc, err := retrieveCloudConfig(ctx, provider) + if err != nil { + return fmt.Errorf("failed to retrieve cloud config: %w", err) + } - cc, err := retrieveCloudConfig(ctx, provider) - if err != nil { - return fmt.Errorf("failed to retrieve cloud config: %w", err) + if err = processCloudConfig(cfg, cc); err != nil { + return fmt.Errorf("failed to process cloud config: %w", err) + } + } else { + logger.Printf("unsupported user data provider, we extract and calculate initdata hash only.\n") } - if err = processCloudConfig(cfg, cc); err != nil { - return fmt.Errorf("failed to process cloud config: %w", err) + if err := extractInitdataAndHash(cfg); err != nil { + return fmt.Errorf("failed to extract initdata hash: %w", err) } return nil diff --git a/src/cloud-api-adaptor/pkg/userdata/provision_test.go b/src/cloud-api-adaptor/pkg/userdata/provision_test.go index 3d264bf71..7c6d2486c 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision_test.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "testing" "time" @@ -60,6 +61,49 @@ name = 'cc_kbc' url = 'http://1.2.3.4:8080' ` +var testAAConfig string = `[token_configs] +[token_configs.coco_as] +url = 'http://127.0.0.1:8080' + +[token_configs.kbs] +url = 'http://127.0.0.1:8080' +` + +var testPolicyConfig string = `package agent_policy + +import future.keywords.in +import future.keywords.every + +import input + +# Default values, returned by OPA when rules cannot be evaluated to true. +default CopyFileRequest := false +default CreateContainerRequest := false +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := false +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default OnlineCPUMemRequest := true +default PullImageRequest := true +default ReadStreamRequest := false +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StatsContainerRequest := true +default TtyWinResizeRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := false +` + +var testCheckSum = "14980c75860de9adcba2e0e494fc612f0f4fe3d86f5dc8e238a3255acfdf43bf82b9ccfc21da95d639ff0c98cc15e05e" + +var cc_init_data = "YWxnb3JpdGhtID0gInNoYTM4NCIKdmVyc2lvbiA9ICIwLjEuMCIKCltkYXRhXQoiYWEudG9tbCIgPSAnJycKW3Rva2VuX2NvbmZpZ3NdClt0b2tlbl9jb25maWdzLmNvY29fYXNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCgpbdG9rZW5fY29uZmlncy5rYnNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCicnJwoKImNkaC50b21sIiAgPSAnJycKc29ja2V0ID0gJ3VuaXg6Ly8vcnVuL2NvbmZpZGVudGlhbC1jb250YWluZXJzL2NkaC5zb2NrJwpjcmVkZW50aWFscyA9IFtdCgpba2JjXQpuYW1lID0gJ2NjX2tiYycKdXJsID0gJ2h0dHA6Ly8xLjIuMy40OjgwODAnCicnJwoKInBvbGljeS5yZWdvIiA9ICcnJwpwYWNrYWdlIGFnZW50X3BvbGljeQoKaW1wb3J0IGZ1dHVyZS5rZXl3b3Jkcy5pbgppbXBvcnQgZnV0dXJlLmtleXdvcmRzLmV2ZXJ5CgppbXBvcnQgaW5wdXQKCiMgRGVmYXVsdCB2YWx1ZXMsIHJldHVybmVkIGJ5IE9QQSB3aGVuIHJ1bGVzIGNhbm5vdCBiZSBldmFsdWF0ZWQgdG8gdHJ1ZS4KZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBDcmVhdGVDb250YWluZXJSZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgQ3JlYXRlU2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IERlc3Ryb3lTYW5kYm94UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgRXhlY1Byb2Nlc3NSZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBPbmxpbmVDUFVNZW1SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNpZ25hbFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydENvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXRzQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVHR5V2luUmVzaXplUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlRXBoZW1lcmFsTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlSW50ZXJmYWNlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlUm91dGVzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV2FpdFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXcml0ZVN0cmVhbVJlcXVlc3QgOj0gZmFsc2UKJycn" + // Test server to simulate the metadata service func startTestServer() *httptest.Server { // Create base64 encoded test data @@ -236,35 +280,14 @@ func indentTextBlock(text string, by int) string { return indented } -// TestProcessCloudConfig tests parsing and provisioning of a daemon config func TestProcessCloudConfig(t *testing.T) { - // create temporary agent config file - tmpAgentConfigFile, err := os.CreateTemp("", "test") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(tmpAgentConfigFile.Name()) + tempDir, _ := os.MkdirTemp("", "tmp_writefiles_root") + defer os.RemoveAll(tempDir) - // create temporary daemon config file - tmpDaemonConfigFile, err := os.CreateTemp("", "test") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(tmpDaemonConfigFile.Name()) - - // create temporary auth json file - tmpAuthJsonFile, err := os.CreateTemp("", "test") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(tmpAuthJsonFile.Name()) - - // create temporary cdh config file - tmpCDHConfigFile, err := os.CreateTemp("", "test") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(tmpCDHConfigFile.Name()) + var agentCfgPath = filepath.Join(tempDir, "agent-config.toml") + var daemonPath = filepath.Join(tempDir, "daemon.json") + var authPath = filepath.Join(tempDir, "auth.json") + var initdataPath = filepath.Join(tempDir, "initdata") content := fmt.Sprintf(`#cloud-config write_files: @@ -281,14 +304,14 @@ write_files: content: | %s `, - tmpAgentConfigFile.Name(), + agentCfgPath, indentTextBlock(testAgentConfig, 4), - tmpDaemonConfigFile.Name(), + daemonPath, indentTextBlock(testDaemonConfig, 4), - tmpCDHConfigFile.Name(), - indentTextBlock(testCDHConfig, 4), - tmpAuthJsonFile.Name(), - indentTextBlock(testAuthJson, 4)) + authPath, + indentTextBlock(testAuthJson, 4), + initdataPath, + indentTextBlock(cc_init_data, 4)) provider := TestProvider{content: content} @@ -298,52 +321,51 @@ write_files: } cfg := Config{ - paths: paths{ - agentConfig: tmpAgentConfigFile.Name(), - daemonConfig: tmpDaemonConfigFile.Name(), - cdhConfig: tmpCDHConfigFile.Name(), - authJson: tmpAuthJsonFile.Name(), - }, + fetchTimeout: 180, + digestPath: "", + initdataPath: initdataPath, + parentPath: tempDir, + writeFiles: WriteFilesList, + initdataFiles: nil, } if err := processCloudConfig(&cfg, cc); err != nil { t.Fatalf("failed to process cloud config file: %v", err) } // check if files have been written correctly - data, _ := os.ReadFile(tmpAgentConfigFile.Name()) + data, _ := os.ReadFile(agentCfgPath) fileContent := string(data) if fileContent != testAgentConfig { t.Fatalf("file content does not match daemon config fixture: got %q", fileContent) } - data, _ = os.ReadFile(tmpDaemonConfigFile.Name()) + data, _ = os.ReadFile(daemonPath) fileContent = string(data) if fileContent != testDaemonConfig { t.Fatalf("file content does not match daemon config fixture: got %q", fileContent) } - data, _ = os.ReadFile(tmpCDHConfigFile.Name()) + data, _ = os.ReadFile(authPath) fileContent = string(data) - if fileContent != testCDHConfig { - t.Fatalf("file content does not match cdh config fixture: got %q", fileContent) + if fileContent != testAuthJson { + t.Fatalf("file content does not match auth json fixture: got %q", fileContent) } - data, _ = os.ReadFile(tmpAuthJsonFile.Name()) + data, _ = os.ReadFile(initdataPath) fileContent = string(data) - if fileContent != testAuthJson { - t.Fatalf("file content does not match auth json fixture: got %q", fileContent) + if fileContent != cc_init_data+"\n" { + t.Fatalf("file content does not match initdata fixture: got %q", fileContent) } } -func TestProcessWithOptionalEntries(t *testing.T) { - tmpAgentConfigFile, _ := os.CreateTemp("", "test") - defer os.Remove(tmpAgentConfigFile.Name()) - tmpDaemonConfigFile, _ := os.CreateTemp("", "test") - defer os.Remove(tmpDaemonConfigFile.Name()) - tmpAuthJsonFile, _ := os.CreateTemp("", "test") - defer os.Remove(tmpAuthJsonFile.Name()) - tmpCDHConfigFile, _ := os.CreateTemp("", "test") - os.Remove(tmpCDHConfigFile.Name()) +func TestProcessCloudConfigWithMalicious(t *testing.T) { + tempDir, _ := os.MkdirTemp("", "tmp_writefiles_root") + defer os.RemoveAll(tempDir) + + var agentCfgPath = filepath.Join(tempDir, "agent-config.toml") + var daemonPath = filepath.Join(tempDir, "daemon.json") + var authPath = filepath.Join(tempDir, "auth.json") + var malicious = filepath.Join(tempDir, "malicious") content := fmt.Sprintf(`#cloud-config write_files: @@ -353,11 +375,22 @@ write_files: - path: %s content: | %s +- path: %s + content: | +%s +- path: %s + content: | +%s `, - tmpAgentConfigFile.Name(), + agentCfgPath, indentTextBlock(testAgentConfig, 4), - tmpDaemonConfigFile.Name(), - indentTextBlock(testDaemonConfig, 4)) + daemonPath, + indentTextBlock(testDaemonConfig, 4), + authPath, + indentTextBlock(testAuthJson, 4), + malicious, + indentTextBlock("malicious", 4)) + provider := TestProvider{content: content} cc, err := retrieveCloudConfig(context.TODO(), &provider) @@ -366,19 +399,39 @@ write_files: } cfg := Config{ - paths: paths{ - agentConfig: tmpAgentConfigFile.Name(), - daemonConfig: tmpDaemonConfigFile.Name(), - cdhConfig: tmpCDHConfigFile.Name(), - }, + fetchTimeout: 180, + digestPath: "", + initdataPath: "", + parentPath: tempDir, + writeFiles: WriteFilesList, + initdataFiles: nil, } if err := processCloudConfig(&cfg, cc); err != nil { t.Fatalf("failed to process cloud config file: %v", err) } - _, err = os.Stat(tmpCDHConfigFile.Name()) - if !os.IsNotExist(err) { - t.Fatalf("CDH config file shouldn't exist") + // check if files have been written correctly + data, _ := os.ReadFile(agentCfgPath) + fileContent := string(data) + if fileContent != testAgentConfig { + t.Fatalf("file content does not match daemon config fixture: got %q", fileContent) + } + + data, _ = os.ReadFile(daemonPath) + fileContent = string(data) + if fileContent != testDaemonConfig { + t.Fatalf("file content does not match daemon config fixture: got %q", fileContent) + } + + data, _ = os.ReadFile(authPath) + fileContent = string(data) + if fileContent != testAuthJson { + t.Fatalf("file content does not match auth json fixture: got %q", fileContent) + } + + data, _ = os.ReadFile(malicious) + if data != nil { + t.Fatalf("file content should be nil but: got %q", string(data)) } } @@ -403,3 +456,94 @@ func TestFailPlainTextUserData(t *testing.T) { } } + +func TestExtractInitdataAndHash(t *testing.T) { + tempDir, _ := os.MkdirTemp("", "tmp_initdata_root") + defer os.RemoveAll(tempDir) + + var initdataPath = filepath.Join(tempDir, "initdata") + var aaPath = filepath.Join(tempDir, "aa.toml") + var cdhPath = filepath.Join(tempDir, "cdh.toml") + var policyPath = filepath.Join(tempDir, "policy.rego") + var digestPath = filepath.Join(tempDir, "initdata.digest") + cfg := Config{ + fetchTimeout: 180, + digestPath: digestPath, + initdataPath: initdataPath, + parentPath: tempDir, + writeFiles: nil, + initdataFiles: InitdDataFilesList, + } + + _ = writeFile(initdataPath, []byte(cc_init_data)) + err := extractInitdataAndHash(&cfg) + if err != nil { + t.Fatalf("extractInitdataAndHash returned err: %v", err) + } + + bytes, _ := os.ReadFile(aaPath) + aaStr := string(bytes) + if testAAConfig != aaStr { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", aaStr, testAAConfig) + } + + bytes, _ = os.ReadFile(cdhPath) + cdhStr := string(bytes) + if testCDHConfig != cdhStr { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", cdhStr, testCDHConfig) + } + + bytes, _ = os.ReadFile(policyPath) + regoStr := string(bytes) + if testPolicyConfig != regoStr { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", regoStr, testPolicyConfig) + } + + bytes, _ = os.ReadFile(digestPath) + sum := string(bytes) + if testCheckSum != sum { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", sum, testCheckSum) + } +} + +func TestExtractInitdataWithMalicious(t *testing.T) { + tempDir, _ := os.MkdirTemp("", "tmp_initdata_root") + defer os.RemoveAll(tempDir) + + var initdataPath = filepath.Join(tempDir, "initdata") + var aaPath = filepath.Join(tempDir, "aa.toml") + var cdhPath = filepath.Join(tempDir, "cdh.toml") + var policyPath = filepath.Join(tempDir, "malicious.rego") + var digestPath = filepath.Join(tempDir, "initdata.digest") + cfg := Config{ + fetchTimeout: 180, + digestPath: digestPath, + initdataPath: initdataPath, + parentPath: tempDir, + writeFiles: nil, + initdataFiles: InitdDataFilesList, + } + + _ = writeFile(initdataPath, []byte(cc_init_data)) + err := extractInitdataAndHash(&cfg) + if err != nil { + t.Fatalf("extractInitdataAndHash returned err: %v", err) + } + + bytes, _ := os.ReadFile(aaPath) + aaStr := string(bytes) + if testAAConfig != aaStr { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", aaStr, testAAConfig) + } + + bytes, _ = os.ReadFile(cdhPath) + cdhStr := string(bytes) + if testCDHConfig != cdhStr { + t.Fatalf("extractInitdataAndHash returned: %s does not match %s", cdhStr, testCDHConfig) + } + + bytes, _ = os.ReadFile(policyPath) + if bytes != nil { + t.Fatalf("Should not read malicious file but got %s", string(bytes)) + } +} diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go index b2ba396af..775aa430d 100644 --- a/src/cloud-api-adaptor/pkg/util/cloud.go +++ b/src/cloud-api-adaptor/pkg/util/cloud.go @@ -69,6 +69,11 @@ func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64) return vcpuInt, memoryInt } +// Method to get initdata from annotation +func GetInitdataFromAnnotation(annotations map[string]string) string { + return annotations["io.katacontainers.config.runtime.cc_init_data"] +} + // Method to check if a string exists in a slice func Contains(slice []string, s string) bool { for _, item := range slice { diff --git a/src/cloud-api-adaptor/podvm/files/etc/systemd/system/confidential-data-hub.service b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/confidential-data-hub.service index 38b4eae3c..3f49b7595 100644 --- a/src/cloud-api-adaptor/podvm/files/etc/systemd/system/confidential-data-hub.service +++ b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/confidential-data-hub.service @@ -5,8 +5,8 @@ After=network.target cloud-init.service process-user-data.service [Service] Type=simple ExecStart=/bin/bash -c \ - 'if [ -f /run/confidential-containers/cdh.toml ]; \ - then /usr/local/bin/confidential-data-hub -c /run/confidential-containers/cdh.toml; \ + 'if [ -f /run/peerpod/cdh.toml ]; \ + then /usr/local/bin/confidential-data-hub -c /run/peerpod/cdh.toml; \ else /usr/local/bin/confidential-data-hub; \ fi' RestartSec=1 diff --git a/src/cloud-api-adaptor/podvm/files/etc/systemd/system/process-user-data.service b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/process-user-data.service index 26bad136c..3bb4fa7f5 100644 --- a/src/cloud-api-adaptor/podvm/files/etc/systemd/system/process-user-data.service +++ b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/process-user-data.service @@ -3,7 +3,8 @@ [Unit] Description=Process user data -After=network.target +# some providers use cloud-init to provision config files, it does not matter if cloud-init disabled +After=network.target cloud-init.service DefaultDependencies=no diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index 20c62e5e6..8aa3293d1 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -35,7 +35,7 @@ git: reference: v0.9.0 kata-containers: url: https://github.com/kata-containers/kata-containers - reference: 3.7.0 + reference: a113fc93c8fe75cf58b9b6edadd07a3ebfd665f3 umoci: url: https://github.com/opencontainers/umoci reference: v0.4.7