initdata: use annotation to provision config files

Fixes: #1895

Signed-off-by: Qi Feng Huo <[email protected]>

src/cloud-api-adaptor/cmd/process-user-data/main.go

@@ -7,9 +7,6 @@ import (
 	"os"
 
 	cmdUtil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
-	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/aa"
-	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh"
-	daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/userdata"
 	"github.com/spf13/cobra"
 )
@@ -18,8 +15,6 @@ const (
 	programName   = "process-user-data"
 	providerAzure = "azure"
 	providerAws   = "aws"
-
-	defaultAuthJsonPath = "/run/peerpod/auth.json"
 )
 
 var versionFlag bool
@@ -36,19 +31,14 @@ var rootCmd = &cobra.Command{
 }
 
 func init() {
-	var aaConfigPath, cdhConfigPath, daemonConfigPath string
 	var fetchTimeout int
-
 	rootCmd.PersistentFlags().BoolVarP(&versionFlag, "version", "v", false, "Print the version")
-	rootCmd.PersistentFlags().StringVarP(&daemonConfigPath, "daemon-config-path", "d", daemon.DefaultConfigPath, "Path to a daemon config file")
-	rootCmd.PersistentFlags().StringVarP(&aaConfigPath, "aa-config-path", "a", aa.DefaultAaConfigPath, "Path to a AA config file")
-	rootCmd.PersistentFlags().StringVarP(&cdhConfigPath, "cdh-config-path", "c", cdh.ConfigFilePath, "Path to a CDH config file")
 
 	var provisionFilesCmd = &cobra.Command{
 		Use:   "provision-files",
 		Short: "Provision required files based on user data",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := userdata.NewConfig(aaConfigPath, defaultAuthJsonPath, daemonConfigPath, cdhConfigPath, fetchTimeout)
+			cfg := userdata.NewConfig(fetchTimeout)
 			return userdata.ProvisionFiles(cfg)
 		},
 		SilenceUsage: true, // Silence usage on error

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -5,6 +5,7 @@ package cloud
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -19,12 +20,11 @@ import (
 	"github.com/containerd/containerd/pkg/cri/annotations"
 	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
 
-	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/aa"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/k8sops"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
-	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/userdata"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util"
 	provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
 	putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util"
@@ -267,24 +267,33 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
-	if s.aaKBCParams != "" {
-		logger.Printf("aaKBCParams: %s, support cc_kbc::*", s.aaKBCParams)
-		toml, err := cdh.CreateConfigFile(s.aaKBCParams)
+	initdataStr := util.GetInitdataFromAnnotation(req.Annotations)
+	logger.Printf("initdata: %s", initdataStr)
+	if initdataStr != "" {
+		decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr)
 		if err != nil {
-			return nil, fmt.Errorf("creating CDH config: %w", err)
+			return nil, fmt.Errorf("Error base64 decode initdata: %w", err)
 		}
-		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
-			Path:    cdh.ConfigFilePath,
-			Content: toml,
-		})
-
-		toml, err = aa.CreateConfigFile(s.aaKBCParams)
+		initdata := userdata.InitData{}
+		err = json.Unmarshal(decodedBytes, &initdata)
+		if err != nil {
+			return nil, fmt.Errorf("Error unmarshalling initdata: %w", err)
+		}
+		for key, value := range initdata.Data {
+			cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
+				Path:    filepath.Join(userdata.ConfigParent, key),
+				Content: value,
+			})
+		}
+		cloned := initdata
+		cloned.Data = nil
+		clonedJSON, err := json.Marshal(cloned) // Data field omitted
 		if err != nil {
-			return nil, fmt.Errorf("creating attestation agent config: %w", err)
+			return nil, fmt.Errorf("Error Marshal cloned initdata: %w", err)
 		}
 		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
-			Path:    aa.DefaultAaConfigPath,
-			Content: toml,
+			Path:    userdata.AlgorithmPath,
+			Content: string(clonedJSON),
 		})
 	}
 

src/cloud-api-adaptor/pkg/userdata/provision.go

@@ -2,11 +2,15 @@ package userdata
 
 import (
 	"context"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/avast/retry-go/v4"
@@ -17,23 +21,25 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+const (
+	AlgorithmPath  = "/run/peerpod/initdata.json"
+	AuthJsonPath   = "/run/peerpod/auth.json"
+	CheckSumPath   = "/run/peerpod/checksum.txt"
+	ConfigParent   = "/run/peerpod"
+	DaemonJsonName = "daemon.json"
+)
+
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
 
-type paths struct {
-	aaConfig     string
-	authJson     string
-	cdhConfig    string
-	daemonConfig string
-}
+var StaticFiles = []string{"/run/peerpod/aa.toml", "/run/peerpod/cdh.toml", "/run/peerpod/policy.rego"}
 
 type Config struct {
+	parentPath   string
 	fetchTimeout int
-	paths        paths
 }
 
-func NewConfig(aaConfigPath, authJsonPath, daemonConfigPath, cdhConfig string, fetchTimeout int) *Config {
-	cfgPaths := paths{aaConfigPath, authJsonPath, cdhConfig, daemonConfigPath}
-	return &Config{fetchTimeout, cfgPaths}
+func NewConfig(fetchTimeout int) *Config {
+	return &Config{ConfigParent, fetchTimeout}
 }
 
 type WriteFile struct {
@@ -45,6 +51,12 @@ type CloudConfig struct {
 	WriteFiles []WriteFile `yaml:"write_files"`
 }
 
+type InitData struct {
+	Algorithom string            `json:"algorithm"`
+	Version    string            `json:"version"`
+	Data       map[string]string `json:"data,omitempty"`
+}
+
 type UserDataProvider interface {
 	GetUserData(ctx context.Context) ([]byte, error)
 	GetRetryDelay() time.Duration
@@ -150,16 +162,6 @@ func parseDaemonConfig(content []byte) (*daemon.Config, error) {
 	return &dc, nil
 }
 
-func findConfigEntry(path string, cc *CloudConfig) []byte {
-	for _, wf := range cc.WriteFiles {
-		if wf.Path != path {
-			continue
-		}
-		return []byte(wf.Content)
-	}
-	return nil
-}
-
 func writeFile(path string, bytes []byte) error {
 	// Ensure the parent directory exists
 	err := os.MkdirAll(filepath.Dir(path), 0755)
@@ -169,42 +171,87 @@ func writeFile(path string, bytes []byte) error {
 
 	err = os.WriteFile(path, bytes, 0644)
 	if err != nil {
-		return fmt.Errorf("failed to write file: %w", err)
+		return fmt.Errorf("failed to write file %s: %w", path, err)
 	}
 	logger.Printf("Wrote %s\n", path)
 	return nil
 }
 
 func processCloudConfig(cfg *Config, cc *CloudConfig) error {
-	bytes := findConfigEntry(cfg.paths.daemonConfig, cc)
-	if bytes == nil {
-		return fmt.Errorf("failed to find daemon config entry in cloud config")
+	for _, wf := range cc.WriteFiles {
+		path := wf.Path
+		bytes := []byte(wf.Content)
+		if strings.HasPrefix(path, cfg.parentPath) { // handle files in "/run/peerpod"
+			if strings.HasSuffix(path, DaemonJsonName) { // handle daemon.json file
+				if bytes == nil {
+					return fmt.Errorf("failed to find daemon config entry in cloud config")
+				}
+				daemonConfig, err := parseDaemonConfig(bytes)
+				if err != nil {
+					return fmt.Errorf("failed to parse daemon config %s: %w", path, err)
+				}
+				if err = writeFile(path, bytes); err != nil {
+					return fmt.Errorf("failed to write daemon config file %s: %w", path, err)
+				}
+				if daemonConfig.AuthJson != "" { // handle auth json file
+					bytes := []byte(daemonConfig.AuthJson)
+					if err = writeFile(AuthJsonPath, bytes); err != nil {
+						return fmt.Errorf("failed to write auth json file %s: %w", AuthJsonPath, err)
+					}
+				}
+			} else { // handle other config files
+				if err := writeFile(path, bytes); err != nil {
+					return fmt.Errorf("failed to write config file %s: %w", path, err)
+				}
+			}
+		} else {
+			return fmt.Errorf("failed to write config file, path %s does not in folder %s", path, cfg.parentPath)
+		}
 	}
-	daemonConfig, err := parseDaemonConfig(bytes)
+	return nil
+}
+
+func calculateUserDataHash() error {
+	initJson, err := os.ReadFile(AlgorithmPath)
 	if err != nil {
-		return fmt.Errorf("failed to parse daemon config: %w", err)
+		return err
 	}
-	if err = writeFile(cfg.paths.daemonConfig, bytes); err != nil {
-		return fmt.Errorf("failed to write daemon config file: %w", err)
+	var initdata InitData
+	err = json.Unmarshal(initJson, &initdata)
+	if err != nil {
+		return err
 	}
 
-	if bytes := findConfigEntry(cfg.paths.aaConfig, cc); bytes != nil {
-		if err = writeFile(cfg.paths.aaConfig, bytes); err != nil {
-			return fmt.Errorf("failed to write aa config file: %w", err)
+	checksumStr := ""
+	var byteData []byte
+	for _, file := range StaticFiles {
+		if _, err := os.Stat(file); err == nil {
+			logger.Printf("calculateUserDataHash and reading file %s\n", file)
+			bytes, err := os.ReadFile(file)
+			if err != nil {
+				return fmt.Errorf("Error reading file %s: %v", file, err)
+			}
+			byteData = append(byteData, bytes...)
 		}
 	}
 
-	if bytes := findConfigEntry(cfg.paths.cdhConfig, cc); bytes != nil {
-		if err = writeFile(cfg.paths.cdhConfig, bytes); err != nil {
-			return fmt.Errorf("failed to write cdh config file: %w", err)
-		}
+	switch initdata.Algorithom {
+	case "sha256":
+		hash := sha256.Sum256(byteData)
+		checksumStr = hex.EncodeToString(hash[:])
+	case "sha384":
+		hash := sha512.Sum384(byteData)
+		checksumStr = hex.EncodeToString(hash[:])
+	case "sha512":
+		hash := sha512.Sum512(byteData)
+		checksumStr = hex.EncodeToString(hash[:])
+	default:
+		return fmt.Errorf("Error creating initdata hash, the algorothom %s not supported", initdata.Algorithom)
 	}
 
-	if daemonConfig.AuthJson != "" {
-		bytes := []byte(daemonConfig.AuthJson)
-		if err = writeFile(cfg.paths.authJson, bytes); err != nil {
-			return fmt.Errorf("failed to write auth json file: %w", err)
-		}
+	err = os.WriteFile(CheckSumPath, []byte(checksumStr), 0644) // the hash in CheckSumPath will also be used by attester
+	if err != nil {
+		return fmt.Errorf("failed to write file %s: %w", CheckSumPath, err)
 	}
 
 	return nil
@@ -216,18 +263,25 @@ func ProvisionFiles(cfg *Config) error {
 	ctx, cancel := context.WithTimeout(bg, duration)
 	defer cancel()
 
+	// some providers provision config files via process-user-data
+	// some providers rely on cloud-init provisin config files
+	// all providers need calculate the hash value for attesters usage
 	provider, err := newProvider(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to create UserData provider: %w", err)
-	}
+	if provider != nil {
+		cc, err := retrieveCloudConfig(ctx, provider)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve cloud config: %w", err)
+		}
 
-	cc, err := retrieveCloudConfig(ctx, provider)
-	if err != nil {
-		return fmt.Errorf("failed to retrieve cloud config: %w", err)
+		if err = processCloudConfig(cfg, cc); err != nil {
+			return fmt.Errorf("failed to process cloud config: %w", err)
+		}
+	} else {
+		logger.Printf("unsupported user data provider %s, we calculate initdata hash only.\n")
 	}
 
-	if err = processCloudConfig(cfg, cc); err != nil {
-		return fmt.Errorf("failed to process cloud config: %w", err)
+	if err = calculateUserDataHash(); err != nil {
+		return fmt.Errorf("failed to calculate initdata hash: %w", err)
 	}
 
 	return nil

src/cloud-api-adaptor/pkg/util/cloud.go

@@ -69,6 +69,11 @@ func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64)
 	return vcpuInt, memoryInt
 }
 
+// Method to get initdata from annotation
+func GetInitdataFromAnnotation(annotations map[string]string) string {
+	return annotations["io.katacontainers.config.runtime.cc_init_data"]
+}
+
 // Method to check if a string exists in a slice
 func Contains(slice []string, s string) bool {
 	for _, item := range slice {

src/cloud-api-adaptor/podvm/files/etc/systemd/system/process-user-data.service

@@ -3,7 +3,8 @@
 
 [Unit]
 Description=Process user data
-After=network.target
+# some providers use cloud-init to provision config files, it does not matter if cloud-init disabled
+After=network.target cloud-init.service
 DefaultDependencies=no