upgrade to spec generator mechanism (#230)

pkg/engine/backend/backend.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/gocty"
+
 	backendInit "kusionstack.io/kusion/pkg/engine/backend/init"
 	"kusionstack.io/kusion/pkg/engine/states"
 	"kusionstack.io/kusion/pkg/engine/states/local"

pkg/engine/backend/backend_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/zclconf/go-cty/cty"
+
 	_ "kusionstack.io/kusion/pkg/engine/backend/init"
 	"kusionstack.io/kusion/pkg/engine/states"
 	"kusionstack.io/kusion/pkg/engine/states/local"

pkg/engine/kcl.go

@@ -15,10 +15,10 @@ import (
 
 const MaxLogLength = 3751
 
-func ConvertKCLResult2Resources(resourceYAMLs []kcl.KCLResult) (*models.Spec, error) {
+func ResourcesYAML2Spec(resourcesYAML []kcl.KCLResult) (*models.Spec, error) {
 	resources := []models.Resource{}
 
-	for _, resourcesYamlMap := range resourceYAMLs {
+	for _, resourcesYamlMap := range resourcesYAML {
 		// Convert kcl result to yaml string
 		msg := jsonUtil.MustMarshal2String(resourcesYamlMap)
 		if len(msg) > MaxLogLength {

pkg/engine/operation/watch_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	k8sWatch "k8s.io/apimachinery/pkg/watch"
-
 	"kusionstack.io/kusion/pkg/engine/models"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/runtime"

pkg/kusionctl/cmd/apply/options.go

@@ -10,14 +10,15 @@ import (
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/pterm/pterm"
 
-	"kusionstack.io/kusion/pkg/compile"
 	"kusionstack.io/kusion/pkg/engine/backend"
 	_ "kusionstack.io/kusion/pkg/engine/backend/init"
 	"kusionstack.io/kusion/pkg/engine/models"
 	"kusionstack.io/kusion/pkg/engine/operation"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/states"
 	previewcmd "kusionstack.io/kusion/pkg/kusionctl/cmd/preview"
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/spec"
+	"kusionstack.io/kusion/pkg/kusionctl/generator"
 	"kusionstack.io/kusion/pkg/log"
 	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
@@ -64,8 +65,8 @@ func (o *ApplyOptions) Run() error {
 		return err
 	}
 
-	// Get compile result
-	planResources, err := compile.GenerateSpec(&compile.Options{
+	// generate Spec
+	sp, err := spec.GenerateSpecWithSpinner(&generator.Options{
 		WorkDir:     o.WorkDir,
 		Filenames:   o.Filenames,
 		Settings:    o.Settings,
@@ -74,13 +75,13 @@ func (o *ApplyOptions) Run() error {
 		DisableNone: o.DisableNone,
 		OverrideAST: o.OverrideAST,
 		NoStyle:     o.NoStyle,
-	}, stack)
+	}, project, stack)
 	if err != nil {
 		return err
 	}
 
 	// return immediately if no resource found in stack
-	if planResources == nil || len(planResources.Resources) == 0 {
+	if sp == nil || len(sp.Resources) == 0 {
 		fmt.Println(pretty.GreenBold("\nNo resource found in this stack."))
 		return nil
 	}
@@ -92,7 +93,7 @@ func (o *ApplyOptions) Run() error {
 	}
 
 	// Compute changes for preview
-	changes, err := previewcmd.Preview(&o.PreviewOptions, stateStorage, planResources, project, stack)
+	changes, err := previewcmd.Preview(&o.PreviewOptions, stateStorage, sp, project, stack)
 	if err != nil {
 		return err
 	}
@@ -136,7 +137,7 @@ func (o *ApplyOptions) Run() error {
 	}
 
 	fmt.Println("Start applying diffs ...")
-	if err := Apply(o, stateStorage, planResources, changes, os.Stdout); err != nil {
+	if err := Apply(o, stateStorage, sp, changes, os.Stdout); err != nil {
 		return err
 	}
 
@@ -148,7 +149,7 @@ func (o *ApplyOptions) Run() error {
 
 	if o.Watch {
 		fmt.Println("\nStart watching changes ...")
-		if err := Watch(o, planResources, changes, os.Stdout); err != nil {
+		if err := Watch(o, sp, changes, os.Stdout); err != nil {
 			return err
 		}
 	}

pkg/kusionctl/cmd/apply/options_test.go

@@ -1,6 +1,3 @@
-//go:build !arm64
-// +build !arm64
-
 package apply
 
 import (
@@ -15,13 +12,14 @@ import (
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/stretchr/testify/assert"
 
-	"kusionstack.io/kusion/pkg/compile"
 	"kusionstack.io/kusion/pkg/engine"
 	"kusionstack.io/kusion/pkg/engine/models"
 	"kusionstack.io/kusion/pkg/engine/operation"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/engine/states/local"
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/spec"
+	"kusionstack.io/kusion/pkg/kusionctl/generator"
 	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 )
@@ -81,7 +79,7 @@ func mockDetectProjectAndStack() {
 }
 
 func mockGenerateSpec() {
-	monkey.Patch(compile.GenerateSpec, func(o *compile.Options, stack *projectstack.Stack) (*models.Spec, error) {
+	monkey.Patch(spec.GenerateSpecWithSpinner, func(o *generator.Options, project *projectstack.Project, stack *projectstack.Stack) (*models.Spec, error) {
 		return &models.Spec{Resources: []models.Resource{sa1, sa2, sa3}}, nil
 	})
 }

pkg/kusionctl/cmd/check/check_test.go

@@ -1,39 +1,25 @@
-//go:build !arm64
-// +build !arm64
-
 package check
 
 import (
 	"testing"
 
-	kcl "kusionstack.io/kclvm-go"
-
 	"bou.ke/monkey"
 	"github.com/stretchr/testify/assert"
-	"kusionstack.io/kusion/pkg/compile"
+
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/compile"
 )
 
 func TestNewCmdCheck(t *testing.T) {
 	t.Run("", func(t *testing.T) {
-		monkey.Patch(
-			compile.Compile,
-			func(workDir string,
-				filenames, settings, arguments, overrides []string,
-				disableNone bool,
-				overrideAST bool,
-			) (*compile.CompileResult, error) {
-				return &compile.CompileResult{
-					Documents: []kcl.KCLResult{
-						map[string]interface{}{
-							"str":    "v1",
-							"int":    2,
-							"bool":   false,
-							"struct": struct{}{},
-						},
-					},
-				}, nil
-			},
-		)
+		defer monkey.UnpatchAll()
+
+		monkey.Patch((*compile.CompileOptions).Complete, func(o *compile.CompileOptions, args []string) {
+			o.Output = "stdout"
+		})
+		monkey.Patch((*compile.CompileOptions).Run, func(*compile.CompileOptions) error {
+			return nil
+		})
+
 		cmd := NewCmdCheck()
 		err := cmd.Execute()
 		assert.Nil(t, err)

pkg/kusionctl/cmd/compile/compile_test.go

@@ -1,44 +1,23 @@
-//go:build !arm64
-// +build !arm64
-
 package compile
 
 import (
 	"testing"
 
 	"bou.ke/monkey"
 	"github.com/stretchr/testify/assert"
-	kcl "kusionstack.io/kclvm-go"
-
-	"kusionstack.io/kusion/pkg/compile"
 )
 
 func TestNewCmdCompile(t *testing.T) {
-	t.Run("compile success", func(t *testing.T) {
-		monkey.Patch(
-			compile.Compile,
-			func(workDir string,
-				filenames, settings, arguments, overrides []string,
-				disableNone bool,
-				overrideAST bool,
-			) (*compile.CompileResult, error) {
-				return &compile.CompileResult{
-					Documents: []kcl.KCLResult{
-						map[string]interface{}{
-							"str":    "v1",
-							"int":    2,
-							"bool":   false,
-							"struct": struct{}{},
-						},
-					},
-				}, nil
-			},
-		)
-		monkey.Patch((*CompileOptions).Complete, func(o *CompileOptions, args []string) {
-			o.Output = "stdout"
-		})
-		defer monkey.UnpatchAll()
+	defer monkey.UnpatchAll()
 
+	monkey.Patch((*CompileOptions).Complete, func(o *CompileOptions, args []string) {
+		o.Output = "stdout"
+	})
+	monkey.Patch((*CompileOptions).Run, func(*CompileOptions) error {
+		return nil
+	})
+
+	t.Run("compile success", func(t *testing.T) {
 		cmd := NewCmdCompile()
 		err := cmd.Execute()
 		assert.Nil(t, err)

pkg/kusionctl/cmd/compile/options.go

@@ -5,7 +5,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"kusionstack.io/kusion/pkg/compile"
+	yamlv3 "gopkg.in/yaml.v3"
+
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/spec"
+	"kusionstack.io/kusion/pkg/kusionctl/generator"
 	"kusionstack.io/kusion/pkg/projectstack"
 )
 
@@ -57,8 +60,22 @@ func (o *CompileOptions) Validate() error {
 }
 
 func (o *CompileOptions) Run() error {
-	// Compile
-	compileResult, err := compile.Compile(o.WorkDir, o.Filenames, o.Settings, o.Arguments, o.Overrides, o.DisableNone, o.OverrideAST)
+	// Parse project and stack of work directory
+	project, stack, err := projectstack.DetectProjectAndStack(o.WorkDir)
+	if err != nil {
+		return err
+	}
+
+	sp, err := spec.GenerateSpecWithSpinner(&generator.Options{
+		WorkDir:     o.WorkDir,
+		Filenames:   o.Filenames,
+		Settings:    o.Settings,
+		Arguments:   o.Arguments,
+		Overrides:   o.Overrides,
+		DisableNone: o.DisableNone,
+		OverrideAST: o.OverrideAST,
+	}, project, stack)
+
 	if o.IsCheck {
 		if err != nil {
 			fmt.Print(err)
@@ -68,16 +85,18 @@ func (o *CompileOptions) Run() error {
 			return err
 		}
 
-		// Output
-		yaml := compileResult.RawYAML()
+		yaml, err := yamlv3.Marshal(sp.Resources)
+		if err != nil {
+			return err
+		}
 		if o.Output == Stdout {
-			fmt.Print(yaml)
+			fmt.Print(string(yaml))
 		} else {
 			if o.WorkDir != "" {
 				o.Output = filepath.Join(o.WorkDir, o.Output)
 			}
 
-			err := os.WriteFile(o.Output, []byte(yaml), 0o666)
+			err := os.WriteFile(o.Output, yaml, 0o666)
 			if err != nil {
 				return err
 			}

pkg/kusionctl/cmd/destroy/options.go

@@ -9,13 +9,14 @@ import (
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/pterm/pterm"
 
-	"kusionstack.io/kusion/pkg/compile"
 	"kusionstack.io/kusion/pkg/engine/backend"
 	"kusionstack.io/kusion/pkg/engine/models"
 	"kusionstack.io/kusion/pkg/engine/operation"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/states"
 	compilecmd "kusionstack.io/kusion/pkg/kusionctl/cmd/compile"
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/spec"
+	"kusionstack.io/kusion/pkg/kusionctl/generator"
 	"kusionstack.io/kusion/pkg/log"
 	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
@@ -54,15 +55,15 @@ func (o *DestroyOptions) Run() error {
 	}
 
 	// Get compile result
-	planResources, err := compile.GenerateSpec(&compile.Options{
+	planResources, err := spec.GenerateSpecWithSpinner(&generator.Options{
 		WorkDir:     o.WorkDir,
 		Filenames:   o.Filenames,
 		Settings:    o.Settings,
 		Arguments:   o.Arguments,
 		Overrides:   o.Overrides,
 		DisableNone: o.DisableNone,
 		OverrideAST: o.OverrideAST,
-	}, stack)
+	}, project, stack)
 	if err != nil {
 		return err
 	}

pkg/kusionctl/cmd/destroy/options_test.go

@@ -1,6 +1,3 @@
-//go:build !arm64
-// +build !arm64
-
 package destroy
 
 import (
@@ -14,13 +11,14 @@ import (
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/stretchr/testify/assert"
 
-	"kusionstack.io/kusion/pkg/compile"
 	"kusionstack.io/kusion/pkg/engine"
 	"kusionstack.io/kusion/pkg/engine/models"
 	"kusionstack.io/kusion/pkg/engine/operation"
 	opsmodels "kusionstack.io/kusion/pkg/engine/operation/models"
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/engine/states/local"
+	"kusionstack.io/kusion/pkg/kusionctl/cmd/spec"
+	"kusionstack.io/kusion/pkg/kusionctl/generator"
 	"kusionstack.io/kusion/pkg/projectstack"
 	"kusionstack.io/kusion/pkg/status"
 )
@@ -90,7 +88,7 @@ func mockDetectProjectAndStack() {
 }
 
 func mockGenerateSpec() {
-	monkey.Patch(compile.GenerateSpec, func(o *compile.Options, stack *projectstack.Stack) (*models.Spec, error) {
+	monkey.Patch(spec.GenerateSpecWithSpinner, func(o *generator.Options, project *projectstack.Project, stack *projectstack.Stack) (*models.Spec, error) {
 		return &models.Spec{Resources: []models.Resource{sa1}}, nil
 	})
 }