Skip to content

Commit

Permalink
feat: add KCL based customized health check
Browse files Browse the repository at this point in the history
  • Loading branch information
Yangyang96 committed Aug 20, 2024
1 parent f2ee452 commit a950a01
Show file tree
Hide file tree
Showing 7 changed files with 401 additions and 8 deletions.
4 changes: 4 additions & 0 deletions pkg/apis/api.kusion.io/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@ const (
EnvViettelCloudProjectID = "VIETTEL_CLOUD_PROJECT_ID"

FieldImportedResources = "importedResources"
FieldHealthPolicy = "healthPolicy"
FieldKCLHealthCheckKCL = "health.kcl"
// kind field in kubernetes resource Attributes
FieldKind = "kind"
)

// BackendConfigs contains the configuration of multiple backends and the current backend.
Expand Down
41 changes: 38 additions & 3 deletions pkg/cmd/apply/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
"k8s.io/cli-runtime/pkg/genericiooptions"
"k8s.io/kubectl/pkg/util/templates"

"gopkg.in/yaml.v3"

apiv1 "kusionstack.io/kusion/pkg/apis/api.kusion.io/v1"
v1 "kusionstack.io/kusion/pkg/apis/status/v1"
"kusionstack.io/kusion/pkg/cmd/generate"
Expand All @@ -48,6 +50,7 @@ import (
runtimeinit "kusionstack.io/kusion/pkg/engine/runtime/init"
"kusionstack.io/kusion/pkg/log"
"kusionstack.io/kusion/pkg/util/i18n"
"kusionstack.io/kusion/pkg/util/kcl"
"kusionstack.io/kusion/pkg/util/pretty"
"kusionstack.io/kusion/pkg/util/signal"
"kusionstack.io/kusion/pkg/util/terminal"
Expand Down Expand Up @@ -758,7 +761,8 @@ func Watch(

// Setup a go-routine to concurrently watch K8s and TF resources.
if res.Type == apiv1.Kubernetes {
go watchK8sResources(id, w.Watchers, table, tables, dryRun)
healthPolicy, kind := getResourceInfo(&res)
go watchK8sResources(id, kind, w.Watchers, table, tables, dryRun, healthPolicy)
} else if res.Type == apiv1.Terraform {
go watchTFResources(id, w.TFWatcher, table, dryRun)
} else {
Expand Down Expand Up @@ -884,11 +888,12 @@ func prompt(ui *terminal.UI) (string, error) {
}

func watchK8sResources(
id string,
id, kind string,
chs []<-chan watch.Event,
table *printers.Table,
tables map[string]*printers.Table,
dryRun bool,
healthPolicy interface{},
) {
defer func() {
var err error
Expand Down Expand Up @@ -930,7 +935,22 @@ func watchK8sResources(
} else {
// Restore to actual type
target := printers.Convert(o)
detail, ready = printers.Generate(target)
// Check reconcile status with customized health policy for specific resource
if healthPolicy != nil && kind == o.GetObjectKind().GroupVersionKind().Kind {
if code, ok := kcl.ConvertKCLCode(healthPolicy); ok {
resByte, err := yaml.Marshal(o.Object)
if err != nil {
log.Error(err)
return
}
detail, ready = printers.PrintCustomizedHealthCheck(code, resByte)
} else {
detail, ready = printers.Generate(target)
}
} else {
// Check reconcile status with default setup
detail, ready = printers.Generate(target)
}
}

// Mark ready for breaking loop
Expand Down Expand Up @@ -1039,3 +1059,18 @@ func printTable(w *io.Writer, id string, tables map[string]*printers.Table) {
WithHasHeader().WithSeparator(" ").WithData(data).WithWriter(*w).Render()
}
}

// getResourceInfo get health policy and kind from resource for customized health check purpose
func getResourceInfo(res *apiv1.Resource) (healthPolicy interface{}, kind string) {
var ok bool
if res.Extensions != nil {
healthPolicy = res.Extensions[apiv1.FieldHealthPolicy]
}
if res.Attributes == nil {
panic(fmt.Errorf("resource has no Attributes field in the Spec: %s", res))
}
if kind, ok = res.Attributes[apiv1.FieldKind].(string); !ok {
panic(fmt.Errorf("failed to get kind from resource attributes: %s", res.Attributes))
}
return healthPolicy, kind
}
112 changes: 110 additions & 2 deletions pkg/cmd/apply/apply_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,7 @@ func TestPrompt(t *testing.T) {
}

func TestWatchK8sResources(t *testing.T) {
t.Run("successfully apply K8s resources", func(t *testing.T) {
t.Run("successfully apply default K8s resources", func(t *testing.T) {
id := "v1:Namespace:example"
chs := make([]<-chan watch.Event, 1)
events := []watch.Event{
Expand Down Expand Up @@ -415,7 +415,48 @@ func TestWatchK8sResources(t *testing.T) {
id: table,
}

watchK8sResources(id, chs, table, tables, true)
watchK8sResources(id, "", chs, table, tables, true, nil)

assert.Equal(t, true, table.AllCompleted())
})
t.Run("successfully apply customized K8s resources", func(t *testing.T) {
id := "v1:Deployment:example"
chs := make([]<-chan watch.Event, 1)
events := []watch.Event{
{
Type: watch.Added,
Object: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "example",
"generation": 1,
},
"spec": map[string]interface{}{},
},
},
},
}

out := make(chan watch.Event, 10)
for _, e := range events {
out <- e
}
chs[0] = out
table := &printers.Table{
IDs: []string{id},
Rows: map[string]*printers.Row{},
}
tables := map[string]*printers.Table{
id: table,
}
var policyInterface interface{}
healthPolicy := map[string]interface{}{
"health.kcl": "assert res.metadata.generation == 1",
}
policyInterface = healthPolicy
watchK8sResources(id, "Deployment", chs, table, tables, false, policyInterface)

assert.Equal(t, true, table.AllCompleted())
})
Expand Down Expand Up @@ -472,3 +513,70 @@ func TestPrintTable(t *testing.T) {
assert.Contains(t, w.(*bytes.Buffer).String(), tableStr)
})
}

func TestGetResourceInfo(t *testing.T) {
tests := []struct {
name string
resource *apiv1.Resource
expectedKind string
expectPanic bool
}{
{
name: "with valid resource",
resource: &apiv1.Resource{
Attributes: map[string]interface{}{
apiv1.FieldKind: "Service",
},
Extensions: map[string]interface{}{
apiv1.FieldHealthPolicy: "policyValue",
},
},
expectedKind: "Service",
expectPanic: false,
},
{
name: "with nil Attributes",
resource: &apiv1.Resource{
Attributes: nil,
Extensions: map[string]interface{}{
apiv1.FieldHealthPolicy: "policyValue",
},
},
expectPanic: true,
},
{
name: "with non-string kind",
resource: &apiv1.Resource{
Attributes: map[string]interface{}{
apiv1.FieldKind: 123,
},
Extensions: map[string]interface{}{
apiv1.FieldHealthPolicy: "policyValue",
},
},
expectPanic: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.expectPanic {
defer func() {
if r := recover(); r == nil {
t.Errorf("expected panic for test case '%s', but got none", tt.name)
}
}()
}

healthPolicy, kind := getResourceInfo(tt.resource)
if !tt.expectPanic {
if kind != tt.expectedKind {
t.Errorf("expected kind '%s', but got '%s'", tt.expectedKind, kind)
}
if healthPolicy != "policyValue" && !tt.expectPanic {
t.Errorf("expected healthPolicy to be 'policyValue', but got '%v'", healthPolicy)
}
}
})
}
}
25 changes: 25 additions & 0 deletions pkg/engine/printers/generate.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package printers

import (
"fmt"
"strings"

"k8s.io/apimachinery/pkg/runtime"

"kusionstack.io/kusion/pkg/engine/printers/printer"
"kusionstack.io/kusion/pkg/util/kcl"
)

var tg = printer.NewTableGenerator()
Expand All @@ -15,3 +19,24 @@ func init() {
func Generate(obj runtime.Object) (string, bool) {
return tg.GenerateTable(obj)
}

// PrintCustomizedHealthCheck prints customized health check result defined in the `extensions` field of the resource in the Spec.
func PrintCustomizedHealthCheck(healthPolicyCode string, resource []byte) (string, bool) {
// Skip when health policy is empty
if healthPolicyCode == "" {
return "No health policy, skip", true
}
err := kcl.RunKCLHealthCheck(healthPolicyCode, resource)
// Skip when health policy syntax is invalid
if err == kcl.ErrInvalidSyntax {
return fmt.Sprintf("health policy err: %s, skip", strings.TrimSpace(err.Error())), true
}
// Keep reconciling when health policy assertion failed
if err == kcl.ErrEvaluationError {
return "Reconciling...", false
}
if err != nil {
return fmt.Sprintf("health policy err: %s", strings.TrimSpace(err.Error())), false
}
return "Reconciled", true
}
19 changes: 16 additions & 3 deletions pkg/modules/generators/app_configurations_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]
if err != nil {
return nil, nil, nil, err
}

healthPolicy := config.platformConfig[v1.FieldHealthPolicy]
// parse module result
// if only one resource exists in the workload module, it is the workload
if workloadKey == t && len(response.Resources) == 1 {
Expand All @@ -443,7 +443,6 @@ func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]
if err != nil {
return nil, nil, nil, err
}

// filter out workload
if workloadKey == t && temp.Extensions[isWorkload] == "true" {
workload = temp
Expand All @@ -452,7 +451,10 @@ func (g *appConfigurationGenerator) callModules(projectModuleConfigs map[string]
}
}
}

// Add healthPolicy to workload extensions
if healthPolicy != nil && workload != nil {
patchHealthPolicy(workload, healthPolicy)
}
// parse patcher
temp := &v1.Patcher{}
if response.Patcher != nil {
Expand Down Expand Up @@ -670,3 +672,14 @@ func patchImportedResources(resources v1.Resources, projectImportedResources map

return nil
}

// patchHealthPolicy patch the health policy to the `extensions` field of the resource in the Spec.
func patchHealthPolicy(resource *v1.Resource, healthPolicy any) {
healthPolicyMap := make(map[string]any)
if hp, ok := healthPolicy.(v1.GenericConfig); ok {
for k, v := range hp {
healthPolicyMap[k] = v
}
resource.Extensions[v1.FieldHealthPolicy] = healthPolicyMap
}
}
80 changes: 80 additions & 0 deletions pkg/util/kcl/kcl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package kcl

import (
"errors"
"fmt"
"strings"

"kcl-lang.io/kcl-go/pkg/kcl"
"kcl-lang.io/kcl-go/pkg/tools/format"
v1 "kusionstack.io/kusion/pkg/apis/api.kusion.io/v1"
)

const (
ref = "res"
EvaluationErrorStr = "EvaluationError"
InvalidSyntaxErrStr = "InvalidSyntax"
UndefinedTypeErrStr = "UndefinedType"
)

var (
ErrEvaluationError = errors.New("health check fail error")
ErrInvalidSyntax = errors.New("invalid syntax error")
)

// Validate if the KCL health policy has invalid syntax.
func validateKCLHealthCheck(healthPolicyCode string) error {
_, err := kcl.Run("", kcl.WithCode(healthPolicyCode))
if err != nil {
if strings.Contains(err.Error(), InvalidSyntaxErrStr) {
return ErrInvalidSyntax
}
}
return nil
}

// Assemble and format the whole KCL code with yaml integration.
func assembleKCLHealthCheck(healthPolicyCode string, resource []byte) (string, error) {
yamlStr := fmt.Sprintf(`
import yaml
%s = yaml.decode(%q)
`, ref, resource)
kclCode := yamlStr + healthPolicyCode
kclFormatted, err := format.FormatCode(kclCode)
if err != nil {
return "", err
}
return string(kclFormatted), nil
}

// Run health check with KCL health policy during apply.
func RunKCLHealthCheck(healthPolicyCode string, resource []byte) error {
err := validateKCLHealthCheck(healthPolicyCode)
if err != nil {
return err
}
kclCode, err := assembleKCLHealthCheck(healthPolicyCode, resource)
if err != nil {
return err
}
_, err = kcl.Run("", kcl.WithCode(kclCode))
if err != nil && strings.Contains(err.Error(), EvaluationErrorStr) {
if strings.Contains(err.Error(), UndefinedTypeErrStr) {
// Distinguish the undefined error from the evaluation error.
errStr := strings.ReplaceAll(err.Error(), EvaluationErrorStr, "")
return errors.New(errStr)
}
return ErrEvaluationError
}
return err
}

// Get KCL code from extensions of the resource in the Spec.
func ConvertKCLCode(healthPolicy any) (string, bool) {
if hp, ok := healthPolicy.(map[string]any); ok {
if code, ok := hp[v1.FieldKCLHealthCheckKCL].(string); ok {
return code, true
}
}
return "", false
}
Loading

0 comments on commit a950a01

Please sign in to comment.