Merge pull request #11 from zong-zhe/fix-compile-failed

feat: add resource status and fix the compilation failure

.github/workflows/test.yaml

@@ -22,9 +22,3 @@ jobs:
             **/go.mod
       - name: Run tests
         run: make test
-      - name: Check if working tree is dirty
-        run: |
-          if [[ $(git diff --stat) != '' ]]; then
-            echo 'run make test and commit changes'
-            exit 1
-          fi

api/v1alpha1/kclrun_types.go

@@ -36,6 +36,12 @@ type KCLRunSpec struct {
 type KCLRunStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
+
+	// LastAttemptedRevision is the revision of the last reconciliation attempt.
+	// +optional
+	LastAttemptedRevision string `json:"lastAttemptedRevision,omitempty"`
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
 }
 
 //+kubebuilder:object:root=true
@@ -62,3 +68,13 @@ type KCLRunList struct {
 func init() {
 	SchemeBuilder.Register(&KCLRun{}, &KCLRunList{})
 }
+
+// GetConditions returns the status conditions of the object.
+func (in KCLRun) GetConditions() []metav1.Condition {
+	return in.Status.Conditions
+}
+
+// SetConditions sets the status conditions on the object.
+func (in KCLRun) SetConditions(conditions []metav1.Condition) {
+	in.Status.Conditions = conditions
+}

config/crd/bases/krm.kcl.dev.fluxcd_kclruns.yaml

@@ -66,6 +66,79 @@ spec:
             type: object
           status:
             description: KCLRunStatus defines the observed state of KCLRun
+            properties:
+              conditions:
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource. --- This struct is intended for direct
+                    use as an array at the field path .status.conditions.  For example,
+                    \n type FooStatus struct{ // Represents the observations of a
+                    foo's current state. // Known .status.conditions.type are: \"Available\",
+                    \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
+                    // +listType=map // +listMapKey=type Conditions []metav1.Condition
+                    `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\"
+                    protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
+                  properties:
+                    lastTransitionTime:
+                      description: lastTransitionTime is the last time the condition
+                        transitioned from one status to another. This should be when
+                        the underlying condition changed.  If that is not known, then
+                        using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: message is a human readable message indicating
+                        details about the transition. This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: observedGeneration represents the .metadata.generation
+                        that the condition was set based upon. For instance, if .metadata.generation
+                        is currently 12, but the .status.conditions[x].observedGeneration
+                        is 9, the condition is out of date with respect to the current
+                        state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: reason contains a programmatic identifier indicating
+                        the reason for the condition's last transition. Producers
+                        of specific condition types may define expected values and
+                        meanings for this field, and whether the values are considered
+                        a guaranteed API. The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        --- Many .condition.type values are consistent across resources
+                        like Available, but because arbitrary conditions can be useful
+                        (see .node.status.conditions), the ability to deconflict is
+                        important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              lastAttemptedRevision:
+                description: LastAttemptedRevision is the revision of the last reconciliation
+                  attempt.
+                type: string
             type: object
         type: object
     served: true

go.mod

@@ -100,6 +100,7 @@ require (
 	google.golang.org/grpc v1.56.3 // indirect
 	gopkg.in/evanphx/json-patch.v5 v5.7.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gotest.tools/v3 v3.5.1 // indirect
 	k8s.io/cli-runtime v0.28.4 // indirect
 	k8s.io/kubectl v0.28.4 // indirect
 	kcl-lang.io/lib v0.7.8 // indirect
@@ -116,7 +117,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
-	github.com/fluxcd/pkg/apis/meta v1.2.0 // indirect
+	github.com/fluxcd/pkg/apis/meta v1.2.0
 	github.com/fluxcd/source-watcher v1.0.0
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
@@ -144,7 +145,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/go-digest/blake3 v0.0.0-20231025023718-d50d2fec9c98 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
@@ -165,7 +166,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.28.4 // indirect
+	k8s.io/api v0.28.4
 	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect

go.sum

@@ -481,7 +481,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
 k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
 k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=

internal/controller/kclrun_controller.go

@@ -21,10 +21,12 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"kcl-lang.io/kcl-go/pkg/kcl"
+	kclcli "kcl-lang.io/kpm/pkg/client"
 	"kcl-lang.io/kpm/pkg/opt"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -35,7 +37,9 @@ import (
 	// "sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/http/fetch"
+	"github.com/fluxcd/pkg/runtime/conditions"
 	"github.com/fluxcd/pkg/runtime/predicates"
 
 	// "github.com/fluxcd/pkg/runtime/predicates"
@@ -45,7 +49,6 @@ import (
 	sw "github.com/fluxcd/source-watcher/controllers"
 	"github.com/kcl-lang/kcl-controller/api/v1alpha1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	kclapi "kcl-lang.io/kpm/pkg/api"
 )
 
 // KCLRunReconciler reconciles a KCLRun object
@@ -82,40 +85,46 @@ func (r *KCLRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-
+	fmt.Printf("source: %v\n", source)
 	artifact := source.GetArtifact()
-	log.Info(fmt.Sprintf("new revision detected %s", artifact.Revision))
+	progressingMsg := fmt.Sprintf("new revision detected %s", artifact.Revision)
+	log.Info(progressingMsg)
+	conditions.MarkUnknown(&kclRun, meta.ReadyCondition, meta.ProgressingReason, "Reconciliation in progress")
+	conditions.MarkReconciling(&kclRun, meta.ProgressingReason, progressingMsg)
 
 	// create tmp dir
 	tmpDir, err := os.MkdirTemp("", kclRun.Name)
 	if err != nil {
+		conditions.MarkFalse(&kclRun, meta.ReadyCondition, sourcev1.DirCreationFailedReason, err.Error())
 		return ctrl.Result{}, fmt.Errorf("failed to create temp dir, error: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
-
+	log.Info(fmt.Sprintf("fetching......"))
 	// download and extract artifact
 	if err := r.artifactFetcher.Fetch(artifact.URL, artifact.Digest, tmpDir); err != nil {
+		conditions.MarkFalse(&kclRun, meta.ReadyCondition, "failed fetch artifacts", err.Error())
 		log.Error(err, "unable to fetch artifact")
 		return ctrl.Result{}, err
 	}
 
 	// compile the KCL source code into the kubenretes manifests
-	res, err := kclapi.RunWithOpts(
-		opt.WithNoSumCheck(true),
-		opt.WithKclOption(kcl.WithWorkDir(tmpDir)),
-	)
+	res, err := CompileKclPackage(tmpDir)
 
 	if err != nil {
+		conditions.MarkFalse(&kclRun, meta.ReadyCondition, "FetchFailed", err.Error())
 		log.Error(err, "failed to compile the KCL source code")
 		return ctrl.Result{}, err
 	}
 
 	u, err := ssa.ReadObjects(bytes.NewReader(([]byte(res.GetRawYamlResult()))))
 	if err != nil {
+		conditions.MarkFalse(&kclRun, meta.ReadyCondition, "CompileFailed", err.Error())
 		log.Error(err, "failed to compile the yaml str into kubernetes manifests")
 		return ctrl.Result{}, err
 	}
 
+	log.Info(fmt.Sprintf("compile result %s", res.GetRawYamlResult()))
+
 	rm := ssa.NewResourceManager(r.Client, nil, ssa.Owner{
 		Field: "kcl-controler",
 		Group: kclRun.GroupVersionKind().Group,
@@ -127,11 +136,22 @@ func (r *KCLRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	_, err = rm.ApplyAll(ctx, u, ssa.DefaultApplyOptions())
 	if err != nil {
+		conditions.MarkFalse(&kclRun, meta.ReadyCondition, "ApplyFailed", err.Error())
 		err = fmt.Errorf("failed to run server-side apply: %w", err)
 		return ctrl.Result{}, err
 	}
 
 	log.Info("successfully applied kcl resources")
+	kclRun.Status.LastAttemptedRevision = artifact.Revision
+
+	conditions.MarkTrue(&kclRun,
+		meta.ReadyCondition,
+		"ReconciliationSucceeded",
+		fmt.Sprintf("Applied revision: %s", artifact.Revision))
+
+	if err := r.Status().Update(ctx, &kclRun); err != nil {
+		return ctrl.Result{}, err
+	}
 
 	return ctrl.Result{}, nil
 }
@@ -203,10 +223,56 @@ func (r *KCLRunReconciler) requestsForRevisionChangeOf() handler.MapFunc {
 			return nil
 		}
 
-		// TODO: Only the same name and namespace is supported for now
-		reqs := make([]reconcile.Request, 1)
-		reqs[0].NamespacedName.Name = obj.GetName()
-		reqs[0].NamespacedName.Namespace = obj.GetNamespace()
+		var list v1alpha1.KCLRunList
+		if err := r.List(ctx, &list); err != nil {
+			log.Error(err, "failed to list objects for revision change")
+			return nil
+		}
+		log.Info(fmt.Sprintf("found %d objects for revision change", len(list.Items)))
+
+		var reqs []reconcile.Request
+		for i, d := range list.Items {
+			log.Info(fmt.Sprintf("d: %v\n", d))
+			// If the Kustomization is ready and the revision of the artifact equals
+			// to the last attempted revision, we should not make a request for this Kustomization
+			if conditions.IsReady(&list.Items[i]) &&
+				repo.Name == d.Spec.SourceRef.Name &&
+				repo.Namespace == d.Spec.SourceRef.Namespace &&
+				repo.GetArtifact().HasRevision(d.Status.LastAttemptedRevision) {
+				continue
+			}
+			log.Info(fmt.Sprintf("revision of %s/%s changed", repo.GetArtifact().Revision, d.Status.LastAttemptedRevision))
+			log.Info(fmt.Sprintf("enqueueing %s/%s for revision change", d.GetNamespace(), d.GetName()))
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: d.GetNamespace(),
+					Name:      d.GetName(),
+				},
+			})
+		}
+
 		return reqs
 	}
 }
+
+// TODO: This is a temporary function to compile the KCL source code into kubernetes manifests
+// The api of compiling the KCL source code will be updated in the future to fix the issue
+func CompileKclPackage(pkgPath string) (*kcl.KCLResultList, error) {
+	kpmcli, _ := kclcli.NewKpmClient()
+	opts := opt.DefaultCompileOptions()
+
+	pkgPath, err := filepath.Abs(pkgPath)
+	if err != nil {
+		return nil, err
+	}
+	opts.SetPkgPath(pkgPath)
+	// check if the kcl.yaml exists in the pkgPath
+	kclconf := filepath.Join(pkgPath, "kcl.yaml")
+	_, err = os.Stat(kclconf)
+	if err == nil {
+		opts.Option.Merge(kcl.WithSettings(kclconf))
+		opts.SetHasSettingsYaml(true)
+	}
+
+	return kpmcli.CompileWithOpts(opts)
+}