diff --git a/go.mod b/go.mod index 1d9dfbd40032..e58e9281153d 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,7 @@ require ( github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc github.com/rancher/remotedialer v0.3.0 github.com/rancher/wharfie v0.5.3 - github.com/rancher/wrangler v1.1.1 + github.com/rancher/wrangler v1.1.1-0.20230419173538-80fdf092be3b github.com/robfig/cron/v3 v3.0.1 github.com/rootless-containers/rootlesskit v1.0.1 github.com/sirupsen/logrus v1.9.0 diff --git a/go.sum b/go.sum index d4a7da0069bf..4c790d87c9ea 100644 --- a/go.sum +++ b/go.sum @@ -934,8 +934,8 @@ github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcp github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= github.com/rancher/wharfie v0.5.3 h1:6hiO26H7YTgChbLAE6JppxFRjaH3tbKfMItv/LqV0Q0= github.com/rancher/wharfie v0.5.3/go.mod h1:Ebpai7digxegLroBseeC54XRBt5we3DgFS6kAE2ho+o= -github.com/rancher/wrangler v1.1.1 h1:wmqUwqc2M7ADfXnBCJTFkTB5ZREWpD78rnZMzmxwMvM= -github.com/rancher/wrangler v1.1.1/go.mod h1:ioVbKupzcBOdzsl55MvEDN0R1wdGggj8iNCYGFI5JvM= +github.com/rancher/wrangler v1.1.1-0.20230419173538-80fdf092be3b h1:rs3WYld8iaRcSzCmM/CrCIVz9uVgfd96o7FsufIdoVI= +github.com/rancher/wrangler v1.1.1-0.20230419173538-80fdf092be3b/go.mod h1:D6Tu6oVX8aGtCHsMCtYaysgVK3ad920MTSeAu7rzb5U= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/pkg/apis/k3s.cattle.io/v1/types.go b/pkg/apis/k3s.cattle.io/v1/types.go index d1f1fed6cb5e..79ae04f77a0e 100644 --- a/pkg/apis/k3s.cattle.io/v1/types.go +++ b/pkg/apis/k3s.cattle.io/v1/types.go @@ -2,7 +2,6 @@ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" ) // +genclient @@ -12,15 +11,10 @@ type Addon struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec AddonSpec `json:"spec,omitempty"` - Status AddonStatus `json:"status,omitempty"` + Spec AddonSpec `json:"spec,omitempty"` } type AddonSpec struct { Source string `json:"source,omitempty"` Checksum string `json:"checksum,omitempty"` } - -type AddonStatus struct { - GVKs []schema.GroupVersionKind `json:"gvks,omitempty"` -} diff --git a/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go b/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go index 2573519e9f2b..69011aa78d22 100644 --- a/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go +++ b/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go @@ -23,7 +23,6 @@ package v1 import ( runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -32,7 +31,6 @@ func (in *Addon) DeepCopyInto(out *Addon) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - in.Status.DeepCopyInto(&out.Status) return } @@ -102,24 +100,3 @@ func (in *AddonSpec) DeepCopy() *AddonSpec { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AddonStatus) DeepCopyInto(out *AddonStatus) { - *out = *in - if in.GVKs != nil { - in, out := &in.GVKs, &out.GVKs - *out = make([]schema.GroupVersionKind, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus. -func (in *AddonStatus) DeepCopy() *AddonStatus { - if in == nil { - return nil - } - out := new(AddonStatus) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/crd/crds.go b/pkg/crd/crds.go new file mode 100644 index 000000000000..634f555087ea --- /dev/null +++ b/pkg/crd/crds.go @@ -0,0 +1,15 @@ +package crd + +import ( + v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + "github.com/rancher/wrangler/pkg/crd" +) + +func List() []crd.CRD { + addon := crd.NamespacedType("Addon.k3s.cattle.io/v1"). + WithSchemaFromStruct(v1.Addon{}). + WithColumn("Source", ".spec.source"). + WithColumn("Checksum", ".spec.checksum") + + return []crd.CRD{addon} +} diff --git a/pkg/deploy/controller.go b/pkg/deploy/controller.go index 6245e46566bd..0b296eaaebec 100644 --- a/pkg/deploy/controller.go +++ b/pkg/deploy/controller.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "fmt" "io" "os" "path/filepath" @@ -20,6 +21,7 @@ import ( pkgutil "github.com/k3s-io/k3s/pkg/util" errors2 "github.com/pkg/errors" "github.com/rancher/wrangler/pkg/apply" + "github.com/rancher/wrangler/pkg/kv" "github.com/rancher/wrangler/pkg/merr" "github.com/rancher/wrangler/pkg/objectset" "github.com/sirupsen/logrus" @@ -38,7 +40,9 @@ import ( const ( ControllerName = "deploy" + GVKAnnotation = "addon.k3s.cattle.io/gvks" startKey = "_start_" + gvkSep = ";" ) // WatchFiles sets up an OnChange callback to start a periodic goroutine to watch files for changes once the controller has started up. @@ -206,11 +210,17 @@ func (w *watcher) deploy(path string, compareChecksum bool) error { } // Merge GVK list early for validation - addon.Status.GVKs = append(addon.Status.GVKs, objects.GVKs()...) + addonGVKs := objects.GVKs() + for _, gvkString := range strings.Split(addon.Annotations[GVKAnnotation], gvkSep) { + if gvk, err := getGVK(gvkString); err == nil { + addonGVKs = append(addonGVKs, *gvk) + } + } // Ensure that we don't try to prune using GVKs that the server doesn't have. // This can happen when CRDs are removed or when core types are removed - PodSecurityPolicy, for example. - if err := w.validateGVKs(&addon); err != nil { + addonGVKs, err = w.validateGVKs(addonGVKs) + if err != nil { w.recorder.Eventf(&addon, corev1.EventTypeWarning, "ValidateManifestFailed", "Validate GVKs for manifest at %q failed: %v", path, err) return err } @@ -222,15 +232,18 @@ func (w *watcher) deploy(path string, compareChecksum bool) error { // doesn't know to search that GVK for owner references, it won't find and delete them. w.recorder.Eventf(&addon, corev1.EventTypeNormal, "ApplyingManifest", "Applying manifest at %q", path) - if err := w.apply.WithOwner(&addon).WithGVK(addon.Status.GVKs...).Apply(objects); err != nil { + if err := w.apply.WithOwner(&addon).WithGVK(addonGVKs...).Apply(objects); err != nil { w.recorder.Eventf(&addon, corev1.EventTypeWarning, "ApplyManifestFailed", "Applying manifest at %q failed: %v", path, err) return err } // Emit event, Update Addon checksum and GVKs only if apply was successful w.recorder.Eventf(&addon, corev1.EventTypeNormal, "AppliedManifest", "Applied manifest at %q", path) + if addon.Annotations == nil { + addon.Annotations = map[string]string{} + } addon.Spec.Checksum = checksum - addon.Status.GVKs = objects.GVKs() + addon.Annotations[GVKAnnotation] = getGVKString(objects.GVKs()) _, err = w.addons.Update(&addon) return err } @@ -244,6 +257,13 @@ func (w *watcher) delete(path string) error { return err } + addonGVKs := []schema.GroupVersionKind{} + for _, gvkString := range strings.Split(addon.Annotations[GVKAnnotation], gvkSep) { + if gvk, err := getGVK(gvkString); err == nil { + addonGVKs = append(addonGVKs, *gvk) + } + } + content, err := os.ReadFile(path) if err != nil { w.recorder.Eventf(&addon, corev1.EventTypeWarning, "ReadManifestFailed", "Read manifest at %q failed: %v", path, err) @@ -253,13 +273,14 @@ func (w *watcher) delete(path string) error { } else { // Search for objects using both GVKs currently listed in the file, as well as GVKs previously applied. // This ensures that any conflicts between competing deploy controllers are handled properly. - addon.Status.GVKs = append(addon.Status.GVKs, o.GVKs()...) + addonGVKs = append(addonGVKs, o.GVKs()...) } } // Ensure that we don't try to delete using GVKs that the server doesn't have. // This can happen when CRDs are removed or when core types are removed - PodSecurityPolicy, for example. - if err := w.validateGVKs(&addon); err != nil { + addonGVKs, err = w.validateGVKs(addonGVKs) + if err != nil { return err } @@ -271,7 +292,7 @@ func (w *watcher) delete(path string) error { } // apply an empty set with owner & gvk data to delete - if err := w.apply.WithOwner(&addon).WithGVK(addon.Status.GVKs...).ApplyObjects(); err != nil { + if err := w.apply.WithOwner(&addon).WithGVK(addonGVKs...).ApplyObjects(); err != nil { return err } @@ -290,22 +311,19 @@ func (w *watcher) getOrCreateAddon(name string) (apisv1.Addon, error) { return *addon, nil } -// validateGVKs removes from the Addon status any GVKs that the server does not support -func (w *watcher) validateGVKs(addon *apisv1.Addon) error { +// validateGVKs removes from the list any GVKs that the server does not support +func (w *watcher) validateGVKs(addonGVKs []schema.GroupVersionKind) ([]schema.GroupVersionKind, error) { gvks := []schema.GroupVersionKind{} - for _, gvk := range addon.Status.GVKs { + for _, gvk := range addonGVKs { found, err := w.serverHasGVK(gvk) if err != nil { - return err + return gvks, err } if found { gvks = append(gvks, gvk) - } else { - logrus.Warnf("Pruned unknown GVK from %s %s/%s: %s", addon.TypeMeta.GroupVersionKind(), addon.Namespace, addon.Name, gvk) } } - addon.Status.GVKs = gvks - return nil + return gvks, nil } // serverHasGVK uses a positive cache of GVKs that the cluster is known to have supported at some @@ -462,3 +480,22 @@ func shouldDisableFile(base, fileName string, disables map[string]bool) bool { baseName := strings.TrimSuffix(baseFile, suffix) return disables[baseName] } + +func getGVK(s string) (*schema.GroupVersionKind, error) { + parts := strings.Split(s, ", Kind=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid GVK format: %s", s) + } + gvk := &schema.GroupVersionKind{} + gvk.Group, gvk.Version = kv.Split(parts[0], "/") + gvk.Kind = parts[1] + return gvk, nil +} + +func getGVKString(gvks []schema.GroupVersionKind) string { + strs := make([]string, len(gvks)) + for i, gvk := range gvks { + strs[i] = gvk.String() + } + return strings.Join(strs, gvkSep) +} diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/addon.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/addon.go index 3af9c27f9445..6f1bf5871af8 100644 --- a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/addon.go +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/addon.go @@ -40,7 +40,6 @@ type AddonsGetter interface { type AddonInterface interface { Create(ctx context.Context, addon *v1.Addon, opts metav1.CreateOptions) (*v1.Addon, error) Update(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (*v1.Addon, error) - UpdateStatus(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (*v1.Addon, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Addon, error) @@ -136,22 +135,6 @@ func (c *addons) Update(ctx context.Context, addon *v1.Addon, opts metav1.Update return } -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *addons) UpdateStatus(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (result *v1.Addon, err error) { - result = &v1.Addon{} - err = c.client.Put(). - Namespace(c.ns). - Resource("addons"). - Name(addon.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(addon). - Do(ctx). - Into(result) - return -} - // Delete takes name of the addon and deletes it. Returns an error if one occurs. func (c *addons) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return c.client.Delete(). diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_addon.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_addon.go index ac1a4c58b6f9..10730ca8bcbb 100644 --- a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_addon.go +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_addon.go @@ -102,18 +102,6 @@ func (c *FakeAddons) Update(ctx context.Context, addon *k3scattleiov1.Addon, opt return obj.(*k3scattleiov1.Addon), err } -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeAddons) UpdateStatus(ctx context.Context, addon *k3scattleiov1.Addon, opts v1.UpdateOptions) (*k3scattleiov1.Addon, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(addonsResource, "status", c.ns, addon), &k3scattleiov1.Addon{}) - - if obj == nil { - return nil, err - } - return obj.(*k3scattleiov1.Addon), err -} - // Delete takes name of the addon and deletes it. Returns an error if one occurs. func (c *FakeAddons) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/addon.go b/pkg/generated/controllers/k3s.cattle.io/v1/addon.go index ba056beef630..9171951e347f 100644 --- a/pkg/generated/controllers/k3s.cattle.io/v1/addon.go +++ b/pkg/generated/controllers/k3s.cattle.io/v1/addon.go @@ -25,10 +25,7 @@ import ( v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" "github.com/rancher/lasso/pkg/client" "github.com/rancher/lasso/pkg/controller" - "github.com/rancher/wrangler/pkg/apply" - "github.com/rancher/wrangler/pkg/condition" "github.com/rancher/wrangler/pkg/generic" - "github.com/rancher/wrangler/pkg/kv" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -58,7 +55,7 @@ type AddonController interface { type AddonClient interface { Create(*v1.Addon) (*v1.Addon, error) Update(*v1.Addon) (*v1.Addon, error) - UpdateStatus(*v1.Addon) (*v1.Addon, error) + Delete(namespace, name string, options *metav1.DeleteOptions) error Get(namespace, name string, options metav1.GetOptions) (*v1.Addon, error) List(namespace string, opts metav1.ListOptions) (*v1.AddonList, error) @@ -187,11 +184,6 @@ func (c *addonController) Update(obj *v1.Addon) (*v1.Addon, error) { return result, c.client.Update(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) } -func (c *addonController) UpdateStatus(obj *v1.Addon) (*v1.Addon, error) { - result := &v1.Addon{} - return result, c.client.UpdateStatus(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) -} - func (c *addonController) Delete(namespace, name string, options *metav1.DeleteOptions) error { if options == nil { options = &metav1.DeleteOptions{} @@ -262,115 +254,3 @@ func (c *addonCache) GetByIndex(indexName, key string) (result []*v1.Addon, err } return result, nil } - -type AddonStatusHandler func(obj *v1.Addon, status v1.AddonStatus) (v1.AddonStatus, error) - -type AddonGeneratingHandler func(obj *v1.Addon, status v1.AddonStatus) ([]runtime.Object, v1.AddonStatus, error) - -func RegisterAddonStatusHandler(ctx context.Context, controller AddonController, condition condition.Cond, name string, handler AddonStatusHandler) { - statusHandler := &addonStatusHandler{ - client: controller, - condition: condition, - handler: handler, - } - controller.AddGenericHandler(ctx, name, FromAddonHandlerToHandler(statusHandler.sync)) -} - -func RegisterAddonGeneratingHandler(ctx context.Context, controller AddonController, apply apply.Apply, - condition condition.Cond, name string, handler AddonGeneratingHandler, opts *generic.GeneratingHandlerOptions) { - statusHandler := &addonGeneratingHandler{ - AddonGeneratingHandler: handler, - apply: apply, - name: name, - gvk: controller.GroupVersionKind(), - } - if opts != nil { - statusHandler.opts = *opts - } - controller.OnChange(ctx, name, statusHandler.Remove) - RegisterAddonStatusHandler(ctx, controller, condition, name, statusHandler.Handle) -} - -type addonStatusHandler struct { - client AddonClient - condition condition.Cond - handler AddonStatusHandler -} - -func (a *addonStatusHandler) sync(key string, obj *v1.Addon) (*v1.Addon, error) { - if obj == nil { - return obj, nil - } - - origStatus := obj.Status.DeepCopy() - obj = obj.DeepCopy() - newStatus, err := a.handler(obj, obj.Status) - if err != nil { - // Revert to old status on error - newStatus = *origStatus.DeepCopy() - } - - if a.condition != "" { - if errors.IsConflict(err) { - a.condition.SetError(&newStatus, "", nil) - } else { - a.condition.SetError(&newStatus, "", err) - } - } - if !equality.Semantic.DeepEqual(origStatus, &newStatus) { - if a.condition != "" { - // Since status has changed, update the lastUpdatedTime - a.condition.LastUpdated(&newStatus, time.Now().UTC().Format(time.RFC3339)) - } - - var newErr error - obj.Status = newStatus - newObj, newErr := a.client.UpdateStatus(obj) - if err == nil { - err = newErr - } - if newErr == nil { - obj = newObj - } - } - return obj, err -} - -type addonGeneratingHandler struct { - AddonGeneratingHandler - apply apply.Apply - opts generic.GeneratingHandlerOptions - gvk schema.GroupVersionKind - name string -} - -func (a *addonGeneratingHandler) Remove(key string, obj *v1.Addon) (*v1.Addon, error) { - if obj != nil { - return obj, nil - } - - obj = &v1.Addon{} - obj.Namespace, obj.Name = kv.RSplit(key, "/") - obj.SetGroupVersionKind(a.gvk) - - return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). - WithOwner(obj). - WithSetID(a.name). - ApplyObjects() -} - -func (a *addonGeneratingHandler) Handle(obj *v1.Addon, status v1.AddonStatus) (v1.AddonStatus, error) { - if !obj.DeletionTimestamp.IsZero() { - return status, nil - } - - objs, newStatus, err := a.AddonGeneratingHandler(obj, status) - if err != nil { - return newStatus, err - } - - return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). - WithOwner(obj). - WithSetID(a.name). - ApplyObjects(objs...) -} diff --git a/pkg/server/context.go b/pkg/server/context.go index 22408cac783f..0ad06230dc54 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -6,7 +6,9 @@ import ( "os" "runtime" + helmcrd "github.com/k3s-io/helm-controller/pkg/crd" "github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io" + addoncrd "github.com/k3s-io/k3s/pkg/crd" "github.com/k3s-io/k3s/pkg/deploy" "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io" "github.com/k3s-io/k3s/pkg/version" @@ -82,10 +84,8 @@ func crds(ctx context.Context, config *rest.Config) error { return err } - factory.BatchCreateCRDs(ctx, crd.NamespacedTypes( - "Addon.k3s.cattle.io/v1", - "HelmChart.helm.cattle.io/v1", - "HelmChartConfig.helm.cattle.io/v1")...) + types := append(helmcrd.List(), addoncrd.List()...) + factory.BatchCreateCRDs(ctx, types...) return factory.BatchWait() }