Skip to content

Commit

Permalink
feat(zone): create Zone resources on zone automatically
Browse files Browse the repository at this point in the history
Signed-off-by: Jakub Dyszkiewicz <[email protected]>
  • Loading branch information
jakubdyszkiewicz committed Dec 11, 2023
1 parent fac9cd9 commit 32319ca
Show file tree
Hide file tree
Showing 14 changed files with 155 additions and 83 deletions.
7 changes: 7 additions & 0 deletions pkg/config/app/kuma-cp/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,13 @@ func (c Config) IsNonFederatedZoneCP() bool {
return c.Mode == core.Zone && !c.IsFederatedZoneCP()
}

func (c Config) ZoneName() string {
if c.Multizone.Zone.Name == "" {
return core.DefaultZoneName
}
return c.Multizone.Zone.Name
}

func (c *Config) Sanitize() {
c.General.Sanitize()
c.Store.Sanitize()
Expand Down
2 changes: 2 additions & 0 deletions pkg/config/core/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ const (
Standalone CpMode = "standalone"
Zone CpMode = "zone"
Global CpMode = "global"

DefaultZoneName = "default"
)

// ValidateCpMode to check modes of kuma-cp
Expand Down
31 changes: 0 additions & 31 deletions pkg/core/resources/apis/system/location_validator.go

This file was deleted.

30 changes: 16 additions & 14 deletions pkg/defaults/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ func Setup(runtime runtime.Runtime) error {
if !runtime.Config().IsFederatedZoneCP() { // Don't run defaults in Zone connected to global (it's done in Global)
defaultsComponent := NewDefaultsComponent(
runtime.Config().Defaults,
runtime.Config().Environment,
runtime.ResourceManager(),
runtime.ResourceStore(),
runtime.Extensions(),
Expand Down Expand Up @@ -63,10 +62,16 @@ func Setup(runtime runtime.Runtime) error {
}

if runtime.Config().Mode != config_core.Global { // Envoy Admin CA is not synced in multizone and not needed in Global CP.
if err := runtime.Add(&EnvoyAdminCaDefaultComponent{
envoyAdminCaDefault := &EnvoyAdminCaDefaultComponent{
ResManager: runtime.ResourceManager(),
Extensions: runtime.Extensions(),
}); err != nil {
}
zoneDefault := &ZoneDefaultComponent{
ResManager: runtime.ResourceManager(),
Extensions: runtime.Extensions(),
ZoneName: runtime.Config().ZoneName(),
}
if err := runtime.Add(envoyAdminCaDefault, zoneDefault); err != nil {
return err
}
}
Expand All @@ -75,28 +80,25 @@ func Setup(runtime runtime.Runtime) error {

func NewDefaultsComponent(
config *kuma_cp.Defaults,
environment config_core.EnvironmentType,
resManager core_manager.ResourceManager,
resStore store.ResourceStore,
extensions context.Context,
) component.Component {
return &defaultsComponent{
environment: environment,
config: config,
resManager: resManager,
resStore: resStore,
extensions: extensions,
config: config,
resManager: resManager,
resStore: resStore,
extensions: extensions,
}
}

var _ component.Component = &defaultsComponent{}

type defaultsComponent struct {
environment config_core.EnvironmentType
config *kuma_cp.Defaults
resManager core_manager.ResourceManager
resStore store.ResourceStore
extensions context.Context
config *kuma_cp.Defaults
resManager core_manager.ResourceManager
resStore store.ResourceStore
extensions context.Context
}

func (d *defaultsComponent) NeedLeaderElection() bool {
Expand Down
5 changes: 2 additions & 3 deletions pkg/defaults/components_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/kumahq/kuma/api/mesh/v1alpha1"
kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
"github.com/kumahq/kuma/pkg/config/core"
core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
core_model "github.com/kumahq/kuma/pkg/core/resources/model"
Expand All @@ -29,7 +28,7 @@ var _ = Describe("Defaults Component", func() {
}
store := resources_memory.NewStore()
manager = core_manager.NewResourceManager(store)
component = defaults.NewDefaultsComponent(cfg, core.UniversalEnvironment, manager, store, context.Background())
component = defaults.NewDefaultsComponent(cfg, manager, store, context.Background())
})

It("should create default mesh", func() {
Expand Down Expand Up @@ -82,7 +81,7 @@ var _ = Describe("Defaults Component", func() {
}
store := resources_memory.NewStore()
manager = core_manager.NewResourceManager(store)
component = defaults.NewDefaultsComponent(cfg, core.UniversalEnvironment, manager, store, context.Background())
component = defaults.NewDefaultsComponent(cfg, manager, store, context.Background())
})

It("should not create default mesh", func() {
Expand Down
78 changes: 78 additions & 0 deletions pkg/defaults/zone.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package defaults

import (
"context"
"time"

"github.com/go-logr/logr"
"github.com/pkg/errors"
"github.com/sethvargo/go-retry"

"github.com/kumahq/kuma/pkg/core/resources/apis/system"
"github.com/kumahq/kuma/pkg/core/resources/manager"
"github.com/kumahq/kuma/pkg/core/resources/model"
"github.com/kumahq/kuma/pkg/core/resources/store"
"github.com/kumahq/kuma/pkg/core/runtime/component"
"github.com/kumahq/kuma/pkg/core/user"
kuma_log "github.com/kumahq/kuma/pkg/log"
)

type ZoneDefaultComponent struct {
ResManager manager.ResourceManager
Extensions context.Context
ZoneName string
}

var _ component.Component = &ZoneDefaultComponent{}

func (e *ZoneDefaultComponent) Start(stop <-chan struct{}) error {
ctx, cancelFn := context.WithCancel(user.Ctx(context.Background(), user.ControlPlane))
go func() {
<-stop
cancelFn()
}()
return retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
logger := kuma_log.AddFieldsFromCtx(log, ctx, e.Extensions)
if err := EnsureOnlyOneZoneExists(ctx, e.ResManager, e.ZoneName, logger); err != nil {
log.V(1).Info("could not ensure that Zone exists. Retrying.", "err", err)
return retry.RetryableError(err)
}
return nil
})
}

func (e ZoneDefaultComponent) NeedLeaderElection() bool {
return true
}

func EnsureOnlyOneZoneExists(
ctx context.Context,
resManager manager.ResourceManager,
zoneName string,
logger logr.Logger,
) error {
logger.Info("ensuring Zone resource exists", "name", zoneName)
zones := &system.ZoneResourceList{}
if err := resManager.List(ctx, zones); err != nil {
return errors.Wrap(err, "cannot list zones")
}
exists := false
for _, zone := range zones.Items {
if zone.GetMeta().GetName() == zoneName {
exists = true
} else {
logger.Info("detected Zone resource with different name than Zone CP name. Deleting. This might happen if you change the name of the Zone CP", "name", zoneName)
if err := resManager.Delete(ctx, zone); err != nil {
return errors.Wrap(err, "cannot delete old zone")
}
}
}
if !exists {
logger.Info("creating Zone resource", "name", zoneName)
zone := system.NewZoneResource()
if err := resManager.Create(ctx, zone, store.CreateByKey(zoneName, model.NoMesh)); err != nil {
return err
}
}
return nil
}
23 changes: 0 additions & 23 deletions pkg/plugins/runtime/k8s/webhooks/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (

"github.com/kumahq/kuma/pkg/config/core"
core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
"github.com/kumahq/kuma/pkg/core/resources/apis/system"
core_model "github.com/kumahq/kuma/pkg/core/resources/model"
core_registry "github.com/kumahq/kuma/pkg/core/resources/registry"
"github.com/kumahq/kuma/pkg/core/validators"
Expand Down Expand Up @@ -74,10 +73,6 @@ func (h *validatingHandler) Handle(ctx context.Context, req admission.Request) a
case v1.Delete:
return admission.Allowed("")
default:
if resp := h.validateResourceLocation(resType); !resp.Allowed {
return resp
}

coreRes, k8sObj, err := h.decode(req)
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
Expand Down Expand Up @@ -168,24 +163,6 @@ func syncErrorResponse(resType core_model.ResourceType, cpMode core.CpMode) admi
}
}

// validateResourceLocation validates if resources that suppose to be applied on Global are applied on Global and other way around
func (h *validatingHandler) validateResourceLocation(resType core_model.ResourceType) admission.Response {
if err := system.ValidateLocation(resType, h.mode); err != nil {
return admission.Response{
AdmissionResponse: v1.AdmissionResponse{
Allowed: false,
Result: &metav1.Status{
Status: "Failure",
Message: err.Error(),
Reason: "Forbidden",
Code: 403,
},
},
}
}
return admission.Allowed("")
}

func (h *validatingHandler) Supports(admission.Request) bool {
return true
}
Expand Down
18 changes: 6 additions & 12 deletions pkg/plugins/runtime/k8s/webhooks/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ var _ = Describe("Validation", func() {
},
operation: admissionv1.Create,
}),
Entry("should fail validation due to applying Zone on Zone CP", testCase{
Entry("should pass validation due to applying Zone on Zone CP", testCase{
mode: core.Zone,
federatedZone: true,
objTemplate: &system_proto.Zone{},
Expand All @@ -450,18 +450,15 @@ var _ = Describe("Validation", func() {
resp: kube_admission.Response{
AdmissionResponse: admissionv1.AdmissionResponse{
UID: "12345",
Allowed: false,
Allowed: true,
Result: &kube_meta.Status{
Status: "Failure",
Message: "Zone resource can only be applied on CP with mode: [global]",
Reason: "Forbidden",
Code: 403,
Code: 200,
},
},
},
operation: admissionv1.Create,
}),
Entry("should fail validation due to applying Zone on non federated Zone CP", testCase{
Entry("should pass validation due to applying Zone on non federated Zone CP", testCase{
mode: core.Zone,
objTemplate: &system_proto.Zone{},
username: "cli-user",
Expand All @@ -484,12 +481,9 @@ var _ = Describe("Validation", func() {
resp: kube_admission.Response{
AdmissionResponse: admissionv1.AdmissionResponse{
UID: "12345",
Allowed: false,
Allowed: true,
Result: &kube_meta.Status{
Status: "Failure",
Message: "Zone resource can only be applied on CP with mode: [global]",
Reason: "Forbidden",
Code: 403,
Code: 200,
},
},
},
Expand Down
5 changes: 5 additions & 0 deletions test/e2e_env/kubernetes/defaults/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/kumahq/kuma/pkg/config/core"
. "github.com/kumahq/kuma/test/framework"
"github.com/kumahq/kuma/test/framework/envs/kubernetes"
)
Expand Down Expand Up @@ -47,4 +48,8 @@ func Defaults() {
Eventually(policyCreated("circuitbreaker", "circuit-breaker-all-"+meshName), "30s", "1s").Should(BeTrue())
Eventually(policyCreated("retry", "retry-all-"+meshName), "30s", "1s").Should(BeTrue())
})

It("should create a zone", func() {
Eventually(policyCreated("zone", core.DefaultZoneName), "30s", "1s").Should(BeTrue())
})
}
28 changes: 28 additions & 0 deletions test/e2e_env/multizone/defaults/defaults.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package defaults

import (
"encoding/json"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/kumahq/kuma/pkg/core/resources/model/rest/v1alpha1"
"github.com/kumahq/kuma/test/framework/envs/multizone"
)

func Defaults() {
It("should create a Zone resource on every zone", func() {
for _, zone := range multizone.Zones() {
Eventually(func(g Gomega) {
// when
out, err := zone.GetKumactlOptions().RunKumactlAndGetOutput("get", "zone", zone.ZoneName(), "-ojson")

// then
g.Expect(err).ToNot(HaveOccurred())
meta := &v1alpha1.ResourceMeta{}
g.Expect(json.Unmarshal([]byte(out), meta)).To(Succeed())
g.Expect(meta.Name).To(Equal(zone.ZoneName()))
}, "30s", "1s").Should(Succeed())
}
})
}
2 changes: 2 additions & 0 deletions test/e2e_env/multizone/multizone_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/kumahq/kuma/pkg/test"
"github.com/kumahq/kuma/test/e2e_env/multizone/connectivity"
"github.com/kumahq/kuma/test/e2e_env/multizone/defaults"
"github.com/kumahq/kuma/test/e2e_env/multizone/externalservices"
"github.com/kumahq/kuma/test/e2e_env/multizone/gateway"
"github.com/kumahq/kuma/test/e2e_env/multizone/healthcheck"
Expand Down Expand Up @@ -72,4 +73,5 @@ var (
_ = Describe("Advanced LocalityAwareness with MeshLoadBalancingStrategy", localityawarelb.LocalityAwareLB, Ordered)
_ = Describe("Advanced LocalityAwareness with MeshLoadBalancingStrategy with Gateway", localityawarelb.LocalityAwareLBGateway, Ordered)
_ = Describe("Advanced LocalityAwareness with MeshLoadBalancingStrategy and Enabled Egress", localityawarelb.LocalityAwareLBEgress, Ordered)
_ = Describe("Defaults", defaults.Defaults, Ordered)
)
1 change: 1 addition & 0 deletions test/framework/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ type Cluster interface {
GetZoneIngressEnvoyTunnel() envoy_admin.Tunnel
Verbose() bool
Install(fn InstallFunc) error
ZoneName() string

// K8s
GetKubectlOptions(namespace ...string) *k8s.KubectlOptions
Expand Down
4 changes: 4 additions & 0 deletions test/framework/k8s_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -1249,6 +1249,10 @@ func (c *K8sCluster) K8sVersionCompare(otherVersion string, baseMessage string)
return version.Compare(semver.MustParse(otherVersion)), fmt.Sprintf("%s with k8s version %s", baseMessage, version)
}

func (c *K8sCluster) ZoneName() string {
return c.GetKumactlOptions().CPName
}

func printDetailedPodInfo(testingT testing.TestingT, kubectlOptions *k8s.KubectlOptions, podError error, pod v1.Pod) error {
if podError == nil {
return podError
Expand Down
Loading

0 comments on commit 32319ca

Please sign in to comment.